vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-15 07:00:52 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'upstream/1.6' into upstream/latest

Browse Source
This commit is contained in:
Sébastien Delafond
2024-12-29 08:06:32 +01:00
parent 5c4f97f88e 895cf5e5c6
commit 0111596d50
513 changed files with 17881 additions and 5776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+10
View File
@@ -0,0 +1,10 @@
.go/
.git/
obj-x86_64-linux-gnu/
obj-aarch64-linux-gnu/
obj-arm-linux-gnueabihf/
obj-i686-linux-gnu/
unit.out
aptly.test
build/
dpkgs/
.flake8
+2 -2
View File
@@ -1,7 +1,7 @@
[flake8]
max-line-length = 200
max-line-length = 240
ignore = E126,E241,E741,W504
include =
system
exclude =
system/env
system/env
.github/FUNDING.yml
+1
View File
@@ -0,0 +1 @@
github: aptly-dev
.github/workflows/ci.yml
+227 -89
View File
@@ -1,5 +1,3 @@
# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
name: CI
on:
@@ -13,32 +11,17 @@ on:
defaults:
run:
# see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
shell: bash --noprofile --norc -eo pipefail -x {0}
shell: bash --noprofile --norc -eo pipefail {0}
env:
DEBIAN_FRONTEND: noninteractive
jobs:
build:
name: test
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.allow_failure }}
test:
name: "Test (Ubuntu 22.04)"
runs-on: ubuntu-22.04
continue-on-error: false
timeout-minutes: 30
strategy:
fail-fast: true
matrix:
go: [1.16, 1.17, 1.18]
run_long_tests: [no]
allow_failure: [false]
include:
# Disable this for now as it's not clear how to select the latest master branch
# version of go using actions/setup-go@v2.
# - go: master
# run_long_tests: no
# allow_failure: true
- go: 1.17
run_long_tests: yes
allow_failure: false
env:
NO_FTP_ACCESS: yes
@@ -47,101 +30,256 @@ jobs:
GOPROXY: "https://proxy.golang.org"
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:
version: v1.45.0
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install O/S packages
- name: "Install packages"
run: |
sudo apt-get update
sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
- name: Install Python packages
- name: "Checkout repository"
uses: actions/checkout@v4
with:
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: "Run flake8"
run: |
pip install six packaging appdirs virtualenv
pip install -U pip setuptools
pip install -r system/requirements.txt
make flake8
- name: Install Azurite
- name: "Read go version from go.mod"
run: |
gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
echo "Go Version: $gover"
echo "GOVER=$gover" >> $GITHUB_OUTPUT
id: goversion
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: "Install Azurite"
id: azuright
uses: potatoqualitee/azuright@v1.1
with:
directory: ${{ runner.temp }}
- name: Get aptly version
run: |
make version
- name: Make
- name: "Run Unit Tests"
env:
RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
RUN_LONG_TESTS: 'yes'
AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
make
sudo mkdir -p /srv ; sudo chown runner /srv
COVERAGE_DIR=${{ runner.temp }} make test
- name: Upload code coverage
if: matrix.run_long_tests
- name: "Run Benchmark"
run: |
COVERAGE_DIR=${{ runner.temp }} make bench
- name: "Run System Tests"
env:
RUN_LONG_TESTS: 'yes'
AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
sudo mkdir -p /srv ; sudo chown runner /srv
COVERAGE_DIR=${{ runner.temp }} make system-test
- name: "Merge code coverage"
run: |
go install github.com/wadey/gocovmerge@latest
~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
- name: "Upload code coverage"
uses: codecov/codecov-action@v2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: coverage.txt
release:
name: release
needs: build
runs-on: ubuntu-20.04
ci-debian-build:
name: "Build"
needs: test
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
arch: ["amd64", "i386" , "arm64" , "armhf"]
include:
- name: "Debian 13/testing"
suite: trixie
image: debian:trixie-slim
- name: "Debian 12/bookworm"
suite: bookworm
image: debian:bookworm-slim
- name: "Debian 11/bullseye"
suite: bullseye
image: debian:bullseye-slim
- name: "Debian 10/buster"
suite: buster
image: debian:buster-slim
- name: "Ubuntu 24.04"
suite: noble
image: ubuntu:24.04
- name: "Ubuntu 22.04"
suite: jammy
image: ubuntu:22.04
- name: "Ubuntu 20.04"
suite: focal
image: ubuntu:20.04
container:
image: ${{ matrix.image }}
env:
APT_LISTCHANGES_FRONTEND: none
DEBIAN_FRONTEND: noninteractive
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: "Install packages"
run: |
apt-get update
apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf \
libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \
gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: "Checkout repository"
uses: actions/checkout@v4
with:
# fetch the whole repot for `git describe` to
# work and get the nightly verion
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v2
- name: "Read go version from go.mod"
run: |
gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
echo "Go Version: $gover"
echo "GOVER=$gover" >> $GITHUB_OUTPUT
id: goversion
- name: Make Release
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: "Ensure CI build"
if: github.ref == 'refs/heads/master'
run: |
echo "FORCE_CI=true" >> $GITHUB_OUTPUT
id: force_ci
- name: "Build Debian packages"
env:
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
make dpkg DEBARCH=${{ matrix.arch }}
- name: "Check aptly credentials"
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
found=no
if [ -n "$APTLY_USER" ] && [ -n "$APTLY_PASSWORD" ]; then
found=yes
fi
echo "Aptly credentials available: $found"
echo "FOUND=$found" >> $GITHUB_OUTPUT
id: aptlycreds
- name: "Publish CI release to aptly"
if: github.ref == 'refs/heads/master' && steps.aptlycreds.outputs.FOUND == 'yes'
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
.github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
- name: "Publish release to aptly"
if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
.github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
ci-binary-build:
name: "Build"
needs: test
runs-on: ubuntu-latest
strategy:
matrix:
goos: [linux, freebsd, darwin]
goarch: ["386", "amd64", "arm", "arm64"]
exclude:
- goos: darwin
goarch: 386
- goos: darwin
goarch: arm
steps:
- name: "Checkout repository"
uses: actions/checkout@v4
with:
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: "Read go version from go.mod"
run: |
echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
id: goversion
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: "Ensure CI build"
if: github.ref == 'refs/heads/master'
run: |
echo "FORCE_CI=true" >> $GITHUB_OUTPUT
id: force_ci
- name: "Get aptly version"
env:
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
aptlyver=$(make -s version)
echo "Aptly Version: $aptlyver"
echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
id: releaseversion
- name: "Build aptly ${{ matrix.goos }}/${{ matrix.goarch }}"
env:
GOBIN: /usr/local/bin
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
make release
make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
- name: Publish nightly release to aptly
if: github.ref == 'refs/heads/master'
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
./upload-artifacts.sh nightly
- name: Publish release to aptly
- name: "Upload Artifacts"
uses: actions/upload-artifact@v4
if: startsWith(github.event.ref, 'refs/tags')
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
./upload-artifacts.sh release
- name: Upload artifacts to GitHub Release
if: startsWith(github.event.ref, 'refs/tags')
uses: softprops/action-gh-release@v1
with:
body: Release ${{ github.ref }} generated by the CI.
files: build/*
name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
compression-level: 0 # no compression
gh-release:
name: "Github Release"
runs-on: ubuntu-latest
continue-on-error: false
needs: ci-binary-build
if: startsWith(github.event.ref, 'refs/tags')
steps:
- name: "Download Artifacts"
uses: actions/download-artifact@v4
with:
path: out/
- name: "Release"
uses: softprops/action-gh-release@v2
with:
files: "out/**/aptly_*.zip"
.github/workflows/golangci-lint.yml
+72
View File
@@ -0,0 +1,72 @@
name: golangci-lint
on:
push:
branches:
- master
pull_request:
permissions:
contents: read
# Optional: allow read access to pull request. Use with `only-new-issues` option.
# pull-requests: read
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: "Read go version from go.mod"
run: |
echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
id: goversion
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: Create VERSION file
run: |
make -s version | tr -d '\n' > VERSION
shell: sh
- name: Install and initialize swagger
run: |
go install github.com/swaggo/swag/cmd/swag@latest
swag init -q --markdownFiles docs
shell: sh
- name: golangci-lint
uses: golangci/golangci-lint-action@v4
with:
# Require: The version of golangci-lint to use.
# When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
# When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
version: v1.54.1
# Optional: working directory, useful for monorepos
# working-directory: somedir
# Optional: golangci-lint command line arguments.
#
# Note: By default, the `.golangci.yml` file should be at the root of the repository.
# The location of the configuration file can be changed by using `--config=`
# args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true
# Optional: if set to true, then all caching functionality will be completely disabled,
# takes precedence over all other caching options.
# skip-cache: true
# Optional: if set to true, then the action won't cache or restore ~/go/pkg.
# skip-pkg-cache: true
# Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
# skip-build-cache: true
# Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
# install-mode: "goinstall"
.github/workflows/scripts/upload-artifacts.sh
Executable
+167
View File
@@ -0,0 +1,167 @@
#!/bin/sh
set -e
builds=build/
packages=${builds}*.deb
folder=`mktemp -u tmp.XXXXXXXXXXXXXXX`
aptly_user="$APTLY_USER"
aptly_password="$APTLY_PASSWORD"
aptly_api="https://aptly-ops.aptly.info"
version=`make version`
action=$1
dist=$2
usage() {
echo "Usage: $0 ci buster|bullseye|bookworm|focal|jammy|noble" >&2
echo " $0 release" >&2
}
# repos and publish must be created beforehand:
#!/bin/sh
#for dist in buster bullseye bookworm focal jammy noble
#do
# for build in ci release
# do
# echo
# echo "# Creating and publishing $build/$dist"
# aptly repo create -distribution=$dist -component=main aptly-$build-$dist
# aptly publish repo -multi-dist -architectures="amd64,i386,arm64,armhf" -acquire-by-hash -component=main \
# -distribution=$dist -batch -keyring=aptly.pub \
# aptly-$build-$dist \
# s3:repo.aptly.info:$build
# done
#done
if [ -z "$action" ]; then
usage
exit 1
fi
if [ "action" = "ci" ] && [ -z "$dist" ]; then
usage
exit 1
fi
if [ -z "$aptly_user" ] || [ -z "$aptly_password" ]; then
usage
echo Error: please set APTLY_USER and APTLY_PASSWORD
exit 1
fi
echo "Publishing version '$version' to $action for $dist...\n"
upload()
{
echo "\nUploading files:"
for file in $packages; do
echo " - $file"
jsonret=`curl -fsS -X POST -F "file=@$file" -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
done
}
cleanup() {
echo "\nCleanup..."
jsonret=`curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
}
trap cleanup EXIT
wait_task()
{
_id=$1
_success=0
for t in `seq 180`
do
jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id`
_state=`echo $jsonret | jq .State`
if [ "$_state" = "2" ]; then
_success=1
curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id
break
fi
if [ "$_state" = "3" ]; then
echo Error: task failed
return 1
fi
sleep 1
done
if [ "$_success" -ne 1 ]; then
echo Error: task timeout
return 1
fi
return 0
}
add_packages() {
_aptly_repository=$1
_folder=$2
jsonret=`curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$_aptly_repository/file/$_folder?_async=true`
_task_id=`echo $jsonret | jq .ID`
wait_task $_task_id
if [ "$?" -ne 0 ]; then
echo "Error: adding packages to $_aptly_repository failed"
exit 1
fi
}
update_publish() {
_publish=$1
_dist=$2
jsonret=`curl -fsS -X PUT -H 'Content-Type: application/json' --data \
'{"AcquireByHash": true, "MultiDist": true,
"Signing": {"Batch": true, "Keyring": "aptly.repo/aptly.pub", "secretKeyring": "aptly.repo/aptly.sec", "PassphraseFile": "aptly.repo/passphrase"}}' \
-u $aptly_user:$aptly_password ${aptly_api}/api/publish/$_publish/$_dist?_async=true`
_task_id=`echo $jsonret | jq .ID`
wait_task $_task_id
if [ "$?" -ne 0 ]; then
echo "Error: publish failed"
exit 1
fi
}
if [ "$action" = "ci" ]; then
if echo "$version" | grep -vq "+"; then
# skip ci when on release tag
exit 0
fi
aptly_repository=aptly-ci-$dist
aptly_published=s3:repo.aptly.info:ci
elif [ "$action" = "release" ]; then
aptly_repository=aptly-release-$dist
aptly_published=s3:repo.aptly.info:release
fi
upload
echo "\nAdding packages to $aptly_repository ..."
add_packages $aptly_repository $folder
echo "\nUpdating published repo $aptly_published ..."
update_publish $aptly_published $dist
# if [ "$action" = "OBSOLETErelease" ]; then
# aptly_repository=aptly-release
# aptly_snapshot=aptly-$version
# aptly_published=s3:repo.aptly.info:./squeeze
#
# echo "\nAdding packages to $aptly_repository..."
# curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder
# echo
#
# echo "\nCreating snapshot $aptly_snapshot from repo $aptly_repository..."
# curl -fsS -X POST -u $aptly_user:$aptly_password -H 'Content-Type: application/json' --data \
# "{\"Name\":\"$aptly_snapshot\"}" ${aptly_api}/api/repos/$aptly_repository/snapshots
# echo
#
# echo "\nSwitching published repo $aptly_published to use snapshot $aptly_snapshot..."
# curl -fsS -X PUT -H 'Content-Type: application/json' --data \
# "{\"AcquireByHash\": true, \"Snapshots\": [{\"Component\": \"main\", \"Name\": \"$aptly_snapshot\"}],
# \"Signing\": {\"Batch\": true, \"Keyring\": \"aptly.repo/aptly.pub\",
# \"secretKeyring\": \"aptly.repo/aptly.sec\", \"PassphraseFile\": \"aptly.repo/passphrase\"}}" \
# -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$aptly_published
# echo
# fi
.gitignore
+37 -5
View File
@@ -2,11 +2,14 @@
*.o
*.a
*.so
unit.out
# Folders
_obj
_test
tmp/
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
@@ -32,13 +35,42 @@ root/
man/aptly.1.html
man/aptly.1.ronn
.goxc.local.json
system/env/
# created by make build for release artifacts
VERSION
aptly.test
build/
pgp/keyrings/aptly2*.gpg
pgp/keyrings/aptly2*.gpg~
pgp/keyrings/.#*
system/files/aptly2.gpg~
system/files/aptly2_passphrase.gpg~
*.creds
.go/
obj-x86_64-linux-gnu/
obj-aarch64-linux-gnu/
obj-arm-linux-gnueabihf/
obj-i686-linux-gnu/
# debian
debian/.debhelper/
debian/aptly.substvars
debian/aptly/
debian/debhelper-build-stamp
debian/files
debian/aptly-api/
debian/*.debhelper
debian/*.debhelper.log
debian/aptly-api.substvars
debian/aptly-dbg.substvars
debian/aptly-dbg/
usr/bin/aptly
dpkgs/
debian/changelog.dpkg-bak
docs/docs.go
docs/swagger.json
docs/swagger.yaml
docs/swagger.conf
.golangci.yml
-3
View File
@@ -5,7 +5,6 @@ run:
linters:
disable-all: true
enable:
- deadcode
- goconst
- gofmt
- goimports
@@ -14,6 +13,4 @@ linters:
- misspell
- revive
- staticcheck
- structcheck
- varcheck
- vetshadow
.goxc.json
-45
View File
@@ -1,45 +0,0 @@
{
"AppName": "aptly",
"ArtifactsDest": "xc-out/",
"TasksExclude": [
"rmbin",
"go-test",
"go-vet"
],
"TasksAppend": [
"bintray"
],
"TaskSettings": {
"debs": {
"metadata": {
"maintainer": "Andrey Smirnov",
"maintainer-email": "me@smira.ru",
"description": "Debian repository management tool"
},
"metadata-deb": {
"License": "MIT",
"Homepage": "https://www.aptly.info/",
"Depends": "bzip2, xz-utils, gnupg, gpgv",
"Suggests": "graphviz"
},
"other-mapped-files": {
"/": "root/"
}
},
"bintray": {
"repository": "aptly",
"subject": "smira",
"package": "aptly",
"downloadspage": "bintray.md"
}
},
"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
"MainDirsExclude": "_man,vendor",
"BuildSettings": {
"LdFlagsXVars": {
"Version": "main.Version"
}
},
"ConfigVersion": "0.9"
}
.travis.yml
-106
View File
@@ -1,106 +0,0 @@
dist: xenial
sudo: required
language: go
go_import_path: github.com/aptly-dev/aptly
addons:
apt:
packages:
- python-virtualenv
- graphviz
- gnupg2
- gpgv2
env:
global:
- secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
- secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
- NO_FTP_ACCESS: "yes"
- BOTO_CONFIG: /dev/null
- GO111MODULE: "on"
- GOPROXY: https://proxy.golang.org
matrix:
allow_failures:
- go: master
env: RUN_LONG_TESTS=no
fast_finish: true
include:
- go: 1.11.x
env: RUN_LONG_TESTS=no
- go: 1.12.x
env: RUN_LONG_TESTS=no
- go: 1.13.x
env: RUN_LONG_TESTS=no
- go: 1.14.x
env: RUN_LONG_TESTS=yes
- go: 1.15.x
env:
- RUN_LONG_TESTS=yes
- DEPLOY_BINARIES=yes
- APTLY_USER=aptly
- secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
- go: master
env: RUN_LONG_TESTS=no
before_install:
- virtualenv system/env
- . system/env/bin/activate
- pip install six packaging appdirs
- pip install -U pip setuptools
- pip install -r system/requirements.txt
- make version
install:
- make prepare
after_success:
- bash <(curl -s https://codecov.io/bash)
notifications:
webhooks:
urls:
- "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
on_success: change # options: [always|never|change] default: always
on_failure: always # options: [always|never|change] default: always
on_start: false # default: false
before_deploy:
- make release
deploy:
- provider: releases
api_key:
secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
file_glob: true
file: build/*
skip_cleanup: true
on:
tags: true
condition: "$DEPLOY_BINARIES = yes"
- provider: s3
access_key_id:
secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
secret_access_key:
secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
bucket: aptly-nightly
skip_cleanup: true
acl: public-read
local_dir: build
on:
branch: master
condition: "$DEPLOY_BINARIES = yes"
- provider: script
script: bash upload-artifacts.sh nightly
skip_cleanup: true
on:
branch: master
condition: "$DEPLOY_BINARIES = yes"
- provider: script
script: bash upload-artifacts.sh release
skip_cleanup: true
on:
tags: true
condition: "$DEPLOY_BINARIES = yes"
AUTHORS
+19
View File
@@ -49,3 +49,22 @@ List of contributors, in chronological order:
* Samuel Mutel (https://github.com/smutel)
* Russell Greene (https://github.com/russelltg)
* Wade Simmons (https://github.com/wadey)
* Steven Stone (https://github.com/smstone)
* Josh Bayfield (https://github.com/jbayfield)
* Boxjan (https://github.com/boxjan)
* Mauro Regli (https://github.com/reglim)
* Alexander Zubarev (https://github.com/strike)
* Nicolas Dostert (https://github.com/acdn-ndostert)
* Ryan Gonzalez (https://github.com/refi64)
* Paul Cacheux (https://github.com/paulcacheux)
* Nic Waller (https://github.com/sf-nwaller)
* iofq (https://github.com/iofq)
* Noa Resare (https://github.com/nresare)
* Ramon N.Rodriguez (https://github.com/runitonmetal)
* Golf Hu (https://github.com/hudeng-go)
* Cookie Fei (https://github.com/wuhuang26)
* Andrey Loukhnov (https://github.com/aol-nnov)
* Christoph Fiehe (https://github.com/cfiehe)
* Blake Kostner (https://github.com/btkostner)
* Leigh London (https://github.com/leighlondon)
* Gordian Schoenherr (https://github.com/schoenherrg)
CODE_OF_CONDUCT.md
+1 -1
View File
@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
reported by contacting the project team on [Aptly Discussions](https://github.com/aptly-dev/aptly/discussions). All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
CONTRIBUTING.md
+91 -79
View File
@@ -2,7 +2,7 @@
:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
The following is a set of guidelines for contributing to [aptly](https://github.com/aptly-dev/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
## What should I know before I get started?
@@ -11,7 +11,7 @@ These are just guidelines, not rules. Use your best judgment, and feel free to p
This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
By participating, you are expected to uphold this code.
Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discussions](https://github.com/aptly-dev/aptly/discussions)
### List of Repositories
@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
general fixes, clarifications, misspellings, grammar mistakes!
### Your First Code Contribution
### Code Contribution
Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -68,63 +68,117 @@ following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
if that is possible.
### Get the Source
To clone the git repo, run the following commands:
```
git clone git@github.com:aptly-dev/aptly.git
cd aptly
```
## Development Setup
This section describes local setup to start contributing to aptly source.
Working on aptly code can be done locally on the development machine, or for convenience by using docker. The next sections describe the setup process.
### Go & Python
### Docker Development Setup
You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
This section describes the docker setup to start contributing to aptly.
If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
#### Dependencies
Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
to prepend it or to skip this test if you're security conscious.
Install the following on your development machine:
- docker
- make
- git
### Forking and Cloning
##### Docker installation on macOS
1. Install [Docker Desktop on Mac](https://docs.docker.com/desktop/setup/install/mac-install/) (or via [Homebrew](https://brew.sh/))
2. Allow directory sharing
- Open Docker Desktop
- Go to `Settings → Resources → File Sharing → Virtual File Shares`
- Add the aptly git repository path to the shared list (eg. /home/Users/john/aptly)
As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
#### Create docker container
git clone git@github.com:aptly-dev/aptly.git
cd aptly
To build the development docker image, run:
```
make docker-image
```
For main repo under your GitHub user and add it as another Git remote:
#### Build aptly
git remote add <user> git@github.com:<user>/aptly.git
To build the aptly in the development docker container, run:
```
make docker-build
```
That way you can continue to build project as is (you don't need to adjust import paths), but you would need
to specify your remote name when pushing branches:
#### Running aptly commands
git push <user> <your-branch>
To run aptly commands in the development docker container, run:
```
make docker-shell
```
### Dependencies
Example:
```
$ make docker-shell
aptly@b43e8473ef81:/work/src$ aptly version
aptly version: 1.5.0+189+g0fc90dff
```
You would need some additional tools and Python virtual environment to run tests and checks, install them with:
#### Running unit tests
make prepare dev system/env
In order to run aptly unit tests, enter the following:
```
make docker-unit-tests
```
This is usually one-time action.
#### Running system tests
Aptly is using Go modules to manage dependencies, download modules using:
In order to run aptly system tests, enter the following:
```
make docker-system-tests
```
make modules
#### Running golangci-lint
### Building
In order to run aptly unit tests, run:
```
make docker-lint
```
If you want to build aptly binary from your current source tree, run:
#### More info
Run `make help` for more information.
### Local Development Setup
This section describes local setup to start contributing to aptly.
#### Dependencies
Building aptly requires go version 1.22.
On Debian bookworm with backports enabled, go can be installed with:
apt install -t bookworm-backports golang-go
#### Building
To build aptly, run:
make build
Run aptly:
build/aptly
To install aptly into `$GOPATH/bin`, run:
make install
This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
aptly
Or, if it's not on your path:
~/go/bin/aptly
### Unit-tests
#### Unit-tests
aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
@@ -133,7 +187,7 @@ aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.
make test
### Functional Tests
#### Functional Tests
Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
@@ -180,26 +234,6 @@ There are some packages available under `system/files/` directory which are used
this default location. You can run aptly under different user or by using non-default config location with non-default
aptly root directory.
### Style Checks
Style checks could be run with:
make check
aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
for the linter could be found in [.golangci.yml](.golangci.yml) file.
Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
### Vendored Code
aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
packages over packages in `$GOPATH`.
If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
Usually all you need is `dep ensure` or `dep ensure -update`.
### man Page
aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
@@ -217,25 +251,3 @@ Bash and Zsh completion for aptly reside in the same repo under in [completion.d
When new option or command is introduced, bash completion should be updated to reflect that change.
When aptly package is being built, it automatically pulls bash completion and man page into the package.
## Design
This section requires future work.
*TBD*
### Database
### Package Pool
### Package
### PackageList, PackageRefList
### LocalRepo, RemoteRepo, Snapshot
### PublishedRepository
### Context
### Collections, CollectionFactory
Makefile
+193 -55
View File
@@ -1,81 +1,219 @@
GOVERSION=$(shell go version | awk '{print $$3;}')
ifdef TRAVIS_TAG
TAG=$(TRAVIS_TAG)
else
TAG="$(shell git describe --tags --always)"
endif
VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
PACKAGES=context database deb files gpg http query swift s3 utils
GOPATH=$(shell go env GOPATH)
VERSION=$(shell make -s version)
PYTHON?=python3
TESTS?=
BINPATH?=$(GOPATH)/bin
RUN_LONG_TESTS?=yes
GOLANGCI_LINT_VERSION=v1.54.1 # version supporting go 1.19
COVERAGE_DIR?=$(shell mktemp -d)
GOOS=$(shell go env GOHOSTOS)
GOARCH=$(shell go env GOHOSTARCH)
all: modules test bench check system-test
# Uncomment to update system test gold files
# CAPTURE := "--capture"
prepare:
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
help: ## Print this help
@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
modules:
go mod download
prepare: ## Install go module dependencies
# Prepare go modules
go mod verify
go mod tidy -v
# Generate VERSION file
go generate
dev:
go get -u github.com/laher/goxc
releasetype: # Print release type: ci (on any branch/commit), release (on a tag)
@reltype=ci ; \
gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
gittag=`git describe --tags --exact-match 2>/dev/null` ;\
if echo "$$gittag" | grep -q '^v[0-9]'; then \
reltype=release ; \
fi ; \
fi ; \
echo $$reltype
check: system/env
ifeq ($(RUN_LONG_TESTS), yes)
golangci-lint run
system/env/bin/flake8
endif
version: ## Print aptly version
@ci="" ; \
if [ "`make -s releasetype`" = "ci" ]; then \
ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
fi ; \
if which dpkg-parsechangelog > /dev/null 2>&1; then \
echo `dpkg-parsechangelog -S Version`$$ci; \
else \
echo `grep ^aptly -m1 debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
fi
swagger-install:
# Install swag
@test -f $(BINPATH)/swag || GOOS= GOARCH= go install github.com/swaggo/swag/cmd/swag@latest
# Generate swagger.conf
cp docs/swagger.conf.tpl docs/swagger.conf
echo "// @version $(VERSION)" >> docs/swagger.conf
azurite-start:
azurite -l /tmp/aptly-azurite & \
echo $$! > ~/.azurite.pid
azurite-stop:
@kill `cat ~/.azurite.pid`
swagger: swagger-install
# Generate swagger docs
@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
etcd-install:
# Install etcd
test -d /tmp/aptly-etcd || system/t13_etcd/install-etcd.sh
flake8: ## run flake8 on system test python files
flake8 system/
lint: prepare
# Install golangci-lint
@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
# Running lint
@PATH=$(BINPATH)/:$(PATH) golangci-lint run
build: prepare swagger ## Build aptly
go build -o build/aptly
install:
go install -v -ldflags "-X main.Version=$(VERSION)"
@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
# go generate
@go generate
# go install -v
@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
system/env: system/requirements.txt
ifeq ($(RUN_LONG_TESTS), yes)
rm -rf system/env
$(PYTHON) -m venv system/env
system/env/bin/pip install -r system/requirements.txt
endif
test: prepare swagger etcd-install ## Run unit tests
@echo "\e[33m\e[1mStarting etcd ...\e[0m"
@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
@echo "\e[33m\e[1mRunning go test ...\e[0m"
go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
@echo "\e[33m\e[1mStopping etcd ...\e[0m"
@pid=`cat /tmp/etcd.pid`; kill $$pid
@rm -f /tmp/aptly-etcd-data/etcd.log
@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
system-test: install system/env
ifeq ($(RUN_LONG_TESTS), yes)
system-test: prepare swagger etcd-install ## Run system tests
# build coverage binary
go test -v -coverpkg="./..." -c -tags testruncli
# Download fixture-db, fixture-pool, etcd.db
if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
endif
test:
go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
# Run system tests
PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
go test -v ./deb -run=nothing -bench=. -benchmem
serve: prepare swagger-install ## Run development server (auto recompiling)
test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
cp debian/aptly.conf ~/.aptly.conf
sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
dpkg: prepare swagger ## Build debian packages
@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
# set debian version
@if [ "`make -s releasetype`" = "ci" ]; then \
echo CI Build, setting version... ; \
test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog ; \
cp debian/changelog debian/changelog.dpkg-bak ; \
DEBEMAIL="CI <ci@aptly.info>" dch -v `make -s version` "CI build" ; \
fi
# clean
rm -rf obj-i686-linux-gnu obj-arm-linux-gnueabihf obj-aarch64-linux-gnu obj-x86_64-linux-gnu
# Run dpkg-buildpackage
@buildtype="any" ; \
if [ "$(DEBARCH)" = "amd64" ]; then \
buildtype="any,all" ; \
fi ; \
echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
cmd="dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)" ; \
echo "$$cmd" ; \
$$cmd
lintian ../*_$(DEBARCH).changes || true
# cleanup
@test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog; \
mkdir -p build && mv ../*.deb build/ ; \
cd build && ls -l *.deb
binaries: prepare swagger ## Build binary releases (FreeBSD, MacOS, Linux tar)
# build aptly
GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
# install
@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
@cp man/aptly.1 build/tmp/man/
@cp completion.d/aptly build/tmp/completion/bash_completion.d/
@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
@cp README.rst LICENSE AUTHORS build/tmp/
@gzip -f build/tmp/man/aptly.1
@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
rm -rf "build/$$path"; \
mv build/tmp build/"$$path"; \
rm -rf build/tmp; \
cd build; \
zip -r "$$path".zip "$$path" > /dev/null \
&& echo "Built build/$${path}.zip"; \
rm -rf "$$path"
docker-image: ## Build aptly-dev docker image
@docker build -f system/Dockerfile . -t aptly-dev
docker-build: ## Build aptly in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
docker-shell: ## Run aptly and other commands in docker container
@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
docker-deb: ## Build debian packages in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
docker-unit-test: ## Run unit tests in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
test \
azurite-stop
docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
system-test TEST=$(TEST) \
azurite-stop
docker-serve: ## Run development server (auto recompiling) on http://localhost:3142
@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
docker-lint: ## Run golangci-lint in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
docker-binaries: ## Build binary releases (FreeBSD, MacOS, Linux tar) in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
docker-man: ## Create man page in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
mem.png: mem.dat mem.gp
gnuplot mem.gp
open mem.png
goxc: dev
rm -rf root/
mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
cp man/aptly.1 root/usr/share/man/man1
cp completion.d/aptly root/etc/bash_completion.d/
cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
gzip root/usr/share/man/man1/aptly.1
goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
release: goxc
rm -rf build/
mkdir -p build/
mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
man:
man: ## Create man pages
make -C man
version:
@echo $(VERSION)
clean: ## remove local build and module cache
# Clean all generated and build files
test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \;
rm -rf .go/
rm -rf build/ obj-*-linux-gnu* tmp/
rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
.PHONY: man modules version release goxc
.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8
README.rst
+20 -11
View File
@@ -9,7 +9,7 @@ aptly
:target: https://codecov.io/gh/aptly-dev/aptly
.. image:: https://badges.gitter.im/Join Chat.svg
:target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
:target: https://matrix.to/#/#aptly:gitter.im
.. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
:target: https://goreportcard.com/report/aptly-dev/aptly
@@ -39,8 +39,8 @@ Current limitations:
* translations are not supported yet
Download
--------
Install Stable Version
-----------------------
To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
@@ -58,19 +58,28 @@ After that you can install aptly as any other software package::
Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
If you would like to use nightly builds (unstable), please use following repository::
deb http://repo.aptly.info/ nightly main
Other Binaries
~~~~~~~~~~~~~~~~~
Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
If you have Go environment set up, you can build aptly from source by running (go 1.14+ required)::
Install CI Version
--------------------
git clone https://github.com/aptly-dev/aptly
cd aptly
make modules install
More recent versions are available as CI builds (development, might be unstable).
Binary would be installed to ``$GOPATH/bin/aptly``.
Debian GNU/Linux
~~~~~~~~~~~~~~~~~
Install the following APT key::
sudo wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
Contributing
------------
api/api.go
+122 -15
View File
@@ -3,17 +3,18 @@ package api
import (
"fmt"
"log"
"net/http"
"sort"
"strconv"
"strings"
"sync/atomic"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
)
// Lock order acquisition (canonical):
@@ -22,11 +23,67 @@ import (
// 3. SnapshotCollection
// 4. PublishedRepoCollection
// GET /api/version
type aptlyVersion struct {
// Aptly Version
Version string `json:"Version"`
}
// @Summary Aptly Version
// @Description **Get aptly version**
// @Description
// @Description **Example:**
// @Description ```
// @Description $ curl http://localhost:8080/api/version
// @Description {"Version":"0.9~dev"}
// @Description ```
// @Tags Status
// @Produce json
// @Success 200 {object} aptlyVersion
// @Router /api/version [get]
func apiVersion(c *gin.Context) {
c.JSON(200, gin.H{"Version": aptly.Version})
}
type aptlyStatus struct {
// Aptly Status
Status string `json:"Status" example:"'Aptly is ready', 'Aptly is unavailable', 'Aptly is healthy'"`
}
// @Summary Get Ready State
// @Description **Get aptly ready state**
// @Description
// @Description Return aptly ready state:
// @Description - `Aptly is ready` (HTTP 200)
// @Description - `Aptly is unavailable` (HTTP 503)
// @Tags Status
// @Produce json
// @Success 200 {object} aptlyStatus "Aptly is ready"
// @Failure 503 {object} aptlyStatus "Aptly is unavailable"
// @Router /api/ready [get]
func apiReady(isReady *atomic.Value) func(*gin.Context) {
return func(c *gin.Context) {
if isReady == nil || !isReady.Load().(bool) {
c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
return
}
c.JSON(200, gin.H{"Status": "Aptly is ready"})
}
}
// @Summary Get Health State
// @Description **Get aptly health state**
// @Description
// @Description Return aptly health state:
// @Description - `Aptly is healthy` (HTTP 200)
// @Tags Status
// @Produce json
// @Success 200 {object} aptlyStatus
// @Router /api/healthy [get]
func apiHealthy(c *gin.Context) {
c.JSON(200, gin.H{"Status": "Aptly is healthy"})
}
type dbRequestKind int
const (
@@ -137,20 +194,29 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
// Run this task in background if configured globally or per-request
background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
if background {
log.Println("Executing task asynchronously")
log.Debug().Msg("Executing task asynchronously")
task, conflictErr := runTaskInBackground(name, resources, proc)
if conflictErr != nil {
c.AbortWithError(409, conflictErr)
AbortWithJSONError(c, 409, conflictErr)
return
}
c.JSON(202, task)
} else {
log.Println("Executing task synchronously")
out := context.Progress()
detail := task.Detail{}
retValue, err := proc(out, &detail)
log.Debug().Msg("Executing task synchronously")
task, conflictErr := runTaskInBackground(name, resources, proc)
if conflictErr != nil {
AbortWithJSONError(c, 409, conflictErr)
return
}
// wait for task to finish
context.TaskList().WaitForTaskByID(task.ID)
retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
err, _ := context.TaskList().GetTaskErrorByID(task.ID)
context.TaskList().DeleteTaskByID(task.ID)
if err != nil {
c.AbortWithError(retValue.Code, err)
AbortWithJSONError(c, retValue.Code, err)
return
}
if retValue != nil {
@@ -168,7 +234,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -176,7 +242,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
if queryS != "" {
q, err := query.Parse(c.Request.URL.Query().Get("q"))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -193,21 +259,57 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
return
}
}
list.PrepareIndex()
list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList)
list, err = list.Filter(deb.FilterOptions{
Queries: []deb.PackageQuery{q},
WithDependencies: withDeps,
Source: nil,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
})
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
return
}
}
// filter packages by version
if c.Request.URL.Query().Get("maximumVersion") == "1" {
list.PrepareIndex()
list.ForEach(func(p *deb.Package) error {
versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
if err != nil {
fmt.Println("filter packages by version, query string parse err: ", err)
c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
} else {
tmpList, err := list.Filter(deb.FilterOptions{
Queries: []deb.PackageQuery{versionQ},
})
if err == nil {
if tmpList.Len() > 0 {
tmpList.ForEach(func(tp *deb.Package) error {
list.Remove(tp)
return nil
})
list.Add(p)
}
} else {
fmt.Println("filter packages by version, filter err: ", err)
c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
}
}
return nil
})
}
if c.Request.URL.Query().Get("format") == "details" {
list.ForEach(func(p *deb.Package) error {
result = append(result, p)
@@ -219,3 +321,8 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
c.JSON(200, list.Strings())
}
}
func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
return c.AbortWithError(code, err)
}
api/api_test.go
+67 -9
View File
@@ -1,16 +1,20 @@
package api
import (
"bytes"
"encoding/json"
ctx "github.com/aptly-dev/aptly/context"
"github.com/gin-gonic/gin"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/aptly-dev/aptly/aptly"
ctx "github.com/aptly-dev/aptly/context"
"github.com/gin-gonic/gin"
"github.com/smira/flag"
. "gopkg.in/check.v1"
@@ -30,12 +34,13 @@ type ApiSuite struct {
var _ = Suite(&ApiSuite{})
func createTestConfig() *os.File {
file, err := ioutil.TempFile("", "aptly")
file, err := os.CreateTemp("", "aptly")
if err != nil {
return nil
}
jsonString, err := json.Marshal(gin.H{
"architectures": []string{},
"architectures": []string{},
"enableMetricsEndpoint": true,
})
if err != nil {
return nil
@@ -44,9 +49,12 @@ func createTestConfig() *os.File {
return file
}
func (s *ApiSuite) SetUpSuite(c *C) {
func (s *ApiSuite) setupContext() error {
aptly.Version = "testVersion"
file := createTestConfig()
c.Assert(file, NotNil)
if nil == file {
return fmt.Errorf("unable to create the test configuration file")
}
s.configFile = file
flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
@@ -57,10 +65,19 @@ func (s *ApiSuite) SetUpSuite(c *C) {
s.flags = flags
context, err := ctx.NewContext(s.flags)
c.Assert(err, IsNil)
if nil != err {
return err
}
s.context = context
s.router = Router(context)
return nil
}
func (s *ApiSuite) SetUpSuite(c *C) {
err := s.setupContext()
c.Assert(err, IsNil)
}
func (s *ApiSuite) TearDownSuite(c *C) {
@@ -85,11 +102,52 @@ func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*http
return w, nil
}
func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
c.Check(gin.Mode(), Equals, gin.ReleaseMode)
}
func (s *ApiSuite) TestGetVersion(c *C) {
response, err := s.HTTPRequest("GET", "/api/version", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, ".*Version.*")
c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
}
func (s *ApiSuite) TestGetReadiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/ready", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
}
func (s *ApiSuite) TestGetHealthiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/healthy", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
}
func (s *ApiSuite) TestGetMetrics(c *C) {
response, err := s.HTTPRequest("GET", "/api/metrics", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
b := strings.Replace(response.Body.String(), "\n", "", -1)
c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
}
func (s *ApiSuite) TestRepoCreate(c *C) {
body, err := json.Marshal(gin.H{
"Name": "dummy",
})
c.Assert(err, IsNil)
_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
c.Assert(err, IsNil)
}
func (s *ApiSuite) TestTruthy(c *C) {
api/db.go
+10 -2
View File
@@ -11,9 +11,17 @@ import (
"github.com/gin-gonic/gin"
)
// POST /api/db/cleanup
// @Summary DB Cleanup
// @Description **Cleanup Aptly DB**
// @Description Database cleanup removes information about unreferenced packages and deletes files in the package pool that aren’t used by packages anymore.
// @Description It is a good idea to run this command after massive deletion of mirrors, snapshots or local repos.
// @Tags Database
// @Produce json
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} string "Output"
// @Failure 404 {object} Error "Not Found"
// @Router /api/db/cleanup [post]
func apiDbCleanup(c *gin.Context) {
resources := []string{string(task.AllResourcesKey)}
maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
var err error
api/error.go
+5
View File
@@ -0,0 +1,5 @@
package api
type Error struct {
Error string `json:"error"`
}
api/files.go
+109 -29
View File
@@ -6,8 +6,11 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
"github.com/saracen/walker"
)
func verifyPath(path string) bool {
@@ -24,27 +27,37 @@ func verifyPath(path string) bool {
func verifyDir(c *gin.Context) bool {
if !verifyPath(c.Params.ByName("dir")) {
c.AbortWithError(400, fmt.Errorf("wrong dir"))
AbortWithJSONError(c, 400, fmt.Errorf("wrong dir"))
return false
}
return true
}
// GET /files
// @Summary List Directories
// @Description **Get list of upload directories**
// @Description
// @Description **Example:**
// @Description ```
// @Description $ curl http://localhost:8080/api/files
// @Description ["aptly-0.9"]
// @Description ```
// @Tags Files
// @Produce json
// @Success 200 {array} string "List of files"
// @Router /api/files [get]
func apiFilesListDirs(c *gin.Context) {
list := []string{}
listLock := &sync.Mutex{}
err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
if path == context.UploadPath() {
return nil
}
if info.IsDir() {
listLock.Lock()
defer listLock.Unlock()
list = append(list, filepath.Base(path))
return filepath.SkipDir
}
@@ -53,30 +66,50 @@ func apiFilesListDirs(c *gin.Context) {
})
if err != nil && !os.IsNotExist(err) {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
c.JSON(200, list)
}
// POST /files/:dir/
// @Summary Upload Files
// @Description **Upload files to a directory**
// @Description
// @Description - one or more files can be uploaded
// @Description - existing uploaded are overwritten
// @Description
// @Description **Example:**
// @Description ```
// @Description $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9
// @Description ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"]
// @Description ```
// @Tags Files
// @Accept multipart/form-data
// @Param dir path string true "Directory to upload files to. Created if does not exist"
// @Param files formData file true "Files to upload"
// @Produce json
// @Success 200 {array} string "list of uploaded files"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/files/{dir} [post]
func apiFilesUpload(c *gin.Context) {
if !verifyDir(c) {
return
}
path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
err := os.MkdirAll(path, 0777)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -86,7 +119,7 @@ func apiFilesUpload(c *gin.Context) {
for _, file := range files {
src, err := file.Open()
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
defer src.Close()
@@ -94,14 +127,14 @@ func apiFilesUpload(c *gin.Context) {
destPath := filepath.Join(path, filepath.Base(file.Filename))
dst, err := os.Create(destPath)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -109,20 +142,35 @@ func apiFilesUpload(c *gin.Context) {
}
}
apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
c.JSON(200, stored)
}
// GET /files/:dir
// @Summary List Files
// @Description **Show uploaded files in upload directory**
// @Description
// @Description **Example:**
// @Description ```
// @Description $ curl http://localhost:8080/api/files/aptly-0.9
// @Description ["aptly_0.9~dev+217+ge5d646c_i386.deb"]
// @Description ```
// @Tags Files
// @Produce json
// @Param dir path string true "Directory to list"
// @Success 200 {array} string "Files found in directory"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/files/{dir} [get]
func apiFilesListFiles(c *gin.Context) {
if !verifyDir(c) {
return
}
list := []string{}
root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
listLock := &sync.Mutex{}
root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -131,6 +179,8 @@ func apiFilesListFiles(c *gin.Context) {
return nil
}
listLock.Lock()
defer listLock.Unlock()
list = append(list, filepath.Base(path))
return nil
@@ -138,9 +188,9 @@ func apiFilesListFiles(c *gin.Context) {
if err != nil {
if os.IsNotExist(err) {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
} else {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
}
return
}
@@ -148,36 +198,66 @@ func apiFilesListFiles(c *gin.Context) {
c.JSON(200, list)
}
// DELETE /files/:dir
// @Summary Delete Directory
// @Description **Delete upload directory and uploaded files within**
// @Description
// @Description **Example:**
// @Description ```
// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9
// @Description {}
// @Description ```
// @Tags Files
// @Produce json
// @Param dir path string true "Directory"
// @Success 200 {object} string "msg"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/files/{dir} [delete]
func apiFilesDeleteDir(c *gin.Context) {
if !verifyDir(c) {
return
}
err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
err := os.RemoveAll(filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))))
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
c.JSON(200, gin.H{})
}
// DELETE /files/:dir/:name
// @Summary Delete File
// @Description **Delete a uploaded file in upload directory**
// @Description
// @Description **Example:**
// @Description ```
// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb
// @Description {}
// @Description ```
// @Tags Files
// @Produce json
// @Param dir path string true "Directory to delete from"
// @Param name path string true "File to delete"
// @Success 200 {object} string "msg"
// @Failure 400 {object} Error "Bad Request"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/files/{dir}/{name} [delete]
func apiFilesDeleteFile(c *gin.Context) {
if !verifyDir(c) {
return
}
if !verifyPath(c.Params.ByName("name")) {
c.AbortWithError(400, fmt.Errorf("wrong file"))
dir := utils.SanitizePath(c.Params.ByName("dir"))
name := utils.SanitizePath(c.Params.ByName("name"))
if !verifyPath(name) {
AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
return
}
err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
err := os.Remove(filepath.Join(context.UploadPath(), dir, name))
if err != nil {
if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
}
api/gpg.go
+38 -20
View File
@@ -2,31 +2,49 @@ package api
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
)
// POST /api/gpg
func apiGPGAddKey(c *gin.Context) {
var b struct {
Keyserver string
GpgKeyID string
GpgKeyArmor string
Keyring string
}
type gpgAddKeyParams struct {
// Keyserver, when downloading GpgKeyIDs
Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"`
// GpgKeyIDs to download from Keyserver, comma separated list
GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500,8B48AD6246925553"`
// Armored gpg public ket, instead of downloading from keyserver
GpgKeyArmor string `json:"GpgKeyArmor" example:""`
// Keyring for adding the keys (default: trustedkeys.gpg)
Keyring string `json:"Keyring" example:"trustedkeys.gpg"`
}
// @Summary Add GPG Keys
// @Description **Adds GPG keys to aptly keyring**
// @Description
// @Description Add GPG public keys for veryfing remote repositories for mirroring.
// @Tags Mirrors
// @Produce json
// @Success 200 {object} string "OK"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Router /api/gpg [post]
func apiGPGAddKey(c *gin.Context) {
b := gpgAddKeyParams{}
if c.Bind(&b) != nil {
return
}
b.Keyserver = utils.SanitizePath(b.Keyserver)
b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
b.GpgKeyArmor = utils.SanitizePath(b.GpgKeyArmor)
// b.Keyring can be an absolute path
var err error
args := []string{"--no-default-keyring"}
args := []string{"--no-default-keyring", "--allow-non-selfsigned-uid"}
keyring := "trustedkeys.gpg"
if len(b.Keyring) > 0 {
keyring = b.Keyring
@@ -37,9 +55,9 @@ func apiGPGAddKey(c *gin.Context) {
}
if len(b.GpgKeyArmor) > 0 {
var tempdir string
tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
tempdir, err = os.MkdirTemp(os.TempDir(), "aptly")
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
defer os.RemoveAll(tempdir)
@@ -47,11 +65,11 @@ func apiGPGAddKey(c *gin.Context) {
keypath := filepath.Join(tempdir, "key")
keyfile, e := os.Create(keypath)
if e != nil {
c.AbortWithError(400, e)
AbortWithJSONError(c, 400, e)
return
}
if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
c.AbortWithError(400, e)
AbortWithJSONError(c, 400, e)
}
args = append(args, "--import", keypath)
@@ -62,10 +80,10 @@ func apiGPGAddKey(c *gin.Context) {
args = append(args, keys...)
}
finder := pgp.GPG1Finder()
finder := pgp.GPGDefaultFinder()
gpg, _, err := finder.FindGPG()
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -74,11 +92,11 @@ func apiGPGAddKey(c *gin.Context) {
// there is no error handling for such as gpg will do this for us
cmd := exec.Command(gpg, args...)
fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
cmd.Stdout = os.Stdout
if err = cmd.Run(); err != nil {
c.AbortWithError(400, err)
out, err := cmd.CombinedOutput()
if err != nil {
c.JSON(400, string(out))
return
}
c.JSON(200, gin.H{})
c.JSON(200, string(out))
}
api/graph.go
+25 -5
View File
@@ -12,7 +12,27 @@ import (
"github.com/gin-gonic/gin"
)
// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
// @Summary Graph Output
// @Description **Generate dependency graph**
// @Description
// @Description Command graph generates graph of dependencies:
// @Description
// @Description * between snapshots and mirrors (what mirror was used to create each snapshot)
// @Description * between snapshots and local repos (what local repo was used to create snapshot)
// @Description * between snapshots (pulling, merging, etc.)
// @Description * between snapshots, local repos and published repositories (how snapshots were published).
// @Description
// @Description Graph is rendered to PNG file using graphviz package.
// @Description
// @Description Example URL: `http://localhost:8080/api/graph.svg?layout=vertical`
// @Tags Status
// @Produce image/png, image/svg+xml
// @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg."
// @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout."
// @Success 200 {object} []byte "Output"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/graph.{ext} [get]
func apiGraph(c *gin.Context) {
var (
err error
@@ -43,25 +63,25 @@ func apiGraph(c *gin.Context) {
stdin, err := command.StdinPipe()
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
_, err = io.Copy(stdin, buf)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
err = stdin.Close()
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
output, err = command.Output()
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
return
}
api/metrics.go
+116
View File
@@ -0,0 +1,116 @@
package api
import (
"fmt"
"runtime"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/deb"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/rs/zerolog/log"
)
var (
apiRequestsInFlightGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_api_http_requests_in_flight",
Help: "Number of concurrent HTTP api requests currently handled.",
},
[]string{"method", "path"},
)
apiRequestsTotalCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "aptly_api_http_requests_total",
Help: "Total number of api requests.",
},
[]string{"code", "method", "path"},
)
apiRequestSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_size_bytes",
Help: "Api HTTP request size in bytes.",
},
[]string{"code", "method", "path"},
)
apiResponseSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_response_size_bytes",
Help: "Api HTTP response size in bytes.",
},
[]string{"code", "method", "path"},
)
apiRequestsDurationSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_duration_seconds",
Help: "Duration of api requests in seconds.",
},
[]string{"code", "method", "path"},
)
apiVersionGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_build_info",
Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
},
[]string{"version", "goversion"},
)
apiFilesUploadedCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "aptly_api_files_uploaded_total",
Help: "Total number of uploaded files labeled by upload directory.",
},
[]string{"directory"},
)
apiReposPackageCountGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_repos_package_count",
Help: "Current number of published packages labeled by source, distribution and component.",
},
[]string{"source", "distribution", "component"},
)
)
type metricsCollectorRegistrar struct {
hasRegistered bool
}
func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
if !r.hasRegistered {
apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
r.hasRegistered = true
}
}
var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
func countPackagesByRepos() {
err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
if err != nil {
msg := fmt.Sprintf(
"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
err, repo.StoragePrefix(), repo.Distribution, repo.Components())
log.Warn().Msg(msg)
return err
}
components := repo.Components()
for _, c := range components {
count := float64(len(repo.RefList(c).Refs))
apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
}
return nil
})
if err != nil {
msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
log.Warn().Msg(msg)
}
}
api/middleware.go
+58 -45
View File
@@ -9,59 +9,36 @@ import (
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
apiRequestsInFlightGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_api_http_requests_in_flight",
Help: "Number of concurrent HTTP api requests currently handled.",
},
[]string{"method", "path"},
)
apiRequestsTotalCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "aptly_api_http_requests_total",
Help: "Total number of api requests.",
},
[]string{"code", "method", "path"},
)
apiRequestSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_size_bytes",
Help: "Api HTTP request size in bytes.",
},
[]string{"code", "method", "path"},
)
apiResponseSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_response_size_bytes",
Help: "Api HTTP response size in bytes.",
},
[]string{"code", "method", "path"},
)
apiRequestsDurationSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_duration_seconds",
Help: "Duration of api requests in seconds.",
},
[]string{"code", "method", "path"},
)
"github.com/rs/zerolog/log"
)
// Only use base path as label value (e.g.: /api/repos) because of time series cardinality
// See https://prometheus.io/docs/practices/naming/#labels
func getBasePath(c *gin.Context) string {
return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
segment0, err := getURLSegment(c.Request.URL.Path, 0)
if err != nil {
return "/"
}
segment1, err := getURLSegment(c.Request.URL.Path, 1)
if err != nil {
return *segment0
}
return *segment0 + *segment1
}
func getURLSegment(url string, idx int) string {
var urlSegments = strings.Split(url, "/")
func getURLSegment(url string, idx int) (*string, error) {
urlSegments := strings.Split(url, "/")
// Remove segment at index 0 because it's an empty string
var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
return fmt.Sprintf("/%s", segmentAtIndex)
urlSegments = urlSegments[1:cap(urlSegments)]
if len(urlSegments) <= idx {
return nil, fmt.Errorf("index %d out of range, only has %d url segments", idx, len(urlSegments))
}
segmentAtIndex := urlSegments[idx]
s := fmt.Sprintf("/%s", segmentAtIndex)
return &s, nil
}
func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
@@ -101,3 +78,39 @@ func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Co
obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
}
}
// JSONLogger is a gin middleware that takes an instance of Logger and uses it for writing access
// logs that include error messages if there are any.
func JSONLogger() gin.HandlerFunc {
return func(c *gin.Context) {
// Start timer
start := time.Now()
path := c.Request.URL.Path
raw := c.Request.URL.RawQuery
// Process request
c.Next()
ts := time.Now()
if raw != "" {
path = path + "?" + raw
}
errorMessage := strings.TrimSuffix(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
l := log.With().Str("remote", c.ClientIP()).Logger().
With().Str("method", c.Request.Method).Logger().
With().Str("path", path).Logger().
With().Str("protocol", c.Request.Proto).Logger().
With().Str("code", fmt.Sprint(c.Writer.Status())).Logger().
With().Str("latency", ts.Sub(start).String()).Logger().
With().Str("agent", c.Request.UserAgent()).Logger()
if c.Writer.Status() >= 400 && c.Writer.Status() < 500 {
l.Warn().Msg(errorMessage)
} else if c.Writer.Status() >= 500 {
l.Error().Msg(errorMessage)
} else {
l.Info().Msg(errorMessage)
}
}
}
api/middleware_test.go
+255
View File
@@ -0,0 +1,255 @@
package api
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"sync/atomic"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
. "gopkg.in/check.v1"
)
type MiddlewareSuite struct {
router http.Handler
context *gin.Context
logReader *os.File
logWriter *os.File
}
var _ = Suite(&MiddlewareSuite{})
func (s *MiddlewareSuite) SetUpTest(c *C) {
r, w, err := os.Pipe()
c.Assert(err, IsNil)
utils.SetupJSONLogger("debug", w)
mw := JSONLogger()
router := gin.New()
router.UseRawPath = true
router.Use(mw)
router.Use(gin.Recovery(), gin.ErrorLogger())
root := router.Group("/api")
isReady := &atomic.Value{}
isReady.Store(false)
root.GET("/ready", apiReady(isReady))
root.GET("/healthy", apiHealthy)
s.router = router
s.logReader = r
s.logWriter = w
}
func (s *MiddlewareSuite) TearDownTest(c *C) {
s.router = nil
s.context = nil
s.logReader = nil
s.logWriter = nil
}
func (s *MiddlewareSuite) HTTPRequest(method string, url string, body io.Reader) {
recorder := httptest.NewRecorder()
s.context, _ = gin.CreateTestContext(recorder)
req, _ := http.NewRequestWithContext(s.context, method, url, body)
s.context.Request = req
req.Header.Add("Content-Type", "application/json")
s.router.ServeHTTP(httptest.NewRecorder(), req)
}
func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "warn")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["method"]; ok {
c.Check(val, Equals, "GET")
} else {
c.Errorf("Log message didn't have a 'method' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["path"]; ok {
c.Check(val, Equals, "/")
} else {
c.Errorf("Log message didn't have a 'path' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["protocol"]; ok {
c.Check(val, Equals, "HTTP/1.1")
} else {
c.Errorf("Log message didn't have a 'protocol' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["code"]; ok {
c.Check(val, Equals, "404")
} else {
c.Errorf("Log message didn't have a 'code' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["remote"]; !ok {
c.Errorf("Log message didn't have a 'remote' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["latency"]; !ok {
c.Errorf("Log message didn't have a 'latency' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["agent"]; !ok {
c.Errorf("Log message didn't have a 'agent' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["time"]; !ok {
c.Errorf("Log message didn't have a 'time' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "info")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/ready", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "error")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
fmt.Println(capturedOutput)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "info")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestGetBasePath(c *C) {
s.HTTPRequest(http.MethodGet, "", nil)
path := getBasePath(s.context)
c.Check(path, Equals, "/")
s.HTTPRequest(http.MethodGet, "/", nil)
path = getBasePath(s.context)
c.Check(path, Equals, "/")
s.HTTPRequest(http.MethodGet, "/api", nil)
path = getBasePath(s.context)
c.Check(path, Equals, "/api")
s.HTTPRequest(http.MethodGet, "/api/repos/testRepo", nil)
path = getBasePath(s.context)
c.Check(path, Equals, "/api/repos")
}
func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
url := "/"
segment, err := getURLSegment(url, 0)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/")
_, err = getURLSegment(url, 1)
if err == nil {
c.Error("Invalid return value")
}
url = "/api"
segment, err = getURLSegment(url, 0)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/api")
_, err = getURLSegment(url, 1)
if err == nil {
c.Error("Invalid return value")
}
url = "/api/repos/testRepo"
segment, err = getURLSegment(url, 0)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/api")
segment, err = getURLSegment(url, 1)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/repos")
}
api/mirror.go
+210 -101
View File
@@ -2,8 +2,8 @@ package api
import (
"fmt"
"log"
"net/http"
"os"
"sort"
"strings"
"sync"
@@ -14,19 +14,16 @@ import (
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
)
func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
if ignoreSignatures {
return nil, nil
}
func getVerifier(keyRings []string) (pgp.Verifier, error) {
verifier := context.GetVerifier()
for _, keyRing := range keyRings {
verifier.AddKeyring(keyRing)
}
err := verifier.InitKeyring()
err := verifier.InitKeyring(false)
if err != nil {
return nil, err
}
@@ -34,7 +31,13 @@ func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error)
return verifier, nil
}
// GET /api/mirrors
// @Summary List Mirrors
// @Description **Show list of currently available mirrors**
// @Description Each mirror is returned as in “show” API.
// @Tags Mirrors
// @Produce json
// @Success 200 {array} deb.RemoteRepo
// @Router /api/mirrors [get]
func apiMirrorsList(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
@@ -48,24 +51,49 @@ func apiMirrorsList(c *gin.Context) {
c.JSON(200, result)
}
// POST /api/mirrors
type mirrorCreateParams struct {
// Name of mirror to be created
Name string `binding:"required" json:"Name" example:"mirror2"`
// Url of the archive to mirror
ArchiveURL string `binding:"required" json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Distribution name to mirror
Distribution string ` json:"Distribution" example:"'buster', for flat repositories use './'"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Gpg keyring(s) for verifying Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to mirror installer files
DownloadInstaller bool ` json:"DownloadInstaller"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to skip if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip the verification of architectures
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
}
// @Summary Create Mirror
// @Description **Create a mirror of a remote repository**
// @Tags Mirrors
// @Consume json
// @Param request body mirrorCreateParams true "Parameters"
// @Produce json
// @Success 200 {object} deb.RemoteRepo
// @Failure 400 {object} Error "Bad Request"
// @Router /api/mirrors [post]
func apiMirrorsCreate(c *gin.Context) {
var err error
var b struct {
Name string `binding:"required"`
ArchiveURL string `binding:"required"`
Distribution string
Filter string
Components []string
Architectures []string
Keyrings []string
DownloadSources bool
DownloadUdebs bool
DownloadInstaller bool
FilterWithDeps bool
SkipComponentCheck bool
IgnoreSignatures bool
}
var b mirrorCreateParams
b.DownloadSources = context.Config().DownloadSourcePackages
b.IgnoreSignatures = context.Config().GpgDisableVerify
@@ -81,7 +109,7 @@ func apiMirrorsCreate(c *gin.Context) {
if strings.HasPrefix(b.ArchiveURL, "ppa:") {
b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
}
@@ -89,7 +117,7 @@ func apiMirrorsCreate(c *gin.Context) {
if b.Filter != "" {
_, err = query.Parse(b.Filter)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
return
}
}
@@ -98,39 +126,51 @@ func apiMirrorsCreate(c *gin.Context) {
b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
return
}
repo.Filter = b.Filter
repo.FilterWithDeps = b.FilterWithDeps
repo.SkipComponentCheck = b.SkipComponentCheck
repo.SkipArchitectureCheck = b.SkipArchitectureCheck
repo.DownloadSources = b.DownloadSources
repo.DownloadUdebs = b.DownloadUdebs
verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
verifier, err := getVerifier(b.Keyrings)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
return
}
downloader := context.NewDownloader(nil)
err = repo.Fetch(downloader, verifier)
err = repo.Fetch(downloader, verifier, b.IgnoreSignatures)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to fetch mirror: %s", err))
return
}
err = collection.Add(repo)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
return
}
c.JSON(201, repo)
}
// DELETE /api/mirrors/:name
// @Summary Delete Mirror
// @Description **Delete a mirror**
// @Tags Mirrors
// @Param name path string true "mirror name"
// @Param force query int true "force: 1 to enable"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue
// @Failure 404 {object} Error "Mirror not found"
// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
// @Failure 500 {object} Error "Unable to delete"
// @Router /api/mirrors/{name} [delete]
func apiMirrorsDrop(c *gin.Context) {
name := c.Params.ByName("name")
force := c.Request.URL.Query().Get("force") == "1"
@@ -141,13 +181,13 @@ func apiMirrorsDrop(c *gin.Context) {
repo, err := mirrorCollection.ByName(name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to drop: %s", err))
return
}
resources := []string{string(repo.Key())}
taskName := fmt.Sprintf("Delete mirror %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := repo.CheckLock()
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
@@ -157,7 +197,7 @@ func apiMirrorsDrop(c *gin.Context) {
snapshots := snapshotCollection.ByRemoteRepoSource(repo)
if len(snapshots) > 0 {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
}
}
@@ -169,7 +209,15 @@ func apiMirrorsDrop(c *gin.Context) {
})
}
// GET /api/mirrors/:name
// @Summary Get Mirror Info
// @Description **Get mirror information by name**
// @Tags Mirrors
// @Param name path string true "mirror name"
// @Produce json
// @Success 200 {object} deb.RemoteRepo
// @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name} [get]
func apiMirrorsShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
@@ -177,19 +225,30 @@ func apiMirrorsShow(c *gin.Context) {
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
}
c.JSON(200, repo)
}
// GET /api/mirrors/:name/packages
// @Summary List Mirror Packages
// @Description **Get a list of packages from a mirror**
// @Tags Mirrors
// @Param name path string true "mirror name"
// @Param q query string false "search query"
// @Param format query string false "format: `details` for more detailed information"
// @Produce json
// @Success 200 {array} deb.Package "List of Packages"
// @Failure 400 {object} Error "Unable to determine list of architectures"
// @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name}/packages [get]
func apiMirrorsPackages(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
@@ -197,17 +256,17 @@ func apiMirrorsPackages(c *gin.Context) {
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
}
if repo.LastDownloadDate.IsZero() {
c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
AbortWithJSONError(c, 404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
return
}
@@ -216,7 +275,7 @@ func apiMirrorsPackages(c *gin.Context) {
list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -224,7 +283,7 @@ func apiMirrorsPackages(c *gin.Context) {
if queryS != "" {
q, err := query.Parse(c.Request.URL.Query().Get("q"))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -241,17 +300,21 @@ func apiMirrorsPackages(c *gin.Context) {
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
return
}
}
list.PrepareIndex()
list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList)
list, err = list.Filter(deb.FilterOptions{
Queries: []deb.PackageQuery{q},
WithDependencies: withDeps,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
})
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
}
}
@@ -267,36 +330,66 @@ func apiMirrorsPackages(c *gin.Context) {
}
}
// PUT /api/mirrors/:name
type mirrorUpdateParams struct {
// Change mirror name to `Name`
Name string ` json:"Name" example:"mirror1"`
// Url of the archive to mirror
ArchiveURL string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Gpg keyring(s) for verifing Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to skip checking if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip checking if the given architectures are in the Release file
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to ignore checksum errors
IgnoreChecksums bool ` json:"IgnoreChecksums"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
ForceUpdate bool ` json:"ForceUpdate"`
// Set "true" to skip downloading already downloaded packages
SkipExistingPackages bool ` json:"SkipExistingPackages"`
}
// @Summary Update Mirror
// @Description **Update Mirror and download packages**
// @Tags Mirrors
// @Param name path string true "mirror name to update"
// @Consume json
// @Param request body mirrorUpdateParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
// @Success 202 {object} task.Task "Mirror is being updated"
// @Failure 400 {object} Error "Unable to determine list of architectures"
// @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name} [put]
func apiMirrorsUpdate(c *gin.Context) {
var (
err error
remote *deb.RemoteRepo
b mirrorUpdateParams
)
var b struct {
Name string
ArchiveURL string
Filter string
Architectures []string
Components []string
Keyrings []string
FilterWithDeps bool
DownloadSources bool
DownloadUdebs bool
SkipComponentCheck bool
IgnoreChecksums bool
IgnoreSignatures bool
ForceUpdate bool
SkipExistingPackages bool
}
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
remote, err = collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -304,12 +397,14 @@ func apiMirrorsUpdate(c *gin.Context) {
b.DownloadUdebs = remote.DownloadUdebs
b.DownloadSources = remote.DownloadSources
b.SkipComponentCheck = remote.SkipComponentCheck
b.SkipArchitectureCheck = remote.SkipArchitectureCheck
b.FilterWithDeps = remote.FilterWithDeps
b.Filter = remote.Filter
b.Architectures = remote.Architectures
b.Components = remote.Components
b.IgnoreSignatures = context.Config().GpgDisableVerify
log.Printf("%s: Starting mirror update\n", b.Name)
log.Info().Msgf("%s: Starting mirror update", b.Name)
if c.Bind(&b) != nil {
return
@@ -318,14 +413,14 @@ func apiMirrorsUpdate(c *gin.Context) {
if b.Name != remote.Name {
_, err = collection.ByName(b.Name)
if err == nil {
c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
return
}
}
if b.DownloadUdebs != remote.DownloadUdebs {
if remote.IsFlat() && b.DownloadUdebs {
c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
return
}
}
@@ -338,14 +433,15 @@ func apiMirrorsUpdate(c *gin.Context) {
remote.DownloadUdebs = b.DownloadUdebs
remote.DownloadSources = b.DownloadSources
remote.SkipComponentCheck = b.SkipComponentCheck
remote.SkipArchitectureCheck = b.SkipArchitectureCheck
remote.FilterWithDeps = b.FilterWithDeps
remote.Filter = b.Filter
remote.Architectures = b.Architectures
remote.Components = b.Components
verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
verifier, err := getVerifier(b.Keyrings)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
return
}
@@ -353,7 +449,7 @@ func apiMirrorsUpdate(c *gin.Context) {
maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
downloader := context.NewDownloader(out)
err := remote.Fetch(downloader, verifier)
err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
@@ -365,7 +461,7 @@ func apiMirrorsUpdate(c *gin.Context) {
}
}
err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
@@ -458,7 +554,7 @@ func apiMirrorsUpdate(c *gin.Context) {
}
}()
log.Printf("%s: Spawning background processes...\n", b.Name)
log.Info().Msgf("%s: Spawning background processes...", b.Name)
var wg sync.WaitGroup
for i := 0; i < context.Config().DownloadConcurrency; i++ {
wg.Add(1)
@@ -476,7 +572,16 @@ func apiMirrorsUpdate(c *gin.Context) {
var e error
// provision download location
task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
} else {
var file *os.File
file, e = os.CreateTemp("", task.File.Filename)
if e == nil {
task.TempDownPath = file.Name()
file.Close()
}
}
if e != nil {
pushError(e)
continue
@@ -494,6 +599,20 @@ func apiMirrorsUpdate(c *gin.Context) {
continue
}
// and import it back to the pool
task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
if err != nil {
//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
pushError(err)
continue
}
// update "attached" files if any
for _, additionalAtask := range task.Additional {
additionalAtask.File.PoolPath = task.File.PoolPath
additionalAtask.File.Checksums = task.File.Checksums
}
task.Done = true
taskFinished <- task
case <-context.Done():
@@ -505,32 +624,22 @@ func apiMirrorsUpdate(c *gin.Context) {
}
// Wait for all download goroutines to finish
log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
wg.Wait()
log.Printf("%s: Background processes finished\n", b.Name)
log.Info().Msgf("%s: Background processes finished", b.Name)
close(taskFinished)
for idx := range queue {
defer func() {
for _, task := range queue {
if task.TempDownPath == "" {
continue
}
atask := &queue[idx]
if !atask.Done {
// download not finished yet
continue
if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
}
}
// and import it back to the pool
atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
}
// update "attached" files if any
for _, additionalAtask := range atask.Additional {
additionalAtask.File.PoolPath = atask.File.PoolPath
additionalAtask.File.Checksums = atask.File.Checksums
}
}
}()
select {
case <-context.Done():
@@ -539,18 +648,18 @@ func apiMirrorsUpdate(c *gin.Context) {
}
if len(errors) > 0 {
log.Printf("%s: Unable to update because of previous errors\n", b.Name)
log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n %s", strings.Join(errors, "\n "))
}
log.Printf("%s: Finalizing download\n", b.Name)
log.Info().Msgf("%s: Finalizing download...", b.Name)
remote.FinalizeDownload(collectionFactory, out)
err = collectionFactory.RemoteRepoCollection().Update(remote)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
log.Printf("%s: Mirror updated successfully!\n", b.Name)
log.Info().Msgf("%s: Mirror updated successfully", b.Name)
return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
})
}
api/packages.go
+26 -2
View File
@@ -1,17 +1,41 @@
package api
import (
_ "github.com/aptly-dev/aptly/deb" // for swagger
"github.com/gin-gonic/gin"
)
// GET /api/packages/:key
// @Summary Get Package Info
// @Description **Show information about package by package key**
// @Description Package keys could be obtained from various GET .../packages APIs.
// @Tags Packages
// @Produce json
// @Param key path string true "package key (unique package identifier)"
// @Success 200 {object} deb.Package "OK"
// @Failure 404 {object} Error "Not Found"
// @Router /api/packages/{key} [get]
func apiPackagesShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
c.JSON(200, p)
}
// @Summary List Packages
// @Description **Get list of packages**
// @Tags Packages
// @Consume json
// @Produce json
// @Param q query string false "search query"
// @Param format query string false "format: `details` for more detailed information"
// @Success 200 {array} string "List of packages"
// @Router /api/packages [get]
func apiPackages(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PackageCollection()
showPackages(c, collection.AllPackageRefs(), collectionFactory)
}
api/packages_test.go
+18
View File
@@ -0,0 +1,18 @@
package api
import (
. "gopkg.in/check.v1"
)
type PackagesSuite struct {
ApiSuite
}
var _ = Suite(&PackagesSuite{})
func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Equals, "[]")
}
api/publish.go
+822 -160
View File
File diff suppressed because it is too large Load Diff
api/repos.go
+476 -67
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"text/template"
@@ -17,7 +18,59 @@ import (
"github.com/gin-gonic/gin"
)
// GET /api/repos
// @Summary Serve HTML Listing
// @Description If ServeInAPIMode is enabled in aptly config,
// @Description this endpoint is enabled which returns an HTML listing of each repo that can be browsed
// @Tags Repos
// @Produce html
// @Success 200 {object} string "HTML"
// @Router /api/repos [get]
func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
c.Writer.Flush()
c.Writer.WriteString("<pre>\n")
if len(localRepos) == 0 {
c.Writer.WriteString("<a href=\"-/\">default</a>\n")
}
for publishPrefix := range localRepos {
c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
}
c.Writer.WriteString("</pre>")
c.Writer.Flush()
}
}
// @Summary Serve Packages
// @Description If ServeInAPIMode is enabled in aptly config,
// @Description this api serves a specified package from storage
// @Tags Repos
// @Param storage path string true "Storage"
// @Param pkgPath path string true "Package Path" allowReserved=true
// @Produce json
// @Success 200 ""
// @Router /api/{storage}/{pkgPath} [get]
func reposServeInAPIMode(c *gin.Context) {
pkgpath := c.Param("pkgPath")
storage := c.Param("storage")
if storage == "-" {
storage = ""
} else {
storage = "filesystem:" + storage
}
publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
c.FileFromFS(pkgpath, http.Dir(publicPath))
}
// @Summary List Repositories
// @Description **Get list of available repos**
// @Description Each repo is returned as in “show” API.
// @Tags Repos
// @Produce json
// @Success 200 {array} deb.LocalRepo
// @Router /api/repos [get]
func apiReposList(c *gin.Context) {
result := []*deb.LocalRepo{}
@@ -31,14 +84,39 @@ func apiReposList(c *gin.Context) {
c.JSON(200, result)
}
// POST /api/repos
type repoCreateParams struct {
// Name of repository to create
Name string `binding:"required" json:"Name" example:"repo1"`
// Text describing the repository (optional)
Comment string ` json:"Comment" example:"this is a repo"`
// Default distribution when publishing from this local repo
DefaultDistribution string ` json:"DefaultDistribution" example:"stable"`
// Default component when publishing from this local repo
DefaultComponent string ` json:"DefaultComponent" example:"main"`
// Snapshot name to create repoitory from (optional)
FromSnapshot string ` json:"FromSnapshot" example:""`
}
// @Summary Create Repository
// @Description **Create a local repository**
// @Description
// @Description Distribution and component are used as defaults when publishing repo either directly or via snapshot.
// @Description
// @Description ```
// @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Name": "aptly-repo"}' http://localhost:8080/api/repos
// @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
// @Description ```
// @Tags Repos
// @Produce json
// @Consume json
// @Param request body repoCreateParams true "Parameters"
// @Success 201 {object} deb.LocalRepo
// @Failure 404 {object} Error "Source snapshot not found"
// @Failure 409 {object} Error "Local repo already exists"
// @Failure 500 {object} Error "Internal error"
// @Router /api/repos [post]
func apiReposCreate(c *gin.Context) {
var b struct {
Name string `binding:"required"`
Comment string
DefaultDistribution string
DefaultComponent string
}
var b repoCreateParams
if c.Bind(&b) != nil {
return
@@ -49,25 +127,65 @@ func apiReposCreate(c *gin.Context) {
repo.DefaultDistribution = b.DefaultDistribution
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
err := collection.Add(repo)
if err != nil {
c.AbortWithError(400, err)
if b.FromSnapshot != "" {
var snapshot *deb.Snapshot
snapshotCollection := collectionFactory.SnapshotCollection()
snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err))
return
}
err = snapshotCollection.LoadComplete(snapshot)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err))
return
}
repo.UpdateRefList(snapshot.RefList())
}
localRepoCollection := collectionFactory.LocalRepoCollection()
if _, err := localRepoCollection.ByName(b.Name); err == nil {
AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name))
return
}
c.JSON(201, repo)
}
// PUT /api/repos/:name
func apiReposEdit(c *gin.Context) {
var b struct {
Name *string
Comment *string
DefaultDistribution *string
DefaultComponent *string
err := localRepoCollection.Add(repo)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
c.JSON(http.StatusCreated, repo)
}
type reposEditParams struct {
// Name of repository to modify
Name *string `binding:"required" json:"Name" example:"repo1"`
// Change Comment of repository
Comment *string ` json:"Comment" example:"example repo"`
// Change Default Distribution for publishing
DefaultDistribution *string ` json:"DefaultDistribution" example:""`
// Change Devault Component for publishing
DefaultComponent *string ` json:"DefaultComponent" example:""`
}
// @Summary Update Repository
// @Description **Update local repository meta information**
// @Tags Repos
// @Produce json
// @Param request body reposEditParams true "Parameters"
// @Success 200 {object} deb.LocalRepo "msg"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/repos/{name} [put]
func apiReposEdit(c *gin.Context) {
var b reposEditParams
if c.Bind(&b) != nil {
return
}
@@ -77,7 +195,7 @@ func apiReposEdit(c *gin.Context) {
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -85,7 +203,7 @@ func apiReposEdit(c *gin.Context) {
_, err := collection.ByName(*b.Name)
if err == nil {
// already exists
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
repo.Name = *b.Name
@@ -102,7 +220,7 @@ func apiReposEdit(c *gin.Context) {
err = collection.Update(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -110,20 +228,39 @@ func apiReposEdit(c *gin.Context) {
}
// GET /api/repos/:name
// @Summary Get Repository Info
// @Description Returns basic information about local repository.
// @Tags Repos
// @Produce json
// @Param name path string true "Repository name"
// @Success 200 {object} deb.LocalRepo
// @Failure 404 {object} Error "Repository not found"
// @Router /api/repos/{name} [get]
func apiReposShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
c.JSON(200, repo)
}
// DELETE /api/repos/:name
// @Summary Delete Repository
// @Description Drop/delete a repo
// @Description Cannot drop repos that are published.
// @Description Needs force=1 to drop repos used as source by other repos.
// @Tags Repos
// @Produce json
// @Param _async query bool false "Run in background and return task object"
// @Param force query int false "force: 1 to enable"
// @Success 200 {object} task.ProcessReturnValue "Repo object"
// @Failure 404 {object} Error "Not Found"
// @Failure 404 {object} Error "Repo Conflict"
// @Router /api/repos/{name} [delete]
func apiReposDrop(c *gin.Context) {
force := c.Request.URL.Query().Get("force") == "1"
name := c.Params.ByName("name")
@@ -135,13 +272,13 @@ func apiReposDrop(c *gin.Context) {
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(repo.Key())}
taskName := fmt.Sprintf("Delete repo %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
published := publishedCollection.ByLocalRepo(repo)
if len(published) > 0 {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
@@ -158,31 +295,54 @@ func apiReposDrop(c *gin.Context) {
})
}
// GET /api/repos/:name/packages
// @Summary List Repo Packages
// @Description **Return a list of packages present in the repo**
// @Description
// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q
// @Description
// @Description **Example:**
// @Description ```
// @Description $ curl http://localhost:8080/api/repos/aptly-repo/packages
// @Description ["Pi386 aptly 0.8 966561016b44ed80"]
// @Description ```
// @Tags Repos
// @Produce json
// @Param name path string true "Snapshot to search"
// @Param q query string true "Package query (e.g Name%20(~%20matlab))"
// @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
// @Param format query string true "Set to 'details' to return extra info about each package"
// @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
// @Success 200 {object} string "msg"
// @Failure 404 {object} Error "Not Found"
// @Failure 404 {object} Error "Internal Server Error"
// @Router /api/repos/{name}/packages [get]
func apiReposPackagesShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
showPackages(c, repo.RefList(), collectionFactory)
}
type reposPackagesAddDeleteParams struct {
// Package Refs
PackageRefs []string `binding:"required" json:"PackageRefs" example:""`
}
// Handler for both add and delete
func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) {
var b struct {
PackageRefs []string
}
var b reposPackagesAddDeleteParams
if c.Bind(&b) != nil {
return
@@ -193,18 +353,18 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(repo.Key())}
maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.LoadComplete(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
out.Printf("Loading packages...\n")
list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
if err != nil {
@@ -239,7 +399,21 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
})
}
// POST /repos/:name/packages
// @Summary Add Packages by Key
// @Description **Add packages to local repository by package keys.**
// @Description
// @Description Any package can be added that is present in the aptly database (from any mirror, snapshot, local repository). This API combined with package list (search) APIs allows one to implement importing, copying, moving packages around.
// @Description
// @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
// @Tags Repos
// @Produce json
// @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} string "msg"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 400 {object} Error "Internal Server Error"
// @Router /api/repos/{name}/packages [post]
func apiReposPackagesAdd(c *gin.Context) {
apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
out.Printf("Adding package %s\n", p.Name)
@@ -247,7 +421,19 @@ func apiReposPackagesAdd(c *gin.Context) {
})
}
// DELETE /repos/:name/packages
// @Summary Delete Packages by Key
// @Description **Remove packages from local repository by package keys.**
// @Description
// @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
// @Tags Repos
// @Produce json
// @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} string "msg"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 400 {object} Error "Internal Server Error"
// @Router /api/repos/{name}/packages [delete]
func apiReposPackagesDelete(c *gin.Context) {
apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
out.Printf("Removing package %s\n", p.Name)
@@ -256,13 +442,43 @@ func apiReposPackagesDelete(c *gin.Context) {
})
}
// POST /repos/:name/file/:dir/:file
// @Summary Add Uploaded File
// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
// @Description Adding same package to local repository is not an error.
// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
// @Tags Repos
// @Param name path string true "Repository name"
// @Param dir path string true "Directory of packages"
// @Param file path string false "Filename (optional)"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {string} string "OK"
// @Failure 400 {object} Error "wrong file"
// @Failure 404 {object} Error "Repository not found"
// @Failure 500 {object} Error "Error adding files"
// @Router /api/repos/{name}/file/{dir}/{file} [post]
func apiReposPackageFromFile(c *gin.Context) {
// redirect all work to dir method
apiReposPackageFromDir(c)
}
// POST /repos/:name/file/:dir
// @Summary Add Uploaded Directory
// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
// @Description Adding same package to local repository is not an error.
// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
// @Tags Repos
// @Param name path string true "Repository name"
// @Param dir path string true "Directory to add"
// @Consume json
// @Param noRemove query string false "when value is set to 1, don’t remove any files"
// @Param forceReplace query string false "when value is set to 1, remove packages conflicting with package being added (in local repository)"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {string} string "OK"
// @Failure 400 {object} Error "wrong file"
// @Failure 404 {object} Error "Repository not found"
// @Failure 500 {object} Error "Error adding files"
// @Router /api/repos/{name}/file/{dir} [post]
func apiReposPackageFromDir(c *gin.Context) {
forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
noRemove := c.Request.URL.Query().Get("noRemove") == "1"
@@ -271,10 +487,10 @@ func apiReposPackageFromDir(c *gin.Context) {
return
}
dirParam := c.Params.ByName("dir")
fileParam := c.Params.ByName("file")
dirParam := utils.SanitizePath(c.Params.ByName("dir"))
fileParam := utils.SanitizePath(c.Params.ByName("file"))
if fileParam != "" && !verifyPath(fileParam) {
c.AbortWithError(400, fmt.Errorf("wrong file"))
AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
return
}
@@ -284,13 +500,7 @@ func apiReposPackageFromDir(c *gin.Context) {
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -306,7 +516,12 @@ func apiReposPackageFromDir(c *gin.Context) {
resources := []string{string(repo.Key())}
resources = append(resources, sources...)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.LoadComplete(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
verifier := context.GetVerifier()
var (
@@ -382,13 +597,208 @@ func apiReposPackageFromDir(c *gin.Context) {
})
}
// POST /repos/:name/include/:dir/:file
type reposCopyPackageParams struct {
// Copy also dependencies
WithDeps bool `json:"with-deps,omitempty"`
// Do not perform operations
DryRun bool `json:"dry-run,omitempty"`
}
// @Summary Copy Package
// @Description Copies a package from a source to destination repository
// @Tags Repos
// @Produce json
// @Param name path string true "Source repo"
// @Param src path string true "Destination repo"
// @Param file path string true "File/packages to copy"
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} task.ProcessReturnValue "msg"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 422 {object} Error "Unprocessable Entity"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/repos/{name}/copy/{src}/{file} [post]
func apiReposCopyPackage(c *gin.Context) {
dstRepoName := c.Params.ByName("name")
srcRepoName := c.Params.ByName("src")
fileName := c.Params.ByName("file")
jsonBody := reposCopyPackageParams{
WithDeps: false,
DryRun: false,
}
err := c.Bind(&jsonBody)
if err != nil {
return
}
collectionFactory := context.NewCollectionFactory()
dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
if err != nil {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest repo error: %s", err))
return
}
var srcRepo *deb.LocalRepo
srcRepo, err = collectionFactory.LocalRepoCollection().ByName(srcRepoName)
if err != nil {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("src repo error: %s", err))
return
}
if srcRepo.UUID == dstRepo.UUID {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest and source are identical"))
return
}
taskName := fmt.Sprintf("Copy packages from repo %s to repo %s", srcRepoName, dstRepoName)
resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
}
err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
}
srcRefList := srcRepo.RefList()
reporter := &aptly.RecordingResultReporter{
Warnings: []string{},
AddedLines: []string{},
RemovedLines: []string{},
}
dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
}
srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
}
srcList.PrepareIndex()
var architecturesList []string
if jsonBody.WithDeps {
dstList.PrepareIndex()
// Calculate architectures
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = dstList.Architectures(false)
}
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to determine list of architectures, please specify explicitly")
}
}
// srcList.Filter|FilterWithProgress only accept query list
queries := make([]deb.PackageQuery, 1)
queries[0], err = query.Parse(fileName)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err)
}
toProcess, err := srcList.Filter(deb.FilterOptions{
Queries: queries,
WithDependencies: jsonBody.WithDeps,
Source: dstList,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
Progress: context.Progress(),
})
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err)
}
if toProcess.Len() == 0 {
return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("no package found for filter: '%s'", fileName)
}
err = toProcess.ForEach(func(p *deb.Package) error {
err = dstList.Add(p)
if err != nil {
return err
}
name := fmt.Sprintf("added %s-%s(%s)", p.Name, p.Version, p.Architecture)
reporter.AddedLines = append(reporter.AddedLines, name)
return nil
})
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("error processing dest add: %s", err)
}
if jsonBody.DryRun {
reporter.Warning("Changes not saved, as dry run has been requested")
} else {
dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
err = collectionFactory.LocalRepoCollection().Update(dstRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
}
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
"Report": reporter,
}}, nil
})
}
// @Summary Include File from Directory
// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
// @Tags Repos
// @Produce json
// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} string "msg"
// @Failure 404 {object} Error "Not Found"
// @Router /api/repos/{name}/include/{dir}/{file} [post]
func apiReposIncludePackageFromFile(c *gin.Context) {
// redirect all work to dir method
apiReposIncludePackageFromDir(c)
}
// POST /repos/:name/include/:dir
type reposIncludePackageFromDirReport struct {
Warnings []string
Added []string
Deleted []string
}
type reposIncludePackageFromDirResponse struct {
Report reposIncludePackageFromDirReport
FailedFiles []string
}
// @Summary Include Directory
// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
// @Tags Repos
// @Produce json
// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} reposIncludePackageFromDirResponse "Response"
// @Failure 404 {object} Error "Not Found"
// @Router /api/repos/{name}/include/{dir} [post]
func apiReposIncludePackageFromDir(c *gin.Context) {
forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
@@ -404,10 +814,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
var sources []string
var taskName string
dirParam := c.Params.ByName("dir")
fileParam := c.Params.ByName("file")
dirParam := utils.SanitizePath(c.Params.ByName("dir"))
fileParam := utils.SanitizePath(c.Params.ByName("file"))
if fileParam != "" && !verifyPath(fileParam) {
c.AbortWithError(400, fmt.Errorf("wrong file"))
AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
return
}
@@ -421,7 +831,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
repoTemplate, err := template.New("repo").Parse(repoTemplateString)
if err != nil {
c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("error parsing repo template: %s", err))
return
}
@@ -432,7 +842,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
// repo template string is simple text so only use resource key of specific repository
repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -440,7 +850,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
}
resources = append(resources, sources...)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
var (
err error
verifier = context.GetVerifier()
@@ -490,6 +900,5 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
"Report": reporter,
"FailedFiles": failedFiles,
}}, nil
})
}
api/router.go
+145 -70
View File
@@ -2,36 +2,91 @@ package api
import (
"net/http"
"os"
"sync/atomic"
"github.com/aptly-dev/aptly/aptly"
ctx "github.com/aptly-dev/aptly/context"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rs/zerolog/log"
"github.com/aptly-dev/aptly/docs"
swaggerFiles "github.com/swaggo/files"
ginSwagger "github.com/swaggo/gin-swagger"
)
var context *ctx.AptlyContext
func apiMetricsGet() gin.HandlerFunc {
return func(c *gin.Context) {
countPackagesByRepos()
promhttp.Handler().ServeHTTP(c.Writer, c.Request)
}
}
func redirectSwagger(c *gin.Context) {
if c.Request.URL.Path == "/docs/index.html" {
c.Redirect(http.StatusMovedPermanently, "/docs.html")
return
}
if c.Request.URL.Path == "/docs/" {
c.Redirect(http.StatusMovedPermanently, "/docs.html")
return
}
if c.Request.URL.Path == "/docs" {
c.Redirect(http.StatusMovedPermanently, "/docs.html")
return
}
c.Next()
}
// Router returns prebuilt with routes http.Handler
func Router(c *ctx.AptlyContext) http.Handler {
context = c
router := gin.Default()
router.UseRawPath = true
router.Use(gin.ErrorLogger())
if c.Config().EnableMetricsEndpoint {
router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
if aptly.EnableDebug {
gin.SetMode(gin.DebugMode)
} else {
gin.SetMode(gin.ReleaseMode)
}
router := gin.New()
context = c
router.UseRawPath = true
if c.Config().LogFormat == "json" {
c.StructuredLogging(true)
utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
router.Use(JSONLogger())
} else {
c.StructuredLogging(false)
utils.SetupDefaultLogger(c.Config().LogLevel)
router.Use(gin.Logger())
}
router.Use(gin.Recovery(), gin.ErrorLogger())
if c.Config().EnableSwaggerEndpoint {
router.GET("docs.html", func(c *gin.Context) {
c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
})
router.Use(redirectSwagger)
url := ginSwagger.URL("/docs/doc.json")
router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
}
if c.Config().EnableMetricsEndpoint {
MetricsCollectorRegistrar.Register(router)
}
if c.Config().ServeInAPIMode {
router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
}
api := router.Group("/api")
if context.Flags().Lookup("no-lock").Value.Get().(bool) {
// We use a goroutine to count the number of
// concurrent requests. When no more requests are
@@ -40,7 +95,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
go acquireDatabase()
router.Use(func(c *gin.Context) {
api.Use(func(c *gin.Context) {
var err error
errCh := make(chan error)
@@ -48,7 +103,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
err = <-errCh
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -56,7 +111,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
dbRequests <- dbRequest{releasedb, errCh}
err = <-errCh
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
}
}()
@@ -64,99 +119,119 @@ func Router(c *ctx.AptlyContext) http.Handler {
})
}
root := router.Group("/api")
{
if c.Config().EnableMetricsEndpoint {
root.GET("/metrics", apiMetricsGet())
api.GET("/metrics", apiMetricsGet())
}
root.GET("/version", apiVersion)
api.GET("/version", apiVersion)
api.GET("/storage", apiDiskFree)
isReady := &atomic.Value{}
isReady.Store(false)
defer isReady.Store(true)
api.GET("/ready", apiReady(isReady))
api.GET("/healthy", apiHealthy)
}
{
root.GET("/repos", apiReposList)
root.POST("/repos", apiReposCreate)
root.GET("/repos/:name", apiReposShow)
root.PUT("/repos/:name", apiReposEdit)
root.DELETE("/repos/:name", apiReposDrop)
api.GET("/repos", apiReposList)
api.POST("/repos", apiReposCreate)
api.GET("/repos/:name", apiReposShow)
api.PUT("/repos/:name", apiReposEdit)
api.DELETE("/repos/:name", apiReposDrop)
root.GET("/repos/:name/packages", apiReposPackagesShow)
root.POST("/repos/:name/packages", apiReposPackagesAdd)
root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
api.GET("/repos/:name/packages", apiReposPackagesShow)
api.POST("/repos/:name/packages", apiReposPackagesAdd)
api.DELETE("/repos/:name/packages", apiReposPackagesDelete)
root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
api.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
api.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
api.POST("/repos/:name/copy/:src/:file", apiReposCopyPackage)
root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
api.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
api.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
api.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
}
{
root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
api.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
}
{
root.GET("/mirrors", apiMirrorsList)
root.GET("/mirrors/:name", apiMirrorsShow)
root.GET("/mirrors/:name/packages", apiMirrorsPackages)
root.POST("/mirrors", apiMirrorsCreate)
root.PUT("/mirrors/:name", apiMirrorsUpdate)
root.DELETE("/mirrors/:name", apiMirrorsDrop)
api.GET("/mirrors", apiMirrorsList)
api.GET("/mirrors/:name", apiMirrorsShow)
api.GET("/mirrors/:name/packages", apiMirrorsPackages)
api.POST("/mirrors", apiMirrorsCreate)
api.PUT("/mirrors/:name", apiMirrorsUpdate)
api.DELETE("/mirrors/:name", apiMirrorsDrop)
}
{
root.POST("/gpg/key", apiGPGAddKey)
api.POST("/gpg/key", apiGPGAddKey)
}
{
root.GET("/files", apiFilesListDirs)
root.POST("/files/:dir", apiFilesUpload)
root.GET("/files/:dir", apiFilesListFiles)
root.DELETE("/files/:dir", apiFilesDeleteDir)
root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
api.GET("/s3", apiS3List)
}
{
root.GET("/publish", apiPublishList)
root.POST("/publish", apiPublishRepoOrSnapshot)
root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
api.GET("/files", apiFilesListDirs)
api.POST("/files/:dir", apiFilesUpload)
api.GET("/files/:dir", apiFilesListFiles)
api.DELETE("/files/:dir", apiFilesDeleteDir)
api.DELETE("/files/:dir/:name", apiFilesDeleteFile)
}
{
root.GET("/snapshots", apiSnapshotsList)
root.POST("/snapshots", apiSnapshotsCreate)
root.PUT("/snapshots/:name", apiSnapshotsUpdate)
root.GET("/snapshots/:name", apiSnapshotsShow)
root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
root.DELETE("/snapshots/:name", apiSnapshotsDrop)
root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
api.GET("/publish", apiPublishList)
api.GET("/publish/:prefix/:distribution", apiPublishShow)
api.POST("/publish", apiPublishRepoOrSnapshot)
api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
api.POST("/publish/:prefix/:distribution/sources", apiPublishAddSource)
api.GET("/publish/:prefix/:distribution/sources", apiPublishListChanges)
api.PUT("/publish/:prefix/:distribution/sources", apiPublishSetSources)
api.DELETE("/publish/:prefix/:distribution/sources", apiPublishDropChanges)
api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishUpdateSource)
api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishRemoveSource)
api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate)
}
{
root.GET("/packages/:key", apiPackagesShow)
api.GET("/snapshots", apiSnapshotsList)
api.POST("/snapshots", apiSnapshotsCreate)
api.PUT("/snapshots/:name", apiSnapshotsUpdate)
api.GET("/snapshots/:name", apiSnapshotsShow)
api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
api.DELETE("/snapshots/:name", apiSnapshotsDrop)
api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
api.POST("/snapshots/:name/pull", apiSnapshotsPull)
}
{
root.GET("/graph.:ext", apiGraph)
api.GET("/packages/:key", apiPackagesShow)
api.GET("/packages", apiPackages)
}
{
api.GET("/graph.:ext", apiGraph)
}
{
root.POST("/db/cleanup", apiDbCleanup)
api.POST("/db/cleanup", apiDbCleanup)
}
{
root.GET("/tasks", apiTasksList)
root.POST("/tasks-clear", apiTasksClear)
root.GET("/tasks-wait", apiTasksWait)
root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
root.GET("/tasks/:id/output", apiTasksOutputShow)
root.GET("/tasks/:id/detail", apiTasksDetailShow)
root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
root.GET("/tasks/:id", apiTasksShow)
root.DELETE("/tasks/:id", apiTasksDelete)
root.POST("/tasks-dummy", apiTasksDummy)
api.GET("/tasks", apiTasksList)
api.POST("/tasks-clear", apiTasksClear)
api.GET("/tasks-wait", apiTasksWait)
api.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
api.GET("/tasks/:id/output", apiTasksOutputShow)
api.GET("/tasks/:id/detail", apiTasksDetailShow)
api.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
api.GET("/tasks/:id", apiTasksShow)
api.DELETE("/tasks/:id", apiTasksDelete)
}
return router
api/s3.go
+21
View File
@@ -0,0 +1,21 @@
package api
import (
"github.com/gin-gonic/gin"
)
// @Summary S3 buckets
// @Description **Get list of S3 buckets**
// @Description
// @Description List configured S3 buckets.
// @Tags Status
// @Produce json
// @Success 200 {array} string "List of S3 buckets"
// @Router /api/s3 [get]
func apiS3List(c *gin.Context) {
keys := []string{}
for k := range context.Config().S3PublishRoots {
keys = append(keys, k)
}
c.JSON(200, keys)
}
api/snapshot.go
+475 -57
View File
@@ -3,15 +3,25 @@ package api
import (
"fmt"
"net/http"
"sort"
"strings"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
)
// GET /api/snapshots
// @Summary List Snapshots
// @Description **Get list of snapshots**
// @Description
// @Description Each snapshot is returned as in “show” API.
// @Tags Snapshots
// @Produce json
// @Success 200 {array} deb.Snapshot
// @Router /api/snapshots [get]
func apiSnapshotsList(c *gin.Context) {
SortMethodString := c.Request.URL.Query().Get("sort")
@@ -31,19 +41,34 @@ func apiSnapshotsList(c *gin.Context) {
c.JSON(200, result)
}
// POST /api/mirrors/:name/snapshots/
type snapshotsCreateFromMirrorParams struct {
// Name of snapshot to create
Name string `binding:"required" json:"Name" example:"snap1"`
// Description of snapshot
Description string ` json:"Description"`
}
// @Summary Snapshot Mirror
// @Description **Create a snapshot of a mirror**
// @Tags Snapshots
// @Produce json
// @Param request body snapshotsCreateFromMirrorParams true "Parameters"
// @Param name path string true "Mirror name"
// @Param _async query bool false "Run in background and return task object"
// @Success 201 {object} deb.Snapshot "Created Snapshot"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Mirror Not Found"
// @Failure 409 {object} Error "Conflicting snapshot"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/mirrors/{name}/snapshots [post]
func apiSnapshotsCreateFromMirror(c *gin.Context) {
var (
err error
repo *deb.RemoteRepo
snapshot *deb.Snapshot
b snapshotsCreateFromMirrorParams
)
var b struct {
Name string `binding:"required"`
Description string
}
if c.Bind(&b) != nil {
return
}
@@ -55,14 +80,14 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
repo, err = collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
// including snapshot resource key
resources := []string{string(repo.Key()), "S" + b.Name}
taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := repo.CheckLock()
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
@@ -90,20 +115,37 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
})
}
// POST /api/snapshots
type snapshotsCreateParams struct {
// Name of snapshot to create
Name string `binding:"required" json:"Name" example:"snap2"`
// Description of snapshot
Description string ` json:"Description"`
// List of source snapshots
SourceSnapshots []string ` json:"SourceSnapshots" example:"snap1"`
// List of package refs
PackageRefs []string ` json:"PackageRefs" example:""`
}
// @Summary Snapshot Packages
// @Description **Create a snapshot from package refs**
// @Description
// @Description Refs can be obtained from snapshots, local repos, or mirrors
// @Tags Snapshots
// @Param request body snapshotsCreateParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 201 {object} deb.Snapshot "Created snapshot"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Source snapshot or package refs not found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/snapshots [post]
func apiSnapshotsCreate(c *gin.Context) {
var (
err error
snapshot *deb.Snapshot
b snapshotsCreateParams
)
var b struct {
Name string `binding:"required"`
Description string
SourceSnapshots []string
PackageRefs []string
}
if c.Bind(&b) != nil {
return
}
@@ -123,20 +165,21 @@ func apiSnapshotsCreate(c *gin.Context) {
for i := range b.SourceSnapshots {
sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
if err != nil {
c.AbortWithError(404, err)
return
}
err = snapshotCollection.LoadComplete(sources[i])
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 404, err)
return
}
resources = append(resources, string(sources[i].ResourceKey()))
}
maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
for i := range sources {
err = snapshotCollection.LoadComplete(sources[i])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
}
list := deb.NewPackageList()
// verify package refs and build package list
@@ -160,23 +203,39 @@ func apiSnapshotsCreate(c *gin.Context) {
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
})
}
// POST /api/repos/:name/snapshots
type snapshotsCreateFromRepositoryParams struct {
// Name of snapshot to create
Name string `binding:"required" json:"Name" example:"snap1"`
// Description of snapshot
Description string ` json:"Description"`
}
// @Summary Snapshot Repository
// @Description **Create a snapshot of a repository by name**
// @Tags Snapshots
// @Param name path string true "Repository name"
// @Consume json
// @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
// @Param name path string true "Name of the snapshot"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 201 {object} deb.Snapshot "Created snapshot object"
// @Failure 400 {object} Error "Bad Request"
// @Failure 500 {object} Error "Internal Server Error"
// @Failure 404 {object} Error "Repo Not Found"
// @Router /api/repos/{name}/snapshots [post]
func apiSnapshotsCreateFromRepository(c *gin.Context) {
var (
err error
repo *deb.LocalRepo
snapshot *deb.Snapshot
b snapshotsCreateFromRepositoryParams
)
var b struct {
Name string `binding:"required"`
Description string
}
if c.Bind(&b) != nil {
return
}
@@ -188,14 +247,14 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
repo, err = collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
// including snapshot resource key
resources := []string{string(repo.Key()), "S" + b.Name}
taskName := fmt.Sprintf("Create snapshot of repo %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := collection.LoadComplete(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
@@ -218,18 +277,32 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
})
}
// PUT /api/snapshots/:name
type snapshotsUpdateParams struct {
// Change Name of snapshot
Name string ` json:"Name" example:"snap2"`
// Change Description of snapshot
Description string `json:"Description"`
}
// @Summary Update Snapshot
// @Description **Update snapshot metadata (Name, Description)**
// @Tags Snapshots
// @Param request body snapshotsUpdateParams true "Parameters"
// @Param name path string true "Snapshot name"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} deb.Snapshot "Updated snapshot object"
// @Failure 404 {object} Error "Snapshot Not Found"
// @Failure 409 {object} Error "Conflicting snapshot"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/snapshots/{name} [put]
func apiSnapshotsUpdate(c *gin.Context) {
var (
err error
snapshot *deb.Snapshot
b snapshotsUpdateParams
)
var b struct {
Name string
Description string
}
if c.Bind(&b) != nil {
return
}
@@ -240,13 +313,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
snapshot, err = collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
taskName := fmt.Sprintf("Update snapshot %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
_, err := collection.ByName(b.Name)
if err == nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
@@ -268,27 +341,48 @@ func apiSnapshotsUpdate(c *gin.Context) {
})
}
// GET /api/snapshots/:name
// @Summary Get Snapshot Info
// @Description **Query detailed information about a snapshot by name**
// @Tags Snapshots
// @Param name path string true "Name of the snapshot"
// @Produce json
// @Success 200 {object} deb.Snapshot "msg"
// @Failure 404 {object} Error "Snapshot Not Found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/snapshots/{name} [get]
func apiSnapshotsShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.SnapshotCollection()
snapshot, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(snapshot)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
c.JSON(200, snapshot)
}
// DELETE /api/snapshots/:name
// @Summary Delete Snapshot
// @Description **Delete snapshot by name**
// @Description Cannot drop snapshots that are published.
// @Description Needs force=1 to drop snapshots used as source by other snapshots.
// @Tags Snapshots
// @Param name path string true "Snapshot name"
// @Param force query string false "Force operation"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 ""
// @Failure 404 {object} Error "Snapshot Not Found"
// @Failure 409 {object} Error "Snapshot in use"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/snapshots/{name} [delete]
func apiSnapshotsDrop(c *gin.Context) {
name := c.Params.ByName("name")
force := c.Request.URL.Query().Get("force") == "1"
@@ -299,13 +393,13 @@ func apiSnapshotsDrop(c *gin.Context) {
snapshot, err := snapshotCollection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(snapshot.ResourceKey())}
taskName := fmt.Sprintf("Delete snapshot %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
published := publishedCollection.BySnapshot(snapshot)
if len(published) > 0 {
@@ -327,7 +421,19 @@ func apiSnapshotsDrop(c *gin.Context) {
})
}
// GET /api/snapshots/:name/diff/:withSnapshot
// @Summary Snapshot diff
// @Description **Return the diff between two snapshots (name & withSnapshot)**
// @Description Provide `onlyMatching=1` to return only packages present in both snapshots.
// @Description Otherwise, returns a `left` and `right` result providing packages only in the first and second snapshots
// @Tags Snapshots
// @Produce json
// @Param name path string true "Snapshot name"
// @Param withSnapshot path string true "Snapshot name to diff against"
// @Param onlyMatching query string false "Only return packages present in both snapshots"
// @Success 200 {array} deb.PackageDiff "Package Diff"
// @Failure 404 {object} Error "Snapshot Not Found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/snapshots/{name}/diff/{withSnapshot} [get]
func apiSnapshotsDiff(c *gin.Context) {
onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
@@ -336,32 +442,32 @@ func apiSnapshotsDiff(c *gin.Context) {
snapshotA, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(snapshotA)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
err = collection.LoadComplete(snapshotB)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
// Calculate diff
diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -378,22 +484,334 @@ func apiSnapshotsDiff(c *gin.Context) {
c.JSON(200, result)
}
// GET /api/snapshots/:name/packages
// @Summary List Snapshot Packages
// @Description **List all packages in snapshot or perform search on snapshot contents and return results**
// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q
// @Tags Snapshots
// @Produce json
// @Param name path string true "Snapshot to search"
// @Param q query string false "Package query (e.g Name%20(~%20matlab))"
// @Param withDeps query string false "Set to 1 to include dependencies when evaluating package query"
// @Param format query string false "Set to 'details' to return extra info about each package"
// @Param maximumVersion query string false "Set to 1 to only return the highest version for each package name"
// @Success 200 {array} string "Package info"
// @Failure 404 {object} Error "Snapshot Not Found"
// @Failure 500 {object} Error "Internal Server Error"
// @Router /api/snapshots/{name}/packages [get]
func apiSnapshotsSearchPackages(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.SnapshotCollection()
snapshot, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(snapshot)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
showPackages(c, snapshot.RefList(), collectionFactory)
}
type snapshotsMergeParams struct {
// List of snapshot names to be merged
Sources []string `binding:"required" json:"Sources" example:"snapshot1"`
}
// @Summary Snapshot Merge
// @Description **Merge several source snapshots into a new snapshot**
// @Description
// @Description Merge happens from left to right. By default, packages with the same name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
// @Description
// @Description If only one snapshot is specified, merge copies source into destination.
// @Tags Snapshots
// @Consume json
// @Produce json
// @Param name path string true "Name of the snapshot to be created"
// @Param latest query int false "merge only the latest version of each package"
// @Param no-remove query int false "all versions of packages are preserved during merge"
// @Param request body snapshotsMergeParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Success 201 {object} deb.Snapshot "Resulting snapshot object"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/snapshots/{name}/merge [post]
func apiSnapshotsMerge(c *gin.Context) {
var (
err error
snapshot *deb.Snapshot
body snapshotsMergeParams
)
name := c.Params.ByName("name")
if c.Bind(&body) != nil {
return
}
if len(body.Sources) < 1 {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
return
}
latest := c.Request.URL.Query().Get("latest") == "1"
noRemove := c.Request.URL.Query().Get("no-remove") == "1"
overrideMatching := !latest && !noRemove
if noRemove && latest {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("no-remove and latest are mutually exclusive"))
return
}
collectionFactory := context.NewCollectionFactory()
snapshotCollection := collectionFactory.SnapshotCollection()
sources := make([]*deb.Snapshot, len(body.Sources))
resources := make([]string, len(sources))
for i := range body.Sources {
sources[i], err = snapshotCollection.ByName(body.Sources[i])
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
resources[i] = string(sources[i].ResourceKey())
}
maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = snapshotCollection.LoadComplete(sources[0])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
result := sources[0].RefList()
for i := 1; i < len(sources); i++ {
err = snapshotCollection.LoadComplete(sources[i])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
result = result.Merge(sources[i].RefList(), overrideMatching, false)
}
if latest {
result.FilterLatestRefs()
}
sourceDescription := make([]string, len(sources))
for i, s := range sources {
sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
}
snapshot = deb.NewSnapshotFromRefList(name, sources, result,
fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
err = collectionFactory.SnapshotCollection().Add(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
})
}
type snapshotsPullParams struct {
// Source name to be searched for packages and dependencies
Source string `binding:"required" json:"Source" example:"source-snapshot"`
// Name of the snapshot to be created
Destination string `binding:"required" json:"Destination" example:"idestination-snapshot"`
// List of package queries (i.e. name of package to be pulled from `Source`)
Queries []string `binding:"required" json:"Queries" example:"xserver-xorg"`
// List of architectures (optional)
Architectures []string ` json:"Architectures" example:"amd64, armhf"`
}
// @Summary Snapshot Pull
// @Description **Pulls new packages and dependencies from a source snapshot into a new snapshot**
// @Description
// @Description May also upgrade package versions if name snapshot already contains packages being pulled. New snapshot `Destination` is created as result of this process.
// @Description If architectures are limited (with config architectures or parameter `Architectures`, only mentioned architectures are processed, otherwise aptly will process all architectures in the snapshot.
// @Description If following dependencies by source is enabled (using dependencyFollowSource config), pulling binary packages would also pull corresponding source packages as well.
// @Description By default aptly would remove packages matching name and architecture while importing: e.g. when importing software_1.3_amd64, package software_1.2.9_amd64 would be removed.
// @Description
// @Description With flag `no-remove` both package versions would stay in the snapshot.
// @Description
// @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled.
// @Tags Snapshots
// @Param request body snapshotsPullParams true "Parameters"
// @Param name path string true "Name of the snapshot to be created"
// @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable"
// @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable"
// @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable"
// @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable"
// @Param _async query bool false "Run in background and return task object"
// @Consume json
// @Produce json
// @Success 200 {object} deb.Snapshot "Resulting Snapshot object"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/snapshots/{name}/pull [post]
func apiSnapshotsPull(c *gin.Context) {
var (
err error
destinationSnapshot *deb.Snapshot
body snapshotsPullParams
)
name := c.Params.ByName("name")
if err = c.BindJSON(&body); err != nil {
AbortWithJSONError(c, http.StatusBadRequest, err)
return
}
allMatches := c.Request.URL.Query().Get("all-matches") == "1"
dryRun := c.Request.URL.Query().Get("dry-run") == "1"
noDeps := c.Request.URL.Query().Get("no-deps") == "1"
noRemove := c.Request.URL.Query().Get("no-remove") == "1"
collectionFactory := context.NewCollectionFactory()
// Load <name> snapshot
toSnapshot, err := collectionFactory.SnapshotCollection().ByName(name)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
// Load <Source> snapshot
sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// convert snapshots to package list
toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
toPackageList.PrepareIndex()
sourcePackageList.PrepareIndex()
var architecturesList []string
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = toPackageList.Architectures(false)
}
architecturesList = append(architecturesList, body.Architectures...)
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
for _, arch := range architecturesList {
archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
}
queries := make([]deb.PackageQuery, len(body.Queries))
for i, q := range body.Queries {
queries[i], err = query.Parse(q)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Add architecture filter
queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
}
// Filter with dependencies as requested
destinationPackageList, err := sourcePackageList.Filter(deb.FilterOptions{
Queries: queries,
WithDependencies: !noDeps,
Source: toPackageList,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
Progress: context.Progress(),
})
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
destinationPackageList.PrepareIndex()
removedPackages := []string{}
addedPackages := []string{}
alreadySeen := map[string]bool{}
destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key]
// If we haven't seen such name-architecture pair and were instructed to remove, remove it
if !noRemove && !seen {
// Remove all packages with the same name and architecture
packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
for _, p := range packageSearchResults {
toPackageList.Remove(p)
removedPackages = append(removedPackages, p.String())
}
}
// If !allMatches, add only first matching name-arch package
if !seen || allMatches {
toPackageList.Add(pkg)
addedPackages = append(addedPackages, pkg.String())
}
alreadySeen[key] = true
return nil
})
alreadySeen = nil
if dryRun {
response := struct {
AddedPackages []string `json:"added_packages"`
RemovedPackages []string `json:"removed_packages"`
}{
AddedPackages: addedPackages,
RemovedPackages: removedPackages,
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
}
// Create <destination> snapshot
destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
})
}
api/storage.go
+45
View File
@@ -0,0 +1,45 @@
package api
import (
"fmt"
"syscall"
"github.com/gin-gonic/gin"
)
type diskFree struct {
// Storage size [MiB]
Total uint64
// Available Storage [MiB]
Free uint64
// Percentage Full
PercentFull float32
}
// @Summary Get Storage Utilization
// @Description **Get disk free information of aptly storage**
// @Description
// @Description Units in MiB.
// @Tags Status
// @Produce json
// @Success 200 {object} diskFree "Storage information"
// @Failure 400 {object} Error "Internal Error"
// @Router /api/storage [get]
func apiDiskFree(c *gin.Context) {
var df diskFree
fs := context.Config().GetRootDir()
var stat syscall.Statfs_t
err := syscall.Statfs(fs, &stat)
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
return
}
df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
c.JSON(200, df)
}
api/task.go
+84 -36
View File
@@ -1,155 +1,203 @@
package api
import (
"fmt"
"net/http"
"strconv"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
)
// GET /tasks
// @Summary List Tasks
// @Description **Get list of available tasks. Each task is returned as in “show” API**
// @Tags Tasks
// @Produce json
// @Success 200 {array} task.Task
// @Router /api/tasks [get]
func apiTasksList(c *gin.Context) {
list := context.TaskList()
c.JSON(200, list.GetTasks())
}
// POST /tasks-clear
// @Summary Clear Tasks
// @Description **Removes finished and failed tasks from internal task list**
// @Tags Tasks
// @Produce json
// @Success 200 ""
// @Router /api/tasks-clear [post]
func apiTasksClear(c *gin.Context) {
list := context.TaskList()
list.Clear()
c.JSON(200, gin.H{})
}
// GET /tasks-wait
// @Summary Wait for all Tasks
// @Description **Waits for and returns when all running tasks are complete**
// @Tags Tasks
// @Produce json
// @Success 200 ""
// @Router /api/tasks-wait [get]
func apiTasksWait(c *gin.Context) {
list := context.TaskList()
list.Wait()
c.JSON(200, gin.H{})
}
// GET /tasks/:id/wait
// @Summary Wait for Task
// @Description **Waits for and returns when given Task ID is complete**
// @Tags Tasks
// @Produce json
// @Param id path int true "Task ID"
// @Success 200 {object} task.Task
// @Failure 500 {object} Error "invalid syntax, bad id?"
// @Failure 400 {object} Error "Task Not Found"
// @Router /api/tasks/{id}/wait [get]
func apiTasksWaitForTaskByID(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
task, err := list.WaitForTaskByID(int(id))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
c.JSON(200, task)
}
// GET /tasks/:id
// @Summary Get Task Info
// @Description **Return task information for a given ID**
// @Tags Tasks
// @Produce plain
// @Param id path int true "Task ID"
// @Success 200 {object} task.Task
// @Failure 500 {object} Error "invalid syntax, bad id?"
// @Failure 404 {object} Error "Task Not Found"
// @Router /api/tasks/{id} [get]
func apiTasksShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var task task.Task
task, err = list.GetTaskByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
c.JSON(200, task)
}
// GET /tasks/:id/output
// @Summary Get Task Output
// @Description **Return task output for a given ID**
// @Tags Tasks
// @Produce plain
// @Param id path int true "Task ID"
// @Success 200 {object} string "Task output"
// @Failure 500 {object} Error "invalid syntax, bad ID?"
// @Failure 404 {object} Error "Task Not Found"
// @Router /api/tasks/{id}/output [get]
func apiTasksOutputShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var output string
output, err = list.GetTaskOutputByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
c.JSON(200, output)
}
// GET /tasks/:id/detail
// @Summary Get Task Details
// @Description **Return task detail for a given ID**
// @Tags Tasks
// @Produce json
// @Param id path int true "Task ID"
// @Success 200 {object} string "Task detail"
// @Failure 500 {object} Error "invalid syntax, bad ID?"
// @Failure 404 {object} Error "Task Not Found"
// @Router /api/tasks/{id}/detail [get]
func apiTasksDetailShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var detail interface{}
detail, err = list.GetTaskDetailByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
c.JSON(200, detail)
}
// GET /tasks/:id/return_value
// @Summary Get Task Return Value
// @Description **Return task return value (status code) by given ID**
// @Tags Tasks
// @Produce plain
// @Param id path int true "Task ID"
// @Success 200 {object} string "msg"
// @Failure 500 {object} Error "invalid syntax, bad ID?"
// @Failure 404 {object} Error "Not Found"
// @Router /api/tasks/{id}/return_value [get]
func apiTasksReturnValueShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
output, err := list.GetTaskReturnValueByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
c.JSON(200, output)
}
// DELETE /tasks/:id
// @Summary Delete Task
// @Description **Delete completed task by given ID. Does not stop task execution**
// @Tags Tasks
// @Produce json
// @Param id path int true "Task ID"
// @Success 200 {object} task.Task
// @Failure 500 {object} Error "invalid syntax, bad ID?"
// @Failure 400 {object} Error "Task in progress or not found"
// @Router /api/tasks/{id} [delete]
func apiTasksDelete(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var delTask task.Task
delTask, err = list.DeleteTaskByID(int(id))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
c.JSON(200, delTask)
}
// POST /tasks-dummy
func apiTasksDummy(c *gin.Context) {
resources := []string{"dummy"}
taskName := fmt.Sprintf("Dummy task")
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
out.Printf("Dummy task started\n")
detail.Store([]int{1, 2, 3})
out.Printf("Dummy task finished\n")
return &task.ProcessReturnValue{Code: http.StatusTeapot, Value: []int{1, 2, 3}}, nil
})
}
api/task_test.go
-75
View File
@@ -1,75 +0,0 @@
package api
import (
"encoding/json"
"fmt"
"time"
"github.com/aptly-dev/aptly/task"
. "gopkg.in/check.v1"
)
type TaskSuite struct {
ApiSuite
}
var _ = Suite(&TaskSuite{})
func (s *TaskSuite) TestTasksDummy(c *C) {
response, _ := s.HTTPRequest("POST", "/api/tasks-dummy", nil)
c.Check(response.Code, Equals, 418)
c.Check(response.Body.String(), Equals, "[1,2,3]")
}
func (s *TaskSuite) TestTasksDummyAsync(c *C) {
response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
c.Check(response.Code, Equals, 202)
var t task.Task
err := json.Unmarshal(response.Body.Bytes(), &t)
c.Assert(err, IsNil)
c.Check(t.Name, Equals, "Dummy task")
response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/wait", t.ID), nil)
err = json.Unmarshal(response.Body.Bytes(), &t)
c.Assert(err, IsNil)
c.Check(t.State, Equals, task.SUCCEEDED)
response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/detail", t.ID), nil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Equals, "[1,2,3]")
response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/output", t.ID), nil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "\"Dummy task started.*")
}
func (s *TaskSuite) TestTaskDelete(c *C) {
response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
c.Check(response.Code, Equals, 202)
c.Check(response.Body.String(), Equals, "{\"Name\":\"Dummy task\",\"ID\":1,\"State\":0}")
// Give the task time to start
time.Sleep(time.Second)
response, _ = s.HTTPRequest("DELETE", "/api/tasks/1", nil)
c.Check(response.Code, Equals, 200)
}
func (s *TaskSuite) TestTasksClear(c *C) {
response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
c.Check(response.Code, Equals, 202)
var t task.Task
err := json.Unmarshal(response.Body.Bytes(), &t)
c.Assert(err, IsNil)
c.Check(t.Name, Equals, "Dummy task")
response, _ = s.HTTPRequest("GET", "/api/tasks-wait", nil)
c.Check(response.Code, Equals, 200)
response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
c.Check(response.Code, Equals, 200)
var ts []task.Task
err = json.Unmarshal(response.Body.Bytes(), &ts)
c.Assert(err, IsNil)
c.Check(len(ts), Equals, 1)
c.Check(ts[0].State, Equals, task.SUCCEEDED)
response, _ = s.HTTPRequest("POST", "/api/tasks-clear", nil)
c.Check(response.Code, Equals, 200)
response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Equals, "null")
}
aptly-api.service
-12
View File
@@ -1,12 +0,0 @@
[Unit]
Description=APT repository API
After=network.target
Documentation=man:aptly(1)
Documentation=https://www.aptly.info/doc/api/
[Service]
Type=simple
ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
[Install]
WantedBy=multi-user.target
aptly.service
-12
View File
@@ -1,12 +0,0 @@
[Unit]
Description=APT repository server
After=network.target
Documentation=man:aptly(1)
Documentation=https://www.aptly.info/doc/commands/
[Service]
Type=simple
ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
[Install]
WantedBy=multi-user.target
aptly/conf.go
+4
View File
@@ -0,0 +1,4 @@
package aptly
// Default aptly.conf (filled in at link time)
var AptlyConf []byte
aptly/const.go
+3
View File
@@ -0,0 +1,3 @@
package aptly
var DistributionFocal = "focal"
aptly/interfaces.go
+5 -3
View File
@@ -35,8 +35,8 @@ type PackagePool interface {
Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
// Stat returns Unix stat(2) info
Stat(path string) (os.FileInfo, error)
// Size returns the size of the given file in bytes.
Size(path string) (size int64, err error)
// Open returns ReadSeekerCloser to access the file
Open(path string) (ReadSeekerCloser, error)
// FilepathList returns file paths of all the files in the pool
@@ -47,6 +47,8 @@ type PackagePool interface {
// LocalPackagePool is implemented by PackagePools residing on the same filesystem
type LocalPackagePool interface {
// Stat returns Unix stat(2) info
Stat(path string) (os.FileInfo, error)
// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
GenerateTempPath(filename string) (string, error)
// Link generates hardlink to destination path
@@ -70,7 +72,7 @@ type PublishedStorage interface {
// Remove removes single file under public path
Remove(path string) error
// LinkFromPool links package file from pool to dist's pool location
LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
// Filelist returns list of files under prefix
Filelist(prefix string) ([]string, error)
// RenameFile renames (moves) file
azure/azure.go
+135 -1
View File
@@ -1,2 +1,136 @@
// Package azure handles publishing to Azure Storage
package azure
// Package azure handles publishing to Azure Storage
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/aptly-dev/aptly/aptly"
)
func isBlobNotFound(err error) bool {
var respErr *azcore.ResponseError
if errors.As(err, &respErr) {
return respErr.StatusCode == 404 // BlobNotFound
}
return false
}
type azContext struct {
client *azblob.Client
container string
prefix string
}
func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
if err != nil {
return nil, err
}
if endpoint == "" {
endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
}
serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
if err != nil {
return nil, err
}
result := &azContext{
client: serviceClient,
container: container,
prefix: prefix,
}
return result, nil
}
func (az *azContext) blobPath(path string) string {
return filepath.Join(az.prefix, path)
}
func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
const delimiter = "/"
paths = make([]string, 0, 1024)
md5s = make([]string, 0, 1024)
prefix = filepath.Join(az.prefix, prefix)
if prefix != "" {
prefix += delimiter
}
ctx := context.Background()
maxResults := int32(1)
pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
Prefix: &prefix,
MaxResults: &maxResults,
Include: azblob.ListBlobsInclude{Metadata: true},
})
// Iterate over each page
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
}
for _, blob := range page.Segment.BlobItems {
if prefix == "" {
paths = append(paths, *blob.Name)
} else {
name := *blob.Name
paths = append(paths, name[len(prefix):])
}
b := *blob
md5 := b.Properties.ContentMD5
md5s = append(md5s, fmt.Sprintf("%x", md5))
}
if progress != nil {
time.Sleep(time.Duration(500) * time.Millisecond)
progress.AddBar(1)
}
}
return paths, md5s, nil
}
func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
uploadOptions := &azblob.UploadFileOptions{
BlockSize: 4 * 1024 * 1024,
Concurrency: 8,
}
path := az.blobPath(blobName)
if len(sourceMD5) > 0 {
decodedMD5, err := hex.DecodeString(sourceMD5)
if err != nil {
return err
}
uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
BlobContentMD5: decodedMD5,
}
}
var err error
if file, ok := source.(*os.File); ok {
_, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
}
return err
}
// String
func (az *azContext) String() string {
return fmt.Sprintf("Azure: %s/%s", az.container, az.prefix)
}
azure/package_pool.go
+215
View File
@@ -0,0 +1,215 @@
package azure
import (
"context"
"os"
"path/filepath"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
"github.com/pkg/errors"
)
type PackagePool struct {
az *azContext
}
// Check interface
var (
_ aptly.PackagePool = (*PackagePool)(nil)
)
// NewPackagePool creates published storage from Azure storage credentials
func NewPackagePool(accountName, accountKey, container, prefix, endpoint string) (*PackagePool, error) {
azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
if err != nil {
return nil, err
}
return &PackagePool{az: azctx}, nil
}
// String
func (pool *PackagePool) String() string {
return pool.az.String()
}
func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string {
hash := checksums.SHA256
// Use the same path as the file pool, for compat reasons.
return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
}
func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
targetChecksums, err := checksumStorage.Get(poolPath)
if err != nil {
return nil, err
}
if targetChecksums == nil {
// we don't have checksums stored yet for this file
download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
if err != nil {
if isBlobNotFound(err) {
return nil, nil
}
return nil, errors.Wrapf(err, "error downloading blob at %s", poolPath)
}
targetChecksums = &utils.ChecksumInfo{}
*targetChecksums, err = utils.ChecksumsForReader(download.Body)
if err != nil {
return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
}
err = checksumStorage.Update(poolPath, targetChecksums)
if err != nil {
return nil, err
}
}
return targetChecksums, nil
}
func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
if progress != nil {
progress.InitBar(0, false, aptly.BarGeneralBuildFileList)
defer progress.ShutdownBar()
}
paths, _, err := pool.az.internalFilelist("", progress)
return paths, err
}
func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) {
return "", errors.New("Azure package pool does not support legacy paths")
}
func (pool *PackagePool) Size(path string) (int64, error) {
serviceClient := pool.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(pool.az.container)
blobClient := containerClient.NewBlobClient(path)
props, err := blobClient.GetProperties(context.TODO(), nil)
if err != nil {
return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
}
return *props.ContentLength, nil
}
func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
temp, err := os.CreateTemp("", "blob-download")
if err != nil {
return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
}
defer os.Remove(temp.Name())
_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
if err != nil {
return nil, errors.Wrapf(err, "error downloading blob %s", path)
}
return temp, nil
}
func (pool *PackagePool) Remove(path string) (int64, error) {
serviceClient := pool.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(pool.az.container)
blobClient := containerClient.NewBlobClient(path)
props, err := blobClient.GetProperties(context.TODO(), nil)
if err != nil {
return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
}
_, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
if err != nil {
return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
}
return *props.ContentLength, nil
}
func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" {
// need to update checksums, MD5 and SHA256 should be always defined
var err error
*checksums, err = utils.ChecksumsForFile(srcPath)
if err != nil {
return "", err
}
}
path := pool.buildPoolPath(basename, checksums)
targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
if err != nil {
return "", err
} else if targetChecksums != nil {
// target already exists
*checksums = *targetChecksums
return path, nil
}
source, err := os.Open(srcPath)
if err != nil {
return "", err
}
defer source.Close()
err = pool.az.putFile(path, source, checksums.MD5)
if err != nil {
return "", err
}
if !checksums.Complete() {
// need full checksums here
*checksums, err = utils.ChecksumsForFile(srcPath)
if err != nil {
return "", err
}
}
err = checksumStorage.Update(path, checksums)
if err != nil {
return "", err
}
return path, nil
}
func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
if poolPath == "" {
if checksums.SHA256 != "" {
poolPath = pool.buildPoolPath(basename, checksums)
} else {
// No checksums or pool path, so no idea what file to look for.
return "", false, nil
}
}
size, err := pool.Size(poolPath)
if err != nil {
return "", false, err
} else if size != checksums.Size {
return "", false, nil
}
targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage)
if err != nil {
return "", false, err
} else if targetChecksums == nil {
return "", false, nil
}
if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
// wrong file?
return "", false, nil
}
// fill back checksums
*checksums = *targetChecksums
return poolPath, true, nil
}
azure/package_pool_test.go
+257
View File
@@ -0,0 +1,257 @@
package azure
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
type PackagePoolSuite struct {
accountName, accountKey, endpoint string
pool, prefixedPool *PackagePool
debFile string
cs aptly.ChecksumStorage
}
var _ = Suite(&PackagePoolSuite{})
func (s *PackagePoolSuite) SetUpSuite(c *C) {
s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
if s.accountName == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCOUNT not set.")
}
s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
if s.accountKey == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
}
s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
}
func (s *PackagePoolSuite) SetUpTest(c *C) {
container := randContainer()
prefix := "lala"
var err error
s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil)
publicAccessType := azblob.PublicAccessTypeContainer
_, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
c.Assert(err, IsNil)
s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
c.Assert(err, IsNil)
_, _File, _, _ := runtime.Caller(0)
s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
s.cs = files.NewMockChecksumStorage()
}
func (s *PackagePoolSuite) TestFilepathList(c *C) {
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
list, err = s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{
"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb",
"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb",
})
}
func (s *PackagePoolSuite) TestRemove(c *C) {
s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
c.Check(err, IsNil)
c.Check(size, Equals, int64(2738))
_, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"})
}
func (s *PackagePoolSuite) TestImportOk(c *C) {
var checksum utils.ChecksumInfo
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// SHA256 should be automatically calculated
c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
// checksum storage is filled with new checksum
c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
size, err := s.pool.Size(path)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(2738))
// import as different name
checksum = utils.ChecksumInfo{}
path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
// checksum storage is filled with new checksum
c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
// double import, should be ok
checksum = utils.ChecksumInfo{}
path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// checksum is filled back based on checksum storage
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// clear checksum storage, and do double-import
delete(s.cs.(*files.MockChecksumStorage).Store, path)
checksum = utils.ChecksumInfo{}
path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// checksum is filled back based on re-calculation of file in the pool
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// import under new name, but with path-relevant checksums already filled in
checksum = utils.ChecksumInfo{SHA256: checksum.SHA256}
path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
// checksum is filled back based on re-calculation of source file
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
}
func (s *PackagePoolSuite) TestVerify(c *C) {
// file doesn't exist yet
ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
// import file
checksum := utils.ChecksumInfo{}
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// check existence
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, ppath)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence with fixed path
checksum = utils.ChecksumInfo{Size: checksum.Size}
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, but with checksums missing (that aren't needed to find the path)
checksum.SHA512 = ""
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
// checksum is filled back based on checksum storage
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, with missing checksum info but correct path and size available
checksum = utils.ChecksumInfo{Size: checksum.Size}
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
// checksum is filled back based on checksum storage
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, with wrong checksum info but correct path and size available
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{
SHA256: "abc",
Size: checksum.Size,
}, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
// check existence, with missing checksums (that aren't needed to find the path)
// and no info in checksum storage
delete(s.cs.(*files.MockChecksumStorage).Store, path)
checksum.SHA512 = ""
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
// checksum is filled back based on re-calculation
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, with wrong size
checksum = utils.ChecksumInfo{Size: 13455}
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
// check existence, with empty checksum info
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
}
func (s *PackagePoolSuite) TestImportNotExist(c *C) {
_, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs)
c.Check(err, ErrorMatches, ".*no such file or directory")
}
func (s *PackagePoolSuite) TestSize(c *C) {
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
c.Check(err, IsNil)
size, err := s.pool.Size(path)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(2738))
_, err = s.pool.Size("do/es/ntexist")
c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
}
func (s *PackagePoolSuite) TestOpen(c *C) {
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
c.Check(err, IsNil)
f, err := s.pool.Open(path)
c.Assert(err, IsNil)
contents, err := ioutil.ReadAll(f)
c.Assert(err, IsNil)
c.Check(len(contents), Equals, 2738)
c.Check(f.Close(), IsNil)
_, err = s.pool.Open("do/es/ntexist")
c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
}
azure/public.go
+90 -178
View File
@@ -2,27 +2,25 @@ package azure
import (
"context"
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
"github.com/pborman/uuid"
"github.com/pkg/errors"
)
// PublishedStorage abstract file system with published files (actually hosted on Azure)
type PublishedStorage struct {
container azblob.ContainerURL
prefix string
pathCache map[string]string
az *azContext
pathCache map[string]map[string]string
}
// Check interface
@@ -30,60 +28,23 @@ var (
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
)
func isEmulatorEndpoint(endpoint string) bool {
if h, _, err := net.SplitHostPort(endpoint); err == nil {
endpoint = h
}
if endpoint == "localhost" {
return true
}
// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
// In this case, eliminate the '[' and ']' in the URL.
// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
endpoint = endpoint[1 : len(endpoint)-1]
}
return net.ParseIP(endpoint) != nil
}
// NewPublishedStorage creates published storage from Azure storage credentials
func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
if err != nil {
return nil, err
}
if endpoint == "" {
endpoint = "blob.core.windows.net"
}
var url *url.URL
if isEmulatorEndpoint(endpoint) {
url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
} else {
url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
}
if err != nil {
return nil, err
}
containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
result := &PublishedStorage{
container: containerURL,
prefix: prefix,
}
return result, nil
return &PublishedStorage{az: azctx}, nil
}
// String
func (storage *PublishedStorage) String() string {
return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
return storage.az.String()
}
// MkDir creates directory recursively under public path
func (storage *PublishedStorage) MkDir(path string) error {
func (storage *PublishedStorage) MkDir(_ string) error {
// no op for Azure
return nil
}
@@ -106,7 +67,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
}
defer source.Close()
err = storage.putFile(path, source, sourceMD5)
err = storage.az.putFile(path, source, sourceMD5)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
}
@@ -114,46 +75,17 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
return err
}
// putFile uploads file-like object to
func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
path = filepath.Join(storage.prefix, path)
blob := storage.container.NewBlockBlobURL(path)
uploadOptions := azblob.UploadStreamToBlockBlobOptions{
BufferSize: 4 * 1024 * 1024,
MaxBuffers: 8,
}
if len(sourceMD5) > 0 {
decodedMD5, err := hex.DecodeString(sourceMD5)
if err != nil {
return err
}
uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
ContentMD5: decodedMD5,
}
}
_, err := azblob.UploadStreamToBlockBlob(
context.Background(),
source,
blob,
uploadOptions,
)
return err
}
// RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
path = storage.az.blobPath(path)
filelist, err := storage.Filelist(path)
if err != nil {
return err
}
for _, filename := range filelist {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
blob := filepath.Join(path, filename)
_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
if err != nil {
return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
}
@@ -164,8 +96,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
// Remove removes single file under public path
func (storage *PublishedStorage) Remove(path string) error {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
path = storage.az.blobPath(path)
_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
}
@@ -174,31 +106,38 @@ func (storage *PublishedStorage) Remove(path string) error {
// LinkFromPool links package file from pool to dist's pool location
//
// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
// publishedPrefix is desired prefix for the location in the pool.
// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is filepath to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
relPath := filepath.Join(publishedDirectory, fileName)
poolPath := filepath.Join(storage.prefix, relPath)
relFilePath := filepath.Join(publishedRelPath, fileName)
prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
poolPath := storage.az.blobPath(prefixRelFilePath)
if storage.pathCache == nil {
paths, md5s, err := storage.internalFilelist("")
storage.pathCache = make(map[string]map[string]string)
}
pathCache := storage.pathCache[publishedPrefix]
if pathCache == nil {
paths, md5s, err := storage.az.internalFilelist(publishedPrefix, nil)
if err != nil {
return fmt.Errorf("error caching paths under prefix: %s", err)
}
storage.pathCache = make(map[string]string, len(paths))
pathCache = make(map[string]string, len(paths))
for i := range paths {
storage.pathCache[paths[i]] = md5s[i]
pathCache[paths[i]] = md5s[i]
}
storage.pathCache[publishedPrefix] = pathCache
}
destinationMD5, exists := storage.pathCache[relPath]
destinationMD5, exists := pathCache[relFilePath]
sourceMD5 := sourceChecksums.MD5
if exists {
@@ -221,9 +160,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
}
defer source.Close()
err = storage.putFile(relPath, source, sourceMD5)
err = storage.az.putFile(relFilePath, source, sourceMD5)
if err == nil {
storage.pathCache[relPath] = sourceMD5
pathCache[relFilePath] = sourceMD5
} else {
err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
}
@@ -231,98 +170,65 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
return err
}
func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
const delimiter = "/"
paths = make([]string, 0, 1024)
md5s = make([]string, 0, 1024)
prefix = filepath.Join(storage.prefix, prefix)
if prefix != "" {
prefix += delimiter
}
for marker := (azblob.Marker{}); marker.NotDone(); {
listBlob, err := storage.container.ListBlobsFlatSegment(
context.Background(), marker, azblob.ListBlobsSegmentOptions{
Prefix: prefix,
MaxResults: 1000,
Details: azblob.BlobListingDetails{Metadata: true}})
if err != nil {
return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
}
marker = listBlob.NextMarker
for _, blob := range listBlob.Segment.BlobItems {
if prefix == "" {
paths = append(paths, blob.Name)
} else {
paths = append(paths, blob.Name[len(prefix):])
}
md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
}
}
return paths, md5s, nil
}
// Filelist returns list of files under prefix
func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
paths, _, err := storage.internalFilelist(prefix)
paths, _, err := storage.az.internalFilelist(prefix, nil)
return paths, err
}
// Internal copy or move implementation
func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
const leaseDuration = 30
leaseID := uuid.NewRandom().String()
dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
if err != nil || leaseResp.StatusCode() != http.StatusCreated {
return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
serviceClient := storage.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(storage.az.container)
srcBlobClient := containerClient.NewBlobClient(src)
blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
if err != nil {
return fmt.Errorf("error acquiring lease on source blob %s", src)
}
defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{})
srcBlobLeaseID := leaseResp.LeaseID()
copyResp, err := dstBlobURL.StartCopyFromURL(
context.Background(),
srcBlobURL.URL(),
metadata,
azblob.ModifiedAccessConditions{},
azblob.BlobAccessConditions{},
azblob.DefaultAccessTier,
nil)
_, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
if err != nil {
return fmt.Errorf("error acquiring lease on source blob %s", src)
}
defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
dstBlobClient := containerClient.NewBlobClient(dst)
copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
Metadata: metadata,
})
if err != nil {
return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
}
copyStatus := copyResp.CopyStatus()
copyStatus := *copyResp.CopyStatus
for {
if copyStatus == azblob.CopyStatusSuccess {
if copyStatus == blob.CopyStatusTypeSuccess {
if move {
_, err = srcBlobURL.Delete(
context.Background(),
azblob.DeleteSnapshotsOptionNone,
azblob.BlobAccessConditions{
LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
})
_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
AccessConditions: &blob.AccessConditions{
LeaseAccessConditions: &blob.LeaseAccessConditions{
LeaseID: &leaseID,
},
},
})
return err
}
return nil
} else if copyStatus == azblob.CopyStatusPending {
} else if copyStatus == blob.CopyStatusTypePending {
time.Sleep(1 * time.Second)
blobPropsResp, err := dstBlobURL.GetProperties(
context.Background(),
azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
azblob.ClientProvidedKeyOptions{})
getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
if err != nil {
return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
return fmt.Errorf("error getting copy progress %s", dst)
}
copyStatus = blobPropsResp.CopyStatus()
copyStatus = *getMetadata.CopyStatus
_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
_, err = blobLeaseClient.RenewLease(context.Background(), nil)
if err != nil {
return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
return fmt.Errorf("error renewing source blob lease %s", src)
}
} else {
return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
@@ -337,7 +243,9 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
// SymLink creates a copy of src file and adds link information as meta data
func (storage *PublishedStorage) SymLink(src string, dst string) error {
return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
metadata := make(map[string]*string)
metadata["SymLink"] = &src
return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
}
// HardLink using symlink functionality as hard links do not exist
@@ -347,29 +255,33 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
// FileExists returns true if path exists
func (storage *PublishedStorage) FileExists(path string) (bool, error) {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
serviceClient := storage.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(storage.az.container)
blobClient := containerClient.NewBlobClient(path)
_, err := blobClient.GetProperties(context.Background(), nil)
if err != nil {
storageError, ok := err.(azblob.StorageError)
if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
if isBlobNotFound(err) {
return false, nil
}
return false, err
} else if resp.StatusCode() == http.StatusOK {
return true, nil
return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
}
return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
return true, nil
}
// ReadLink returns the symbolic link pointed to by path.
// This simply reads text file created with SymLink
func (storage *PublishedStorage) ReadLink(path string) (string, error) {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
serviceClient := storage.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(storage.az.container)
blobClient := containerClient.NewBlobClient(path)
props, err := blobClient.GetProperties(context.Background(), nil)
if err != nil {
return "", err
} else if resp.StatusCode() != http.StatusOK {
return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
return "", fmt.Errorf("failed to get blob properties: %v", err)
}
return resp.NewMetadata()["SymLink"], nil
metadata := props.Metadata
if originalBlob, exists := metadata["original_blob"]; exists {
return *originalBlob, nil
}
return "", fmt.Errorf("error reading link %s: %v", path, err)
}
azure/public_test.go
+36 -33
View File
@@ -7,8 +7,11 @@ import (
"io/ioutil"
"os"
"path/filepath"
"bytes"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
@@ -43,14 +46,14 @@ func randString(n int) string {
func (s *PublishedStorageSuite) SetUpSuite(c *C) {
s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
if s.accountName == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println("Please set the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCOUNT not set.")
}
s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
if s.accountKey == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println("Please set the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
@@ -66,8 +69,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil)
cnt := s.storage.container
_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
publicAccessType := azblob.PublicAccessTypeContainer
_, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
c.Assert(err, IsNil)
s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -75,41 +80,39 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
}
func (s *PublishedStorageSuite) TearDownTest(c *C) {
cnt := s.storage.container
_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
_, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
blob := s.storage.container.NewBlobURL(path)
resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
c.Assert(err, IsNil)
body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
data, err := ioutil.ReadAll(body)
data, err := ioutil.ReadAll(resp.Body)
c.Assert(err, IsNil)
return data
}
func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
_, err := s.storage.container.NewBlobURL(path).GetProperties(
context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
serviceClient := s.storage.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(s.storage.az.container)
blobClient := containerClient.NewBlobClient(path)
_, err := blobClient.GetProperties(context.Background(), nil)
c.Assert(err, NotNil)
storageError, ok := err.(azblob.StorageError)
storageError, ok := err.(*azcore.ResponseError)
c.Assert(ok, Equals, true)
c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
c.Assert(storageError.StatusCode, Equals, 404)
}
func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
hash := md5.Sum(data)
_, err := azblob.UploadBufferToBlockBlob(
context.Background(),
data,
s.storage.container.NewBlockBlobURL(path),
azblob.UploadToBlockBlobOptions{
BlobHTTPHeaders: azblob.BlobHTTPHeaders{
ContentMD5: hash[:],
},
})
uploadOptions := &azblob.UploadStreamOptions{
HTTPHeaders: &blob.HTTPHeaders{
BlobContentMD5: hash[:],
},
}
reader := bytes.NewReader(data)
_, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
c.Assert(err, IsNil)
}
@@ -129,7 +132,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
c.Check(err, IsNil)
c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.az.prefix, filename)), DeepEquals, content)
}
func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
@@ -300,45 +303,45 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Assert(err, IsNil)
// first link from pool
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// duplicate link from pool
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with conflict
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
c.Check(err, ErrorMatches, ".*file already exists and is different.*")
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with conflict and force
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
// for prefixed storage:
// first link from pool
err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
// 2nd link from pool, providing wrong path for source file
//
// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
// this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
s.prefixedStorage.pathCache = nil
err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with nested file name
err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
cmd/api_serve.go
+24 -9
View File
@@ -1,11 +1,15 @@
package cmd
import (
stdcontext "context"
"errors"
"fmt"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"syscall"
"github.com/aptly-dev/aptly/api"
"github.com/aptly-dev/aptly/systemd/activation"
@@ -30,7 +34,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
// anything else must fail.
// E.g.: Running the service under a different user may lead to a rootDir
// that exists but is not usable due to access permissions.
err = utils.DirIsAccessible(context.Config().RootDir)
err = utils.DirIsAccessible(context.Config().GetRootDir())
if err != nil {
return err
}
@@ -55,6 +59,19 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
listen := context.Flags().Lookup("listen").Value.String()
fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
server := http.Server{Handler: api.Router(context)}
sigchan := make(chan os.Signal, 1)
signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
go (func() {
if _, ok := <-sigchan; ok {
fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
context.TaskList().Wait()
server.Shutdown(stdcontext.Background())
}
})()
defer close(sigchan)
listenURL, err := url.Parse(listen)
if err == nil && listenURL.Scheme == "unix" {
file := listenURL.Path
@@ -67,19 +84,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
}
defer listener.Close()
err = http.Serve(listener, api.Router(context))
if err != nil {
return fmt.Errorf("unable to serve: %s", err)
}
return nil
err = server.Serve(listener)
} else {
server.Addr = listen
err = server.ListenAndServe()
}
err = http.ListenAndServe(listen, api.Router(context))
if err != nil {
if err != nil && !errors.Is(err, http.ErrServerClosed) {
return fmt.Errorf("unable to serve: %s", err)
}
return err
return nil
}
func makeCmdAPIServe() *commander.Command {
cmd/cmd.go
+1 -1
View File
@@ -118,7 +118,7 @@ package environment to new version.`,
cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
cmd.Flag.String("config", "", "location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)")
cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
if aptly.EnableDebug {
cmd/config_show.go
+18 -6
View File
@@ -5,19 +5,30 @@ import (
"fmt"
"github.com/smira/commander"
"gopkg.in/yaml.v3"
)
func aptlyConfigShow(cmd *commander.Command, args []string) error {
func aptlyConfigShow(_ *commander.Command, _ []string) error {
showYaml := context.Flags().Lookup("yaml").Value.Get().(bool)
config := context.Config()
prettyJSON, err := json.MarshalIndent(config, "", " ")
if err != nil {
return fmt.Errorf("unable to dump the config file: %s", err)
if showYaml {
yamlData, err := yaml.Marshal(&config)
if err != nil {
return fmt.Errorf("error marshaling to YAML: %s", err)
}
fmt.Println(string(yamlData))
} else {
prettyJSON, err := json.MarshalIndent(config, "", " ")
if err != nil {
return fmt.Errorf("unable to dump the config file: %s", err)
}
fmt.Println(string(prettyJSON))
}
fmt.Println(string(prettyJSON))
return nil
}
@@ -35,5 +46,6 @@ Example:
`,
}
cmd.Flag.Bool("yaml", false, "show yaml config")
return cmd
}
cmd/graph.go
+4 -17
View File
@@ -4,13 +4,11 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/utils"
@@ -36,7 +34,7 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
buf := bytes.NewBufferString(graph.String())
tempfile, err := ioutil.TempFile("", "aptly-graph")
tempfile, err := os.CreateTemp("", "aptly-graph")
if err != nil {
return err
}
@@ -80,10 +78,6 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
return err
}
defer func() {
_ = os.Remove(tempfilename)
}()
if output != "" {
err = utils.CopyFile(tempfilename, output)
if err != nil {
@@ -91,23 +85,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
}
fmt.Printf("Output saved to %s\n", output)
_ = os.Remove(tempfilename)
} else {
command := getOpenCommand()
fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
fmt.Printf("Displaying %s file: %s %s\n", format, command, tempfilename)
args := strings.Split(command, " ")
viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
viewer.Stderr = os.Stderr
if err = viewer.Start(); err == nil {
// Wait for a second so that the visualizer has a chance to
// open the input file. This needs to be done even if we're
// waiting for the visualizer as it can be just a wrapper that
// spawns a browser tab and returns right away.
defer func(t <-chan time.Time) {
<-t
}(time.After(time.Second))
}
err = viewer.Start()
}
return err
cmd/mirror.go
+5 -5
View File
@@ -9,18 +9,18 @@ import (
)
func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
return nil, nil
}
keyRings := flags.Lookup("keyring").Value.Get().([]string)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
verifier := context.GetVerifier()
for _, keyRing := range keyRings {
verifier.AddKeyring(keyRing)
}
err := verifier.InitKeyring()
err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
if err != nil {
return nil, err
}
cmd/mirror_create.go
+7 -3
View File
@@ -20,6 +20,10 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
var (
mirrorName, archiveURL, distribution string
@@ -42,7 +46,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to create mirror: %s", err)
}
repo.Filter = context.Flags().Lookup("filter").Value.String()
repo.Filter = context.Flags().Lookup("filter").Value.String() // allows file/stdin with @
repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
@@ -59,7 +63,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
return fmt.Errorf("unable to fetch mirror: %s", err)
}
@@ -99,7 +103,7 @@ Example:
cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
cmd.Flag.String("filter", "", "filter packages in mirror")
AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
cmd/mirror_edit.go
+6 -3
View File
@@ -28,10 +28,11 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
}
fetchMirror := false
ignoreSignatures := context.Config().GpgDisableVerify
context.Flags().Visit(func(flag *flag.Flag) {
switch flag.Name {
case "filter":
repo.Filter = flag.Value.String()
repo.Filter = flag.Value.String() // allows file/stdin with @
case "filter-with-deps":
repo.FilterWithDeps = flag.Value.Get().(bool)
case "with-installer":
@@ -43,6 +44,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
case "archive-url":
repo.SetArchiveRoot(flag.Value.String())
fetchMirror = true
case "ignore-signatures":
ignoreSignatures = true
}
})
@@ -69,7 +72,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
@@ -101,7 +104,7 @@ Example:
}
cmd.Flag.String("archive-url", "", "archive url is the root of archive")
cmd.Flag.String("filter", "", "filter packages in mirror")
AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
cmd/mirror_list.go
+2 -2
View File
@@ -24,7 +24,7 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
return aptlyMirrorListTxt(cmd, args)
}
func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -65,7 +65,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
var err error
repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
cmd/mirror_show.go
+2 -2
View File
@@ -27,7 +27,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
return aptlyMirrorShowTxt(cmd, args)
}
func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
var err error
name := args[0]
@@ -93,7 +93,7 @@ func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
var err error
name := args[0]
cmd/mirror_update.go
+32 -6
View File
@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"os"
"strings"
"sync"
@@ -41,20 +42,24 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
}
}
ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
ignoreChecksums := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
verifier, err := getVerifier(context.Flags())
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("Downloading & parsing package files...\n")
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreSignatures, ignoreChecksums)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
@@ -161,7 +166,16 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
var e error
// provision download location
task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
} else {
var file *os.File
file, e = os.CreateTemp("", task.File.Filename)
if e == nil {
task.TempDownPath = file.Name()
file.Close()
}
}
if e != nil {
pushError(e)
continue
@@ -173,7 +187,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
repo.PackageURL(task.File.DownloadURL()).String(),
task.TempDownPath,
&task.File.Checksums,
ignoreMismatch)
ignoreChecksums)
if e != nil {
pushError(e)
continue
@@ -197,6 +211,18 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: %s", err)
}
defer func() {
for _, task := range queue {
if task.TempDownPath == "" {
continue
}
if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
}
}
}()
// Import downloaded files
context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
@@ -241,7 +267,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
context.Progress().Printf("\nMirror `%s` has been updated successfully.\n", repo.Name)
return err
}
cmd/package_search.go
+6 -1
View File
@@ -21,7 +21,11 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
}
if len(args) == 1 {
q, err = query.Parse(args[0])
value, err := GetStringOrFileContent(args[0])
if err != nil {
return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
}
q, err = query.Parse(value)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
@@ -49,6 +53,7 @@ func makeCmdPackageSearch() *commander.Command {
Long: `
Command search displays list of packages in whole DB that match package query.
Use '@file' to read query from file or '@-' for stdin.
If query is not specified, all the packages are displayed.
Example:
cmd/package_show.go
+7 -1
View File
@@ -66,7 +66,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
return commander.ErrCommandError
}
q, err := query.Parse(args[0])
value, err := GetStringOrFileContent(args[0])
if err != nil {
return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
}
q, err := query.Parse(value)
if err != nil {
return fmt.Errorf("unable to show: %s", err)
}
@@ -130,6 +134,8 @@ matching query. Information from Debian control file is displayed.
Optionally information about package files and
inclusion into mirrors/snapshots/local repos is shown.
Use '@file' to read query from file or '@-' for stdin.
Example:
$ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'
cmd/publish.go
+17 -1
View File
@@ -34,10 +34,26 @@ func makeCmdPublish() *commander.Command {
makeCmdPublishDrop(),
makeCmdPublishList(),
makeCmdPublishRepo(),
makeCmdPublishShow(),
makeCmdPublishSnapshot(),
makeCmdPublishSource(),
makeCmdPublishSwitch(),
makeCmdPublishUpdate(),
makeCmdPublishShow(),
},
}
}
func makeCmdPublishSource() *commander.Command {
return &commander.Command{
UsageLine: "source",
Short: "manage sources of published repository",
Subcommands: []*commander.Command{
makeCmdPublishSourceAdd(),
makeCmdPublishSourceDrop(),
makeCmdPublishSourceList(),
makeCmdPublishSourceRemove(),
makeCmdPublishSourceReplace(),
makeCmdPublishSourceUpdate(),
},
}
}
cmd/publish_list.go
+3 -3
View File
@@ -25,7 +25,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
return aptlyPublishListTxt(cmd, args)
}
func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -34,7 +34,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
e := collectionFactory.PublishedRepoCollection().LoadShallow(repo, collectionFactory)
if e != nil {
fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
repo.StoragePrefix(), repo.Distribution, repo.Components())
@@ -77,7 +77,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
var err error
repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
cmd/publish_repo.go
+2
View File
@@ -48,8 +48,10 @@ Example:
cmd.Flag.String("butautomaticupgrades", "", "set value for ButAutomaticUpgrades field")
cmd.Flag.String("label", "", "label to publish")
cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/publish_show.go
+4 -3
View File
@@ -24,7 +24,7 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
return aptlyPublishShowTxt(cmd, args)
}
func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
var err error
distribution := args[0]
@@ -52,7 +52,8 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
fmt.Printf("Sources:\n")
for component, sourceID := range repo.Sources {
for _, component := range repo.Components() {
sourceID := repo.Sources[component]
var name string
if repo.SourceKind == deb.SourceSnapshot {
source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
@@ -76,7 +77,7 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
func aptlyPublishShowJSON(_ *commander.Command, args []string) error {
var err error
distribution := args[0]
cmd/publish_snapshot.go
+11 -4
View File
@@ -116,8 +116,9 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
origin := context.Flags().Lookup("origin").Value.String()
notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory, multiDist)
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
}
@@ -132,6 +133,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
}
published.Label = context.Flags().Lookup("label").Value.String()
published.Suite = context.Flags().Lookup("suite").Value.String()
published.Codename = context.Flags().Lookup("codename").Value.String()
published.SkipContents = context.Config().SkipContentsPublishing
@@ -148,6 +150,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
}
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
if duplicate != nil {
collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
@@ -161,11 +167,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
if forceOverwrite {
context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
"the same package pool.\n")
context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
}
@@ -239,8 +244,10 @@ Example:
cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
cmd.Flag.String("label", "", "label to publish")
cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/publish_source_add.go
+94
View File
@@ -0,0 +1,94 @@
package cmd
import (
"fmt"
"strings"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander"
"github.com/smira/flag"
)
func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error {
if len(args) < 2 {
cmd.Usage()
return commander.ErrCommandError
}
distribution := args[0]
names := args[1:]
components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
if len(names) != len(components) {
return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
}
prefix := context.Flags().Lookup("prefix").Value.String()
storage, prefix := deb.ParsePrefix(prefix)
collectionFactory := context.NewCollectionFactory()
published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to add: %s", err)
}
err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if err != nil {
return fmt.Errorf("unable to add: %s", err)
}
revision := published.ObtainRevision()
sources := revision.Sources
for i, component := range components {
name := names[i]
_, exists := sources[component]
if exists {
return fmt.Errorf("unable to add: component '%s' has already been added", component)
}
context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
sources[component] = name
}
err = collectionFactory.PublishedRepoCollection().Update(published)
if err != nil {
return fmt.Errorf("unable to save to DB: %s", err)
}
context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
distribution, published.StoragePrefix())
return err
}
func makeCmdPublishSourceAdd() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishSourceAdd,
UsageLine: "add <distribution> <source>",
Short: "add source components to a published repo",
Long: `
The command adds components of a snapshot or local repository to be published.
This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
The flag -component is mandatory. Use a comma-separated list of components, if
multiple components should be modified. The number of given components must be
equal to the number of given sources, e.g.:
aptly publish source add -component=main,contrib wheezy wheezy-main wheezy-contrib
Example:
$ aptly publish source add -component=contrib wheezy ppa wheezy-contrib
This command assigns the snapshot wheezy-contrib to the component contrib and
adds it to published repository revision of ppa/wheezy.
`,
Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
}
cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
return cmd
}
cmd/publish_source_drop.go
+62
View File
@@ -0,0 +1,62 @@
package cmd
import (
"fmt"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander"
"github.com/smira/flag"
)
func aptlyPublishSourceDrop(cmd *commander.Command, args []string) error {
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
prefix := context.Flags().Lookup("prefix").Value.String()
distribution := args[0]
storage, prefix := deb.ParsePrefix(prefix)
collectionFactory := context.NewCollectionFactory()
published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to drop: %s", err)
}
err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if err != nil {
return fmt.Errorf("unable to drop: %s", err)
}
published.DropRevision()
err = collectionFactory.PublishedRepoCollection().Update(published)
if err != nil {
return fmt.Errorf("unable to save to DB: %s", err)
}
context.Progress().Printf("Source changes have been removed successfully.\n")
return err
}
func makeCmdPublishSourceDrop() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishSourceDrop,
UsageLine: "drop <distribution>",
Short: "drop pending source component changes of a published repository",
Long: `
Remove all pending changes what would be applied with a subsequent 'aptly publish update'.
Example:
$ aptly publish source drop wheezy
`,
Flag: *flag.NewFlagSet("aptly-publish-source-drop", flag.ExitOnError),
}
cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
return cmd
}
cmd/publish_source_list.go
+89
View File
@@ -0,0 +1,89 @@
package cmd
import (
"encoding/json"
"fmt"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander"
"github.com/smira/flag"
)
func aptlyPublishSourceList(cmd *commander.Command, args []string) error {
if len(args) != 1 {
cmd.Usage()
return commander.ErrCommandError
}
prefix := context.Flags().Lookup("prefix").Value.String()
distribution := args[0]
storage, prefix := deb.ParsePrefix(prefix)
published, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to list: %s", err)
}
err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(published, context.NewCollectionFactory())
if err != nil {
return err
}
if published.Revision == nil {
return fmt.Errorf("unable to list: no source changes exist")
}
jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
if jsonFlag {
return aptlyPublishSourceListJSON(published)
}
return aptlyPublishSourceListTxt(published)
}
func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error {
revision := published.Revision
fmt.Printf("Sources:\n")
for _, component := range revision.Components() {
name := revision.Sources[component]
fmt.Printf(" %s: %s [%s]\n", component, name, published.SourceKind)
}
return nil
}
func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error {
revision := published.Revision
output, err := json.MarshalIndent(revision.SourceList(), "", " ")
if err != nil {
return fmt.Errorf("unable to list: %s", err)
}
fmt.Println(string(output))
return nil
}
func makeCmdPublishSourceList() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishSourceList,
UsageLine: "list <distribution>",
Short: "lists revision of published repository",
Long: `
Command lists sources of a published repository.
Example:
$ aptly publish source list wheezy
`,
Flag: *flag.NewFlagSet("aptly-publish-source-list", flag.ExitOnError),
}
cmd.Flag.Bool("json", false, "display record in JSON format")
cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
return cmd
}
cmd/publish_source_remove.go
+86
View File
@@ -0,0 +1,86 @@
package cmd
import (
"fmt"
"strings"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander"
"github.com/smira/flag"
)
func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error {
if len(args) < 1 {
cmd.Usage()
return commander.ErrCommandError
}
distribution := args[0]
components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
if len(components) == 0 {
return fmt.Errorf("unable to remove: missing components, specify at least one component")
}
prefix := context.Flags().Lookup("prefix").Value.String()
storage, prefix := deb.ParsePrefix(prefix)
collectionFactory := context.NewCollectionFactory()
published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
revision := published.ObtainRevision()
sources := revision.Sources
for _, component := range components {
name, exists := sources[component]
if !exists {
return fmt.Errorf("unable to remove: component '%s' does not exist", component)
}
context.Progress().Printf("Removing component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
delete(sources, component)
}
err = collectionFactory.PublishedRepoCollection().Update(published)
if err != nil {
return fmt.Errorf("unable to save to DB: %s", err)
}
context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
distribution, published.StoragePrefix())
return err
}
func makeCmdPublishSourceRemove() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishSourceRemove,
UsageLine: "remove <distribution> [[<endpoint>:]<prefix>] <source>",
Short: "remove source components from a published repo",
Long: `
The command removes source components (snapshot / local repo) from a published repository.
This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
The flag -component is mandatory. Use a comma-separated list of components, if
multiple components should be removed, e.g.:
Example:
$ aptly publish source remove -component=contrib,non-free wheezy filesystem:symlink:debian
`,
Flag: *flag.NewFlagSet("aptly-publish-source-remove", flag.ExitOnError),
}
cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)")
return cmd
}
cmd/publish_source_replace.go
+89
View File
@@ -0,0 +1,89 @@
package cmd
import (
"fmt"
"strings"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander"
"github.com/smira/flag"
)
func aptlyPublishSourceReplace(cmd *commander.Command, args []string) error {
if len(args) < 2 {
cmd.Usage()
return commander.ErrCommandError
}
distribution := args[0]
names := args[1:]
components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
if len(names) != len(components) {
return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
}
prefix := context.Flags().Lookup("prefix").Value.String()
storage, prefix := deb.ParsePrefix(prefix)
collectionFactory := context.NewCollectionFactory()
published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to add: %s", err)
}
err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if err != nil {
return fmt.Errorf("unable to add: %s", err)
}
revision := published.ObtainRevision()
sources := revision.Sources
context.Progress().Printf("Replacing source list...\n")
clear(sources)
for i, component := range components {
name := names[i]
context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
sources[component] = name
}
err = collectionFactory.PublishedRepoCollection().Update(published)
if err != nil {
return fmt.Errorf("unable to save to DB: %s", err)
}
context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
distribution, published.StoragePrefix())
return err
}
func makeCmdPublishSourceReplace() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishSourceReplace,
UsageLine: "replace <distribution> <source>",
Short: "replace the source components of a published repository",
Long: `
The command replaces the source components of a snapshot or local repository to be published.
This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
The flag -component is mandatory. Use a comma-separated list of components, if
multiple components should be modified. The number of given components must be
equal to the number of given sources, e.g.:
aptly publish source replace -component=main,contrib wheezy wheezy-main wheezy-contrib
Example:
$ aptly publish source replace -component=contrib wheezy ppa wheezy-contrib
`,
Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
}
cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
return cmd
}
cmd/publish_source_update.go
+91
View File
@@ -0,0 +1,91 @@
package cmd
import (
"fmt"
"strings"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander"
"github.com/smira/flag"
)
func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error {
if len(args) < 2 {
cmd.Usage()
return commander.ErrCommandError
}
distribution := args[0]
names := args[1:]
components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
if len(names) != len(components) {
return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
}
prefix := context.Flags().Lookup("prefix").Value.String()
storage, prefix := deb.ParsePrefix(prefix)
collectionFactory := context.NewCollectionFactory()
published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
revision := published.ObtainRevision()
sources := revision.Sources
for i, component := range components {
name := names[i]
_, exists := sources[component]
if !exists {
return fmt.Errorf("unable to update: component '%s' does not exist", component)
}
context.Progress().Printf("Updating component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
sources[component] = name
}
err = collectionFactory.PublishedRepoCollection().Update(published)
if err != nil {
return fmt.Errorf("unable to save to DB: %s", err)
}
context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
distribution, published.StoragePrefix())
return err
}
func makeCmdPublishSourceUpdate() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishSourceUpdate,
UsageLine: "update <distribution> <source>",
Short: "update the source components of a published repository",
Long: `
The command updates the source components of a snapshot or local repository to be published.
This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
The flag -component is mandatory. Use a comma-separated list of components, if
multiple components should be modified. The number of given components must be
equal to the number of given sources, e.g.:
aptly publish source update -component=main,contrib wheezy wheezy-main wheezy-contrib
Example:
$ aptly publish source update -component=contrib wheezy ppa wheezy-contrib
`,
Flag: *flag.NewFlagSet("aptly-publish-source-update", flag.ExitOnError),
}
cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
return cmd
}
cmd/publish_switch.go
+24 -21
View File
@@ -11,7 +11,10 @@ import (
)
func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
var err error
var (
err error
names []string
)
components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
@@ -23,11 +26,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
distribution := args[0]
param := "."
var (
names []string
snapshot *deb.Snapshot
)
if len(args) == len(components)+2 {
param = args[1]
names = args[2:]
@@ -42,16 +40,16 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
collectionFactory := context.NewCollectionFactory()
published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
return fmt.Errorf("unable to switch: %s", err)
}
if published.SourceKind != deb.SourceSnapshot {
return fmt.Errorf("unable to update: not a snapshot publish")
return fmt.Errorf("unable to switch: not a published snapshot repository")
}
err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
return fmt.Errorf("unable to switch: %s", err)
}
publishedComponents := published.Components()
@@ -63,17 +61,18 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
}
snapshotCollection := collectionFactory.SnapshotCollection()
for i, component := range components {
if !utils.StrSliceHasItem(publishedComponents, component) {
return fmt.Errorf("unable to switch: component %s is not in published repository", component)
return fmt.Errorf("unable to switch: component %s does not exist in published repository", component)
}
snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
snapshot, err := snapshotCollection.ByName(names[i])
if err != nil {
return fmt.Errorf("unable to switch: %s", err)
}
err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
err = snapshotCollection.LoadComplete(snapshot)
if err != nil {
return fmt.Errorf("unable to switch: %s", err)
}
@@ -100,7 +99,11 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
}
@@ -112,14 +115,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
if !skipCleanup {
err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
context.GetPublishedStorage(storage), collectionFactory, context.Progress())
err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress())
if err != nil {
return fmt.Errorf("unable to update: %s", err)
return fmt.Errorf("unable to switch: %s", err)
}
}
context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
context.Progress().Printf("\nPublished %s repository %s has been successfully switched to new source.\n", published.SourceKind, published.String())
return err
}
@@ -127,15 +129,15 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
func makeCmdPublishSwitch() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishSwitch,
UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
Short: "update published repository by switching to new snapshot",
UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-source>",
Short: "update published repository by switching to new source",
Long: `
Command switches in-place published snapshots with new snapshot contents. All
Command switches in-place published snapshots with new source contents. All
publishing parameters are preserved (architecture list, distribution,
component).
For multiple component repositories, flag -component should be given with
list of components to update. Corresponding snapshots should be given in the
list of components to update. Corresponding sources should be given in the
same order, e.g.:
aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
@@ -161,6 +163,7 @@ This command would switch published repository (with one component) named ppa/wh
cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/publish_update.go
+26 -18
View File
@@ -31,18 +31,14 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: %s", err)
}
if published.SourceKind != deb.SourceLocalRepo {
return fmt.Errorf("unable to update: not a local repository publish")
}
err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
components := published.Components()
for _, component := range components {
published.UpdateLocalRepo(component)
result, err := published.Update(collectionFactory, context.Progress())
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
signer, err := getSigner(context.Flags())
@@ -64,7 +60,11 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
}
@@ -76,14 +76,15 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
if !skipCleanup {
err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
context.GetPublishedStorage(storage), collectionFactory, context.Progress())
cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress())
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
context.Progress().Printf("\nPublished %s repository %s has been updated successfully.\n", published.SourceKind, published.String())
return err
}
@@ -92,15 +93,21 @@ func makeCmdPublishUpdate() *commander.Command {
cmd := &commander.Command{
Run: aptlyPublishUpdate,
UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
Short: "update published local repository",
Short: "update published repository",
Long: `
Command re-publishes (updates) published local repository. <distribution>
and <prefix> should be occupied with local repository published
using command aptly publish repo. Update happens in-place with
minimum possible downtime for published repository.
The command updates updates a published repository after applying pending changes to the sources.
For multiple component published repositories, all local repositories
are updated.
For published local repositories:
* update to match local repository contents
For published snapshots:
* switch components to new snapshot
The update happens in-place with minimum possible downtime for published repository.
For multiple component published repositories, all local repositories are updated.
Example:
@@ -119,6 +126,7 @@ Example:
cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/repo_add.go
+1 -1
View File
@@ -91,7 +91,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
func makeCmdRepoAdd() *commander.Command {
cmd := &commander.Command{
Run: aptlyRepoAdd,
UsageLine: "add <name> <package file.deb>|<directory> ...",
UsageLine: "add <name> (<package file.deb>|<directory>)...",
Short: "add packages to local repository",
Long: `
Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
cmd/repo_include.go
+6 -3
View File
@@ -29,7 +29,10 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
collectionFactory := context.NewCollectionFactory()
@@ -83,14 +86,14 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
func makeCmdRepoInclude() *commander.Command {
cmd := &commander.Command{
Run: aptlyRepoInclude,
UsageLine: "include <file.changes>|<directory> ...",
UsageLine: "include (<file.changes>|<directory>)...",
Short: "add packages to local repositories based on .changes files",
Long: `
Command include looks for .changes files in list of arguments or specified directories. Each
.changes file is verified, parsed, referenced files are put into separate temporary directory
and added into local repository. Successfully imported files are removed by default.
Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
Additionally uploads could be restricted with 'uploaders.json' file. Rules in this file control
uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
Example:
cmd/repo_list.go
+2 -2
View File
@@ -24,7 +24,7 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
return aptlyRepoListTxt(cmd, args)
}
func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -71,7 +71,7 @@ func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
var err error
repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
cmd/repo_move.go
+15 -2
View File
@@ -110,13 +110,24 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
queries := make([]deb.PackageQuery, len(args)-2)
for i := 0; i < len(args)-2; i++ {
queries[i], err = query.Parse(args[i+2])
value, err := GetStringOrFileContent(args[i+2])
if err != nil {
return fmt.Errorf("unable to read package query from file %s: %w", args[i+2], err)
}
queries[i], err = query.Parse(value)
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
}
toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
toProcess, err := srcList.Filter(deb.FilterOptions{
Queries: queries,
WithDependencies: withDeps,
Source: dstList,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
Progress: context.Progress(),
})
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
@@ -179,6 +190,8 @@ func makeCmdRepoMove() *commander.Command {
Command move moves packages matching <package-query> from local repo
<src-name> to local repo <dst-name>.
Use '@file' to read package queries from file or '@-' for stdin.
Example:
$ aptly repo move testing stable 'myapp (=0.1.12)'
cmd/repo_remove.go
+8 -2
View File
@@ -38,14 +38,18 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
queries := make([]deb.PackageQuery, len(args)-1)
for i := 0; i < len(args)-1; i++ {
queries[i], err = query.Parse(args[i+1])
value, err := GetStringOrFileContent(args[i+1])
if err != nil {
return fmt.Errorf("unable to read package query from file %s: %w", args[i+1], err)
}
queries[i], err = query.Parse(value)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
}
list.PrepareIndex()
toRemove, err := list.Filter(queries, false, nil, 0, nil)
toRemove, err := list.Filter(deb.FilterOptions{Queries: queries})
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
@@ -81,6 +85,8 @@ Commands removes packages matching <package-query> from local repository
snapshots, they can be removed completely (including files) by running
'aptly db cleanup'.
Use '@file' to read package queries from file or '@-' for stdin.
Example:
$ aptly repo remove testing 'myapp (=0.1.12)'
cmd/repo_show.go
+2 -2
View File
@@ -25,7 +25,7 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
return aptlyRepoShowTxt(cmd, args)
}
func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
var err error
name := args[0]
@@ -58,7 +58,7 @@ func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
var err error
name := args[0]
cmd/serve.go
+1 -1
View File
@@ -29,7 +29,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
// anything else must fail.
// E.g.: Running the service under a different user may lead to a rootDir
// that exists but is not usable due to access permissions.
err = utils.DirIsAccessible(context.Config().RootDir)
err = utils.DirIsAccessible(context.Config().GetRootDir())
if err != nil {
return err
}
cmd/snapshot_create.go
+1 -1
View File
@@ -84,7 +84,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
func makeCmdSnapshotCreate() *commander.Command {
cmd := &commander.Command{
Run: aptlySnapshotCreate,
UsageLine: "create <name> from mirror <mirror-name> | from repo <repo-name> | empty",
UsageLine: "create <name> (from mirror <mirror-name> | from repo <repo-name> | empty)",
Short: "creates snapshot of mirror (local repository) contents",
Long: `
Command create <name> from mirror makes persistent immutable snapshot of remote
cmd/snapshot_filter.go
+15 -2
View File
@@ -60,14 +60,25 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
// Initial queries out of arguments
queries := make([]deb.PackageQuery, len(args)-2)
for i, arg := range args[2:] {
queries[i], err = query.Parse(arg)
value, err := GetStringOrFileContent(arg)
if err != nil {
return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
}
queries[i], err = query.Parse(value)
if err != nil {
return fmt.Errorf("unable to parse query: %s", err)
}
}
// Filter with dependencies as requested
result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
result, err := packageList.Filter(deb.FilterOptions{
Queries: queries,
WithDependencies: withDeps,
Source: nil,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
Progress: context.Progress(),
})
if err != nil {
return fmt.Errorf("unable to filter: %s", err)
}
@@ -96,6 +107,8 @@ Command filter does filtering in snapshot <source>, producing another
snapshot <destination>. Packages could be specified simply
as 'package-name' or as package queries.
Use '@file' syntax to read package queries from file and '@-' to read from stdin.
Example:
$ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'
cmd/snapshot_list.go
+2 -2
View File
@@ -23,7 +23,7 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
return aptlySnapshotListTxt(cmd, args)
}
func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -59,7 +59,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
var err error
sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
cmd/snapshot_pull.go
+16 -3
View File
@@ -88,7 +88,11 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
// Initial queries out of arguments
queries := make([]deb.PackageQuery, len(args)-3)
for i, arg := range args[3:] {
queries[i], err = query.Parse(arg)
value, err := GetStringOrFileContent(arg)
if err != nil {
return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
}
queries[i], err = query.Parse(value)
if err != nil {
return fmt.Errorf("unable to parse query: %s", err)
}
@@ -97,7 +101,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
}
// Filter with dependencies as requested
result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
result, err := sourcePackageList.Filter(deb.FilterOptions{
Queries: queries,
WithDependencies: !noDeps,
Source: packageList,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
Progress: context.Progress(),
})
if err != nil {
return fmt.Errorf("unable to pull: %s", err)
}
@@ -112,7 +123,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
// If we haven't seen such name-architecture pair and were instructed to remove, remove it
if !noRemove && !seen {
// Remove all packages with the same name and architecture
pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
for _, p := range pS {
packageList.Remove(p)
context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
@@ -160,6 +171,8 @@ versions from <source> following dependencies. New snapshot <destination>
is created as a result of this process. Packages could be specified simply
as 'package-name' or as package queries.
Use '@file' syntax to read package queries from file and '@-' to read from stdin.
Example:
$ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server
cmd/snapshot_search.go
+14 -3
View File
@@ -78,7 +78,11 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
list.PrepareIndex()
if len(args) == 2 {
q, err = query.Parse(args[1])
value, err := GetStringOrFileContent(args[1])
if err != nil {
return fmt.Errorf("unable to read package query from file %s: %w", args[1], err)
}
q, err = query.Parse(value)
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
@@ -103,8 +107,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
}
}
result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList, context.Progress())
result, err := list.Filter(deb.FilterOptions{
Queries: []deb.PackageQuery{q},
WithDependencies: withDeps,
DependencyOptions: context.DependencyOptions(),
Architectures: architecturesList,
Progress: context.Progress(),
})
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
@@ -129,6 +138,8 @@ Command search displays list of packages in snapshot that match package query
If query is not specified, all the packages are displayed.
Use '@file' syntax to read package query from file and '@-' to read from stdin.
Example:
$ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
cmd/snapshot_show.go
+2 -2
View File
@@ -25,7 +25,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
return aptlySnapshotShowTxt(cmd, args)
}
func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
var err error
name := args[0]
collectionFactory := context.NewCollectionFactory()
@@ -85,7 +85,7 @@ func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
var err error
name := args[0]
cmd/string_or_file_flag.go
+55
View File
@@ -0,0 +1,55 @@
package cmd
import (
"io"
"os"
"strings"
"github.com/smira/flag"
)
// StringOrFileFlag is a custom flag type that can handle both string input and file input.
// If the input starts with '@', it is treated as a filename and the contents are read from the file.
// If the input is '@-', the contents are read from stdin.
type StringOrFileFlag struct {
value string
}
func (s *StringOrFileFlag) String() string {
return s.value
}
func (s *StringOrFileFlag) Set(value string) error {
var err error
s.value, err = GetStringOrFileContent(value)
return err
}
func (s *StringOrFileFlag) Get() any {
return s.value
}
func AddStringOrFileFlag(flagSet *flag.FlagSet, name string, value string, usage string) *StringOrFileFlag {
result := &StringOrFileFlag{value: value}
flagSet.Var(result, name, usage)
return result
}
func GetStringOrFileContent(value string) (string, error) {
if !strings.HasPrefix(value, "@") {
return value, nil
}
filename := strings.TrimPrefix(value, "@")
var data []byte
var err error
if filename == "-" { // Read from stdin
data, err = io.ReadAll(os.Stdin)
} else {
data, err = os.ReadFile(filename)
}
if err != nil {
return "", err
}
return string(data), nil
}
cmd/task_run.go
+4 -5
View File
@@ -62,11 +62,10 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
text, _ = reader.ReadString('\n')
if text == "\n" {
break
} else {
text = strings.TrimSpace(text) + ","
parsedArgs, _ := shellwords.Parse(text)
cmdArgs = append(cmdArgs, parsedArgs...)
}
text = strings.TrimSpace(text) + ","
parsedArgs, _ := shellwords.Parse(text)
cmdArgs = append(cmdArgs, parsedArgs...)
}
if len(cmdArgs) == 0 {
@@ -132,7 +131,7 @@ func formatCommands(args []string) [][]string {
func makeCmdTaskRun() *commander.Command {
cmd := &commander.Command{
Run: aptlyTaskRun,
UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
UsageLine: "run (-filename=<filename> | <commands>...)",
Short: "run aptly tasks",
Long: `
Command helps organise multiple aptly commands in one single aptly task, running as single thread.
codecov.yml
+1 -1
View File
@@ -3,5 +3,5 @@ coverage:
project:
default:
target: auto
threshold: 0%
threshold: 2%
if_ci_failed: error
completion.d/_aptly
+1
View File
@@ -457,6 +457,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
"-distribution=[distribution name to publish]:distribution:($dists)"
"-label=[label to publish]:label: "
"-suite=[suite to publish]:suite: "
"-codename=[codename to publish]:codename: "
"-notautomatic=[set value for NotAutomatic field]:notautomatic: "
"-origin=[origin name to publish]:origin: "
${components_options[@]}
completion.d/aptly
+28 -4
View File
@@ -1,5 +1,3 @@
#!/bin/bash
# (The MIT License)
#
# Copyright (c) 2014 Andrey Smirnov
@@ -56,12 +54,14 @@ _aptly()
{
cur="${COMP_WORDS[COMP_CWORD]}"
prev="${COMP_WORDS[COMP_CWORD-1]}"
prevprev="${COMP_WORDS[COMP_CWORD-2]}"
commands="api config db graph mirror package publish repo serve snapshot task version"
options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
db_subcommands="cleanup recover"
mirror_subcommands="create drop edit show list rename search update"
publish_subcommands="drop list repo snapshot switch update"
publish_subcommands="drop list repo snapshot switch update source"
publish_source_subcommands="drop list add remove update replace"
snapshot_subcommands="create diff drop filter list merge pull rename search show verify"
repo_subcommands="add copy create drop edit import include list move remove rename search show"
package_subcommands="search show"
@@ -150,6 +150,17 @@ _aptly()
esac
fi
case "$prevprev" in
"publish")
case "$prev" in
"source")
COMPREPLY=($(compgen -W "${publish_source_subcommands}" -- ${cur}))
return 0
;;
esac
;;
esac
case "$cmd" in
"mirror")
case "$subcmd" in
@@ -503,7 +514,7 @@ _aptly()
"snapshot"|"repo")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing -multi-dist" -- ${cur}))
else
if [[ "$subcmd" == "snapshot" ]]; then
COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
@@ -577,6 +588,19 @@ _aptly()
;;
esac
;;
"source")
case "$subcmd" in
"add")
return 0
;;
"list")
if [[ $numargs -eq 0 ]]; then
COMPREPLY=($(compgen -W "-raw" -- ${cur}))
return 0
fi
;;
esac
;;
"package")
case "$subcmd" in
"search")
console/progress.go
+71 -14
View File
@@ -7,6 +7,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/cheggaaa/pb"
"github.com/rs/zerolog/log"
"github.com/wsxiaoys/terminal/color"
)
@@ -34,6 +35,7 @@ type Progress struct {
queue chan printTask
bar *pb.ProgressBar
barShown bool
worker ProgressWorker
}
// Check interface
@@ -42,16 +44,19 @@ var (
)
// NewProgress creates new progress instance
func NewProgress() *Progress {
return &Progress{
func NewProgress(structuredLogging bool) *Progress {
p := &Progress{
stopped: make(chan bool),
queue: make(chan printTask, 100),
}
p.worker = progressWorkerFactroy(structuredLogging, p)
return p
}
// Start makes progress start its work
func (p *Progress) Start() {
go p.worker()
go p.worker.run()
}
// Shutdown shuts down progress display
@@ -69,7 +74,7 @@ func (p *Progress) Flush() {
}
// InitBar starts progressbar for count bytes or count items
func (p *Progress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
if p.bar != nil {
panic("bar already initialized")
}
@@ -173,42 +178,94 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
}
}
func (p *Progress) worker() {
type ProgressWorker interface {
run()
}
func progressWorkerFactroy(structuredLogging bool, progress *Progress) ProgressWorker {
if structuredLogging {
worker := loggerProgressWorker{progress: progress}
return &worker
}
worker := standardProgressWorker{progress: progress}
return &worker
}
type standardProgressWorker struct {
progress *Progress
}
func (w *standardProgressWorker) run() {
hasBar := false
for {
task := <-p.queue
task := <-w.progress.queue
switch task.code {
case codeBarEnabled:
hasBar = true
case codeBarDisabled:
hasBar = false
case codePrint:
if p.barShown {
if w.progress.barShown {
fmt.Print("\r\033[2K")
p.barShown = false
w.progress.barShown = false
}
fmt.Print(task.message)
case codePrintStdErr:
if p.barShown {
if w.progress.barShown {
fmt.Print("\r\033[2K")
p.barShown = false
w.progress.barShown = false
}
fmt.Fprint(os.Stderr, task.message)
case codeProgress:
if hasBar {
fmt.Print("\r" + task.message)
p.barShown = true
w.progress.barShown = true
}
case codeHideProgress:
if p.barShown {
if w.progress.barShown {
fmt.Print("\r\033[2K")
p.barShown = false
w.progress.barShown = false
}
case codeFlush:
task.reply <- true
case codeStop:
p.stopped <- true
w.progress.stopped <- true
return
}
}
}
type loggerProgressWorker struct {
progress *Progress
}
func (w *loggerProgressWorker) run() {
hasBar := false
for {
task := <-w.progress.queue
switch task.code {
case codeBarEnabled:
hasBar = true
case codeBarDisabled:
hasBar = false
case codePrint, codePrintStdErr:
log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
case codeProgress:
if hasBar {
log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
w.progress.barShown = true
}
case codeHideProgress:
if w.progress.barShown {
w.progress.barShown = false
}
case codeFlush:
task.reply <- true
case codeStop:
w.progress.stopped <- true
return
}
}
console/progress_test.go
+24
View File
@@ -0,0 +1,24 @@
package console
import (
"fmt"
"testing"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) {
TestingT(t)
}
type ProgressSuite struct {}
var _ = Suite(&ProgressSuite{})
func (s *ProgressSuite) TestNewProgress(c *C) {
p := NewProgress(false)
c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &standardProgressWorker{}))
p = NewProgress(true)
c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &loggerProgressWorker{}))
}
context/context.go
+69 -24
View File
@@ -12,12 +12,14 @@ import (
"runtime/pprof"
"strings"
"sync"
"syscall"
"time"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/azure"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/etcddb"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/files"
@@ -48,6 +50,7 @@ type AptlyContext struct {
publishedStorages map[string]aptly.PublishedStorage
dependencyOptions int
architecturesList []string
structuredLogging bool
// Debug features
fileCPUProfile *os.File
fileMemProfile *os.File
@@ -93,13 +96,14 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
Fatal(err)
}
} else {
configLocations := []string{
filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
"/etc/aptly.conf",
}
homeLocation := filepath.Join(os.Getenv("HOME"), ".aptly.conf")
configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
for _, configLocation := range configLocations {
err = utils.LoadConfig(configLocation, &utils.Config)
if os.IsPermission(err) || os.IsNotExist(err) {
continue
}
if err == nil {
break
}
@@ -109,11 +113,13 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
}
if err != nil {
fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
// as this is fresh aptly installation, we don't need to support legacy pool locations
utils.Config.SkipLegacyPool = true
utils.SaveConfig(configLocations[0], &utils.Config)
utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
err = utils.LoadConfig(homeLocation, &utils.Config)
if err != nil {
Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
}
}
}
@@ -195,7 +201,7 @@ func (context *AptlyContext) Progress() aptly.Progress {
func (context *AptlyContext) _progress() aptly.Progress {
if context.progress == nil {
context.progress = console.NewProgress()
context.progress = console.NewProgress(context.structuredLogging)
context.progress.Start()
}
@@ -272,7 +278,7 @@ func (context *AptlyContext) DBPath() string {
// DBPath builds path to database
func (context *AptlyContext) dbPath() string {
return filepath.Join(context.config().RootDir, "db")
return filepath.Join(context.config().GetRootDir(), "db")
}
// Database opens and returns current instance of database
@@ -286,8 +292,18 @@ func (context *AptlyContext) Database() (database.Storage, error) {
func (context *AptlyContext) _database() (database.Storage, error) {
if context.database == nil {
var err error
context.database, err = goleveldb.NewDB(context.dbPath())
switch context.config().DatabaseBackend.Type {
case "leveldb":
dbPath := filepath.Join(context.config().GetRootDir(), "db")
if len(context.config().DatabaseBackend.DbPath) != 0 {
dbPath = context.config().DatabaseBackend.DbPath
}
context.database, err = goleveldb.NewDB(dbPath)
case "etcd":
context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL)
default:
context.database, err = goleveldb.NewDB(context.dbPath())
}
if err != nil {
return nil, fmt.Errorf("can't instantiate database: %s", err)
}
@@ -344,10 +360,7 @@ func (context *AptlyContext) ReOpenDatabase() error {
// NewCollectionFactory builds factory producing all kinds of collections
func (context *AptlyContext) NewCollectionFactory() *deb.CollectionFactory {
context.Lock()
defer context.Unlock()
db, err := context._database()
db, err := context.Database()
if err != nil {
Fatal(err)
}
@@ -360,7 +373,26 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
defer context.Unlock()
if context.packagePool == nil {
context.packagePool = files.NewPackagePool(context.config().RootDir, !context.config().SkipLegacyPool)
storageConfig := context.config().PackagePoolStorage
if storageConfig.Azure != nil {
var err error
context.packagePool, err = azure.NewPackagePool(
storageConfig.Azure.AccountName,
storageConfig.Azure.AccountKey,
storageConfig.Azure.Container,
storageConfig.Azure.Prefix,
storageConfig.Azure.Endpoint)
if err != nil {
Fatal(err)
}
} else {
poolRoot := context.config().PackagePoolStorage.Local.Path
if poolRoot == "" {
poolRoot = filepath.Join(context.config().GetRootDir(), "pool")
}
context.packagePool = files.NewPackagePool(poolRoot, !context.config().SkipLegacyPool)
}
}
return context.packagePool
@@ -374,7 +406,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
publishedStorage, ok := context.publishedStorages[name]
if !ok {
if name == "" {
publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().RootDir, "public"), "hardlink", "")
publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().GetRootDir(), "public"), "hardlink", "")
} else if strings.HasPrefix(name, "filesystem:") {
params, ok := context.config().FileSystemPublishRoots[name[11:]]
if !ok {
@@ -393,7 +425,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
params.ForceSigV2, params.Debug)
params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
if err != nil {
Fatal(err)
}
@@ -432,7 +464,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
// UploadPath builds path to upload storage
func (context *AptlyContext) UploadPath() string {
return filepath.Join(context.Config().RootDir, "upload")
return filepath.Join(context.Config().GetRootDir(), "upload")
}
func (context *AptlyContext) pgpProvider() string {
@@ -456,7 +488,7 @@ func (context *AptlyContext) pgpProvider() string {
return provider
}
func (context *AptlyContext) getGPGFinder(provider string) pgp.GPGFinder {
func (context *AptlyContext) getGPGFinder() pgp.GPGFinder {
switch context.pgpProvider() {
case "gpg1":
return pgp.GPG1Finder()
@@ -479,7 +511,7 @@ func (context *AptlyContext) GetSigner() pgp.Signer {
return &pgp.GoSigner{}
}
return pgp.NewGpgSigner(context.getGPGFinder(provider))
return pgp.NewGpgSigner(context.getGPGFinder())
}
// GetVerifier returns Verifier with respect to provider
@@ -492,7 +524,12 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier {
return &pgp.GoVerifier{}
}
return pgp.NewGpgVerifier(context.getGPGFinder(provider))
return pgp.NewGpgVerifier(context.getGPGFinder())
}
// SkelPath builds the local skeleton folder
func (context *AptlyContext) SkelPath() string {
return filepath.Join(context.config().GetRootDir(), "skel")
}
// UpdateFlags sets internal copy of flags in the context
@@ -526,7 +563,7 @@ func (context *AptlyContext) GoContextHandleSignals() {
// Catch ^C
sigch := make(chan os.Signal, 1)
signal.Notify(sigch, os.Interrupt)
signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
var cancel gocontext.CancelFunc
@@ -540,6 +577,11 @@ func (context *AptlyContext) GoContextHandleSignals() {
}()
}
// StructuredLogging allows to set the structuredLogging flag
func (context *AptlyContext) StructuredLogging(structuredLogging bool) {
context.structuredLogging = structuredLogging
}
// Shutdown shuts context down
func (context *AptlyContext) Shutdown() {
context.Lock()
@@ -561,6 +603,9 @@ func (context *AptlyContext) Shutdown() {
context.fileMemProfile = nil
}
}
if context.taskList != nil {
context.taskList.Stop()
}
if context.database != nil {
context.database.Close()
context.database = nil
context/context_test.go
+4 -1
View File
@@ -1,6 +1,8 @@
package context
import (
"fmt"
"os"
"reflect"
"testing"
@@ -82,5 +84,6 @@ func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
// storage never exists.
c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") },
FatalErrorPanicMatches,
&FatalError{ReturnCode: 1, Message: "published local storage fuji not configured"})
&FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)",
os.Getenv("HOME"))})
}
database/etcddb/batch.go
+53
View File
@@ -0,0 +1,53 @@
package etcddb
import (
"github.com/aptly-dev/aptly/database"
clientv3 "go.etcd.io/etcd/client/v3"
)
type EtcDBatch struct {
s *EtcDStorage
ops []clientv3.Op
}
type WriteOptions struct {
NoWriteMerge bool
Sync bool
}
func (b *EtcDBatch) Put(key []byte, value []byte) (err error) {
b.ops = append(b.ops, clientv3.OpPut(string(key), string(value)))
return
}
func (b *EtcDBatch) Delete(key []byte) (err error) {
b.ops = append(b.ops, clientv3.OpDelete(string(key)))
return
}
func (b *EtcDBatch) Write() (err error) {
kv := clientv3.NewKV(b.s.db)
batchSize := 128
for i := 0; i < len(b.ops); i += batchSize {
txn := kv.Txn(Ctx)
end := i + batchSize
if end > len(b.ops) {
end = len(b.ops)
}
batch := b.ops[i:end]
txn.Then(batch...)
_, err = txn.Commit()
if err != nil {
panic(err)
}
}
return
}
// batch should implement database.Batch
var (
_ database.Batch = &EtcDBatch{}
)
database/etcddb/database.go
+32
View File
@@ -0,0 +1,32 @@
package etcddb
import (
"context"
"time"
"github.com/aptly-dev/aptly/database"
clientv3 "go.etcd.io/etcd/client/v3"
)
var Ctx = context.TODO()
func internalOpen(url string) (cli *clientv3.Client, err error) {
cfg := clientv3.Config{
Endpoints: []string{url},
DialTimeout: 30 * time.Second,
MaxCallSendMsgSize: 2147483647, // (2048 * 1024 * 1024) - 1
MaxCallRecvMsgSize: 2147483647,
DialKeepAliveTimeout: 7200 * time.Second,
}
cli, err = clientv3.New(cfg)
return
}
func NewDB(url string) (database.Storage, error) {
cli, err := internalOpen(url)
if err != nil {
return nil, err
}
return &EtcDStorage{url, cli, ""}, nil
}
database/etcddb/database_test.go
+158
View File
@@ -0,0 +1,158 @@
package etcddb_test
import (
"testing"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/etcddb"
. "gopkg.in/check.v1"
)
// Launch gocheck tests
func Test(t *testing.T) {
TestingT(t)
}
type EtcDDBSuite struct {
url string
db database.Storage
}
var _ = Suite(&EtcDDBSuite{})
func (s *EtcDDBSuite) SetUpTest(c *C) {
var err error
s.db, err = etcddb.NewDB("127.0.0.1:2379")
c.Assert(err, IsNil)
}
func (s *EtcDDBSuite) TestSetUpTest(c *C) {
var err error
s.db, err = etcddb.NewDB("127.0.0.1:2379")
c.Assert(err, IsNil)
}
func (s *EtcDDBSuite) TestGetPut(c *C) {
var (
key = []byte("key")
value = []byte("value")
)
var err error
err = s.db.Put(key, value)
c.Assert(err, IsNil)
result, err := s.db.Get(key)
c.Assert(err, IsNil)
c.Assert(result, DeepEquals, value)
}
func (s *EtcDDBSuite) TestDelete(c *C) {
var (
key = []byte("key")
value = []byte("value")
)
err := s.db.Put(key, value)
c.Assert(err, IsNil)
_, err = s.db.Get(key)
c.Assert(err, IsNil)
err = s.db.Delete(key)
c.Assert(err, IsNil)
}
func (s *EtcDDBSuite) TestByPrefix(c *C) {
//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
keys := [][]byte{}
values := [][]byte{}
c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
keys = append(keys, append([]byte(nil), k...))
values = append(values, append([]byte(nil), v...))
return nil
}), IsNil)
c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
return database.ErrNotFound
}), Equals, database.ErrNotFound)
c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
return database.ErrNotFound
}), IsNil)
c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
}
func (s *EtcDDBSuite) TestHasPrefix(c *C) {
//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
}
func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
var (
key = []byte("key")
key2 = []byte("key2")
value = []byte("value")
value2 = []byte("value2")
)
transaction, err := s.db.OpenTransaction()
err = s.db.Put(key, value)
c.Assert(err, IsNil)
c.Assert(err, IsNil)
transaction.Put(key2, value2)
v, err := s.db.Get(key)
c.Check(v, DeepEquals, value)
err = transaction.Delete(key)
c.Assert(err, IsNil)
_, err = transaction.Get(key2)
c.Assert(err, IsNil)
v2, err := transaction.Get(key2)
c.Check(err, IsNil)
c.Check(v2, DeepEquals, value2)
_, err = transaction.Get(key)
c.Assert(err, IsNil)
err = transaction.Commit()
c.Check(err, IsNil)
v2, err = transaction.Get(key2)
c.Check(err, IsNil)
c.Check(v2, DeepEquals, value2)
_, err = transaction.Get(key)
c.Assert(err, NotNil)
}
database/etcddb/storage.go
+202
View File
@@ -0,0 +1,202 @@
package etcddb
import (
"github.com/aptly-dev/aptly/database"
"github.com/pborman/uuid"
clientv3 "go.etcd.io/etcd/client/v3"
"fmt"
)
type EtcDStorage struct {
url string
db *clientv3.Client
tmpPrefix string // prefix for temporary DBs
}
// CreateTemporary creates new DB of the same type in temp dir
func (s *EtcDStorage) CreateTemporary() (database.Storage, error) {
tmp := uuid.NewRandom().String()
return &EtcDStorage{
url: s.url,
db: s.db,
tmpPrefix: tmp,
}, nil
}
func (s *EtcDStorage) applyPrefix(key []byte) []byte {
if len(s.tmpPrefix) != 0 {
return append([]byte(s.tmpPrefix+"/"), key...)
}
return key
}
// Get key value from etcd
func (s *EtcDStorage) Get(key []byte) (value []byte, err error) {
realKey := s.applyPrefix(key)
getResp, err := s.db.Get(Ctx, string(realKey))
if err != nil {
return
}
for _, kv := range getResp.Kvs {
value = kv.Value
break
}
if len(value) == 0 {
err = database.ErrNotFound
return
}
return
}
// Put saves key to etcd, if key has the same value in DB already, it is not saved
func (s *EtcDStorage) Put(key []byte, value []byte) (err error) {
realKey := s.applyPrefix(key)
_, err = s.db.Put(Ctx, string(realKey), string(value))
if err != nil {
return
}
return
}
// Delete removes key from etcd
func (s *EtcDStorage) Delete(key []byte) (err error) {
realKey := s.applyPrefix(key)
_, err = s.db.Delete(Ctx, string(realKey))
if err != nil {
return
}
return
}
// KeysByPrefix returns all keys that start with prefix
func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte {
realPrefix := s.applyPrefix(prefix)
result := make([][]byte, 0, 20)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return nil
}
for _, ev := range getResp.Kvs {
key := ev.Key
keyc := make([]byte, len(key))
copy(keyc, key)
result = append(result, key)
}
return result
}
// FetchByPrefix returns all values with keys that start with prefix
func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte {
realPrefix := s.applyPrefix(prefix)
result := make([][]byte, 0, 20)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return nil
}
for _, kv := range getResp.Kvs {
valc := make([]byte, len(kv.Value))
copy(valc, kv.Value)
result = append(result, kv.Value)
}
return result
}
// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
func (s *EtcDStorage) HasPrefix(prefix []byte) bool {
realPrefix := s.applyPrefix(prefix)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return false
}
return getResp.Count > 0
}
// ProcessByPrefix iterates through all entries where key starts with prefix and calls
// StorageProcessor on key value pair
func (s *EtcDStorage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
realPrefix := s.applyPrefix(prefix)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return err
}
for _, kv := range getResp.Kvs {
err := proc(kv.Key, kv.Value)
if err != nil {
return err
}
}
return nil
}
// Close finishes etcd connect
func (s *EtcDStorage) Close() error {
// do not close temporary db
if len(s.tmpPrefix) != 0 {
return nil
}
if s.db == nil {
return nil
}
err := s.db.Close()
s.db = nil
return err
}
// Reopen tries to open (re-open) the database
func (s *EtcDStorage) Open() error {
if s.db != nil {
return nil
}
var err error
s.db, err = internalOpen(s.url)
return err
}
// CreateBatch creates a Batch object
func (s *EtcDStorage) CreateBatch() database.Batch {
if s.db == nil {
return nil
}
return &EtcDBatch{
s: s,
}
}
// OpenTransaction creates new transaction.
func (s *EtcDStorage) OpenTransaction() (database.Transaction, error) {
tmpdb, err := s.CreateTemporary()
if err != nil {
return nil, err
}
return &transaction{s: s, tmpdb: tmpdb}, nil
}
// CompactDB does nothing for etcd
func (s *EtcDStorage) CompactDB() error {
return nil
}
// Drop removes only temporary DBs with etcd (i.e. remove all prefixed keys)
func (s *EtcDStorage) Drop() error {
if len(s.tmpPrefix) != 0 {
getResp, err := s.db.Get(Ctx, s.tmpPrefix, clientv3.WithPrefix())
if err != nil {
return nil
}
for _, kv := range getResp.Kvs {
_, err = s.db.Delete(Ctx, string(kv.Key))
if err != nil {
return fmt.Errorf("cannot delete tempdb entry: %s", kv.Key)
}
}
}
return nil
}
// Check interface
var (
_ database.Storage = &EtcDStorage{}
)
database/etcddb/transaction.go
+75
View File
@@ -0,0 +1,75 @@
package etcddb
import (
"github.com/aptly-dev/aptly/database"
clientv3 "go.etcd.io/etcd/client/v3"
)
type transaction struct {
s *EtcDStorage
tmpdb database.Storage
ops []clientv3.Op
}
// Get implements database.Reader interface.
func (t *transaction) Get(key []byte) (value []byte, err error) {
value, err = t.tmpdb.Get(key)
// if not found, search main db
if err != nil {
value, err = t.s.Get(key)
}
return
}
// Put implements database.Writer interface.
func (t *transaction) Put(key, value []byte) (err error) {
err = t.tmpdb.Put(key, value)
if err != nil {
return
}
t.ops = append(t.ops, clientv3.OpPut(string(key), string(value)))
return
}
// Delete implements database.Writer interface.
func (t *transaction) Delete(key []byte) (err error) {
err = t.tmpdb.Delete(key)
if err != nil {
return
}
t.ops = append(t.ops, clientv3.OpDelete(string(key)))
return
}
func (t *transaction) Commit() (err error) {
kv := clientv3.NewKV(t.s.db)
batchSize := 128
for i := 0; i < len(t.ops); i += batchSize {
txn := kv.Txn(Ctx)
end := i + batchSize
if end > len(t.ops) {
end = len(t.ops)
}
batch := t.ops[i:end]
txn.Then(batch...)
_, err = txn.Commit()
if err != nil {
panic(err)
}
}
t.ops = []clientv3.Op{}
return
}
// Discard is safe to call after Commit(), it would be no-op
func (t *transaction) Discard() {
t.ops = []clientv3.Op{}
t.tmpdb.Drop()
return
}
// transaction should implement database.Transaction
var _ database.Transaction = &transaction{}
database/goleveldb/storage.go
+1 -2
View File
@@ -3,7 +3,6 @@ package goleveldb
import (
"bytes"
"errors"
"io/ioutil"
"os"
"github.com/syndtr/goleveldb/leveldb"
@@ -19,7 +18,7 @@ type storage struct {
// CreateTemporary creates new DB of the same type in temp dir
func (s *storage) CreateTemporary() (database.Storage, error) {
tempdir, err := ioutil.TempDir("", "aptly")
tempdir, err := os.MkdirTemp("", "aptly")
if err != nil {
return nil, err
}
deb/changes.go
+10 -8
View File
@@ -4,16 +4,17 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"text/template"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
"github.com/saracen/walker"
)
// Changes is a result of .changes file parsing
@@ -39,7 +40,7 @@ func NewChanges(path string) (*Changes, error) {
ChangesName: filepath.Base(path),
}
c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
c.TempDir, err = os.MkdirTemp(os.TempDir(), "aptly")
if err != nil {
return nil, err
}
@@ -221,12 +222,12 @@ func (c *Changes) GetField(field string) string {
}
// MatchesDependency implements PackageLike interface
func (c *Changes) MatchesDependency(d Dependency) bool {
func (c *Changes) MatchesDependency(_ Dependency) bool {
return false
}
// MatchesArchitecture implements PackageLike interface
func (c *Changes) MatchesArchitecture(arch string) bool {
func (c *Changes) MatchesArchitecture(_ string) bool {
return false
}
@@ -248,6 +249,8 @@ func (c *Changes) GetArchitecture() string {
// CollectChangesFiles walks filesystem collecting all .changes files
func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
changesFilesLock := &sync.Mutex{}
for _, location := range locations {
info, err2 := os.Stat(location)
if err2 != nil {
@@ -256,15 +259,14 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
continue
}
if info.IsDir() {
err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
if err3 != nil {
return err3
}
err2 = walker.Walk(location, func(path string, info os.FileInfo) error {
if info.IsDir() {
return nil
}
if strings.HasSuffix(info.Name(), ".changes") {
changesFilesLock.Lock()
defer changesFilesLock.Unlock()
changesFiles = append(changesFiles, path)
}

Some files were not shown because too many files have changed in this diff Show More