vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-07-25 13:37:51 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1162 from aptly-dev/feature/176-snapshot-pull-api

Browse Source 
Snapshot Pull API
This commit is contained in:
André Roth
2024-10-03 23:07:27 +02:00
committed by GitHub
parent 880f487093 bce54d5878
commit d6a156b181
6 changed files with 484 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+9 -8
View File
@@ -37,9 +37,8 @@ ifeq ($(RUN_LONG_TESTS), yes)
endif endif
install: install:
@echo Building aptly ... @echo "\e[33m\e[1mBuilding aptly ...\e[0m"
go generate go generate
@echo go install -v
@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi @out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
system/env: system/requirements.txt system/env: system/requirements.txt
@@ -66,7 +65,7 @@ ifeq ($(RUN_LONG_TESTS), yes)
endif endif
docker-test: ## Run system tests docker-test: ## Run system tests
@echo Building aptly.test ... @echo "\e[33m\e[1mBuilding aptly.test ...\e[0m"
@rm -f aptly.test @rm -f aptly.test
go generate go generate
# install and initialize swagger # install and initialize swagger
@@ -74,7 +73,7 @@ docker-test: ## Run system tests
PATH=$(BINPATH)/:$(PATH) swag init -q PATH=$(BINPATH)/:$(PATH) swag init -q
# build coverage binary # build coverage binary
go test -v -coverpkg="./..." -c -tags testruncli go test -v -coverpkg="./..." -c -tags testruncli
@echo Running python tests ... @echo "\e[33m\e[1mRunning python tests ...\e[0m"
@test -e aws.creds && . ./aws.creds; \ @test -e aws.creds && . ./aws.creds; \
export PATH=$(BINPATH)/:$(PATH); \ export PATH=$(BINPATH)/:$(PATH); \
export APTLY_VERSION=$(VERSION); \ export APTLY_VERSION=$(VERSION); \
@@ -82,16 +81,17 @@ docker-test: ## Run system tests
test: prepare ## Run unit tests test: prepare ## Run unit tests
@test -d /srv/etcd || system/t13_etcd/install-etcd.sh @test -d /srv/etcd || system/t13_etcd/install-etcd.sh
@echo "\nStarting etcd ..." @echo "\e[33m\e[1mStarting etcd ...\e[0m"
@mkdir -p /tmp/etcd-data; system/t13_etcd/start-etcd.sh > /tmp/etcd-data/etcd.log 2>&1 & @mkdir -p /tmp/etcd-data; system/t13_etcd/start-etcd.sh > /tmp/etcd-data/etcd.log 2>&1 &
@echo "\nRunning go test ..." @echo "\e[33m\e[1mRunning go test ...\e[0m"
go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
@echo "\nStopping etcd ..." @echo "\e[33m\e[1mStopping etcd ...\e[0m"
@pid=`cat /tmp/etcd.pid`; kill $$pid @pid=`cat /tmp/etcd.pid`; kill $$pid
@rm -f /tmp/etcd-data/etcd.log @rm -f /tmp/etcd-data/etcd.log
@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret @ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
bench: bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
go test -v ./deb -run=nothing -bench=. -benchmem go test -v ./deb -run=nothing -bench=. -benchmem
mem.png: mem.dat mem.gp mem.png: mem.dat mem.gp
@@ -157,7 +157,7 @@ dpkg: ## Build debian packages
if [ "$(DEBARCH)" = "amd64" ]; then \ if [ "$(DEBARCH)" = "amd64" ]; then \
buildtype="any,all" ; \ buildtype="any,all" ; \
fi ; \ fi ; \
echo Building: $$buildtype ; \ echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH) dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)
# cleanup # cleanup
@test -f debian/changelog.dpkg-bak && mv debian/changelog.dpkg-bak debian/changelog || true ; \ @test -f debian/changelog.dpkg-bak && mv debian/changelog.dpkg-bak debian/changelog || true ; \
@@ -221,5 +221,6 @@ flake8: ## run flake8 on system tests
clean: ## remove local build and module cache clean: ## remove local build and module cache
test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true
rm -rf build/ docs/ obj-*-linux-gnu* rm -rf build/ docs/ obj-*-linux-gnu*
rm -f unit.out aptly.test
.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server .PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server
api/mirror.go
+83 -78
View File
@@ -32,9 +32,10 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) {
} }
// @Summary Get mirrors // @Summary Get mirrors
// @Description Show list of currently available mirrors. Each mirror is returned as in “show” API. // @Description **Show list of currently available mirrors**
// @Description Each mirror is returned as in “show” API.
// @Tags Mirrors // @Tags Mirrors
// @Produce json // @Produce json
// @Success 200 {array} deb.RemoteRepo // @Success 200 {array} deb.RemoteRepo
// @Router /api/mirrors [get] // @Router /api/mirrors [get]
func apiMirrorsList(c *gin.Context) { func apiMirrorsList(c *gin.Context) {
@@ -50,45 +51,49 @@ func apiMirrorsList(c *gin.Context) {
c.JSON(200, result) c.JSON(200, result)
} }
type mirrorCreateParams struct {
// Name of mirror to be created
Name string `binding:"required" json:"Name" example:"mirror2"`
// Url of the archive to mirror
ArchiveURL string `binding:"required" json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Distribution name to mirror
Distribution string ` json:"Distribution" example:"'buster', for flat repositories use './'"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Gpg keyring(s) for verifying Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to mirror installer files
DownloadInstaller bool ` json:"DownloadInstaller"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to skip if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip the verification of architectures
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
}
// @Summary Create mirror // @Summary Create mirror
// @Description Create empty mirror with specified parameters. // @Description **Create a mirror**
// @Tags Mirrors // @Tags Mirrors
// @Accept json // @Consume json
// @Produce json // @Param request body mirrorCreateParams true "Parameters"
// @Param Name query string true "mirror name" // @Produce json
// @Param ArchiveURL query string true "url of the archive to mirror e.g. http://deb.debian.org/debian/"
// @Param Distribution query string false "distribution name to mirror e.g. `buster`, for flat repositories use `./` instead of distribution name"
// @Param Filter query string false "package query that is applied to packages in the mirror"
// @Param Components query []string false "components to mirror, if not specified aptly would fetch all components"
// @Param Architectures query []string false "limit mirror to those architectures, if not specified aptly would fetch all architectures"
// @Param Keyrings query []string false "gpg keyring(s) to use when verifying `Release` file"
// @Param DownloadSources query bool false "whether to mirror sources"
// @Param DownloadUdebs query bool false "whether to mirror `.udeb` packages (Debian installer support)"
// @Param DownloadInstaller query bool false "whether to download additional not packaged installer files"
// @Param FilterWithDeps query bool false "when filtering, include dependencies of matching packages as well"
// @Param SkipComponentCheck query bool false "whether to skip if the given components are in the `Release` file"
// @Param IgnoreSignatures query bool false "whether to skip the verification of `Release` file signatures"
// @Success 200 {object} deb.RemoteRepo // @Success 200 {object} deb.RemoteRepo
// @Failure 400 {object} Error "Bad Request" // @Failure 400 {object} Error "Bad Request"
// @Router /api/mirrors [post] // @Router /api/mirrors [post]
func apiMirrorsCreate(c *gin.Context) { func apiMirrorsCreate(c *gin.Context) {
var err error var err error
var b struct { var b mirrorCreateParams
Name string `binding:"required"`
ArchiveURL string `binding:"required"`
Distribution string
Filter string
Components []string
Architectures []string
Keyrings []string
DownloadSources bool
DownloadUdebs bool
DownloadInstaller bool
FilterWithDeps bool
SkipComponentCheck bool
SkipArchitectureCheck bool
IgnoreSignatures bool
}
b.DownloadSources = context.Config().DownloadSourcePackages b.DownloadSources = context.Config().DownloadSourcePackages
b.IgnoreSignatures = context.Config().GpgDisableVerify b.IgnoreSignatures = context.Config().GpgDisableVerify
@@ -155,12 +160,11 @@ func apiMirrorsCreate(c *gin.Context) {
} }
// @Summary Delete Mirror // @Summary Delete Mirror
// @Description Delete a mirror // @Description **Delete a mirror**
// @Tags Mirrors // @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name" // @Param name path string true "mirror name"
// @Param force query int true "force: 1 to enable" // @Param force query int true "force: 1 to enable"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue // @Success 200 {object} task.ProcessReturnValue
// @Failure 404 {object} Error "Mirror not found" // @Failure 404 {object} Error "Mirror not found"
// @Failure 403 {object} Error "Unable to delete mirror with snapshots" // @Failure 403 {object} Error "Unable to delete mirror with snapshots"
@@ -205,11 +209,10 @@ func apiMirrorsDrop(c *gin.Context) {
} }
// @Summary Show Mirror // @Summary Show Mirror
// @Description Get mirror information by name // @Description **Get mirror information by name**
// @Tags Mirrors // @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name" // @Param name path string true "mirror name"
// @Produce json
// @Success 200 {object} deb.RemoteRepo // @Success 200 {object} deb.RemoteRepo
// @Failure 404 {object} Error "Mirror not found" // @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error" // @Failure 500 {object} Error "Internal Error"
@@ -234,13 +237,12 @@ func apiMirrorsShow(c *gin.Context) {
} }
// @Summary List Mirror Packages // @Summary List Mirror Packages
// @Description Get a list of packages from a mirror // @Description **Get a list of packages from a mirror**
// @Tags Mirrors // @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name" // @Param name path string true "mirror name"
// @Param q query string false "search query" // @Param q query string false "search query"
// @Param format query string false "format: `details` for more detailed information" // @Param format query string false "format: `details` for more detailed information"
// @Produce json
// @Success 200 {array} deb.Package "List of Packages" // @Success 200 {array} deb.Package "List of Packages"
// @Failure 400 {object} Error "Unable to determine list of architectures" // @Failure 400 {object} Error "Unable to determine list of architectures"
// @Failure 404 {object} Error "Mirror not found" // @Failure 404 {object} Error "Mirror not found"
@@ -323,26 +325,46 @@ func apiMirrorsPackages(c *gin.Context) {
} }
} }
type mirrorUpdateParams struct {
// Change mirror name to `Name`
Name string ` json:"Name" example:"mirror1"`
// Url of the archive to mirror
ArchiveURL string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Gpg keyring(s) for verifing Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to skip checking if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip checking if the given architectures are in the Release file
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to ignore checksum errors
IgnoreChecksums bool ` json:"IgnoreChecksums"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
ForceUpdate bool ` json:"ForceUpdate"`
// Set "true" to skip downloading already downloaded packages
SkipExistingPackages bool ` json:"SkipExistingPackages"`
}
// @Summary Update Mirror // @Summary Update Mirror
// @Description Update Mirror and download packages // @Description **Update Mirror and download packages**
// @Tags Mirrors // @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name to update" // @Param name path string true "mirror name to update"
// @Param Name query string false "change mirror name" // @Consume json
// @Param ArchiveURL query string false "ArchiveURL" // @Param request body mirrorUpdateParams true "Parameters"
// @Param Filter query string false "Filter" // @Produce json
// @Param Architectures query []string false "Architectures"
// @Param Components query []string false "Components"
// @Param Keyrings query []string false "Keyrings"
// @Param FilterWithDeps query bool false "FilterWithDeps"
// @Param DownloadSources query bool false "DownloadSources"
// @Param DownloadUdebs query bool false "DownloadUdebs"
// @Param SkipComponentCheck query bool false "SkipComponentCheck"
// @Param IgnoreChecksums query bool false "IgnoreChecksums"
// @Param IgnoreSignatures query bool false "IgnoreSignatures"
// @Param ForceUpdate query bool false "ForceUpdate"
// @Param SkipExistingPackages query bool false "SkipExistingPackages"
// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully" // @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
// @Success 202 {object} task.Task "Mirror is being updated" // @Success 202 {object} task.Task "Mirror is being updated"
// @Failure 400 {object} Error "Unable to determine list of architectures" // @Failure 400 {object} Error "Unable to determine list of architectures"
@@ -353,26 +375,9 @@ func apiMirrorsUpdate(c *gin.Context) {
var ( var (
err error err error
remote *deb.RemoteRepo remote *deb.RemoteRepo
b mirrorUpdateParams
) )
var b struct {
Name string
ArchiveURL string
Filter string
Architectures []string
Components []string
Keyrings []string
FilterWithDeps bool
DownloadSources bool
DownloadUdebs bool
SkipComponentCheck bool
SkipArchitectureCheck bool
IgnoreChecksums bool
IgnoreSignatures bool
ForceUpdate bool
SkipExistingPackages bool
}
collectionFactory := context.NewCollectionFactory() collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection() collection := collectionFactory.RemoteRepoCollection()
api/router.go
+3 -1
View File
@@ -185,6 +185,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
} }
{ {
api.GET("/publish", apiPublishList) api.GET("/publish", apiPublishList)
api.POST("/publish", apiPublishRepoOrSnapshot) api.POST("/publish", apiPublishRepoOrSnapshot)
api.POST("/publish/:prefix", apiPublishRepoOrSnapshot) api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
@@ -200,7 +201,8 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages) api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
api.DELETE("/snapshots/:name", apiSnapshotsDrop) api.DELETE("/snapshots/:name", apiSnapshotsDrop)
api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff) api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
api.POST("/snapshots/merge", apiSnapshotsMerge) api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
api.POST("/snapshots/:name/pull", apiSnapshotsPull)
} }
{ {
api/snapshot.go
+220 -7
View File
@@ -3,11 +3,13 @@ package api
import ( import (
"fmt" "fmt"
"net/http" "net/http"
"sort"
"strings" "strings"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database" "github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/deb" "github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task" "github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
) )
@@ -404,17 +406,37 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
showPackages(c, snapshot.RefList(), collectionFactory) showPackages(c, snapshot.RefList(), collectionFactory)
} }
// POST /api/snapshots/merge type snapshotsMergeParams struct {
// List of snapshot names to be merged
Sources []string `binding:"required" json:"Sources" example:"snapshot1"`
}
// @Summary Snapshot Merge
// @Description **Merge several source snapshots into a new snapshot**
// @Description
// @Description Merge happens from left to right. By default, packages with the same name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
// @Description
// @Description If only one snapshot is specified, merge copies source into destination.
// @Tags Snapshots
// @Param name path string true "Name of the snapshot to be created"
// @Param latest query int false "merge only the latest version of each package"
// @Param no-remove query int false "all versions of packages are preserved during merge"
// @Consume json
// @Param request body snapshotsMergeParams true "Parameters"
// @Produce json
// @Success 200
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/snapshots/{name}/merge [post]
func apiSnapshotsMerge(c *gin.Context) { func apiSnapshotsMerge(c *gin.Context) {
var ( var (
err error err error
snapshot *deb.Snapshot snapshot *deb.Snapshot
body snapshotsMergeParams
) )
var body struct { name := c.Params.ByName("name")
Destination string `binding:"required"`
Sources []string `binding:"required"`
}
if c.Bind(&body) != nil { if c.Bind(&body) != nil {
return return
@@ -454,7 +476,7 @@ func apiSnapshotsMerge(c *gin.Context) {
resources[i] = string(sources[i].ResourceKey()) resources[i] = string(sources[i].ResourceKey())
} }
maybeRunTaskInBackground(c, "Merge snapshot "+body.Destination, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
result := sources[0].RefList() result := sources[0].RefList()
for i := 1; i < len(sources); i++ { for i := 1; i < len(sources); i++ {
result = result.Merge(sources[i].RefList(), overrideMatching, false) result = result.Merge(sources[i].RefList(), overrideMatching, false)
@@ -469,7 +491,7 @@ func apiSnapshotsMerge(c *gin.Context) {
sourceDescription[i] = fmt.Sprintf("'%s'", s.Name) sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
} }
snapshot = deb.NewSnapshotFromRefList(body.Destination, sources, result, snapshot = deb.NewSnapshotFromRefList(name, sources, result,
fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", "))) fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
err = collectionFactory.SnapshotCollection().Add(snapshot) err = collectionFactory.SnapshotCollection().Add(snapshot)
@@ -480,3 +502,194 @@ func apiSnapshotsMerge(c *gin.Context) {
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
}) })
} }
type snapshotsPullParams struct {
// Source name to be searched for packages and dependencies
Source string `binding:"required" json:"Source" example:"source-snapshot"`
// Name of the snapshot to be created
Destination string `binding:"required" json:"Destination" example:"idestination-snapshot"`
// List of package queries (i.e. name of package to be pulled from `Source`)
Queries []string `binding:"required" json:"Queries" example:"xserver-xorg"`
// List of architectures (optional)
Architectures []string ` json:"Architectures" example:"amd64, armhf"`
}
// @Summary Snapshot Pull
// @Description **Pulls new packages and dependencies from a source snapshot into a new snapshot**
// @Description
// @Description May also upgrade package versions if name snapshot already contains packages being pulled. New snapshot `Destination` is created as result of this process.
// @Description If architectures are limited (with config architectures or parameter `Architectures`, only mentioned architectures are processed, otherwise aptly will process all architectures in the snapshot.
// @Description If following dependencies by source is enabled (using dependencyFollowSource config), pulling binary packages would also pull corresponding source packages as well.
// @Description By default aptly would remove packages matching name and architecture while importing: e.g. when importing software_1.3_amd64, package software_1.2.9_amd64 would be removed.
// @Description
// @Description With flag `no-remove` both package versions would stay in the snapshot.
// @Description
// @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled.
// @Tags Snapshots
// @Param name path string true "Name of the snapshot to be created"
// @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable"
// @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable"
// @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable"
// @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable"
// @Consume json
// @Param request body snapshotsPullParams true "Parameters"
// @Produce json
// @Success 200
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/snapshots/{name}/pull [post]
func apiSnapshotsPull(c *gin.Context) {
var (
err error
destinationSnapshot *deb.Snapshot
body snapshotsPullParams
)
name := c.Params.ByName("name")
if err = c.BindJSON(&body); err != nil {
AbortWithJSONError(c, http.StatusBadRequest, err)
return
}
allMatches := c.Request.URL.Query().Get("all-matches") == "1"
dryRun := c.Request.URL.Query().Get("dry-run") == "1"
noDeps := c.Request.URL.Query().Get("no-deps") == "1"
noRemove := c.Request.URL.Query().Get("no-remove") == "1"
collectionFactory := context.NewCollectionFactory()
// Load <name> snapshot
toSnapshot, err := collectionFactory.SnapshotCollection().ByName(name)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
// Load <Source> snapshot
sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// convert snapshots to package list
toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
toPackageList.PrepareIndex()
sourcePackageList.PrepareIndex()
var architecturesList []string
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = toPackageList.Architectures(false)
}
architecturesList = append(architecturesList, body.Architectures...)
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
for _, arch := range architecturesList {
archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
}
queries := make([]deb.PackageQuery, len(body.Queries))
for i, q := range body.Queries {
queries[i], err = query.Parse(q)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Add architecture filter
queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
}
// Filter with dependencies as requested
destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
destinationPackageList.PrepareIndex()
removedPackages := []string{}
addedPackages := []string{}
alreadySeen := map[string]bool{}
destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key]
// If we haven't seen such name-architecture pair and were instructed to remove, remove it
if !noRemove && !seen {
// Remove all packages with the same name and architecture
packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
for _, p := range packageSearchResults {
toPackageList.Remove(p)
removedPackages = append(removedPackages, p.String())
}
}
// If !allMatches, add only first matching name-arch package
if !seen || allMatches {
toPackageList.Add(pkg)
addedPackages = append(addedPackages, pkg.String())
}
alreadySeen[key] = true
return nil
})
alreadySeen = nil
if dryRun {
response := struct {
AddedPackages []string `json:"added_packages"`
RemovedPackages []string `json:"removed_packages"`
}{
AddedPackages: addedPackages,
RemovedPackages: removedPackages,
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
}
// Create <destination> snapshot
destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
})
}
system/t12_api/gpg.py
+1 -1
View File
@@ -29,7 +29,7 @@ class GPGAPITestAddKey(APITest):
with tempfile.NamedTemporaryFile(suffix=".pub") as keyring: with tempfile.NamedTemporaryFile(suffix=".pub") as keyring:
gpgkeyid = "9E3E53F19C7DE460" gpgkeyid = "9E3E53F19C7DE460"
resp = self.post("/api/gpg/key", json={ resp = self.post("/api/gpg/key", json={
"Keyserver": "keyserver.ubuntu.com", "Keyserver": "hkp://keyserver.ubuntu.com:80",
"Keyring": keyring.name, "Keyring": keyring.name,
"GpgKeyID": gpgkeyid "GpgKeyID": gpgkeyid
}) })
system/t12_api/snapshots.py
+168 -10
View File
@@ -7,6 +7,7 @@ class SnapshotsAPITestCreateShowEmpty(APITest):
""" """
GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages
""" """
def check(self): def check(self):
snapshot_name = self.random_name() snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot', snapshot_desc = {'Description': 'fun snapshot',
@@ -35,6 +36,7 @@ class SnapshotsAPITestCreateFromRefs(APITest):
GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages, GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages,
GET /api/snapshots GET /api/snapshots
""" """
def check(self): def check(self):
snapshot_name = self.random_name() snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot', snapshot_desc = {'Description': 'fun snapshot',
@@ -96,6 +98,7 @@ class SnapshotsAPITestCreateFromRepo(APITest):
""" """
POST /api/repos, POST /api/repos/:name/snapshots, GET /api/snapshots/:name POST /api/repos, POST /api/repos/:name/snapshots, GET /api/snapshots/:name
""" """
def check(self): def check(self):
repo_name = self.random_name() repo_name = self.random_name()
snapshot_name = self.random_name() snapshot_name = self.random_name()
@@ -140,6 +143,7 @@ class SnapshotsAPITestCreateUpdate(APITest):
""" """
POST /api/snapshots, PUT /api/snapshots/:name, GET /api/snapshots/:name POST /api/snapshots, PUT /api/snapshots/:name, GET /api/snapshots/:name
""" """
def check(self): def check(self):
snapshot_name = self.random_name() snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot', snapshot_desc = {'Description': 'fun snapshot',
@@ -172,6 +176,7 @@ class SnapshotsAPITestCreateDelete(APITest):
""" """
POST /api/snapshots, DELETE /api/snapshots/:name, GET /api/snapshots/:name POST /api/snapshots, DELETE /api/snapshots/:name, GET /api/snapshots/:name
""" """
def check(self): def check(self):
snapshot_name = self.random_name() snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot', snapshot_desc = {'Description': 'fun snapshot',
@@ -226,6 +231,7 @@ class SnapshotsAPITestSearch(APITest):
""" """
POST /api/snapshots, GET /api/snapshots?sort=name, GET /api/snapshots/:name POST /api/snapshots, GET /api/snapshots?sort=name, GET /api/snapshots/:name
""" """
def check(self): def check(self):
repo_name = self.random_name() repo_name = self.random_name()
@@ -260,6 +266,7 @@ class SnapshotsAPITestDiff(APITest):
""" """
GET /api/snapshot/:name/diff/:name2 GET /api/snapshot/:name/diff/:name2
""" """
def check(self): def check(self):
repos = [self.random_name() for x in range(2)] repos = [self.random_name() for x in range(2)]
snapshots = [self.random_name() for x in range(2)] snapshots = [self.random_name() for x in range(2)]
@@ -301,7 +308,7 @@ class SnapshotsAPITestDiff(APITest):
class SnapshotsAPITestMerge(APITest): class SnapshotsAPITestMerge(APITest):
""" """
POST /api/snapshots, GET /api/snapshots/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name POST /api/snapshots, POST /api/snapshots/:name/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name
""" """
def check(self): def check(self):
@@ -318,9 +325,8 @@ class SnapshotsAPITestMerge(APITest):
# create merge snapshot # create merge snapshot
merged_name = self.random_name() merged_name = self.random_name()
task = self.post_task( task = self.post_task(
"/api/snapshots/merge", f"/api/snapshots/{merged_name}/merge",
json={ json={
"Destination": merged_name,
"Sources": [source["Name"] for source in sources], "Sources": [source["Name"] for source in sources],
}, },
) )
@@ -345,7 +351,7 @@ class SnapshotsAPITestMerge(APITest):
# create merge snapshot without sources # create merge snapshot without sources
merged_name = self.random_name() merged_name = self.random_name()
resp = self.post( resp = self.post(
"/api/snapshots/merge", json={"Destination": merged_name, "Sources": []} f"/api/snapshots/{merged_name}/merge", json={"Sources": []}
) )
self.check_equal(resp.status_code, 400) self.check_equal(resp.status_code, 400)
self.check_equal( self.check_equal(
@@ -357,8 +363,8 @@ class SnapshotsAPITestMerge(APITest):
merged_name = self.random_name() merged_name = self.random_name()
non_existing_source = self.random_name() non_existing_source = self.random_name()
resp = self.post( resp = self.post(
"/api/snapshots/merge", f"/api/snapshots/{merged_name}/merge",
json={"Destination": merged_name, "Sources": [non_existing_source]}, json={"Sources": [non_existing_source]},
) )
self.check_equal( self.check_equal(
resp.json()["error"], f"snapshot with name {non_existing_source} not found" resp.json()["error"], f"snapshot with name {non_existing_source} not found"
@@ -370,8 +376,8 @@ class SnapshotsAPITestMerge(APITest):
# create merge snapshot with used name # create merge snapshot with used name
merged_name = sources[0]["Name"] merged_name = sources[0]["Name"]
resp = self.post( resp = self.post(
"/api/snapshots/merge", f"/api/snapshots/{merged_name}/merge",
json={"Destination": merged_name, "Sources": [source["Name"] for source in sources]}, json={"Sources": [source["Name"] for source in sources]},
) )
self.check_equal( self.check_equal(
resp.json()["error"], resp.json()["error"],
@@ -382,9 +388,8 @@ class SnapshotsAPITestMerge(APITest):
# create merge snapshot with "latest" and "no-remove" flags (should fail) # create merge snapshot with "latest" and "no-remove" flags (should fail)
merged_name = self.random_name() merged_name = self.random_name()
resp = self.post( resp = self.post(
"/api/snapshots/merge", f"/api/snapshots/{merged_name}/merge",
json={ json={
"Destination": merged_name,
"Sources": [source["Name"] for source in sources], "Sources": [source["Name"] for source in sources],
}, },
params={"latest": "1", "no-remove": "1"}, params={"latest": "1", "no-remove": "1"},
@@ -393,3 +398,156 @@ class SnapshotsAPITestMerge(APITest):
resp.json()["error"], "no-remove and latest are mutually exclusive" resp.json()["error"], "no-remove and latest are mutually exclusive"
) )
self.check_equal(resp.status_code, 400) self.check_equal(resp.status_code, 400)
class SnapshotsAPITestPull(APITest):
"""
POST /api/snapshots/:name/pull, POST /api/snapshots, GET /api/snapshots/:name/packages?name=:package_name
"""
def check(self):
repo_with_libboost = self.random_name()
empty_repo = self.random_name()
snapshot_repo_with_libboost = self.random_name()
snapshot_empty_repo = self.random_name()
# create repo with file in it and snapshot of it
self.check_equal(self.post("/api/repos", json={"Name": repo_with_libboost}).status_code, 201)
dir_name = self.random_name()
self.check_equal(self.upload(f"/api/files/{dir_name}",
"libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200)
self.check_equal(self.post(f"/api/repos/{repo_with_libboost}/file/{dir_name}").status_code, 200)
resp = self.post(f"/api/repos/{repo_with_libboost}/snapshots", json={'Name': snapshot_repo_with_libboost})
self.check_equal(resp.status_code, 201)
# create empty repo and snapshot of it
self.check_equal(self.post("/api/repos", json={"Name": empty_repo}).status_code, 201)
resp = self.post(f"/api/repos/{empty_repo}/snapshots", json={'Name': snapshot_empty_repo})
self.check_equal(resp.status_code, 201)
# pull libboost from repo_with_libboost to empty_repo, save into snapshot_pull_libboost
snapshot_pull_libboost = self.random_name()
# dry run first
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull?dry-run=1", json={
'Source': snapshot_repo_with_libboost,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': [
'amd64'
'i386'
]
})
self.check_equal(resp.status_code, 200)
# dry run, all-matches
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull?dry-run=1&all-matches=1", json={
'Source': snapshot_repo_with_libboost,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': [
'amd64'
'i386'
]
})
self.check_equal(resp.status_code, 200)
# missing argument
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull", json={
'Source': snapshot_repo_with_libboost,
'Destination': snapshot_pull_libboost,
})
self.check_equal(resp.status_code, 400)
# dry run, emtpy architectures
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull?dry-run=1", json={
'Source': snapshot_repo_with_libboost,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': []
})
self.check_equal(resp.status_code, 500)
# dry run, non-existing To
resp = self.post("/api/snapshots/asd123/pull", json={
'Source': snapshot_repo_with_libboost,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
]
})
self.check_equal(resp.status_code, 404)
# dry run, non-existing source
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull?dry-run=1", json={
'Source': "asd123",
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
]
})
self.check_equal(resp.status_code, 404)
# snapshot pull
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull", json={
'Source': snapshot_repo_with_libboost,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': [
'amd64'
'i386'
]
})
self.check_equal(resp.status_code, 201)
self.check_subset({
'Name': snapshot_pull_libboost,
'SourceKind': 'snapshot',
'Description': f"Pulled into '{snapshot_empty_repo}' with '{snapshot_repo_with_libboost}' as source, pull request was: 'libboost-program-options-dev'",
}, resp.json())
# check that snapshot_pull_libboost contains libboost
resp = self.get(f"/api/snapshots/{snapshot_pull_libboost}/packages?name=libboost-program-options-dev")
self.check_equal(resp.status_code, 200)
# pull from non-existing source
non_existing_source = self.random_name()
destination = self.random_name()
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull", json={
'Source': non_existing_source,
'Destination': destination,
'Queries': [
'Name (~ *)'
],
'Architectures': [
'all',
]
})
self.check_equal(resp.status_code, 404)
self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_source} not found")
# pull to non-existing snapshot
non_existing_snapshot = self.random_name()
destination = self.random_name()
resp = self.post(f"/api/snapshots/{snapshot_empty_repo}/pull", json={
'Source': non_existing_snapshot,
'Destination': destination,
'Queries': [
'Name (~ *)'
],
'Architectures': [
'all',
]
})
self.check_equal(resp.status_code, 404)
self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_snapshot} not found")