vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-04-20 19:38:39 +00:00
Code Issues Projects Releases Wiki Activity

Feature: Add Pull Snapshot API

Browse Source
This commit is contained in:
Mauro Regli
2023-03-13 08:55:54 +01:00
committed by André Roth
parent aff7b0db50
commit af5b04b24f
4 changed files with 337 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/router.go
View File

@@ -184,6 +184,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
}
{
api.GET("/publish", apiPublishList)
api.POST("/publish", apiPublishRepoOrSnapshot)
api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
@@ -200,6 +201,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.DELETE("/snapshots/:name", apiSnapshotsDrop)
api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
api.POST("/snapshots/merge", apiSnapshotsMerge)
api.POST("/snapshots/pull", apiSnapshotsPull)
}
{

164
api/snapshot.go
View File

@@ -3,11 +3,13 @@ package api
import (
"fmt"
"net/http"
"sort"
"strings"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
)
@@ -480,3 +482,165 @@ func apiSnapshotsMerge(c *gin.Context) {
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
})
}
// POST /api/snapshots/pull
func apiSnapshotsPull(c *gin.Context) {
var (
err error
destinationSnapshot *deb.Snapshot
)
var body struct {
Source string `binding:"required"`
To string `binding:"required"`
Destination string `binding:"required"`
Queries []string `binding:"required"`
Architectures []string
}
if err = c.BindJSON(&body); err != nil {
AbortWithJSONError(c, http.StatusBadRequest, err)
return
}
allMatches := c.Request.URL.Query().Get("all-matches") == "1"
dryRun := c.Request.URL.Query().Get("dry-run") == "1"
noDeps := c.Request.URL.Query().Get("no-deps") == "1"
noRemove := c.Request.URL.Query().Get("no-remove") == "1"
collectionFactory := context.NewCollectionFactory()
// Load <To> snapshot
toSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.To)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
// Load <Source> snapshot
sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, body.To, body.Destination)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// convert snapshots to package list
toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
toPackageList.PrepareIndex()
sourcePackageList.PrepareIndex()
var architecturesList []string
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = toPackageList.Architectures(false)
}
architecturesList = append(architecturesList, body.Architectures...)
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
for _, arch := range architecturesList {
archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
}
queries := make([]deb.PackageQuery, len(body.Queries))
for i, q := range body.Queries {
queries[i], err = query.Parse(q)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Add architecture filter
queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
}
// Filter with dependencies as requested
destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
destinationPackageList.PrepareIndex()
removedPackages := []string{}
addedPackages := []string{}
alreadySeen := map[string]bool{}
destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key]
// If we haven't seen such name-architecture pair and were instructed to remove, remove it
if !noRemove && !seen {
// Remove all packages with the same name and architecture
packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
for _, p := range packageSearchResults {
toPackageList.Remove(p)
removedPackages = append(removedPackages, p.String())
}
}
// If !allMatches, add only first matching name-arch package
if !seen || allMatches {
toPackageList.Add(pkg)
addedPackages = append(addedPackages, pkg.String())
}
alreadySeen[key] = true
return nil
})
alreadySeen = nil
if dryRun {
response := struct {
AddedPackages []string `json:"added_packages"`
RemovedPackages []string `json:"removed_packages"`
}{
AddedPackages: addedPackages,
RemovedPackages: removedPackages,
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
}
// Create <destination> snapshot
destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
})
}

2
system/t12_api/gpg.py
View File

@@ -29,7 +29,7 @@ class GPGAPITestAddKey(APITest):
with tempfile.NamedTemporaryFile(suffix=".pub") as keyring:
gpgkeyid = "9E3E53F19C7DE460"
resp = self.post("/api/gpg/key", json={
"Keyserver": "keyserver.ubuntu.com",
"Keyserver": "hkp://keyserver.ubuntu.com:80",
"Keyring": keyring.name,
"GpgKeyID": gpgkeyid
})

171
system/t12_api/snapshots.py
View File

@@ -7,6 +7,7 @@ class SnapshotsAPITestCreateShowEmpty(APITest):
"""
GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages
"""
def check(self):
snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot',
@@ -35,6 +36,7 @@ class SnapshotsAPITestCreateFromRefs(APITest):
GET /api/snapshots/:name, POST /api/snapshots, GET /api/snapshots/:name/packages,
GET /api/snapshots
"""
def check(self):
snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot',
@@ -96,6 +98,7 @@ class SnapshotsAPITestCreateFromRepo(APITest):
"""
POST /api/repos, POST /api/repos/:name/snapshots, GET /api/snapshots/:name
"""
def check(self):
repo_name = self.random_name()
snapshot_name = self.random_name()
@@ -140,6 +143,7 @@ class SnapshotsAPITestCreateUpdate(APITest):
"""
POST /api/snapshots, PUT /api/snapshots/:name, GET /api/snapshots/:name
"""
def check(self):
snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot',
@@ -172,6 +176,7 @@ class SnapshotsAPITestCreateDelete(APITest):
"""
POST /api/snapshots, DELETE /api/snapshots/:name, GET /api/snapshots/:name
"""
def check(self):
snapshot_name = self.random_name()
snapshot_desc = {'Description': 'fun snapshot',
@@ -226,6 +231,7 @@ class SnapshotsAPITestSearch(APITest):
"""
POST /api/snapshots, GET /api/snapshots?sort=name, GET /api/snapshots/:name
"""
def check(self):
repo_name = self.random_name()
@@ -260,6 +266,7 @@ class SnapshotsAPITestDiff(APITest):
"""
GET /api/snapshot/:name/diff/:name2
"""
def check(self):
repos = [self.random_name() for x in range(2)]
snapshots = [self.random_name() for x in range(2)]
@@ -301,7 +308,7 @@ class SnapshotsAPITestDiff(APITest):
class SnapshotsAPITestMerge(APITest):
"""
POST /api/snapshots, GET /api/snapshots/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name
POST /api/snapshots, POST /api/snapshots/merge, GET /api/snapshots/:name, DELETE /api/snapshots/:name
"""
def check(self):
@@ -393,3 +400,165 @@ class SnapshotsAPITestMerge(APITest):
resp.json()["error"], "no-remove and latest are mutually exclusive"
)
self.check_equal(resp.status_code, 400)
class SnapshotsAPITestPull(APITest):
"""
POST /api/snapshots/pull, POST /api/snapshots, GET /api/snapshots/:name/packages?name=:package_name
"""
def check(self):
repo_with_libboost = self.random_name()
empty_repo = self.random_name()
snapshot_repo_with_libboost = self.random_name()
snapshot_empty_repo = self.random_name()
# create repo with file in it and snapshot of it
self.check_equal(self.post("/api/repos", json={"Name": repo_with_libboost}).status_code, 201)
dir_name = self.random_name()
self.check_equal(self.upload(f"/api/files/{dir_name}",
"libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200)
self.check_equal(self.post(f"/api/repos/{repo_with_libboost}/file/{dir_name}").status_code, 200)
resp = self.post(f"/api/repos/{repo_with_libboost}/snapshots", json={'Name': snapshot_repo_with_libboost})
self.check_equal(resp.status_code, 201)
# create empty repo and snapshot of it
self.check_equal(self.post("/api/repos", json={"Name": empty_repo}).status_code, 201)
resp = self.post(f"/api/repos/{empty_repo}/snapshots", json={'Name': snapshot_empty_repo})
self.check_equal(resp.status_code, 201)
# pull libboost from repo_with_libboost to empty_repo, save into snapshot_pull_libboost
snapshot_pull_libboost = self.random_name()
# dry run first
resp = self.post("/api/snapshots/pull?dry-run=1", json={
'Source': snapshot_repo_with_libboost,
'To': snapshot_empty_repo,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': [
'amd64'
'i386'
]
})
self.check_equal(resp.status_code, 200)
# dry run, all-matches
resp = self.post("/api/snapshots/pull?dry-run=1&all-matches=1", json={
'Source': snapshot_repo_with_libboost,
'To': snapshot_empty_repo,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': [
'amd64'
'i386'
]
})
self.check_equal(resp.status_code, 200)
# missing argument
resp = self.post("/api/snapshots/pull", json={
'Source': snapshot_repo_with_libboost,
'To': snapshot_empty_repo,
'Destination': snapshot_pull_libboost,
})
self.check_equal(resp.status_code, 400)
# dry run, emtpy architectures
resp = self.post("/api/snapshots/pull?dry-run=1", json={
'Source': snapshot_repo_with_libboost,
'To': snapshot_empty_repo,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': []
})
self.check_equal(resp.status_code, 500)
# dry run, non-existing To
resp = self.post("/api/snapshots/pull", json={
'Source': snapshot_repo_with_libboost,
'To': "asd123",
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
]
})
self.check_equal(resp.status_code, 404)
# dry run, non-existing source
resp = self.post("/api/snapshots/pull?dry-run=1", json={
'Source': "asd123",
'To': snapshot_empty_repo,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
]
})
self.check_equal(resp.status_code, 404)
# snapshot pull
resp = self.post("/api/snapshots/pull", json={
'Source': snapshot_repo_with_libboost,
'To': snapshot_empty_repo,
'Destination': snapshot_pull_libboost,
'Queries': [
'libboost-program-options-dev'
],
'Architectures': [
'amd64'
'i386'
]
})
self.check_equal(resp.status_code, 201)
self.check_subset({
'Name': snapshot_pull_libboost,
'SourceKind': 'snapshot',
'Description': f"Pulled into '{snapshot_empty_repo}' with '{snapshot_repo_with_libboost}' as source, pull request was: 'libboost-program-options-dev'",
}, resp.json())
# check that snapshot_pull_libboost contains libboost
resp = self.get(f"/api/snapshots/{snapshot_pull_libboost}/packages?name=libboost-program-options-dev")
self.check_equal(resp.status_code, 200)
# pull from non-existing source
non_existing_source = self.random_name()
destination = self.random_name()
resp = self.post("/api/snapshots/pull", json={
'Source': non_existing_source,
'To': snapshot_empty_repo,
'Destination': destination,
'Queries': [
'Name (~ *)'
],
'Architectures': [
'all',
]
})
self.check_equal(resp.status_code, 404)
self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_source} not found")
# pull to non-existing snapshot
non_existing_snapshot = self.random_name()
destination = self.random_name()
resp = self.post("/api/snapshots/pull", json={
'Source': non_existing_snapshot,
'To': snapshot_empty_repo,
'Destination': destination,
'Queries': [
'Name (~ *)'
],
'Architectures': [
'all',
]
})
self.check_equal(resp.status_code, 404)
self.check_equal(resp.json()['error'], f"snapshot with name {non_existing_snapshot} not found")