Merge pull request #1162 from aptly-dev/feature/176-snapshot-pull-api

Snapshot Pull API

api/mirror.go

@@ -32,9 +32,10 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) {
 }
 
 // @Summary Get mirrors
-// @Description Show list of currently available mirrors. Each mirror is returned as in “show” API.
+// @Description **Show list of currently available mirrors**
+// @Description Each mirror is returned as in “show” API.
 // @Tags Mirrors
-// @Produce  json
+// @Produce json
 // @Success 200 {array} deb.RemoteRepo
 // @Router /api/mirrors [get]
 func apiMirrorsList(c *gin.Context) {
@@ -50,45 +51,49 @@ func apiMirrorsList(c *gin.Context) {
 	c.JSON(200, result)
 }
 
+type mirrorCreateParams struct {
+	// Name of mirror to be created
+	Name string `binding:"required"          json:"Name"              example:"mirror2"`
+	// Url of the archive to mirror
+	ArchiveURL string `binding:"required"    json:"ArchiveURL"        example:"http://deb.debian.org/debian"`
+	// Distribution name to mirror
+	Distribution string `                    json:"Distribution"      example:"'buster', for flat repositories use './'"`
+	// Package query that is applied to mirror packages
+	Filter string `                          json:"Filter"            example:"xserver-xorg"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `                    json:"Components"        example:"main"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `                 json:"Architectures"     example:"amd64"`
+	// Gpg keyring(s) for verifying Release file
+	Keyrings []string `                      json:"Keyrings"          example:"trustedkeys.gpg"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `                   json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `                     json:"DownloadUdebs"`
+	// Set "true" to mirror installer files
+	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `                    json:"FilterWithDeps"`
+	// Set "true" to skip if the given components are in the Release file
+	SkipComponentCheck bool `                json:"SkipComponentCheck"`
+	// Set "true" to skip the verification of architectures
+	SkipArchitectureCheck bool `             json:"SkipArchitectureCheck"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `                  json:"IgnoreSignatures"`
+}
+
 // @Summary Create mirror
-// @Description Create empty mirror with specified parameters.
+// @Description **Create a mirror**
 // @Tags Mirrors
-// @Accept  json
-// @Produce  json
-// @Param Name query string true "mirror name"
-// @Param ArchiveURL query string true "url of the archive to mirror e.g. http://deb.debian.org/debian/"
-// @Param Distribution query string false "distribution name to mirror e.g. `buster`, for flat repositories use `./` instead of distribution name"
-// @Param Filter query string false "package query that is applied to packages in the mirror"
-// @Param Components query []string false "components to mirror, if not specified aptly would fetch all components"
-// @Param Architectures query []string false "limit mirror to those architectures, if not specified aptly would fetch all architectures"
-// @Param Keyrings query []string false "gpg keyring(s) to use when verifying `Release` file"
-// @Param DownloadSources query bool false "whether to mirror sources"
-// @Param DownloadUdebs query bool false "whether to mirror `.udeb` packages (Debian installer support)"
-// @Param DownloadInstaller query bool false "whether to download additional not packaged installer files"
-// @Param FilterWithDeps query bool false "when filtering, include dependencies of matching packages as well"
-// @Param SkipComponentCheck query bool false "whether to skip if the given components are in the `Release` file"
-// @Param IgnoreSignatures query bool false "whether to skip the verification of `Release` file signatures"
+// @Consume json
+// @Param request body mirrorCreateParams true "Parameters"
+// @Produce json
 // @Success 200 {object} deb.RemoteRepo
 // @Failure 400 {object} Error "Bad Request"
 // @Router /api/mirrors [post]
 func apiMirrorsCreate(c *gin.Context) {
 	var err error
-	var b struct {
-		Name                  string `binding:"required"`
-		ArchiveURL            string `binding:"required"`
-		Distribution          string
-		Filter                string
-		Components            []string
-		Architectures         []string
-		Keyrings              []string
-		DownloadSources       bool
-		DownloadUdebs         bool
-		DownloadInstaller     bool
-		FilterWithDeps        bool
-		SkipComponentCheck    bool
-		SkipArchitectureCheck bool
-		IgnoreSignatures      bool
-	}
+	var b mirrorCreateParams
 
 	b.DownloadSources = context.Config().DownloadSourcePackages
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
@@ -155,12 +160,11 @@ func apiMirrorsCreate(c *gin.Context) {
 }
 
 // @Summary Delete Mirror
-// @Description Delete a mirror
+// @Description **Delete a mirror**
 // @Tags Mirrors
-// @Consume  json
-// @Produce  json
 // @Param name path string true "mirror name"
 // @Param force query int true "force: 1 to enable"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue
 // @Failure 404 {object} Error "Mirror not found"
 // @Failure 403 {object} Error "Unable to delete mirror with snapshots"
@@ -205,11 +209,10 @@ func apiMirrorsDrop(c *gin.Context) {
 }
 
 // @Summary Show Mirror
-// @Description Get mirror information by name
+// @Description **Get mirror information by name**
 // @Tags Mirrors
-// @Consume  json
-// @Produce  json
 // @Param name path string true "mirror name"
+// @Produce json
 // @Success 200 {object} deb.RemoteRepo
 // @Failure 404 {object} Error "Mirror not found"
 // @Failure 500 {object} Error "Internal Error"
@@ -234,13 +237,12 @@ func apiMirrorsShow(c *gin.Context) {
 }
 
 // @Summary List Mirror Packages
-// @Description Get a list of packages from a mirror
+// @Description **Get a list of packages from a mirror**
 // @Tags Mirrors
-// @Consume  json
-// @Produce  json
 // @Param name path string true "mirror name"
 // @Param q query string false "search query"
 // @Param format query string false "format: `details` for more detailed information"
+// @Produce json
 // @Success 200 {array} deb.Package "List of Packages"
 // @Failure 400 {object} Error "Unable to determine list of architectures"
 // @Failure 404 {object} Error "Mirror not found"
@@ -323,26 +325,46 @@ func apiMirrorsPackages(c *gin.Context) {
 	}
 }
 
+type mirrorUpdateParams struct {
+	// Change mirror name to `Name`
+	Name string `                 json:"Name"                   example:"mirror1"`
+	// Url of the archive to mirror
+	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
+	// Package query that is applied to mirror packages
+	Filter string `               json:"Filter"                 example:"xserver-xorg"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `      json:"Architectures"          example:"amd64"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `         json:"Components"             example:"main"`
+	// Gpg keyring(s) for verifing Release file
+	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `         json:"FilterWithDeps"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `        json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `          json:"DownloadUdebs"`
+	// Set "true" to skip checking if the given components are in the Release file
+	SkipComponentCheck bool `     json:"SkipComponentCheck"`
+	// Set "true" to skip checking if the given architectures are in the Release file
+	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
+	// Set "true" to ignore checksum errors
+	IgnoreChecksums bool `        json:"IgnoreChecksums"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `       json:"IgnoreSignatures"`
+	// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
+	ForceUpdate bool `            json:"ForceUpdate"`
+	// Set "true" to skip downloading already downloaded packages
+	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+}
+
 // @Summary Update Mirror
-// @Description Update Mirror and download packages
+// @Description **Update Mirror and download packages**
 // @Tags Mirrors
-// @Consume  json
-// @Produce  json
 // @Param name path string true "mirror name to update"
-// @Param Name query string false "change mirror name"
-// @Param ArchiveURL query string false "ArchiveURL"
-// @Param Filter query string false "Filter"
-// @Param Architectures query []string false "Architectures"
-// @Param Components query []string false "Components"
-// @Param Keyrings query []string false "Keyrings"
-// @Param FilterWithDeps query bool false "FilterWithDeps"
-// @Param DownloadSources query bool false "DownloadSources"
-// @Param DownloadUdebs query bool false "DownloadUdebs"
-// @Param SkipComponentCheck query bool false "SkipComponentCheck"
-// @Param IgnoreChecksums query bool false "IgnoreChecksums"
-// @Param IgnoreSignatures query bool false "IgnoreSignatures"
-// @Param ForceUpdate query bool false "ForceUpdate"
-// @Param SkipExistingPackages query bool false "SkipExistingPackages"
+// @Consume json
+// @Param request body mirrorUpdateParams true "Parameters"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
 // @Success 202 {object} task.Task "Mirror is being updated"
 // @Failure 400 {object} Error "Unable to determine list of architectures"
@@ -353,26 +375,9 @@ func apiMirrorsUpdate(c *gin.Context) {
 	var (
 		err    error
 		remote *deb.RemoteRepo
+		b      mirrorUpdateParams
 	)
 
-	var b struct {
-		Name                  string
-		ArchiveURL            string
-		Filter                string
-		Architectures         []string
-		Components            []string
-		Keyrings              []string
-		FilterWithDeps        bool
-		DownloadSources       bool
-		DownloadUdebs         bool
-		SkipComponentCheck    bool
-		SkipArchitectureCheck bool
-		IgnoreChecksums       bool
-		IgnoreSignatures      bool
-		ForceUpdate           bool
-		SkipExistingPackages  bool
-	}
-
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 

api/router.go

@@ -185,6 +185,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 	}
 
 	{
+
 		api.GET("/publish", apiPublishList)
 		api.POST("/publish", apiPublishRepoOrSnapshot)
 		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
@@ -200,7 +201,8 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
 		api.DELETE("/snapshots/:name", apiSnapshotsDrop)
 		api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
-		api.POST("/snapshots/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/:name/pull", apiSnapshotsPull)
 	}
 
 	{

api/snapshot.go

@@ -3,11 +3,13 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"sort"
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
@@ -404,17 +406,37 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
 	showPackages(c, snapshot.RefList(), collectionFactory)
 }
 
-// POST /api/snapshots/merge
+type snapshotsMergeParams struct {
+	// List of snapshot names to be merged
+	Sources []string `binding:"required" json:"Sources"     example:"snapshot1"`
+}
+
+// @Summary Snapshot Merge
+// @Description **Merge several source snapshots into a new snapshot**
+// @Description
+// @Description Merge happens from left to right. By default, packages with the same name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
+// @Description
+// @Description If only one snapshot is specified, merge copies source into destination.
+// @Tags Snapshots
+// @Param name path string true "Name of the snapshot to be created"
+// @Param latest query int false "merge only the latest version of each package"
+// @Param no-remove query int false "all versions of packages are preserved during merge"
+// @Consume json
+// @Param request body snapshotsMergeParams true "Parameters"
+// @Produce  json
+// @Success 200
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/merge [post]
 func apiSnapshotsMerge(c *gin.Context) {
 	var (
 		err      error
 		snapshot *deb.Snapshot
+		body     snapshotsMergeParams
 	)
 
-	var body struct {
-		Destination string   `binding:"required"`
-		Sources     []string `binding:"required"`
-	}
+	name := c.Params.ByName("name")
 
 	if c.Bind(&body) != nil {
 		return
@@ -454,7 +476,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 		resources[i] = string(sources[i].ResourceKey())
 	}
 
-	maybeRunTaskInBackground(c, "Merge snapshot "+body.Destination, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		result := sources[0].RefList()
 		for i := 1; i < len(sources); i++ {
 			result = result.Merge(sources[i].RefList(), overrideMatching, false)
@@ -469,7 +491,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 			sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(body.Destination, sources, result,
+		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
 			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
 		err = collectionFactory.SnapshotCollection().Add(snapshot)
@@ -480,3 +502,194 @@ func apiSnapshotsMerge(c *gin.Context) {
 		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
 	})
 }
+
+type snapshotsPullParams struct {
+	// Source name to be searched for packages and dependencies
+	Source string `binding:"required"      json:"Source"            example:"source-snapshot"`
+	// Name of the snapshot to be created
+	Destination string `binding:"required" json:"Destination"       example:"idestination-snapshot"`
+	// List of package queries (i.e. name of package to be pulled from `Source`)
+	Queries []string `binding:"required"   json:"Queries"           example:"xserver-xorg"`
+	// List of architectures (optional)
+	Architectures []string `               json:"Architectures"     example:"amd64, armhf"`
+}
+
+// @Summary Snapshot Pull
+// @Description **Pulls new packages and dependencies from a source snapshot into a new snapshot**
+// @Description
+// @Description May also upgrade package versions if name snapshot already contains packages being pulled. New snapshot `Destination` is created as result of this process.
+// @Description If architectures are limited (with config architectures or parameter `Architectures`, only mentioned architectures are processed, otherwise aptly will process all architectures in the snapshot.
+// @Description If following dependencies by source is enabled (using dependencyFollowSource config), pulling binary packages would also pull corresponding source packages as well.
+// @Description By default aptly would remove packages matching name and architecture while importing: e.g. when importing software_1.3_amd64, package software_1.2.9_amd64 would be removed.
+// @Description
+// @Description With flag `no-remove` both package versions would stay in the snapshot.
+// @Description
+// @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled.
+// @Tags Snapshots
+// @Param name path string true "Name of the snapshot to be created"
+// @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable"
+// @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable"
+// @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable"
+// @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable"
+// @Consume json
+// @Param request body snapshotsPullParams true "Parameters"
+// @Produce json
+// @Success 200
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/pull [post]
+func apiSnapshotsPull(c *gin.Context) {
+	var (
+		err                 error
+		destinationSnapshot *deb.Snapshot
+		body                snapshotsPullParams
+	)
+
+	name := c.Params.ByName("name")
+
+	if err = c.BindJSON(&body); err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, err)
+		return
+	}
+
+	allMatches := c.Request.URL.Query().Get("all-matches") == "1"
+	dryRun := c.Request.URL.Query().Get("dry-run") == "1"
+	noDeps := c.Request.URL.Query().Get("no-deps") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <name> snapshot
+	toSnapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
+	}
+
+	// Load <Source> snapshot
+	sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
+	}
+
+	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// convert snapshots to package list
+		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		toPackageList.PrepareIndex()
+		sourcePackageList.PrepareIndex()
+
+		var architecturesList []string
+
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = toPackageList.Architectures(false)
+		}
+
+		architecturesList = append(architecturesList, body.Architectures...)
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+		var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+		for _, arch := range architecturesList {
+			archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
+		}
+
+		queries := make([]deb.PackageQuery, len(body.Queries))
+		for i, q := range body.Queries {
+			queries[i], err = query.Parse(q)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// Add architecture filter
+			queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+		}
+
+		// Filter with dependencies as requested
+		destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		destinationPackageList.PrepareIndex()
+
+		removedPackages := []string{}
+		addedPackages := []string{}
+		alreadySeen := map[string]bool{}
+
+		destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
+			key := pkg.Architecture + "_" + pkg.Name
+			_, seen := alreadySeen[key]
+
+			// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+			if !noRemove && !seen {
+				// Remove all packages with the same name and architecture
+				packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+				for _, p := range packageSearchResults {
+					toPackageList.Remove(p)
+					removedPackages = append(removedPackages, p.String())
+				}
+			}
+
+			// If !allMatches, add only first matching name-arch package
+			if !seen || allMatches {
+				toPackageList.Add(pkg)
+				addedPackages = append(addedPackages, pkg.String())
+			}
+
+			alreadySeen[key] = true
+
+			return nil
+		})
+		alreadySeen = nil
+
+		if dryRun {
+			response := struct {
+				AddedPackages   []string `json:"added_packages"`
+				RemovedPackages []string `json:"removed_packages"`
+			}{
+				AddedPackages:   addedPackages,
+				RemovedPackages: removedPackages,
+			}
+
+			return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
+		}
+
+		// Create <destination> snapshot
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
+	})
+}