From bd64232eb6f2da63a1e314888396ed5d09e1ae04 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Wed, 9 Oct 2024 07:38:23 +0200 Subject: [PATCH 01/25] Allow management of components This commit allows to add, remove and update components of published repositories without the need to recreate them. Signed-off-by: Christoph Fiehe --- Makefile | 2 +- api/publish.go | 530 ++++++++++++++++++-- api/router.go | 8 +- cmd/publish.go | 17 +- cmd/publish_show.go | 3 +- cmd/publish_snapshot.go | 4 + cmd/publish_source_add.go | 89 ++++ cmd/publish_source_drop.go | 62 +++ cmd/publish_source_list.go | 89 ++++ cmd/publish_source_remove.go | 81 +++ cmd/publish_source_update.go | 86 ++++ cmd/publish_switch.go | 79 +-- cmd/publish_update.go | 14 +- deb/publish.go | 229 ++++++++- docs/index.go | 2 + system/build-deb | 18 - system/lib.py | 58 ++- system/t06_publish/AzurePublish2Test_gold | 2 +- system/t06_publish/AzurePublish3Test_gold | 2 +- system/t06_publish/PublishSwitch11Test_gold | 2 +- system/t06_publish/PublishSwitch13Test_gold | 2 +- system/t06_publish/PublishSwitch14Test_gold | 2 +- system/t06_publish/PublishSwitch15Test_gold | 2 +- system/t06_publish/PublishSwitch16Test_gold | 2 +- system/t06_publish/PublishSwitch1Test_gold | 2 +- system/t06_publish/PublishSwitch2Test_gold | 2 +- system/t06_publish/PublishSwitch3Test_gold | 2 +- system/t06_publish/PublishSwitch4Test_gold | 2 +- system/t06_publish/PublishSwitch5Test_gold | 2 +- system/t06_publish/PublishSwitch6Test_gold | 2 +- system/t06_publish/PublishSwitch8Test_gold | 2 +- system/t06_publish/PublishUpdate10Test_gold | 2 +- system/t06_publish/PublishUpdate11Test_gold | 2 +- system/t06_publish/PublishUpdate12Test_gold | 2 +- system/t06_publish/PublishUpdate13Test_gold | 2 +- system/t06_publish/PublishUpdate14Test_gold | 2 +- system/t06_publish/PublishUpdate1Test_gold | 2 +- system/t06_publish/PublishUpdate2Test_gold | 2 +- system/t06_publish/PublishUpdate3Test_gold | 2 +- system/t06_publish/PublishUpdate4Test_gold | 2 +- system/t06_publish/PublishUpdate7Test_gold | 2 +- system/t06_publish/PublishUpdate8Test_gold | 2 +- system/t06_publish/S3Publish2Test_gold | 2 +- system/t06_publish/S3Publish3Test_gold | 2 +- system/t06_publish/S3Publish6Test_gold | 2 +- 45 files changed, 1248 insertions(+), 179 deletions(-) create mode 100644 cmd/publish_source_add.go create mode 100644 cmd/publish_source_drop.go create mode 100644 cmd/publish_source_list.go create mode 100644 cmd/publish_source_remove.go create mode 100644 cmd/publish_source_update.go delete mode 100755 system/build-deb diff --git a/Makefile b/Makefile index 363b2101..89aceb22 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ flake8: ## run flake8 on system test python files lint: # Install golangci-lint - go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) + @test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) # Running lint @PATH=$(BINPATH)/:$(PATH) golangci-lint run diff --git a/api/publish.go b/api/publish.go index c7aced61..9e04499c 100644 --- a/api/publish.go +++ b/api/publish.go @@ -3,6 +3,7 @@ package api import ( "fmt" "net/http" + "path/filepath" "strings" "github.com/aptly-dev/aptly/aptly" @@ -13,17 +14,29 @@ import ( "github.com/gin-gonic/gin" ) -// SigningOptions is a shared between publish API GPG options structure -type SigningOptions struct { - Skip bool - GpgKey string - Keyring string - SecretKeyring string - Passphrase string - PassphraseFile string +type signingParams struct { + // Don't sign published repository + Skip bool ` json:"Skip"` + // GPG key ID to use when signing the release, if not specified default key is used + GpgKey string ` json:"GpgKey"` + // GPG keyring to use (instead of default) + Keyring string ` json:"Keyring"` + // GPG secret keyring to use (instead of default) + SecretKeyring string ` json:"SecretKeyring"` + // GPG passphrase to unlock private key (possibly insecure) + Passphrase string ` json:"Passphrase"` + // GPG passphrase file to unlock private key (possibly insecure) + PassphraseFile string ` json:"PassphraseFile"` } -func getSigner(options *SigningOptions) (pgp.Signer, error) { +type sourceParams struct { + // Name of the component + Component string `binding:"required" json:"Component" example:"contrib"` + // Name of the local repository/snapshot + Name string `binding:"required" json:"Name" example:"snap1"` +} + +func getSigner(options *signingParams) (pgp.Signer, error) { if options.Skip { return nil, nil } @@ -54,10 +67,10 @@ func slashEscape(path string) string { return result } -// @Summary Get publish points -// @Description Get list of available publish points. Each publish point is returned as in “show” API. +// @Summary List published repositories +// @Description **Get a list of all published repositories** // @Tags Publish -// @Produce json +// @Produce json // @Success 200 {array} deb.PublishedRepo // @Failure 500 {object} Error "Internal Error" // @Router /api/publish [get] @@ -79,37 +92,97 @@ func apiPublishList(c *gin.Context) { }) if err != nil { - AbortWithJSONError(c, 500, err) + AbortWithJSONError(c, http.StatusInternalServerError, err) return } - c.JSON(200, result) + c.JSON(http.StatusOK, result) } -// POST /publish/:prefix -func apiPublishRepoOrSnapshot(c *gin.Context) { +// @Summary Show published repository +// @Description **Get published repository by name** +// @Tags Publish +// @Consume json +// @Produce json +// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs" +// @Param distribution path string true "distribution name" +// @Success 200 {object} deb.RemoteRepo +// @Failure 404 {object} Error "Published repository not found" +// @Failure 500 {object} Error "Internal Error" +// @Router /api/publish/{prefix}/{distribution} [get] +func apiPublishShow(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) - var b struct { - SourceKind string `binding:"required"` - Sources []struct { - Component string - Name string `binding:"required"` - } `binding:"required"` - Distribution string - Label string - Origin string - NotAutomatic string - ButAutomaticUpgrades string - ForceOverwrite bool - SkipContents *bool - SkipBz2 *bool - Architectures []string - Signing SigningOptions - AcquireByHash *bool - MultiDist bool + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + return } + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + return + } + + c.JSON(http.StatusOK, published) +} + +type publishedRepoCreateParams struct { + // 'local' for local repositories and 'snapshot' for snapshots + SourceKind string `binding:"required" json:"SourceKind" example:"snapshot"` + // List of 'Component/Name' objects, 'Name' is either local repository or snapshot name + Sources []sourceParams `binding:"required" json:"Sources"` + // Distribution name, if missing Aptly would try to guess from sources + Distribution string ` json:"Distribution"` + // Value of Label: field in published repository stanza + Label string ` json:"Label"` + // Value of Origin: field in published repository stanza + Origin string ` json:"Origin"` + // when publishing, overwrite files in pool/ directory without notice + ForceOverwrite bool ` json:"ForceOverwrite"` + // Override list of published architectures + Architectures []string ` json:"Architectures"` + // GPG options + Signing signingParams ` json:"Signing"` + // Setting to yes indicates to the package manager to not install or upgrade packages from the repository without user consent + NotAutomatic string ` json:"NotAutomatic"` + // setting to yes excludes upgrades from the NotAutomic setting + ButAutomaticUpgrades string ` json:"ButAutomaticUpgrades"` + // Don't generate contents indexes + SkipContents *bool ` json:"SkipContents"` + // Don't remove unreferenced files in prefix/component + SkipCleanup *bool ` json:"SkipCleanup"` + // Skip bz2 compression for index files + SkipBz2 *bool ` json:"SkipBz2"` + // Provide index files by hash + AcquireByHash *bool ` json:"AcquireByHash"` + // Enable multiple packages with the same filename in different distributions + MultiDist *bool ` json:"MultiDist"` +} + +// @Summary Create published repository +// @Description Publish local repository or snapshot under specified prefix. Storage might be passed in prefix as well, e.g. `s3:packages/`. +// @Description To supply empty prefix, just remove last part (`POST /api/publish`). +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Consume json +// @Param request body publishedRepoCreateParams true "Parameters" +// @Produce json +// @Success 200 {object} deb.RemoteRepo +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Source not found" +// @Failure 500 {object} Error "Internal Error" +// @Router /api/publish/{prefix} [post] +func apiPublishRepoOrSnapshot(c *gin.Context) { + var b publishedRepoCreateParams + + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) if c.Bind(&b) != nil { return @@ -119,12 +192,12 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { signer, err := getSigner(&b.Signing) if err != nil { - AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG signer: %s", err)) + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err)) return } if len(b.Sources) == 0 { - AbortWithJSONError(c, 400, fmt.Errorf("unable to publish: soures are empty")) + AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to publish: soures are empty")) return } @@ -145,12 +218,11 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { snapshot, err = snapshotCollection.ByName(source.Name) if err != nil { - AbortWithJSONError(c, 404, fmt.Errorf("unable to publish: %s", err)) + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err)) return } resources = append(resources, string(snapshot.ResourceKey())) - sources = append(sources, snapshot) } } else if b.SourceKind == deb.SourceLocalRepo { @@ -164,18 +236,24 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { localRepo, err = localCollection.ByName(source.Name) if err != nil { - AbortWithJSONError(c, 404, fmt.Errorf("unable to publish: %s", err)) + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to publish: %s", err)) return } + resources = append(resources, string(localRepo.Key())) sources = append(sources, localRepo) } } else { - AbortWithJSONError(c, 400, fmt.Errorf("unknown SourceKind")) + AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unknown SourceKind")) return } - taskName := fmt.Sprintf("Publish %s: %s", b.SourceKind, strings.Join(names, ", ")) + taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"", + b.SourceKind, published.StoragePrefix(), published.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`)) + + resources = append(resources, string(published.Key())) + collection := collectionFactory.PublishedRepoCollection() + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { taskDetail := task.PublishDetail{ Detail: detail, @@ -233,7 +311,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { published.AcquireByHash = *b.AcquireByHash } - collection := collectionFactory.PublishedRepoCollection() + if b.MultiDist != nil { + published.MultiDist = *b.MultiDist + } + duplicate := collection.CheckDuplicate(published) if duplicate != nil { collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory) @@ -254,7 +335,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { }) } -// PUT /publish/:prefix/:distribution +// @Summary Update published repository +// @Description Update a published repository. +// @Tags Publish +// @Accept json +// @Produce json +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Success 200 {object} deb.RemoteRepo +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository or source not found" +// @Failure 500 {object} Error "Internal Error" +// @Router /api/publish/{prefix}/{distribution} [put] func apiPublishUpdateSwitch(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) @@ -262,7 +354,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { var b struct { ForceOverwrite bool - Signing SigningOptions + Signing signingParams SkipContents *bool SkipBz2 *bool SkipCleanup *bool @@ -280,7 +372,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { signer, err := getSigner(&b.Signing) if err != nil { - AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG signer: %s", err)) + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err)) return } @@ -290,12 +382,13 @@ func apiPublishUpdateSwitch(c *gin.Context) { published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { - AbortWithJSONError(c, 404, fmt.Errorf("unable to update: %s", err)) + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) return } var updatedComponents []string var updatedSnapshots []string + if published.SourceKind == deb.SourceLocalRepo { if len(b.Snapshots) > 0 { AbortWithJSONError(c, 400, fmt.Errorf("snapshots shouldn't be given when updating local repo")) @@ -335,7 +428,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { var resources []string resources = append(resources, string(published.Key())) - taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", ")) + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.LoadComplete(published, collectionFactory) if err != nil { @@ -362,6 +455,11 @@ func apiPublishUpdateSwitch(c *gin.Context) { } } + result, err := published.Update(collectionFactory, out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) @@ -372,26 +470,53 @@ func apiPublishUpdateSwitch(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } + updatedComponents := result.UpdatedComponents() + removedComponents := result.RemovedComponents() + if b.SkipCleanup == nil || !*b.SkipCleanup { - err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents, - context.GetPublishedStorage(storage), collectionFactory, out) + publishedStorage := context.GetPublishedStorage(storage) + + err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents, publishedStorage, collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } + + if len(removedComponents) > 0 { + // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. + for _, component := range removedComponents { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + } } return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil }) } -// DELETE /publish/:prefix/:distribution +// @Summary Delete published repository +// @Description Delete a published repository. +// @Tags Publish +// @Accept json +// @Produce json +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Param force query int true "force: 1 to enable" +// @Param skipCleanup query int true "skipCleanup: 1 to enable" +// @Success 200 {object} task.ProcessReturnValue +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository not found" +// @Failure 500 {object} Error "Internal Error" +// @Router /api/publish/{prefix}/{distribution} [delete] func apiPublishDrop(c *gin.Context) { force := c.Request.URL.Query().Get("force") == "1" skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1" param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := c.Params.ByName("distribution") + distribution := parseEscapedPath(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -404,7 +529,7 @@ func apiPublishDrop(c *gin.Context) { resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Delete published %s (%s)", prefix, distribution) + taskName := fmt.Sprintf("Delete published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err := collection.Remove(context, storage, prefix, distribution, collectionFactory, out, force, skipCleanup) @@ -415,3 +540,304 @@ func apiPublishDrop(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil }) } + +// @Router /api/publish/{prefix}/{distribution}/sources/{component} [put] +func apiPublishSourceUpdate(c *gin.Context) { + var ( + err error + b sourceParams + ) + + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + component := parseEscapedPath(c.Params.ByName("component")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + return + } + + revision := published.ObtainRevision() + sources := revision.Sources + + b.Component = component + b.Name = revision.Sources[component] + + if c.Bind(&b) != nil { + return + } + + if b.Component != component { + delete(sources, component) + } + sources[b.Component] = b.Name + + resources := []string{string(published.Key())} + + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) +} + +// @Router /api/publish/{prefix}/{distribution}/sources [put] +func apiPublishSourcesUpdate(c *gin.Context) { + var ( + err error + b []sourceParams + ) + + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + return + } + + if c.Bind(&b) != nil { + return + } + + revision := published.ObtainRevision() + sources := make(map[string]string, len(b)) + revision.Sources = sources + + for _, source := range b { + component := source.Component + name := source.Name + sources[component] = name + } + + resources := []string{string(published.Key())} + + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) +} + +// @Router /api/publish/{prefix}/{distribution}/sources [get] +func apiPublishSourceList(c *gin.Context) { + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + return + } + + revision := published.Revision + if revision == nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: No source changes exist")) + return + } + + c.JSON(http.StatusOK, revision.ToJSON()["Sources"]) +} + +// @Router /api/publish/{prefix}/{distribution}/sources/{component} [delete] +func apiPublishSourceDelete(c *gin.Context) { + var err error + + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + component := parseEscapedPath(c.Params.ByName("component")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + return + } + + revision := published.ObtainRevision() + sources := revision.Sources + + delete(sources, component) + + resources := []string{string(published.Key())} + + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) +} + +// @Router /api/publish/{prefix}/{distribution}/sources [delete] +func apiPublishSourcesDelete(c *gin.Context) { + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + return + } + + published.DropRevision() +} + +// @Router /api/publish/{prefix}/{distribution}/update [post] +func apiPublishUpdate(c *gin.Context) { + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + + var b struct { + AcquireByHash *bool + ForceOverwrite bool + Signing signingParams + SkipBz2 *bool + SkipCleanup *bool + SkipContents *bool + } + + if c.Bind(&b) != nil { + return + } + + signer, err := getSigner(&b.Signing) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err)) + return + } + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) + return + } + + if b.SkipContents != nil { + published.SkipContents = *b.SkipContents + } + + if b.SkipBz2 != nil { + published.SkipBz2 = *b.SkipBz2 + } + + if b.AcquireByHash != nil { + published.AcquireByHash = *b.AcquireByHash + } + + resources := []string{string(published.Key())} + + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + result, err := published.Update(collectionFactory, out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + updatedComponents := result.UpdatedComponents() + removedComponents := result.RemovedComponents() + + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + if b.SkipCleanup == nil || !*b.SkipCleanup { + publishedStorage := context.GetPublishedStorage(storage) + + err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents, publishedStorage, collectionFactory, out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + if len(removedComponents) > 0 { + // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. + for _, component := range removedComponents { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + } + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) +} diff --git a/api/router.go b/api/router.go index d5b6e612..66353fc2 100644 --- a/api/router.go +++ b/api/router.go @@ -185,12 +185,18 @@ func Router(c *ctx.AptlyContext) http.Handler { } { - api.GET("/publish", apiPublishList) + api.GET("/publish/:prefix/:distribution", apiPublishShow) api.POST("/publish", apiPublishRepoOrSnapshot) api.POST("/publish/:prefix", apiPublishRepoOrSnapshot) api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch) api.DELETE("/publish/:prefix/:distribution", apiPublishDrop) + api.GET("/publish/:prefix/:distribution/sources", apiPublishSourceList) + api.PUT("/publish/:prefix/:distribution/sources", apiPublishSourcesUpdate) + api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishSourceUpdate) + api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishSourceDelete) + api.DELETE("/publish/:prefix/:distribution/sources", apiPublishSourcesDelete) + api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate) } { diff --git a/cmd/publish.go b/cmd/publish.go index d74384e0..887929d6 100644 --- a/cmd/publish.go +++ b/cmd/publish.go @@ -34,10 +34,25 @@ func makeCmdPublish() *commander.Command { makeCmdPublishDrop(), makeCmdPublishList(), makeCmdPublishRepo(), + makeCmdPublishShow(), makeCmdPublishSnapshot(), + makeCmdPublishSource(), makeCmdPublishSwitch(), makeCmdPublishUpdate(), - makeCmdPublishShow(), + }, + } +} + +func makeCmdPublishSource() *commander.Command { + return &commander.Command{ + UsageLine: "source", + Short: "manage sources of published repository", + Subcommands: []*commander.Command{ + makeCmdPublishSourceAdd(), + makeCmdPublishSourceDrop(), + makeCmdPublishSourceList(), + makeCmdPublishSourceRemove(), + makeCmdPublishSourceUpdate(), }, } } diff --git a/cmd/publish_show.go b/cmd/publish_show.go index d943df3f..a0af446b 100644 --- a/cmd/publish_show.go +++ b/cmd/publish_show.go @@ -52,7 +52,8 @@ func aptlyPublishShowTxt(_ *commander.Command, args []string) error { fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " ")) fmt.Printf("Sources:\n") - for component, sourceID := range repo.Sources { + for _, component := range repo.Components() { + sourceID := repo.Sources[component] var name string if repo.SourceKind == deb.SourceSnapshot { source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID) diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 3d2e43e6..6312c4d4 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -150,6 +150,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool) } + if context.Flags().IsSet("multi-dist") { + published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) + } + duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published) if duplicate != nil { collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory) diff --git a/cmd/publish_source_add.go b/cmd/publish_source_add.go new file mode 100644 index 00000000..52e98000 --- /dev/null +++ b/cmd/publish_source_add.go @@ -0,0 +1,89 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error { + if len(args) < 2 { + cmd.Usage() + return commander.ErrCommandError + } + + distribution := args[0] + names := args[1:] + components := strings.Split(context.Flags().Lookup("component").Value.String(), ",") + + if len(names) != len(components) { + return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names)) + } + + prefix := context.Flags().Lookup("prefix").Value.String() + storage, prefix := deb.ParsePrefix(prefix) + + collectionFactory := context.NewCollectionFactory() + published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to add: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) + if err != nil { + return fmt.Errorf("unable to add: %s", err) + } + + revision := published.ObtainRevision() + sources := revision.Sources + + for i, component := range components { + name := names[i] + _, exists := sources[component] + if exists { + return fmt.Errorf("unable to add: Component %q has already been added", component) + } + context.Progress().Printf("Adding component %q with source %q [%s]...\n", component, name, published.SourceKind) + + sources[component] = names[i] + } + + err = collectionFactory.PublishedRepoCollection().Update(published) + if err != nil { + return fmt.Errorf("unable to save to DB: %s", err) + } + + return err +} + +func makeCmdPublishSourceAdd() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishSourceAdd, + UsageLine: "add ", + Short: "add package source to published repository", + Long: ` +The command adds (in place) one or multiple package sources to a published repository. + +The flag -component is mandatory. Use a comma-separated list of components, if +multiple components should be modified. The number of given components must be +equal to the number of given sources, e.g.: + + aptly publish add -component=main,contrib wheezy wheezy-main wheezy-contrib + +Example: + + $ aptly publish add -component=contrib wheezy ppa wheezy-contrib + +This command assigns the snapshot wheezy-contrib to the component contrib and +adds it to published repository revision of ppa/wheezy. +`, + Flag: *flag.NewFlagSet("aptly-publish-add", flag.ExitOnError), + } + cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") + cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") + + return cmd +} diff --git a/cmd/publish_source_drop.go b/cmd/publish_source_drop.go new file mode 100644 index 00000000..af5c1f84 --- /dev/null +++ b/cmd/publish_source_drop.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "fmt" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishSourceDrop(cmd *commander.Command, args []string) error { + if len(args) != 1 { + cmd.Usage() + return commander.ErrCommandError + } + + prefix := context.Flags().Lookup("prefix").Value.String() + distribution := args[0] + storage, prefix := deb.ParsePrefix(prefix) + + collectionFactory := context.NewCollectionFactory() + published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to drop: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) + if err != nil { + return fmt.Errorf("unable to drop: %s", err) + } + + published.DropRevision() + + err = collectionFactory.PublishedRepoCollection().Update(published) + if err != nil { + return fmt.Errorf("unable to save to DB: %s", err) + } + + context.Progress().Printf("Source changes have been removed successfully.\n") + + return err +} + +func makeCmdPublishSourceDrop() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishSourceDrop, + UsageLine: "drop ", + Short: "drops revision of published repository", + Long: ` +Command drops revision of a published repository. + +Example: + + $ aptly publish revision drop wheezy +`, + Flag: *flag.NewFlagSet("aptly-publish-revision-create", flag.ExitOnError), + } + cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") + cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") + + return cmd +} diff --git a/cmd/publish_source_list.go b/cmd/publish_source_list.go new file mode 100644 index 00000000..2a915439 --- /dev/null +++ b/cmd/publish_source_list.go @@ -0,0 +1,89 @@ +package cmd + +import ( + "encoding/json" + "fmt" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishSourceList(cmd *commander.Command, args []string) error { + if len(args) != 1 { + cmd.Usage() + return commander.ErrCommandError + } + + prefix := context.Flags().Lookup("prefix").Value.String() + distribution := args[0] + storage, prefix := deb.ParsePrefix(prefix) + + published, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to list: %s", err) + } + + err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(published, context.NewCollectionFactory()) + if err != nil { + return err + } + + if published.Revision == nil { + return fmt.Errorf("unable to list: No source changes exist") + } + + jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool) + + if jsonFlag { + return aptlyPublishSourceListJSON(published) + } + + return aptlyPublishSourceListTxt(published) +} + +func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error { + revision := published.Revision + + fmt.Printf("Sources:\n") + for _, component := range revision.Components() { + name := revision.Sources[component] + fmt.Printf(" %s: %s [%s]\n", component, name, published.SourceKind) + } + + return nil +} + +func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error { + revision := published.Revision + + output, err := json.MarshalIndent(revision.ToJSON()["Sources"], "", " ") + if err != nil { + return fmt.Errorf("unable to list: %s", err) + } + + fmt.Println(string(output)) + + return nil +} + +func makeCmdPublishSourceList() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishSourceList, + UsageLine: "list ", + Short: "lists revision of published repository", + Long: ` +Command lists sources of a published repository. + +Example: + + $ aptly publish source list wheezy +`, + Flag: *flag.NewFlagSet("aptly-publish-source-list", flag.ExitOnError), + } + cmd.Flag.Bool("json", false, "display record in JSON format") + cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") + cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") + + return cmd +} diff --git a/cmd/publish_source_remove.go b/cmd/publish_source_remove.go new file mode 100644 index 00000000..6a6136c3 --- /dev/null +++ b/cmd/publish_source_remove.go @@ -0,0 +1,81 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error { + if len(args) < 1 { + cmd.Usage() + return commander.ErrCommandError + } + + distribution := args[0] + components := strings.Split(context.Flags().Lookup("component").Value.String(), ",") + + if len(components) == 0 { + return fmt.Errorf("unable to remove: Missing components, specify at least one component") + } + + prefix := context.Flags().Lookup("prefix").Value.String() + storage, prefix := deb.ParsePrefix(prefix) + + collectionFactory := context.NewCollectionFactory() + published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to remove: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) + if err != nil { + return fmt.Errorf("unable to remove: %s", err) + } + + revision := published.ObtainRevision() + sources := revision.Sources + + for _, component := range components { + name, exists := sources[component] + if !exists { + return fmt.Errorf("unable to remove: Component %q is not part of revision", component) + } + context.Progress().Printf("Removing component %q with source %q [%s]...\n", component, name, published.SourceKind) + + delete(sources, component) + } + + err = collectionFactory.PublishedRepoCollection().Update(published) + if err != nil { + return fmt.Errorf("unable to save to DB: %s", err) + } + + return err +} + +func makeCmdPublishSourceRemove() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishSourceRemove, + UsageLine: "remove [[:]] ", + Short: "remove package source to published repository", + Long: ` +The command removes one or multiple components from a published repository. + +The flag -component is mandatory. Use a comma-separated list of components, if +multiple components should be removed, e.g.: + +Example: + + $ aptly publish remove -component=contrib,non-free wheezy filesystem:symlink:debian +`, + Flag: *flag.NewFlagSet("aptly-publish-remove", flag.ExitOnError), + } + cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") + cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)") + + return cmd +} diff --git a/cmd/publish_source_update.go b/cmd/publish_source_update.go new file mode 100644 index 00000000..6fe90c55 --- /dev/null +++ b/cmd/publish_source_update.go @@ -0,0 +1,86 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error { + if len(args) < 2 { + cmd.Usage() + return commander.ErrCommandError + } + + distribution := args[0] + names := args[1:] + components := strings.Split(context.Flags().Lookup("component").Value.String(), ",") + + if len(names) != len(components) { + return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names)) + } + + prefix := context.Flags().Lookup("prefix").Value.String() + storage, prefix := deb.ParsePrefix(prefix) + + collectionFactory := context.NewCollectionFactory() + published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + revision := published.ObtainRevision() + sources := revision.Sources + + for i, component := range components { + name := names[i] + _, exists := sources[component] + if !exists { + return fmt.Errorf("unable to update: Component %q does not exist", component) + } + context.Progress().Printf("Updating component %q with source %q [%s]...\n", component, name, published.SourceKind) + + sources[component] = name + } + + err = collectionFactory.PublishedRepoCollection().Update(published) + if err != nil { + return fmt.Errorf("unable to save to DB: %s", err) + } + + return err +} + +func makeCmdPublishSourceUpdate() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishSourceUpdate, + UsageLine: "update ", + Short: "update package source to published repository", + Long: ` +The command updates one or multiple components in a published repository. + +The flag -component is mandatory. Use a comma-separated list of components, if +multiple components should be modified. The number of given components must be +equal to the number of given sources, e.g.: + + aptly publish update -component=main,contrib wheezy wheezy-main wheezy-contrib + +Example: + + $ aptly publish update -component=contrib wheezy ppa wheezy-contrib +`, + Flag: *flag.NewFlagSet("aptly-publish-revision-source-update", flag.ExitOnError), + } + cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") + cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") + + return cmd +} diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index 0f1a620b..6d65db72 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -5,12 +5,16 @@ import ( "strings" "github.com/aptly-dev/aptly/deb" + "github.com/aptly-dev/aptly/utils" "github.com/smira/commander" "github.com/smira/flag" ) func aptlyPublishSwitch(cmd *commander.Command, args []string) error { - var err error + var ( + err error + names []string + ) components := strings.Split(context.Flags().Lookup("component").Value.String(), ",") @@ -22,11 +26,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { distribution := args[0] param := "." - var ( - names []string - snapshot *deb.Snapshot - ) - if len(args) == len(components)+2 { param = args[1] names = args[2:] @@ -41,16 +40,12 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { collectionFactory := context.NewCollectionFactory() published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { - return fmt.Errorf("unable to update: %s", err) - } - - if published.SourceKind != deb.SourceSnapshot { - return fmt.Errorf("unable to update: not a snapshot publish") + return fmt.Errorf("unable to switch: %s", err) } err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) if err != nil { - return fmt.Errorf("unable to update: %s", err) + return fmt.Errorf("unable to switch: %s", err) } publishedComponents := published.Components() @@ -62,18 +57,46 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names)) } - for i, component := range components { - snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i]) - if err != nil { - return fmt.Errorf("unable to switch: %s", err) - } + if published.SourceKind == deb.SourceLocalRepo { + localRepoCollection := collectionFactory.LocalRepoCollection() + for i, component := range components { + if !utils.StrSliceHasItem(publishedComponents, component) { + return fmt.Errorf("unable to switch: component %s does not exist in published repository", component) + } - err = collectionFactory.SnapshotCollection().LoadComplete(snapshot) - if err != nil { - return fmt.Errorf("unable to switch: %s", err) - } + localRepo, err := localRepoCollection.ByName(names[i]) + if err != nil { + return fmt.Errorf("unable to switch: %s", err) + } - published.UpdateSnapshot(component, snapshot) + err = localRepoCollection.LoadComplete(localRepo) + if err != nil { + return fmt.Errorf("unable to switch: %s", err) + } + + published.UpdateLocalRepo(component, localRepo) + } + } else if published.SourceKind == deb.SourceSnapshot { + snapshotCollection := collectionFactory.SnapshotCollection() + for i, component := range components { + if !utils.StrSliceHasItem(publishedComponents, component) { + return fmt.Errorf("unable to switch: component %s does not exist in published repository", component) + } + + snapshot, err := snapshotCollection.ByName(names[i]) + if err != nil { + return fmt.Errorf("unable to switch: %s", err) + } + + err = snapshotCollection.LoadComplete(snapshot) + if err != nil { + return fmt.Errorf("unable to switch: %s", err) + } + + published.UpdateSnapshot(component, snapshot) + } + } else { + return fmt.Errorf("unknown published repository type") } signer, err := getSigner(context.Flags()) @@ -114,11 +137,11 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, context.GetPublishedStorage(storage), collectionFactory, context.Progress()) if err != nil { - return fmt.Errorf("unable to update: %s", err) + return fmt.Errorf("unable to switch: %s", err) } } - context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String()) + context.Progress().Printf("\nPublished %s repository %s has been successfully switched to new source.\n", published.SourceKind, published.String()) return err } @@ -126,15 +149,15 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { func makeCmdPublishSwitch() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSwitch, - UsageLine: "switch [[:]] ", - Short: "update published repository by switching to new snapshot", + UsageLine: "switch [[:]] ", + Short: "update published repository by switching to new source", Long: ` -Command switches in-place published snapshots with new snapshot contents. All +Command switches in-place published snapshots with new source contents. All publishing parameters are preserved (architecture list, distribution, component). For multiple component repositories, flag -component should be given with -list of components to update. Corresponding snapshots should be given in the +list of components to update. Corresponding sources should be given in the same order, e.g.: aptly publish switch -component=main,contrib wheezy wh-main wh-contrib diff --git a/cmd/publish_update.go b/cmd/publish_update.go index 28de8c67..decd4e2b 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -31,18 +31,14 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to update: %s", err) } - if published.SourceKind != deb.SourceLocalRepo { - return fmt.Errorf("unable to update: not a local repository publish") - } - err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) if err != nil { return fmt.Errorf("unable to update: %s", err) } - components := published.Components() - for _, component := range components { - published.UpdateLocalRepo(component) + result, err := published.Update(collectionFactory, context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) } signer, err := getSigner(context.Flags()) @@ -80,14 +76,14 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) if !skipCleanup { - err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, + err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, result.UpdatedComponents(), context.GetPublishedStorage(storage), collectionFactory, context.Progress()) if err != nil { return fmt.Errorf("unable to update: %s", err) } } - context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String()) + context.Progress().Printf("\nPublished %s repository %s has been successfully updated.\n", published.SourceKind, published.String()) return err } diff --git a/deb/publish.go b/deb/publish.go index 79c47cb8..e5472093 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -21,6 +21,12 @@ import ( "github.com/aptly-dev/aptly/utils" ) +type PublishedRepoUpdateResult struct { + AddedSources map[string]string + UpdatedSources map[string]string + RemovedSources map[string]string +} + type repoSourceItem struct { // Pointer to snapshot if SourceKind == "snapshot" snapshot *Snapshot @@ -73,6 +79,168 @@ type PublishedRepo struct { // Support multiple distributions MultiDist bool + + // Revision + Revision *PublishedRepoRevision +} + +type PublishedRepoRevision struct { + // Map of sources: component name -> snapshot name/local repo Name + Sources map[string]string +} + +func (result *PublishedRepoUpdateResult) AddedComponents() []string { + components := make([]string, 0, len(result.AddedSources)) + for component := range result.AddedSources { + components = append(components, component) + } + sort.Strings(components) + + return components +} + +func (result *PublishedRepoUpdateResult) UpdatedComponents() []string { + components := make([]string, 0, len(result.UpdatedSources)) + for component := range result.UpdatedSources { + components = append(components, component) + } + sort.Strings(components) + + return components +} + +func (result *PublishedRepoUpdateResult) RemovedComponents() []string { + components := make([]string, 0, len(result.RemovedSources)) + for component := range result.RemovedSources { + components = append(components, component) + } + sort.Strings(components) + + return components +} + +func (revision *PublishedRepoRevision) Components() []string { + components := make([]string, 0, len(revision.Sources)) + for component := range revision.Sources { + components = append(components, component) + } + sort.Strings(components) + + return components +} + +func (revision *PublishedRepoRevision) SourceNames() []string { + sources := revision.Sources + names := make([]string, 0, len(sources)) + for _, component := range revision.Components() { + names = append(names, sources[component]) + } + + return names +} + +func (p *PublishedRepo) DropRevision() *PublishedRepoRevision { + revision := p.Revision + p.Revision = nil + + return revision +} + +func (p *PublishedRepo) ObtainRevision() *PublishedRepoRevision { + revision := p.Revision + if revision == nil { + sources := make(map[string]string, len(p.Sources)) + for _, component := range p.Components() { + item := p.sourceItems[component] + if item.snapshot != nil { + sources[component] = item.snapshot.Name + } else if item.localRepo != nil { + sources[component] = item.localRepo.Name + } else { + panic("no snapshot/localRepo") + } + } + revision = &PublishedRepoRevision{ + Sources: sources, + } + p.Revision = revision + } + return revision +} + +func (p *PublishedRepo) Update(collectionFactory *CollectionFactory, _ aptly.Progress) (*PublishedRepoUpdateResult, error) { + result := &PublishedRepoUpdateResult{ + AddedSources: map[string]string{}, + UpdatedSources: map[string]string{}, + RemovedSources: map[string]string{}, + } + + revision := p.ObtainRevision() + p.DropRevision() + + publishedComponents := p.Components() + + for _, component := range publishedComponents { + name, exists := revision.Sources[component] + if !exists { + p.RemoveComponent(component) + result.RemovedSources[component] = name + } + } + + if p.SourceKind == SourceLocalRepo { + localRepoCollection := collectionFactory.LocalRepoCollection() + for component, name := range revision.Sources { + localRepo, err := localRepoCollection.ByName(name) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + err = localRepoCollection.LoadComplete(localRepo) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + _, exists := p.Sources[component] + if exists { + // Even in the case, when the local repository has not been changed as package source, + // it may contain a modified set of packages that requires (re-)publication. + p.UpdateLocalRepo(component, localRepo) + result.UpdatedSources[component] = name + } else { + p.UpdateLocalRepo(component, localRepo) + result.AddedSources[component] = name + } + } + } else if p.SourceKind == SourceSnapshot { + snapshotCollection := collectionFactory.SnapshotCollection() + for component, name := range revision.Sources { + snapshot, err := snapshotCollection.ByName(name) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + err = snapshotCollection.LoadComplete(snapshot) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + sourceUUID, exists := p.Sources[component] + if exists { + if snapshot.UUID != sourceUUID { + p.UpdateSnapshot(component, snapshot) + result.UpdatedSources[component] = name + } + } else { + p.UpdateSnapshot(component, snapshot) + result.AddedSources[component] = name + } + } + } else { + return result, fmt.Errorf("unknown published repository type") + } + + return result, nil } // ParsePrefix splits [storage:]prefix into components @@ -281,14 +449,42 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri return result, nil } -// MarshalJSON requires object to filled by "LoadShallow" or "LoadComplete" -func (p *PublishedRepo) MarshalJSON() ([]byte, error) { - type sourceInfo struct { - Component, Name string +type sourceInfo struct { + Component, Name string +} + +func (revision *PublishedRepoRevision) ToJSON() map[string]any { + sources := []sourceInfo{} + for _, component := range revision.Components() { + name := revision.Sources[component] + sources = append(sources, sourceInfo{ + Component: component, + Name: name, + }) + } + return map[string]any{"Sources": sources} +} + +func (revision *PublishedRepoRevision) MarshalJSON() ([]byte, error) { + sources := []sourceInfo{} + for _, component := range revision.Components() { + name := revision.Sources[component] + sources = append(sources, sourceInfo{ + Component: component, + Name: name, + }) } + return json.Marshal(map[string]interface{}{ + "Sources": sources, + }) +} + +// MarshalJSON requires object to filled by "LoadShallow" or "LoadComplete" +func (p *PublishedRepo) MarshalJSON() ([]byte, error) { sources := []sourceInfo{} - for component, item := range p.sourceItems { + for _, component := range p.Components() { + item := p.sourceItems[component] name := "" if item.snapshot != nil { name = item.snapshot.Name @@ -444,20 +640,25 @@ func (p *PublishedRepo) SourceNames() []string { return sources } -// UpdateLocalRepo updates content from local repo in component -func (p *PublishedRepo) UpdateLocalRepo(component string) { +// UpdateLocalRepo inserts/updates local repository source for component +func (p *PublishedRepo) UpdateLocalRepo(component string, localRepo *LocalRepo) { if p.SourceKind != SourceLocalRepo { panic("not local repo publish") } - item := p.sourceItems[component] - item.packageRefs = item.localRepo.RefList() + item, exists := p.sourceItems[component] + if !exists { + item = repoSourceItem{} + } + item.localRepo = localRepo + item.packageRefs = localRepo.RefList() p.sourceItems[component] = item + p.Sources[component] = localRepo.UUID p.rePublishing = true } -// UpdateSnapshot switches snapshot for component +// UpdateSnapshot inserts/updates snapshot source for component func (p *PublishedRepo) UpdateSnapshot(component string, snapshot *Snapshot) { if p.SourceKind != SourceSnapshot { panic("not snapshot publish") @@ -474,6 +675,14 @@ func (p *PublishedRepo) UpdateSnapshot(component string, snapshot *Snapshot) { p.rePublishing = true } +// RemoveComponent removes component from published repository +func (p *PublishedRepo) RemoveComponent(component string) { + delete(p.Sources, component) + delete(p.sourceItems, component) + + p.rePublishing = true +} + // Encode does msgpack encoding of PublishedRepo func (p *PublishedRepo) Encode() []byte { var buf bytes.Buffer diff --git a/docs/index.go b/docs/index.go index 5604656f..da325d03 100644 --- a/docs/index.go +++ b/docs/index.go @@ -1 +1,3 @@ package docs + +import _ "github.com/swaggo/swag" // make sure swag is in go.mod diff --git a/system/build-deb b/system/build-deb deleted file mode 100755 index 3c70e0b0..00000000 --- a/system/build-deb +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -e - -usermod -u `stat -c %u /work/src` aptly >/dev/null -chown -R `stat -c %u /work/src` /var/lib/aptly - -su aptly -c 'set -e; cd /work/src; -GOPATH=$PWD/.go go generate -v -# install and initialize swagger -GOPATH=$PWD/.go go install github.com/swaggo/swag/cmd/swag@latest -PATH=$PWD/.go/bin:$PATH swag init -q --markdownFiles docs -git checkout debian/changelog -DEBEMAIL="CI " dch -v `make version` "CI build" -dpkg-buildpackage -us -uc -b -d -mkdir -p build && mv ../*.deb build/ -rm -rf obj-*-linux-gnu* -git checkout debian/changelog -cd build && ls -l *.deb -' diff --git a/system/lib.py b/system/lib.py index e162b964..6f76213f 100644 --- a/system/lib.py +++ b/system/lib.py @@ -319,40 +319,36 @@ class BaseTest(object): return subprocess.Popen(command, stderr=stderr, stdout=stdout, env=environ) def run_cmd(self, command, expected_code=0): - try: - proc = self._start_process(command, stdout=subprocess.PIPE) - raw_output, _ = proc.communicate() + proc = self._start_process(command, stdout=subprocess.PIPE) + raw_output, _ = proc.communicate() - raw_output = raw_output.decode("utf-8", errors='replace') + raw_output = raw_output.decode("utf-8", errors='replace') - returncodes = [proc.returncode] - is_aptly_command = False - if isinstance(command, str): - is_aptly_command = command.startswith("aptly") + returncodes = [proc.returncode] + is_aptly_command = False + if isinstance(command, str): + is_aptly_command = command.startswith("aptly") - if isinstance(command, list): - is_aptly_command = command[0] == "aptly" + if isinstance(command, list): + is_aptly_command = command[0] == "aptly" - if is_aptly_command: - # remove the last two rows as go tests always print PASS/FAIL and coverage in those - # two lines. This would otherwise fail the tests as they would not match gold - matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output) - if not matches: - raise Exception("no matches found in output '%s'" % raw_output) + if is_aptly_command: + # remove the last two rows as go tests always print PASS/FAIL and coverage in those + # two lines. This would otherwise fail the tests as they would not match gold + matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output) + if not matches: + raise Exception("no matches found in command output '%s'" % raw_output) - output, _, returncode = matches[0] - returncodes.append(int(returncode)) - else: - output = raw_output + output, _, returncode = matches[0] + returncodes.append(int(returncode)) + else: + output = raw_output - if expected_code is not None: - if expected_code not in returncodes: - raise Exception("exit code %d != %d (output: %s)" % ( - proc.returncode, expected_code, raw_output)) - return output - except Exception as e: - raise Exception("Running command '%s' failed: %s" % - (command, str(e))) + if expected_code is not None: + if expected_code not in returncodes: + raise Exception("command expected to return %d, but returned %d: \n%s" % ( + expected_code, proc.returncode, raw_output)) + return output def gold_processor(self, gold): return gold @@ -379,6 +375,8 @@ class BaseTest(object): return s def check_output(self): + gold_file = self.get_gold_filename() + print(f"Verifying gold file: {gold_file}") try: self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare) @@ -464,11 +462,11 @@ class BaseTest(object): def check_in(self, item, l): if item not in l: - raise Exception("item %r not in %r", item, l) + raise Exception("expected item: %r\nnot found in: %r" % (item, l)) def check_not_in(self, item, l): if item in l: - raise Exception("item %r in %r", item, l) + raise Exception("unexpected item: %r\n found in: %r" % (item, l)) def check_subset(self, a, b): diff = '' diff --git a/system/t06_publish/AzurePublish2Test_gold b/system/t06_publish/AzurePublish2Test_gold index d9fa9ada..ea8cafb9 100644 --- a/system/t06_publish/AzurePublish2Test_gold +++ b/system/t06_publish/AzurePublish2Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo azure:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository azure:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/AzurePublish3Test_gold b/system/t06_publish/AzurePublish3Test_gold index fbcd2e79..c1855169 100644 --- a/system/t06_publish/AzurePublish3Test_gold +++ b/system/t06_publish/AzurePublish3Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot azure:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository azure:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch11Test_gold b/system/t06_publish/PublishSwitch11Test_gold index 2385a0ed..9d87127b 100644 --- a/system/t06_publish/PublishSwitch11Test_gold +++ b/system/t06_publish/PublishSwitch11Test_gold @@ -7,4 +7,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot ./maverick [i386, source] publishes {main: [snap2]: Snapshot from local repo [local-repo2]} has been successfully switched to new snapshot. +Published snapshot repository ./maverick [i386, source] publishes {main: [snap2]: Snapshot from local repo [local-repo2]} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch13Test_gold b/system/t06_publish/PublishSwitch13Test_gold index 3f440f06..271997f3 100644 --- a/system/t06_publish/PublishSwitch13Test_gold +++ b/system/t06_publish/PublishSwitch13Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch14Test_gold b/system/t06_publish/PublishSwitch14Test_gold index 9b8ce8f1..1cdaf085 100644 --- a/system/t06_publish/PublishSwitch14Test_gold +++ b/system/t06_publish/PublishSwitch14Test_gold @@ -4,4 +4,4 @@ Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch15Test_gold b/system/t06_publish/PublishSwitch15Test_gold index 3f440f06..271997f3 100644 --- a/system/t06_publish/PublishSwitch15Test_gold +++ b/system/t06_publish/PublishSwitch15Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch16Test_gold b/system/t06_publish/PublishSwitch16Test_gold index 122b2882..d3d1cd5b 100644 --- a/system/t06_publish/PublishSwitch16Test_gold +++ b/system/t06_publish/PublishSwitch16Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot ./bookworm (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository ./bookworm (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch1Test_gold b/system/t06_publish/PublishSwitch1Test_gold index 3f440f06..271997f3 100644 --- a/system/t06_publish/PublishSwitch1Test_gold +++ b/system/t06_publish/PublishSwitch1Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch2Test_gold b/system/t06_publish/PublishSwitch2Test_gold index c8464423..7cc00d44 100644 --- a/system/t06_publish/PublishSwitch2Test_gold +++ b/system/t06_publish/PublishSwitch2Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "ppa" components main... -Publish for snapshot ppa/maverick [amd64, i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new snapshot. +Published snapshot repository ppa/maverick [amd64, i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch3Test_gold b/system/t06_publish/PublishSwitch3Test_gold index 3f440f06..271997f3 100644 --- a/system/t06_publish/PublishSwitch3Test_gold +++ b/system/t06_publish/PublishSwitch3Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch4Test_gold b/system/t06_publish/PublishSwitch4Test_gold index 7256431a..899c3d53 100644 --- a/system/t06_publish/PublishSwitch4Test_gold +++ b/system/t06_publish/PublishSwitch4Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "ppa" components main... -Publish for snapshot ppa/maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new snapshot. +Published snapshot repository ppa/maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch5Test_gold b/system/t06_publish/PublishSwitch5Test_gold index c28babdc..498bf248 100644 --- a/system/t06_publish/PublishSwitch5Test_gold +++ b/system/t06_publish/PublishSwitch5Test_gold @@ -1 +1 @@ -ERROR: unable to update: published repo with storage:prefix/distribution ppa/maverick not found +ERROR: unable to switch: published repo with storage:prefix/distribution ppa/maverick not found diff --git a/system/t06_publish/PublishSwitch6Test_gold b/system/t06_publish/PublishSwitch6Test_gold index bb45822f..9253c44d 100644 --- a/system/t06_publish/PublishSwitch6Test_gold +++ b/system/t06_publish/PublishSwitch6Test_gold @@ -1 +1 @@ -ERROR: unable to update: not a snapshot publish +ERROR: unable to switch: local repo with name snap1 not found diff --git a/system/t06_publish/PublishSwitch8Test_gold b/system/t06_publish/PublishSwitch8Test_gold index d9870ff9..a10e335f 100644 --- a/system/t06_publish/PublishSwitch8Test_gold +++ b/system/t06_publish/PublishSwitch8Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components b, c... -Publish for snapshot ./maverick [amd64, i386, source] publishes {a: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {b: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'}, {c: [local2]: Snapshot from local repo [local-repo]} has been successfully switched to new snapshot. +Published snapshot repository ./maverick [amd64, i386, source] publishes {a: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {b: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'}, {c: [local2]: Snapshot from local repo [local-repo]} has been successfully switched to new source. diff --git a/system/t06_publish/PublishUpdate10Test_gold b/system/t06_publish/PublishUpdate10Test_gold index bda2f4dd..082275a3 100644 --- a/system/t06_publish/PublishUpdate10Test_gold +++ b/system/t06_publish/PublishUpdate10Test_gold @@ -7,4 +7,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate11Test_gold b/system/t06_publish/PublishUpdate11Test_gold index 72e92234..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate11Test_gold +++ b/system/t06_publish/PublishUpdate11Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate12Test_gold b/system/t06_publish/PublishUpdate12Test_gold index 3b7a5709..7488eec4 100644 --- a/system/t06_publish/PublishUpdate12Test_gold +++ b/system/t06_publish/PublishUpdate12Test_gold @@ -4,4 +4,4 @@ Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate13Test_gold b/system/t06_publish/PublishUpdate13Test_gold index 72e92234..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate13Test_gold +++ b/system/t06_publish/PublishUpdate13Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate14Test_gold b/system/t06_publish/PublishUpdate14Test_gold index 6bd929ca..e0d3ab8c 100644 --- a/system/t06_publish/PublishUpdate14Test_gold +++ b/system/t06_publish/PublishUpdate14Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate1Test_gold b/system/t06_publish/PublishUpdate1Test_gold index 72e92234..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate1Test_gold +++ b/system/t06_publish/PublishUpdate1Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate2Test_gold b/system/t06_publish/PublishUpdate2Test_gold index 72e92234..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate2Test_gold +++ b/system/t06_publish/PublishUpdate2Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate3Test_gold b/system/t06_publish/PublishUpdate3Test_gold index 72e92234..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate3Test_gold +++ b/system/t06_publish/PublishUpdate3Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate4Test_gold b/system/t06_publish/PublishUpdate4Test_gold index dce245d5..29502a02 100644 --- a/system/t06_publish/PublishUpdate4Test_gold +++ b/system/t06_publish/PublishUpdate4Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo ./maverick [source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate7Test_gold b/system/t06_publish/PublishUpdate7Test_gold index 813d7055..a13b260c 100644 --- a/system/t06_publish/PublishUpdate7Test_gold +++ b/system/t06_publish/PublishUpdate7Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components contrib, main... -Publish for local repo ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate8Test_gold b/system/t06_publish/PublishUpdate8Test_gold index 09aa9845..1a6ebd26 100644 --- a/system/t06_publish/PublishUpdate8Test_gold +++ b/system/t06_publish/PublishUpdate8Test_gold @@ -3,4 +3,4 @@ Generating metadata files and linking package files... Finalizing metadata files... Cleaning up prefix "." components contrib, main... -Publish for local repo ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. diff --git a/system/t06_publish/S3Publish2Test_gold b/system/t06_publish/S3Publish2Test_gold index 12c9c0e6..d6ffd382 100644 --- a/system/t06_publish/S3Publish2Test_gold +++ b/system/t06_publish/S3Publish2Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/S3Publish3Test_gold b/system/t06_publish/S3Publish3Test_gold index 7d609f0f..54c8a83e 100644 --- a/system/t06_publish/S3Publish3Test_gold +++ b/system/t06_publish/S3Publish3Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot s3:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository s3:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/S3Publish6Test_gold b/system/t06_publish/S3Publish6Test_gold index 12c9c0e6..d6ffd382 100644 --- a/system/t06_publish/S3Publish6Test_gold +++ b/system/t06_publish/S3Publish6Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. From dcbb2a06a526f348911f85d4a5b811cc7654bd2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 9 Oct 2024 13:07:33 +0200 Subject: [PATCH 02/25] fix build --- api/publish.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/publish.go b/api/publish.go index 9e04499c..19f9d2ad 100644 --- a/api/publish.go +++ b/api/publish.go @@ -457,12 +457,12 @@ func apiPublishUpdateSwitch(c *gin.Context) { result, err := published.Update(collectionFactory, out) if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err) } err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err) } err = collection.Update(published) @@ -758,6 +758,7 @@ func apiPublishUpdate(c *gin.Context) { SkipBz2 *bool SkipCleanup *bool SkipContents *bool + MultiDist *bool } if c.Bind(&b) != nil { From fa0d2860f089d04f129843db10a40071ea67d645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 9 Oct 2024 14:30:06 +0200 Subject: [PATCH 03/25] fix multidist in publish --- api/publish.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/publish.go b/api/publish.go index 19f9d2ad..0eda0083 100644 --- a/api/publish.go +++ b/api/publish.go @@ -179,11 +179,10 @@ type publishedRepoCreateParams struct { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix} [post] func apiPublishRepoOrSnapshot(c *gin.Context) { - var b publishedRepoCreateParams - param := parseEscapedPath(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) + var b publishedRepoCreateParams if c.Bind(&b) != nil { return } @@ -248,6 +247,11 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return } + multiDist := false + if b.MultiDist != nil { + multiDist = *b.MultiDist + } + taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"", b.SourceKind, published.StoragePrefix(), published.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`)) @@ -279,7 +283,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { } } - published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, b.MultiDist) + published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, multiDist) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err) } @@ -798,6 +802,10 @@ func apiPublishUpdate(c *gin.Context) { published.AcquireByHash = *b.AcquireByHash } + if b.MultiDist != nil { + published.MultiDist = *b.MultiDist + } + resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) From 73cdf5417b0b185a1b0a702aaee75fba01bd0b83 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Wed, 9 Oct 2024 21:41:54 +0200 Subject: [PATCH 04/25] Use POST instead of PUT for source creation. Signed-off-by: Christoph Fiehe --- api/publish.go | 272 ++++++++++++++++++++++++++++++------------------- api/router.go | 5 +- 2 files changed, 168 insertions(+), 109 deletions(-) diff --git a/api/publish.go b/api/publish.go index 0eda0083..c0f3de70 100644 --- a/api/publish.go +++ b/api/publish.go @@ -252,12 +252,11 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { multiDist = *b.MultiDist } - taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"", - b.SourceKind, published.StoragePrefix(), published.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`)) - - resources = append(resources, string(published.Key())) collection := collectionFactory.PublishedRepoCollection() + resources = append(resources, string(published.Key())) + taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"", + b.SourceKind, published.StoragePrefix(), published.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`)) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { taskDetail := task.PublishDetail{ Detail: detail, @@ -430,8 +429,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.MultiDist = *b.MultiDist } - var resources []string - resources = append(resources, string(published.Key())) + resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.LoadComplete(published, collectionFactory) @@ -532,7 +530,6 @@ func apiPublishDrop(c *gin.Context) { } resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Delete published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err := collection.Remove(context, storage, prefix, distribution, @@ -545,8 +542,8 @@ func apiPublishDrop(c *gin.Context) { }) } -// @Router /api/publish/{prefix}/{distribution}/sources/{component} [put] -func apiPublishSourceUpdate(c *gin.Context) { +// @Router /api/publish/{prefix}/{distribution}/sources [post] +func apiPublishSourcesCreate(c *gin.Context) { var ( err error b sourceParams @@ -555,7 +552,57 @@ func apiPublishSourceUpdate(c *gin.Context) { param := parseEscapedPath(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := parseEscapedPath(c.Params.ByName("distribution")) - component := parseEscapedPath(c.Params.ByName("component")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to create: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to create: %s", err)) + return + } + + if c.Bind(&b) != nil { + return + } + + revision := published.ObtainRevision() + sources := revision.Sources + + component := b.Component + name := b.Name + + _, exists := sources[component] + if exists { + AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component %q already exists", component)) + return + } + + sources[component] = name + + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) +} + +// @Router /api/publish/{prefix}/{distribution}/sources [get] +func apiPublishSourcesList(c *gin.Context) { + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -572,32 +619,13 @@ func apiPublishSourceUpdate(c *gin.Context) { return } - revision := published.ObtainRevision() - sources := revision.Sources - - b.Component = component - b.Name = revision.Sources[component] - - if c.Bind(&b) != nil { + revision := published.Revision + if revision == nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: No source changes exist")) return } - if b.Component != component { - delete(sources, component) - } - sources[b.Component] = b.Name - - resources := []string{string(published.Key())} - - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.Update(published) - if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) - } - - return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil - }) + c.JSON(http.StatusOK, revision.ToJSON()["Sources"]) } // @Router /api/publish/{prefix}/{distribution}/sources [put] @@ -641,79 +669,6 @@ func apiPublishSourcesUpdate(c *gin.Context) { } resources := []string{string(published.Key())} - - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.Update(published) - if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) - } - - return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil - }) -} - -// @Router /api/publish/{prefix}/{distribution}/sources [get] -func apiPublishSourceList(c *gin.Context) { - param := parseEscapedPath(c.Params.ByName("prefix")) - storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) - - collectionFactory := context.NewCollectionFactory() - collection := collectionFactory.PublishedRepoCollection() - - published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) - if err != nil { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) - return - } - - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) - return - } - - revision := published.Revision - if revision == nil { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: No source changes exist")) - return - } - - c.JSON(http.StatusOK, revision.ToJSON()["Sources"]) -} - -// @Router /api/publish/{prefix}/{distribution}/sources/{component} [delete] -func apiPublishSourceDelete(c *gin.Context) { - var err error - - param := parseEscapedPath(c.Params.ByName("prefix")) - storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) - component := parseEscapedPath(c.Params.ByName("component")) - - collectionFactory := context.NewCollectionFactory() - collection := collectionFactory.PublishedRepoCollection() - - published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) - if err != nil { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) - return - } - - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) - return - } - - revision := published.ObtainRevision() - sources := revision.Sources - - delete(sources, component) - - resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { err = collection.Update(published) @@ -749,6 +704,110 @@ func apiPublishSourcesDelete(c *gin.Context) { published.DropRevision() } +// @Router /api/publish/{prefix}/{distribution}/sources/{component} [put] +func apiPublishSourceUpdate(c *gin.Context) { + var ( + err error + b sourceParams + ) + + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + component := parseEscapedPath(c.Params.ByName("component")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) + return + } + + revision := published.ObtainRevision() + sources := revision.Sources + + _, exists := sources[component] + if !exists { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component %q does not exist", component)) + return + } + + b.Component = component + b.Name = revision.Sources[component] + + if c.Bind(&b) != nil { + return + } + + if b.Component != component { + delete(sources, component) + } + + component = b.Component + name := b.Name + sources[component] = name + + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) +} + +// @Router /api/publish/{prefix}/{distribution}/sources/{component} [delete] +func apiPublishSourceDelete(c *gin.Context) { + var err error + + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := parseEscapedPath(c.Params.ByName("distribution")) + component := parseEscapedPath(c.Params.ByName("component")) + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + return + } + + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + return + } + + revision := published.ObtainRevision() + sources := revision.Sources + + delete(sources, component) + + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) +} + // @Router /api/publish/{prefix}/{distribution}/update [post] func apiPublishUpdate(c *gin.Context) { param := parseEscapedPath(c.Params.ByName("prefix")) @@ -807,7 +866,6 @@ func apiPublishUpdate(c *gin.Context) { } resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { result, err := published.Update(collectionFactory, out) diff --git a/api/router.go b/api/router.go index 66353fc2..8083966b 100644 --- a/api/router.go +++ b/api/router.go @@ -191,11 +191,12 @@ func Router(c *ctx.AptlyContext) http.Handler { api.POST("/publish/:prefix", apiPublishRepoOrSnapshot) api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch) api.DELETE("/publish/:prefix/:distribution", apiPublishDrop) - api.GET("/publish/:prefix/:distribution/sources", apiPublishSourceList) + api.POST("/publish/:prefix/:distribution/sources", apiPublishSourcesCreate) + api.GET("/publish/:prefix/:distribution/sources", apiPublishSourcesList) api.PUT("/publish/:prefix/:distribution/sources", apiPublishSourcesUpdate) + api.DELETE("/publish/:prefix/:distribution/sources", apiPublishSourcesDelete) api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishSourceUpdate) api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishSourceDelete) - api.DELETE("/publish/:prefix/:distribution/sources", apiPublishSourcesDelete) api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate) } From 14c29ff912be402f490f05c4e0654f6139c9c2ef Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Thu, 10 Oct 2024 08:23:58 +0200 Subject: [PATCH 05/25] Fixing tests. Signed-off-by: Christoph Fiehe --- api/publish.go | 13 ++++- cmd/publish_source_list.go | 2 +- deb/publish.go | 51 ++++++++++--------- system/t06_publish/PublishSwitch12Test_gold | 9 +--- system/t06_publish/PublishUpdate10Test_gold | 2 - ...Test_binary => PublishUpdate11Test_binary} | 0 system/t06_publish/PublishUpdate11Test_gold | 1 - ...st_release => PublishUpdate11Test_release} | 0 ...rig.tar.gz => PublishUpdate11Test_sources} | 0 system/t06_publish/PublishUpdate12Test_gold | 1 + system/t06_publish/PublishUpdate13Test_gold | 2 +- system/t06_publish/PublishUpdate14Test_gold | 8 --- ...e10Test_file => PublishUpdate6Test_binary} | 0 ...est_binary2 => PublishUpdate6Test_binary2} | 0 system/t06_publish/PublishUpdate6Test_gold | 9 +++- ...est_sources => PublishUpdate6Test_sources} | 0 ...st_sources => PublishUpdate6Test_sources2} | 0 system/t06_publish/PublishUpdate7Test_gold | 4 +- .../pyspi_0.6.1-1.5.dsc | 0 .../pyspi_0.6.1.orig.tar.gz} | 0 system/t06_publish/PublishUpdate8Test_gold | 5 +- ...7Test_sources2 => PublishUpdate9Test_file} | 0 system/t06_publish/PublishUpdate9Test_gold | 9 +++- system/t06_publish/switch.py | 13 ++--- system/t06_publish/update.py | 40 +++++---------- 25 files changed, 78 insertions(+), 91 deletions(-) rename system/t06_publish/{PublishUpdate12Test_binary => PublishUpdate11Test_binary} (100%) rename system/t06_publish/{PublishUpdate12Test_release => PublishUpdate11Test_release} (100%) rename system/t06_publish/{PublishUpdate10Test/pyspi_0.6.1.orig.tar.gz => PublishUpdate11Test_sources} (100%) delete mode 100644 system/t06_publish/PublishUpdate14Test_gold rename system/t06_publish/{PublishUpdate10Test_file => PublishUpdate6Test_binary} (100%) rename system/t06_publish/{PublishUpdate7Test_binary2 => PublishUpdate6Test_binary2} (100%) rename system/t06_publish/{PublishUpdate7Test_sources => PublishUpdate6Test_sources} (100%) rename system/t06_publish/{PublishUpdate12Test_sources => PublishUpdate6Test_sources2} (100%) rename system/t06_publish/{PublishUpdate10Test => PublishUpdate8Test}/pyspi_0.6.1-1.5.dsc (100%) rename system/t06_publish/{PublishUpdate7Test_binary => PublishUpdate8Test/pyspi_0.6.1.orig.tar.gz} (100%) rename system/t06_publish/{PublishUpdate7Test_sources2 => PublishUpdate9Test_file} (100%) diff --git a/api/publish.go b/api/publish.go index c0f3de70..01e1d8e5 100644 --- a/api/publish.go +++ b/api/publish.go @@ -429,6 +429,17 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.MultiDist = *b.MultiDist } + revision := published.ObtainRevision() + sources := revision.Sources + + if published.SourceKind == deb.SourceSnapshot { + for _, snapshotInfo := range b.Snapshots { + component := snapshotInfo.Component + name := snapshotInfo.Name + sources[component] = name + } + } + resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { @@ -625,7 +636,7 @@ func apiPublishSourcesList(c *gin.Context) { return } - c.JSON(http.StatusOK, revision.ToJSON()["Sources"]) + c.JSON(http.StatusOK, revision.SourceList()) } // @Router /api/publish/{prefix}/{distribution}/sources [put] diff --git a/cmd/publish_source_list.go b/cmd/publish_source_list.go index 2a915439..a12b5ac9 100644 --- a/cmd/publish_source_list.go +++ b/cmd/publish_source_list.go @@ -57,7 +57,7 @@ func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error { func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error { revision := published.Revision - output, err := json.MarshalIndent(revision.ToJSON()["Sources"], "", " ") + output, err := json.MarshalIndent(revision.SourceList(), "", " ") if err != nil { return fmt.Errorf("unable to list: %s", err) } diff --git a/deb/publish.go b/deb/publish.go index e5472093..b900077d 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -21,6 +21,10 @@ import ( "github.com/aptly-dev/aptly/utils" ) +type SourceEntry struct { + Component, Name string +} + type PublishedRepoUpdateResult struct { AddedSources map[string]string UpdatedSources map[string]string @@ -129,6 +133,21 @@ func (revision *PublishedRepoRevision) Components() []string { return components } +func (revision *PublishedRepoRevision) SourceList() []SourceEntry { + sources := revision.Sources + components := revision.Components() + sourceList := make([]SourceEntry, 0, len(sources)) + for _, component := range components { + name := sources[component] + sourceList = append(sourceList, SourceEntry{ + Component: component, + Name: name, + }) + } + + return sourceList +} + func (revision *PublishedRepoRevision) SourceNames() []string { sources := revision.Sources names := make([]string, 0, len(sources)) @@ -449,40 +468,26 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri return result, nil } -type sourceInfo struct { - Component, Name string -} - -func (revision *PublishedRepoRevision) ToJSON() map[string]any { - sources := []sourceInfo{} - for _, component := range revision.Components() { - name := revision.Sources[component] - sources = append(sources, sourceInfo{ - Component: component, - Name: name, - }) - } - return map[string]any{"Sources": sources} -} - func (revision *PublishedRepoRevision) MarshalJSON() ([]byte, error) { - sources := []sourceInfo{} - for _, component := range revision.Components() { - name := revision.Sources[component] - sources = append(sources, sourceInfo{ + sources := revision.Sources + components := revision.Components() + sourceList := make([]SourceEntry, 0, len(sources)) + for _, component := range components { + name := sources[component] + sourceList = append(sourceList, SourceEntry{ Component: component, Name: name, }) } return json.Marshal(map[string]interface{}{ - "Sources": sources, + "Sources": sourceList, }) } // MarshalJSON requires object to filled by "LoadShallow" or "LoadComplete" func (p *PublishedRepo) MarshalJSON() ([]byte, error) { - sources := []sourceInfo{} + sources := []SourceEntry{} for _, component := range p.Components() { item := p.sourceItems[component] name := "" @@ -493,7 +498,7 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) { } else { panic("no snapshot/local repo") } - sources = append(sources, sourceInfo{ + sources = append(sources, SourceEntry{ Component: component, Name: name, }) diff --git a/system/t06_publish/PublishSwitch12Test_gold b/system/t06_publish/PublishSwitch12Test_gold index 8a50172b..ce8636a4 100644 --- a/system/t06_publish/PublishSwitch12Test_gold +++ b/system/t06_publish/PublishSwitch12Test_gold @@ -1,8 +1 @@ -Loading packages... -Generating metadata files and linking package files... -Finalizing metadata files... -Signing file 'Release' with gpg, please enter your passphrase when prompted: -Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components a, c... - -Publish for snapshot ./maverick [i386] publishes {a: [snap2]: Created as empty}, {b: [snap2]: Created as empty}, {c: [snap3]: Created as empty} has been successfully switched to new snapshot. +ERROR: unable to switch: component c does not exist in published repository diff --git a/system/t06_publish/PublishUpdate10Test_gold b/system/t06_publish/PublishUpdate10Test_gold index 082275a3..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate10Test_gold +++ b/system/t06_publish/PublishUpdate10Test_gold @@ -1,5 +1,3 @@ -WARNING: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool. - Loading packages... Generating metadata files and linking package files... Finalizing metadata files... diff --git a/system/t06_publish/PublishUpdate12Test_binary b/system/t06_publish/PublishUpdate11Test_binary similarity index 100% rename from system/t06_publish/PublishUpdate12Test_binary rename to system/t06_publish/PublishUpdate11Test_binary diff --git a/system/t06_publish/PublishUpdate11Test_gold b/system/t06_publish/PublishUpdate11Test_gold index 05b56b2a..7488eec4 100644 --- a/system/t06_publish/PublishUpdate11Test_gold +++ b/system/t06_publish/PublishUpdate11Test_gold @@ -3,6 +3,5 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate12Test_release b/system/t06_publish/PublishUpdate11Test_release similarity index 100% rename from system/t06_publish/PublishUpdate12Test_release rename to system/t06_publish/PublishUpdate11Test_release diff --git a/system/t06_publish/PublishUpdate10Test/pyspi_0.6.1.orig.tar.gz b/system/t06_publish/PublishUpdate11Test_sources similarity index 100% rename from system/t06_publish/PublishUpdate10Test/pyspi_0.6.1.orig.tar.gz rename to system/t06_publish/PublishUpdate11Test_sources diff --git a/system/t06_publish/PublishUpdate12Test_gold b/system/t06_publish/PublishUpdate12Test_gold index 7488eec4..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate12Test_gold +++ b/system/t06_publish/PublishUpdate12Test_gold @@ -3,5 +3,6 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components main... Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate13Test_gold b/system/t06_publish/PublishUpdate13Test_gold index 05b56b2a..e0d3ab8c 100644 --- a/system/t06_publish/PublishUpdate13Test_gold +++ b/system/t06_publish/PublishUpdate13Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate14Test_gold b/system/t06_publish/PublishUpdate14Test_gold deleted file mode 100644 index e0d3ab8c..00000000 --- a/system/t06_publish/PublishUpdate14Test_gold +++ /dev/null @@ -1,8 +0,0 @@ -Loading packages... -Generating metadata files and linking package files... -Finalizing metadata files... -Signing file 'Release' with gpg, please enter your passphrase when prompted: -Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... - -Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate10Test_file b/system/t06_publish/PublishUpdate6Test_binary similarity index 100% rename from system/t06_publish/PublishUpdate10Test_file rename to system/t06_publish/PublishUpdate6Test_binary diff --git a/system/t06_publish/PublishUpdate7Test_binary2 b/system/t06_publish/PublishUpdate6Test_binary2 similarity index 100% rename from system/t06_publish/PublishUpdate7Test_binary2 rename to system/t06_publish/PublishUpdate6Test_binary2 diff --git a/system/t06_publish/PublishUpdate6Test_gold b/system/t06_publish/PublishUpdate6Test_gold index 98523831..a13b260c 100644 --- a/system/t06_publish/PublishUpdate6Test_gold +++ b/system/t06_publish/PublishUpdate6Test_gold @@ -1 +1,8 @@ -ERROR: unable to update: not a local repository publish +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components contrib, main... + +Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate7Test_sources b/system/t06_publish/PublishUpdate6Test_sources similarity index 100% rename from system/t06_publish/PublishUpdate7Test_sources rename to system/t06_publish/PublishUpdate6Test_sources diff --git a/system/t06_publish/PublishUpdate12Test_sources b/system/t06_publish/PublishUpdate6Test_sources2 similarity index 100% rename from system/t06_publish/PublishUpdate12Test_sources rename to system/t06_publish/PublishUpdate6Test_sources2 diff --git a/system/t06_publish/PublishUpdate7Test_gold b/system/t06_publish/PublishUpdate7Test_gold index a13b260c..1a6ebd26 100644 --- a/system/t06_publish/PublishUpdate7Test_gold +++ b/system/t06_publish/PublishUpdate7Test_gold @@ -1,8 +1,6 @@ Loading packages... Generating metadata files and linking package files... Finalizing metadata files... -Signing file 'Release' with gpg, please enter your passphrase when prompted: -Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components contrib, main... -Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate10Test/pyspi_0.6.1-1.5.dsc b/system/t06_publish/PublishUpdate8Test/pyspi_0.6.1-1.5.dsc similarity index 100% rename from system/t06_publish/PublishUpdate10Test/pyspi_0.6.1-1.5.dsc rename to system/t06_publish/PublishUpdate8Test/pyspi_0.6.1-1.5.dsc diff --git a/system/t06_publish/PublishUpdate7Test_binary b/system/t06_publish/PublishUpdate8Test/pyspi_0.6.1.orig.tar.gz similarity index 100% rename from system/t06_publish/PublishUpdate7Test_binary rename to system/t06_publish/PublishUpdate8Test/pyspi_0.6.1.orig.tar.gz diff --git a/system/t06_publish/PublishUpdate8Test_gold b/system/t06_publish/PublishUpdate8Test_gold index 1a6ebd26..8279ee7d 100644 --- a/system/t06_publish/PublishUpdate8Test_gold +++ b/system/t06_publish/PublishUpdate8Test_gold @@ -1,6 +1,3 @@ Loading packages... Generating metadata files and linking package files... -Finalizing metadata files... -Cleaning up prefix "." components contrib, main... - -Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +ERROR: unable to publish: unable to process packages: error linking file to ${HOME}/.aptly/public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz: file already exists and is different diff --git a/system/t06_publish/PublishUpdate7Test_sources2 b/system/t06_publish/PublishUpdate9Test_file similarity index 100% rename from system/t06_publish/PublishUpdate7Test_sources2 rename to system/t06_publish/PublishUpdate9Test_file diff --git a/system/t06_publish/PublishUpdate9Test_gold b/system/t06_publish/PublishUpdate9Test_gold index 8279ee7d..082275a3 100644 --- a/system/t06_publish/PublishUpdate9Test_gold +++ b/system/t06_publish/PublishUpdate9Test_gold @@ -1,3 +1,10 @@ +WARNING: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool. + Loading packages... Generating metadata files and linking package files... -ERROR: unable to publish: unable to process packages: error linking file to ${HOME}/.aptly/public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz: file already exists and is different +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components main... + +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/switch.py b/system/t06_publish/switch.py index dc777b91..4243a978 100644 --- a/system/t06_publish/switch.py +++ b/system/t06_publish/switch.py @@ -424,22 +424,15 @@ class PublishSwitch11Test(BaseTest): class PublishSwitch12Test(BaseTest): """ - publish switch: add new component to publish + publish switch: wrong component names """ fixtureCmds = [ "aptly snapshot create snap1 empty", "aptly snapshot create snap2 empty", - "aptly snapshot create snap3 empty", "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b snap1 snap2", ] - runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=a,c maverick snap2 snap3" - gold_processor = BaseTest.expand_environ - - def check(self): - super(PublishSwitch12Test, self).check() - - self.check_exists('public/dists/maverick/a/binary-i386/Packages') - self.check_exists('public/dists/maverick/c/binary-i386/Packages') + runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=a,c maverick snap2 snap1" + expectedCode = 1 class PublishSwitch13Test(BaseTest): diff --git a/system/t06_publish/update.py b/system/t06_publish/update.py index 7757715a..5d00d842 100644 --- a/system/t06_publish/update.py +++ b/system/t06_publish/update.py @@ -217,20 +217,6 @@ class PublishUpdate5Test(BaseTest): class PublishUpdate6Test(BaseTest): - """ - publish update: not a local repo - """ - fixtureDB = True - fixturePool = True - fixtureCmds = [ - "aptly snapshot create snap1 from mirror gnuplot-maverick", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1", - ] - runCmd = "aptly publish update maverick" - expectedCode = 1 - - -class PublishUpdate7Test(BaseTest): """ publish update: multiple components, add some packages """ @@ -246,7 +232,7 @@ class PublishUpdate7Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate7Test, self).check() + super(PublishUpdate6Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -280,7 +266,7 @@ class PublishUpdate7Test(BaseTest): self.check_file_contents('public/dists/maverick/contrib/binary-i386/Packages', 'binary2', match_prepare=lambda s: "\n".join(sorted(s.split("\n")))) -class PublishUpdate8Test(BaseTest): +class PublishUpdate7Test(BaseTest): """ publish update: update empty repos to empty repos """ @@ -293,7 +279,7 @@ class PublishUpdate8Test(BaseTest): gold_processor = BaseTest.expand_environ -class PublishUpdate9Test(BaseTest): +class PublishUpdate8Test(BaseTest): """ publish update: conflicting files in the repo """ @@ -309,7 +295,7 @@ class PublishUpdate9Test(BaseTest): gold_processor = BaseTest.expand_environ -class PublishUpdate10Test(BaseTest): +class PublishUpdate9Test(BaseTest): """ publish update: -force-overwrite """ @@ -324,12 +310,12 @@ class PublishUpdate10Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate10Test, self).check() + super(PublishUpdate9Test, self).check() self.check_file_contents("public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz", "file") -class PublishUpdate11Test(BaseTest): +class PublishUpdate10Test(BaseTest): """ publish update: -skip-contents """ @@ -343,7 +329,7 @@ class PublishUpdate11Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate11Test, self).check() + super(PublishUpdate10Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -353,7 +339,7 @@ class PublishUpdate11Test(BaseTest): self.check_not_exists('public/dists/maverick/main/Contents-i386.gz') -class PublishUpdate12Test(BaseTest): +class PublishUpdate11Test(BaseTest): """ publish update: removed some packages skipping cleanup """ @@ -367,7 +353,7 @@ class PublishUpdate12Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate12Test, self).check() + super(PublishUpdate11Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -439,7 +425,7 @@ class PublishUpdate12Test(BaseTest): raise Exception("path seen wrong: %r" % (pathsSeen, )) -class PublishUpdate13Test(BaseTest): +class PublishUpdate12Test(BaseTest): """ publish update: -skip-bz2 """ @@ -453,7 +439,7 @@ class PublishUpdate13Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate13Test, self).check() + super(PublishUpdate12Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -464,7 +450,7 @@ class PublishUpdate13Test(BaseTest): self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2') -class PublishUpdate14Test(BaseTest): +class PublishUpdate13Test(BaseTest): """ publish update: -multi-dist """ @@ -477,7 +463,7 @@ class PublishUpdate14Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate14Test, self).check() + super(PublishUpdate13Test, self).check() self.check_exists('public/dists/bookworm/InRelease') self.check_exists('public/dists/bookworm/Release') From 3057aed5711e1fd9cd55b585a936a77d57ee1a4f Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Thu, 10 Oct 2024 13:49:17 +0200 Subject: [PATCH 06/25] Test cases added. Signed-off-by: Christoph Fiehe --- cmd/publish_source_add.go | 2 +- cmd/publish_source_list.go | 2 +- cmd/publish_source_remove.go | 2 +- cmd/publish_source_update.go | 2 +- system/t06_publish/PublishSourceAdd1Test_gold | 1 + system/t06_publish/PublishSourceAdd2Test_gold | 1 + system/t06_publish/PublishSourceAdd3Test_gold | 1 + .../t06_publish/PublishSourceDrop1Test_gold | 1 + .../t06_publish/PublishSourceList1Test_gold | 1 + .../t06_publish/PublishSourceRemove1Test_gold | 1 + .../t06_publish/PublishSourceRemove2Test_gold | 1 + .../t06_publish/PublishSourceRemove3Test_gold | 1 + .../t06_publish/PublishSourceUpdate1Test_gold | 1 + .../t06_publish/PublishSourceUpdate2Test_gold | 1 + .../t06_publish/PublishSourceUpdate3Test_gold | 1 + system/t06_publish/source.py | 223 ++++++++++++++++++ 16 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 system/t06_publish/PublishSourceAdd1Test_gold create mode 100644 system/t06_publish/PublishSourceAdd2Test_gold create mode 100644 system/t06_publish/PublishSourceAdd3Test_gold create mode 100644 system/t06_publish/PublishSourceDrop1Test_gold create mode 100644 system/t06_publish/PublishSourceList1Test_gold create mode 100644 system/t06_publish/PublishSourceRemove1Test_gold create mode 100644 system/t06_publish/PublishSourceRemove2Test_gold create mode 100644 system/t06_publish/PublishSourceRemove3Test_gold create mode 100644 system/t06_publish/PublishSourceUpdate1Test_gold create mode 100644 system/t06_publish/PublishSourceUpdate2Test_gold create mode 100644 system/t06_publish/PublishSourceUpdate3Test_gold create mode 100644 system/t06_publish/source.py diff --git a/cmd/publish_source_add.go b/cmd/publish_source_add.go index 52e98000..08adeb77 100644 --- a/cmd/publish_source_add.go +++ b/cmd/publish_source_add.go @@ -44,7 +44,7 @@ func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error { name := names[i] _, exists := sources[component] if exists { - return fmt.Errorf("unable to add: Component %q has already been added", component) + return fmt.Errorf("unable to add: component %q has already been added", component) } context.Progress().Printf("Adding component %q with source %q [%s]...\n", component, name, published.SourceKind) diff --git a/cmd/publish_source_list.go b/cmd/publish_source_list.go index a12b5ac9..aa61a2b8 100644 --- a/cmd/publish_source_list.go +++ b/cmd/publish_source_list.go @@ -30,7 +30,7 @@ func aptlyPublishSourceList(cmd *commander.Command, args []string) error { } if published.Revision == nil { - return fmt.Errorf("unable to list: No source changes exist") + return fmt.Errorf("unable to list: no source changes exist") } jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool) diff --git a/cmd/publish_source_remove.go b/cmd/publish_source_remove.go index 6a6136c3..624763b8 100644 --- a/cmd/publish_source_remove.go +++ b/cmd/publish_source_remove.go @@ -19,7 +19,7 @@ func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error { components := strings.Split(context.Flags().Lookup("component").Value.String(), ",") if len(components) == 0 { - return fmt.Errorf("unable to remove: Missing components, specify at least one component") + return fmt.Errorf("unable to remove: missing components, specify at least one component") } prefix := context.Flags().Lookup("prefix").Value.String() diff --git a/cmd/publish_source_update.go b/cmd/publish_source_update.go index 6fe90c55..9a81fcc9 100644 --- a/cmd/publish_source_update.go +++ b/cmd/publish_source_update.go @@ -44,7 +44,7 @@ func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error { name := names[i] _, exists := sources[component] if !exists { - return fmt.Errorf("unable to update: Component %q does not exist", component) + return fmt.Errorf("unable to update: component %q does not exist", component) } context.Progress().Printf("Updating component %q with source %q [%s]...\n", component, name, published.SourceKind) diff --git a/system/t06_publish/PublishSourceAdd1Test_gold b/system/t06_publish/PublishSourceAdd1Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceAdd1Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceAdd2Test_gold b/system/t06_publish/PublishSourceAdd2Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceAdd2Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceAdd3Test_gold b/system/t06_publish/PublishSourceAdd3Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceAdd3Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceDrop1Test_gold b/system/t06_publish/PublishSourceDrop1Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceDrop1Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceList1Test_gold b/system/t06_publish/PublishSourceList1Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceList1Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceRemove1Test_gold b/system/t06_publish/PublishSourceRemove1Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceRemove1Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceRemove2Test_gold b/system/t06_publish/PublishSourceRemove2Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceRemove2Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceRemove3Test_gold b/system/t06_publish/PublishSourceRemove3Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceRemove3Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceUpdate1Test_gold b/system/t06_publish/PublishSourceUpdate1Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceUpdate1Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceUpdate2Test_gold b/system/t06_publish/PublishSourceUpdate2Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceUpdate2Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/PublishSourceUpdate3Test_gold b/system/t06_publish/PublishSourceUpdate3Test_gold new file mode 100644 index 00000000..345e6aef --- /dev/null +++ b/system/t06_publish/PublishSourceUpdate3Test_gold @@ -0,0 +1 @@ +Test diff --git a/system/t06_publish/source.py b/system/t06_publish/source.py new file mode 100644 index 00000000..432f83a8 --- /dev/null +++ b/system/t06_publish/source.py @@ -0,0 +1,223 @@ +from lib import BaseTest + + +class PublishSourceAdd1Test(BaseTest): + """ + publish source add: add single source + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-contrib", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly publish source add -component=contrib wheezy snap2" + ] + runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec" + + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishSourceAdd1Test, self).check() + self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages') + self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.gz') + self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.bz2') + self.check_exists('public/dists/wheezy/contrib/Contents-i386.gz') + self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages') + self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.gz') + self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.bz2') + self.check_exists('public/dists/wheezy/contrib/Contents-amd64.gz') + + release = self.read_file('public/dists/wheezy/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + if ['contrib', 'main'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'contrib main')) + + +class PublishSourceAdd2Test(BaseTest): + """ + publish source add: add multiple sources + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-contrib", + "aptly snapshot create snap3 from mirror wheezy-non-free", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly publish source add -component=contrib,non-free wheezy snap2 snap3" + ] + runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec" + + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishSourceAdd2Test, self).check() + self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages') + self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.gz') + self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.bz2') + self.check_exists('public/dists/wheezy/contrib/Contents-i386.gz') + self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages') + self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.gz') + self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.bz2') + self.check_exists('public/dists/wheezy/contrib/Contents-amd64.gz') + + self.check_exists('public/dists/wheezy/non-free/binary-i386/Packages') + self.check_exists('public/dists/wheezy/non-free/binary-i386/Packages.gz') + self.check_exists('public/dists/wheezy/non-free/binary-i386/Packages.bz2') + self.check_exists('public/dists/wheezy/non-free/Contents-i386.gz') + self.check_exists('public/dists/wheezy/non-free/binary-amd64/Packages') + self.check_exists('public/dists/wheezy/non-free/binary-amd64/Packages.gz') + self.check_exists('public/dists/wheezy/non-free/binary-amd64/Packages.bz2') + self.check_exists('public/dists/wheezy/non-free/Contents-amd64.gz') + + release = self.read_file('public/dists/wheezy/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + if ['contrib', 'main', 'non-free'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'contrib main non-free')) + + +class PublishSourceAdd3Test(BaseTest): + """ + publish source add: (re-)add already added source + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + ] + runCmd = "aptly publish add -component=main wheezy snap2" + expectedCode = 1 + gold_processor = BaseTest.expand_environ + + +class PublishSourceList1Test(BaseTest): + """ + publish source list: show source changes + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-contrib", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + ] + runCmd = "aptly publish source list" + + gold_processor = BaseTest.expand_environ + + +class PublishSourceDrop1Test(BaseTest): + """ + publish source drop: drop source changes + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-contrib", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + ] + runCmd = "aptly publish source drop" + + gold_processor = BaseTest.expand_environ + + +class PublishSourceUpdate1Test(BaseTest): + """ + publish source update: Update single source + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + ] + runCmd = "aptly publish source update -component=main wheezy snap2" + + gold_processor = BaseTest.expand_environ + + +class PublishSourceUpdate2Test(BaseTest): + """ + publish source update: Update multiple sources + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-main", + "aptly snapshot create snap3 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main,test snap1 snap2", + ] + runCmd = "aptly publish source update -component=main,test wheezy snap2 snap3" + + gold_processor = BaseTest.expand_environ + + +class PublishSourceUpdate3Test(BaseTest): + """ + publish source update: Update not existing source + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + ] + runCmd = "aptly publish source update -component=not-existent wheezy snap1" + + gold_processor = BaseTest.expand_environ + + +class PublishSourceRemove1Test(BaseTest): + """ + publish source remove: Remove single source + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-contrib", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main,contrib snap1 snap2", + ] + runCmd = "aptly publish source remove -component=contrib wheezy" + + gold_processor = BaseTest.expand_environ + + +class PublishSourceRemove2Test(BaseTest): + """ + publish source remove: Remove multiple sources + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-contrib", + "aptly snapshot create snap3 from mirror wheezy-non-free", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main,contrib,non-free snap1 snap2 snap3", + ] + runCmd = "aptly publish source remove -component=contrib,non-free wheezy" + + gold_processor = BaseTest.expand_environ + + +class PublishSourceRemove3Test(BaseTest): + """ + publish source remove: Remove not-existing source + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + ] + runCmd = "aptly publish source remove -component=not-existent wheezy" + expectedCode = 1 + gold_processor = BaseTest.expand_environ From 9dffe791ad1141f421a85be2b55a7e62750ba2af Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Thu, 10 Oct 2024 15:50:15 +0200 Subject: [PATCH 07/25] Restoring original test sequence Signed-off-by: Christoph Fiehe --- .../pyspi_0.6.1-1.5.dsc | 0 .../pyspi_0.6.1.orig.tar.gz | 0 ...1Test_sources => PublishUpdate10Test_file} | 0 system/t06_publish/PublishUpdate10Test_gold | 2 + system/t06_publish/PublishUpdate11Test_gold | 1 + ...Test_binary => PublishUpdate12Test_binary} | 0 system/t06_publish/PublishUpdate12Test_gold | 1 - ...st_release => PublishUpdate12Test_release} | 0 ...est_binary => PublishUpdate12Test_sources} | 0 system/t06_publish/PublishUpdate13Test_gold | 2 +- system/t06_publish/PublishUpdate14Test_gold | 8 ++ system/t06_publish/PublishUpdate6Test_gold | 9 +- ...est_sources2 => PublishUpdate7Test_binary} | 0 ...est_binary2 => PublishUpdate7Test_binary2} | 0 system/t06_publish/PublishUpdate7Test_gold | 4 +- ...est_sources => PublishUpdate7Test_sources} | 0 ...9Test_file => PublishUpdate7Test_sources2} | 0 system/t06_publish/PublishUpdate8Test_gold | 5 +- system/t06_publish/PublishUpdate9Test_gold | 9 +- system/t06_publish/SwiftPublish2Test_gold | 2 +- system/t06_publish/SwiftPublish3Test_gold | 2 +- system/t06_publish/update.py | 102 ++++++++++-------- 22 files changed, 81 insertions(+), 66 deletions(-) rename system/t06_publish/{PublishUpdate8Test => PublishUpdate10Test}/pyspi_0.6.1-1.5.dsc (100%) rename system/t06_publish/{PublishUpdate8Test => PublishUpdate10Test}/pyspi_0.6.1.orig.tar.gz (100%) rename system/t06_publish/{PublishUpdate11Test_sources => PublishUpdate10Test_file} (100%) rename system/t06_publish/{PublishUpdate11Test_binary => PublishUpdate12Test_binary} (100%) rename system/t06_publish/{PublishUpdate11Test_release => PublishUpdate12Test_release} (100%) rename system/t06_publish/{PublishUpdate6Test_binary => PublishUpdate12Test_sources} (100%) create mode 100644 system/t06_publish/PublishUpdate14Test_gold rename system/t06_publish/{PublishUpdate6Test_sources2 => PublishUpdate7Test_binary} (100%) rename system/t06_publish/{PublishUpdate6Test_binary2 => PublishUpdate7Test_binary2} (100%) rename system/t06_publish/{PublishUpdate6Test_sources => PublishUpdate7Test_sources} (100%) rename system/t06_publish/{PublishUpdate9Test_file => PublishUpdate7Test_sources2} (100%) diff --git a/system/t06_publish/PublishUpdate8Test/pyspi_0.6.1-1.5.dsc b/system/t06_publish/PublishUpdate10Test/pyspi_0.6.1-1.5.dsc similarity index 100% rename from system/t06_publish/PublishUpdate8Test/pyspi_0.6.1-1.5.dsc rename to system/t06_publish/PublishUpdate10Test/pyspi_0.6.1-1.5.dsc diff --git a/system/t06_publish/PublishUpdate8Test/pyspi_0.6.1.orig.tar.gz b/system/t06_publish/PublishUpdate10Test/pyspi_0.6.1.orig.tar.gz similarity index 100% rename from system/t06_publish/PublishUpdate8Test/pyspi_0.6.1.orig.tar.gz rename to system/t06_publish/PublishUpdate10Test/pyspi_0.6.1.orig.tar.gz diff --git a/system/t06_publish/PublishUpdate11Test_sources b/system/t06_publish/PublishUpdate10Test_file similarity index 100% rename from system/t06_publish/PublishUpdate11Test_sources rename to system/t06_publish/PublishUpdate10Test_file diff --git a/system/t06_publish/PublishUpdate10Test_gold b/system/t06_publish/PublishUpdate10Test_gold index 05b56b2a..082275a3 100644 --- a/system/t06_publish/PublishUpdate10Test_gold +++ b/system/t06_publish/PublishUpdate10Test_gold @@ -1,3 +1,5 @@ +WARNING: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool. + Loading packages... Generating metadata files and linking package files... Finalizing metadata files... diff --git a/system/t06_publish/PublishUpdate11Test_gold b/system/t06_publish/PublishUpdate11Test_gold index 7488eec4..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate11Test_gold +++ b/system/t06_publish/PublishUpdate11Test_gold @@ -3,5 +3,6 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components main... Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate11Test_binary b/system/t06_publish/PublishUpdate12Test_binary similarity index 100% rename from system/t06_publish/PublishUpdate11Test_binary rename to system/t06_publish/PublishUpdate12Test_binary diff --git a/system/t06_publish/PublishUpdate12Test_gold b/system/t06_publish/PublishUpdate12Test_gold index 05b56b2a..7488eec4 100644 --- a/system/t06_publish/PublishUpdate12Test_gold +++ b/system/t06_publish/PublishUpdate12Test_gold @@ -3,6 +3,5 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate11Test_release b/system/t06_publish/PublishUpdate12Test_release similarity index 100% rename from system/t06_publish/PublishUpdate11Test_release rename to system/t06_publish/PublishUpdate12Test_release diff --git a/system/t06_publish/PublishUpdate6Test_binary b/system/t06_publish/PublishUpdate12Test_sources similarity index 100% rename from system/t06_publish/PublishUpdate6Test_binary rename to system/t06_publish/PublishUpdate12Test_sources diff --git a/system/t06_publish/PublishUpdate13Test_gold b/system/t06_publish/PublishUpdate13Test_gold index e0d3ab8c..05b56b2a 100644 --- a/system/t06_publish/PublishUpdate13Test_gold +++ b/system/t06_publish/PublishUpdate13Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate14Test_gold b/system/t06_publish/PublishUpdate14Test_gold new file mode 100644 index 00000000..e0d3ab8c --- /dev/null +++ b/system/t06_publish/PublishUpdate14Test_gold @@ -0,0 +1,8 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components main... + +Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate6Test_gold b/system/t06_publish/PublishUpdate6Test_gold index a13b260c..98523831 100644 --- a/system/t06_publish/PublishUpdate6Test_gold +++ b/system/t06_publish/PublishUpdate6Test_gold @@ -1,8 +1 @@ -Loading packages... -Generating metadata files and linking package files... -Finalizing metadata files... -Signing file 'Release' with gpg, please enter your passphrase when prompted: -Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components contrib, main... - -Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +ERROR: unable to update: not a local repository publish diff --git a/system/t06_publish/PublishUpdate6Test_sources2 b/system/t06_publish/PublishUpdate7Test_binary similarity index 100% rename from system/t06_publish/PublishUpdate6Test_sources2 rename to system/t06_publish/PublishUpdate7Test_binary diff --git a/system/t06_publish/PublishUpdate6Test_binary2 b/system/t06_publish/PublishUpdate7Test_binary2 similarity index 100% rename from system/t06_publish/PublishUpdate6Test_binary2 rename to system/t06_publish/PublishUpdate7Test_binary2 diff --git a/system/t06_publish/PublishUpdate7Test_gold b/system/t06_publish/PublishUpdate7Test_gold index 1a6ebd26..a13b260c 100644 --- a/system/t06_publish/PublishUpdate7Test_gold +++ b/system/t06_publish/PublishUpdate7Test_gold @@ -1,6 +1,8 @@ Loading packages... Generating metadata files and linking package files... Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components contrib, main... -Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate6Test_sources b/system/t06_publish/PublishUpdate7Test_sources similarity index 100% rename from system/t06_publish/PublishUpdate6Test_sources rename to system/t06_publish/PublishUpdate7Test_sources diff --git a/system/t06_publish/PublishUpdate9Test_file b/system/t06_publish/PublishUpdate7Test_sources2 similarity index 100% rename from system/t06_publish/PublishUpdate9Test_file rename to system/t06_publish/PublishUpdate7Test_sources2 diff --git a/system/t06_publish/PublishUpdate8Test_gold b/system/t06_publish/PublishUpdate8Test_gold index 8279ee7d..09aa9845 100644 --- a/system/t06_publish/PublishUpdate8Test_gold +++ b/system/t06_publish/PublishUpdate8Test_gold @@ -1,3 +1,6 @@ Loading packages... Generating metadata files and linking package files... -ERROR: unable to publish: unable to process packages: error linking file to ${HOME}/.aptly/public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz: file already exists and is different +Finalizing metadata files... +Cleaning up prefix "." components contrib, main... + +Publish for local repo ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate9Test_gold b/system/t06_publish/PublishUpdate9Test_gold index 082275a3..8279ee7d 100644 --- a/system/t06_publish/PublishUpdate9Test_gold +++ b/system/t06_publish/PublishUpdate9Test_gold @@ -1,10 +1,3 @@ -WARNING: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool. - Loading packages... Generating metadata files and linking package files... -Finalizing metadata files... -Signing file 'Release' with gpg, please enter your passphrase when prompted: -Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... - -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +ERROR: unable to publish: unable to process packages: error linking file to ${HOME}/.aptly/public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz: file already exists and is different diff --git a/system/t06_publish/SwiftPublish2Test_gold b/system/t06_publish/SwiftPublish2Test_gold index 5beaf677..1ec018b6 100644 --- a/system/t06_publish/SwiftPublish2Test_gold +++ b/system/t06_publish/SwiftPublish2Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for local repo swift:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository swift:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. diff --git a/system/t06_publish/SwiftPublish3Test_gold b/system/t06_publish/SwiftPublish3Test_gold index 16c322a5..eab52653 100644 --- a/system/t06_publish/SwiftPublish3Test_gold +++ b/system/t06_publish/SwiftPublish3Test_gold @@ -5,4 +5,4 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Cleaning up prefix "." components main... -Publish for snapshot swift:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. +Published snapshot repository swift:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. diff --git a/system/t06_publish/update.py b/system/t06_publish/update.py index 5d00d842..88114d92 100644 --- a/system/t06_publish/update.py +++ b/system/t06_publish/update.py @@ -175,37 +175,37 @@ class PublishUpdate3Test(BaseTest): self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') -class PublishUpdate4Test(BaseTest): - """ - publish update: added some packages, but list of published archs doesn't change - """ - fixtureCmds = [ - "aptly repo create local-repo", - "aptly repo add local-repo ${files}/pyspi_0.6.1-1.3.dsc", - "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo", - "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb" - ] - runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" - gold_processor = BaseTest.expand_environ - - def check(self): - super(PublishUpdate4Test, self).check() - - self.check_exists('public/dists/maverick/InRelease') - self.check_exists('public/dists/maverick/Release') - self.check_exists('public/dists/maverick/Release.gpg') - - self.check_not_exists('public/dists/maverick/main/binary-i386/Packages') - self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.gz') - self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2') - self.check_exists('public/dists/maverick/main/source/Sources') - self.check_exists('public/dists/maverick/main/source/Sources.gz') - self.check_exists('public/dists/maverick/main/source/Sources.bz2') - - self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') - self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') - self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') - self.check_not_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') +#class PublishUpdate4Test(BaseTest): +# """ +# publish update: added some packages, but list of published archs doesn't change +# """ +# fixtureCmds = [ +# "aptly repo create local-repo", +# "aptly repo add local-repo ${files}/pyspi_0.6.1-1.3.dsc", +# "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo", +# "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb" +# ] +# runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" +# gold_processor = BaseTest.expand_environ +# +# def check(self): +# super(PublishUpdate4Test, self).check() +# +# self.check_exists('public/dists/maverick/InRelease') +# self.check_exists('public/dists/maverick/Release') +# self.check_exists('public/dists/maverick/Release.gpg') +# +# self.check_not_exists('public/dists/maverick/main/binary-i386/Packages') +# self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.gz') +# self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2') +# self.check_exists('public/dists/maverick/main/source/Sources') +# self.check_exists('public/dists/maverick/main/source/Sources.gz') +# self.check_exists('public/dists/maverick/main/source/Sources.bz2') +# +# self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') +# self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') +# self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') +# self.check_not_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') class PublishUpdate5Test(BaseTest): @@ -217,6 +217,20 @@ class PublishUpdate5Test(BaseTest): class PublishUpdate6Test(BaseTest): + """ + publish update: not a local repo + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1", + ] + runCmd = "aptly publish update maverick" + expectedCode = 1 + + +class PublishUpdate7Test(BaseTest): """ publish update: multiple components, add some packages """ @@ -232,7 +246,7 @@ class PublishUpdate6Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate6Test, self).check() + super(PublishUpdate7Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -266,7 +280,7 @@ class PublishUpdate6Test(BaseTest): self.check_file_contents('public/dists/maverick/contrib/binary-i386/Packages', 'binary2', match_prepare=lambda s: "\n".join(sorted(s.split("\n")))) -class PublishUpdate7Test(BaseTest): +class PublishUpdate8Test(BaseTest): """ publish update: update empty repos to empty repos """ @@ -279,7 +293,7 @@ class PublishUpdate7Test(BaseTest): gold_processor = BaseTest.expand_environ -class PublishUpdate8Test(BaseTest): +class PublishUpdate9Test(BaseTest): """ publish update: conflicting files in the repo """ @@ -295,7 +309,7 @@ class PublishUpdate8Test(BaseTest): gold_processor = BaseTest.expand_environ -class PublishUpdate9Test(BaseTest): +class PublishUpdate10Test(BaseTest): """ publish update: -force-overwrite """ @@ -310,12 +324,12 @@ class PublishUpdate9Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate9Test, self).check() + super(PublishUpdate10Test, self).check() self.check_file_contents("public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz", "file") -class PublishUpdate10Test(BaseTest): +class PublishUpdate11Test(BaseTest): """ publish update: -skip-contents """ @@ -329,7 +343,7 @@ class PublishUpdate10Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate10Test, self).check() + super(PublishUpdate11Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -339,7 +353,7 @@ class PublishUpdate10Test(BaseTest): self.check_not_exists('public/dists/maverick/main/Contents-i386.gz') -class PublishUpdate11Test(BaseTest): +class PublishUpdate12Test(BaseTest): """ publish update: removed some packages skipping cleanup """ @@ -353,7 +367,7 @@ class PublishUpdate11Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate11Test, self).check() + super(PublishUpdate12Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -425,7 +439,7 @@ class PublishUpdate11Test(BaseTest): raise Exception("path seen wrong: %r" % (pathsSeen, )) -class PublishUpdate12Test(BaseTest): +class PublishUpdate13Test(BaseTest): """ publish update: -skip-bz2 """ @@ -439,7 +453,7 @@ class PublishUpdate12Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate12Test, self).check() + super(PublishUpdate13Test, self).check() self.check_exists('public/dists/maverick/InRelease') self.check_exists('public/dists/maverick/Release') @@ -450,7 +464,7 @@ class PublishUpdate12Test(BaseTest): self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2') -class PublishUpdate13Test(BaseTest): +class PublishUpdate14Test(BaseTest): """ publish update: -multi-dist """ @@ -463,7 +477,7 @@ class PublishUpdate13Test(BaseTest): gold_processor = BaseTest.expand_environ def check(self): - super(PublishUpdate13Test, self).check() + super(PublishUpdate14Test, self).check() self.check_exists('public/dists/bookworm/InRelease') self.check_exists('public/dists/bookworm/Release') From d87d8bac92d12209ff2d7dc4f7293fd4545243e0 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Thu, 10 Oct 2024 16:14:25 +0200 Subject: [PATCH 08/25] Fix test cases. Signed-off-by: Christoph Fiehe --- api/publish.go | 76 +-- cmd/publish_source_add.go | 7 +- cmd/publish_source_drop.go | 6 +- cmd/publish_source_remove.go | 9 +- cmd/publish_source_update.go | 7 +- cmd/publish_switch.go | 55 +- cmd/publish_update.go | 12 +- deb/publish.go | 127 ++--- system/t06_publish/PublishSourceAdd1Test_gold | 4 +- system/t06_publish/PublishSourceAdd2Test_gold | 5 +- system/t06_publish/PublishSourceAdd3Test_gold | 2 +- .../t06_publish/PublishSourceDrop1Test_gold | 2 +- .../t06_publish/PublishSourceList1Test_gold | 4 +- .../t06_publish/PublishSourceList2Test_gold | 10 + .../t06_publish/PublishSourceList3Test_gold | 1 + .../t06_publish/PublishSourceRemove1Test_gold | 4 +- .../t06_publish/PublishSourceRemove2Test_gold | 5 +- .../t06_publish/PublishSourceRemove3Test_gold | 2 +- .../t06_publish/PublishSourceUpdate1Test_gold | 4 +- .../t06_publish/PublishSourceUpdate2Test_gold | 5 +- .../t06_publish/PublishSourceUpdate3Test_gold | 2 +- system/t06_publish/PublishSwitch6Test_gold | 2 +- system/t06_publish/PublishUpdate15Test_gold | 8 + system/t06_publish/PublishUpdate16Test_gold | 8 + system/t06_publish/PublishUpdate17Test_gold | 8 + system/t06_publish/PublishUpdate18Test_gold | 8 + system/t06_publish/PublishUpdate8Test_gold | 2 +- system/t06_publish/source.py | 175 +++---- system/t06_publish/update.py | 213 ++++++-- system/t12_api/publish.py | 469 ++++++++++++++++++ 30 files changed, 942 insertions(+), 300 deletions(-) create mode 100644 system/t06_publish/PublishSourceList2Test_gold create mode 100644 system/t06_publish/PublishSourceList3Test_gold create mode 100644 system/t06_publish/PublishUpdate15Test_gold create mode 100644 system/t06_publish/PublishUpdate16Test_gold create mode 100644 system/t06_publish/PublishUpdate17Test_gold create mode 100644 system/t06_publish/PublishUpdate18Test_gold diff --git a/api/publish.go b/api/publish.go index 01e1d8e5..bf34e651 100644 --- a/api/publish.go +++ b/api/publish.go @@ -113,7 +113,7 @@ func apiPublishList(c *gin.Context) { func apiPublishShow(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -179,7 +179,7 @@ type publishedRepoCreateParams struct { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix} [post] func apiPublishRepoOrSnapshot(c *gin.Context) { - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) var b publishedRepoCreateParams @@ -196,7 +196,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { } if len(b.Sources) == 0 { - AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to publish: soures are empty")) + AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to publish: sources are empty")) return } @@ -353,7 +353,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { func apiPublishUpdateSwitch(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := utils.SanitizePath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) var b struct { ForceOverwrite bool @@ -483,24 +483,19 @@ func apiPublishUpdateSwitch(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } - updatedComponents := result.UpdatedComponents() - removedComponents := result.RemovedComponents() - if b.SkipCleanup == nil || !*b.SkipCleanup { publishedStorage := context.GetPublishedStorage(storage) - err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents, publishedStorage, collectionFactory, out) + err = collection.CleanupPrefixComponentFiles(published.Prefix, result.UpdatedComponents(), publishedStorage, collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - if len(removedComponents) > 0 { - // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. - for _, component := range removedComponents { - err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) - if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) - } + // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. + for _, component := range result.RemovedComponents() { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } } } @@ -529,7 +524,7 @@ func apiPublishDrop(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -560,9 +555,9 @@ func apiPublishSourcesCreate(c *gin.Context) { b sourceParams ) - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -605,15 +600,15 @@ func apiPublishSourcesCreate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } - return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + return &task.ProcessReturnValue{Code: http.StatusCreated, Value: published}, nil }) } // @Router /api/publish/{prefix}/{distribution}/sources [get] func apiPublishSourcesList(c *gin.Context) { - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -646,9 +641,9 @@ func apiPublishSourcesUpdate(c *gin.Context) { b []sourceParams ) - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -693,26 +688,37 @@ func apiPublishSourcesUpdate(c *gin.Context) { // @Router /api/publish/{prefix}/{distribution}/sources [delete] func apiPublishSourcesDelete(c *gin.Context) { - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err)) return } err = collection.LoadComplete(published, collectionFactory) if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err)) return } published.DropRevision() + + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + }) } // @Router /api/publish/{prefix}/{distribution}/sources/{component} [put] @@ -722,10 +728,10 @@ func apiPublishSourceUpdate(c *gin.Context) { b sourceParams ) - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) - component := parseEscapedPath(c.Params.ByName("component")) + distribution := slashEscape(c.Params.ByName("distribution")) + component := slashEscape(c.Params.ByName("component")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -782,10 +788,10 @@ func apiPublishSourceUpdate(c *gin.Context) { func apiPublishSourceDelete(c *gin.Context) { var err error - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) - component := parseEscapedPath(c.Params.ByName("component")) + distribution := slashEscape(c.Params.ByName("distribution")) + component := slashEscape(c.Params.ByName("component")) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -821,9 +827,9 @@ func apiPublishSourceDelete(c *gin.Context) { // @Router /api/publish/{prefix}/{distribution}/update [post] func apiPublishUpdate(c *gin.Context) { - param := parseEscapedPath(c.Params.ByName("prefix")) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - distribution := parseEscapedPath(c.Params.ByName("distribution")) + distribution := slashEscape(c.Params.ByName("distribution")) var b struct { AcquireByHash *bool diff --git a/cmd/publish_source_add.go b/cmd/publish_source_add.go index 08adeb77..2ee9e7ff 100644 --- a/cmd/publish_source_add.go +++ b/cmd/publish_source_add.go @@ -56,6 +56,9 @@ func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to save to DB: %s", err) } + context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n", + distribution, published.StoragePrefix()) + return err } @@ -63,9 +66,9 @@ func makeCmdPublishSourceAdd() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceAdd, UsageLine: "add ", - Short: "add package source to published repository", + Short: "add source to staged source list of published repository", Long: ` -The command adds (in place) one or multiple package sources to a published repository. +The command adds sources to the staged source list of the published repository. The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be modified. The number of given components must be diff --git a/cmd/publish_source_drop.go b/cmd/publish_source_drop.go index af5c1f84..d038e0e4 100644 --- a/cmd/publish_source_drop.go +++ b/cmd/publish_source_drop.go @@ -45,13 +45,13 @@ func makeCmdPublishSourceDrop() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceDrop, UsageLine: "drop ", - Short: "drops revision of published repository", + Short: "drops staged source changes of published repository", Long: ` -Command drops revision of a published repository. +Command drops the staged source changes of the published repository. Example: - $ aptly publish revision drop wheezy + $ aptly publish source drop wheezy `, Flag: *flag.NewFlagSet("aptly-publish-revision-create", flag.ExitOnError), } diff --git a/cmd/publish_source_remove.go b/cmd/publish_source_remove.go index 624763b8..1d3892c4 100644 --- a/cmd/publish_source_remove.go +++ b/cmd/publish_source_remove.go @@ -42,7 +42,7 @@ func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error { for _, component := range components { name, exists := sources[component] if !exists { - return fmt.Errorf("unable to remove: Component %q is not part of revision", component) + return fmt.Errorf("unable to remove: component %q does not exist", component) } context.Progress().Printf("Removing component %q with source %q [%s]...\n", component, name, published.SourceKind) @@ -54,6 +54,9 @@ func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to save to DB: %s", err) } + context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n", + distribution, published.StoragePrefix()) + return err } @@ -61,9 +64,9 @@ func makeCmdPublishSourceRemove() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceRemove, UsageLine: "remove [[:]] ", - Short: "remove package source to published repository", + Short: "remove source from staged source list of published repository", Long: ` -The command removes one or multiple components from a published repository. +The command removes sources from the staged source list of the published repository. The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be removed, e.g.: diff --git a/cmd/publish_source_update.go b/cmd/publish_source_update.go index 9a81fcc9..66c27468 100644 --- a/cmd/publish_source_update.go +++ b/cmd/publish_source_update.go @@ -56,6 +56,9 @@ func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to save to DB: %s", err) } + context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n", + distribution, published.StoragePrefix()) + return err } @@ -63,9 +66,9 @@ func makeCmdPublishSourceUpdate() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceUpdate, UsageLine: "update ", - Short: "update package source to published repository", + Short: "update source in staged source list of published repository", Long: ` -The command updates one or multiple components in a published repository. +The command updates sources in the staged source list of the published repository. The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be modified. The number of given components must be diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index 6d65db72..ab11f594 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -43,6 +43,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to switch: %s", err) } + if published.SourceKind != deb.SourceSnapshot { + return fmt.Errorf("unable to switch: not a published snapshot repository") + } + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) if err != nil { return fmt.Errorf("unable to switch: %s", err) @@ -57,46 +61,23 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names)) } - if published.SourceKind == deb.SourceLocalRepo { - localRepoCollection := collectionFactory.LocalRepoCollection() - for i, component := range components { - if !utils.StrSliceHasItem(publishedComponents, component) { - return fmt.Errorf("unable to switch: component %s does not exist in published repository", component) - } - - localRepo, err := localRepoCollection.ByName(names[i]) - if err != nil { - return fmt.Errorf("unable to switch: %s", err) - } - - err = localRepoCollection.LoadComplete(localRepo) - if err != nil { - return fmt.Errorf("unable to switch: %s", err) - } - - published.UpdateLocalRepo(component, localRepo) + snapshotCollection := collectionFactory.SnapshotCollection() + for i, component := range components { + if !utils.StrSliceHasItem(publishedComponents, component) { + return fmt.Errorf("unable to switch: component %s does not exist in published repository", component) } - } else if published.SourceKind == deb.SourceSnapshot { - snapshotCollection := collectionFactory.SnapshotCollection() - for i, component := range components { - if !utils.StrSliceHasItem(publishedComponents, component) { - return fmt.Errorf("unable to switch: component %s does not exist in published repository", component) - } - snapshot, err := snapshotCollection.ByName(names[i]) - if err != nil { - return fmt.Errorf("unable to switch: %s", err) - } - - err = snapshotCollection.LoadComplete(snapshot) - if err != nil { - return fmt.Errorf("unable to switch: %s", err) - } - - published.UpdateSnapshot(component, snapshot) + snapshot, err := snapshotCollection.ByName(names[i]) + if err != nil { + return fmt.Errorf("unable to switch: %s", err) } - } else { - return fmt.Errorf("unknown published repository type") + + err = snapshotCollection.LoadComplete(snapshot) + if err != nil { + return fmt.Errorf("unable to switch: %s", err) + } + + published.UpdateSnapshot(component, snapshot) } signer, err := getSigner(context.Flags()) diff --git a/cmd/publish_update.go b/cmd/publish_update.go index decd4e2b..da2dba57 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "path/filepath" "github.com/aptly-dev/aptly/deb" "github.com/smira/commander" @@ -76,11 +77,20 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) if !skipCleanup { + publishedStorage := context.GetPublishedStorage(storage) err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, result.UpdatedComponents(), - context.GetPublishedStorage(storage), collectionFactory, context.Progress()) + publishedStorage, collectionFactory, context.Progress()) if err != nil { return fmt.Errorf("unable to update: %s", err) } + + // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. + for _, component := range result.RemovedComponents() { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + } } context.Progress().Printf("\nPublished %s repository %s has been successfully updated.\n", published.SourceKind, published.String()) diff --git a/deb/publish.go b/deb/publish.go index b900077d..067f9485 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -194,69 +194,78 @@ func (p *PublishedRepo) Update(collectionFactory *CollectionFactory, _ aptly.Pro RemovedSources: map[string]string{}, } - revision := p.ObtainRevision() - p.DropRevision() - - publishedComponents := p.Components() - - for _, component := range publishedComponents { - name, exists := revision.Sources[component] - if !exists { - p.RemoveComponent(component) - result.RemovedSources[component] = name - } - } - - if p.SourceKind == SourceLocalRepo { - localRepoCollection := collectionFactory.LocalRepoCollection() - for component, name := range revision.Sources { - localRepo, err := localRepoCollection.ByName(name) - if err != nil { - return result, fmt.Errorf("unable to update: %s", err) - } - - err = localRepoCollection.LoadComplete(localRepo) - if err != nil { - return result, fmt.Errorf("unable to update: %s", err) - } - - _, exists := p.Sources[component] - if exists { - // Even in the case, when the local repository has not been changed as package source, - // it may contain a modified set of packages that requires (re-)publication. - p.UpdateLocalRepo(component, localRepo) - result.UpdatedSources[component] = name - } else { - p.UpdateLocalRepo(component, localRepo) - result.AddedSources[component] = name - } - } - } else if p.SourceKind == SourceSnapshot { - snapshotCollection := collectionFactory.SnapshotCollection() - for component, name := range revision.Sources { - snapshot, err := snapshotCollection.ByName(name) - if err != nil { - return result, fmt.Errorf("unable to update: %s", err) - } - - err = snapshotCollection.LoadComplete(snapshot) - if err != nil { - return result, fmt.Errorf("unable to update: %s", err) - } - - sourceUUID, exists := p.Sources[component] - if exists { - if snapshot.UUID != sourceUUID { - p.UpdateSnapshot(component, snapshot) - result.UpdatedSources[component] = name + revision := p.DropRevision() + if revision == nil { + if p.SourceKind == SourceLocalRepo { + // Re-fetch packages from local repository + for component, item := range p.sourceItems { + localRepo := item.localRepo + if localRepo != nil { + p.UpdateLocalRepo(component, localRepo) + result.UpdatedSources[component] = localRepo.Name } - } else { - p.UpdateSnapshot(component, snapshot) - result.AddedSources[component] = name } } } else { - return result, fmt.Errorf("unknown published repository type") + for _, component := range p.Components() { + name, exists := revision.Sources[component] + if !exists { + p.RemoveComponent(component) + result.RemovedSources[component] = name + } + } + + if p.SourceKind == SourceLocalRepo { + localRepoCollection := collectionFactory.LocalRepoCollection() + for component, name := range revision.Sources { + localRepo, err := localRepoCollection.ByName(name) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + err = localRepoCollection.LoadComplete(localRepo) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + _, exists := p.Sources[component] + if exists { + // Even in the case, when the local repository has not been changed as package source, + // it may contain a modified set of packages that requires (re-)publication. + p.UpdateLocalRepo(component, localRepo) + result.UpdatedSources[component] = name + } else { + p.UpdateLocalRepo(component, localRepo) + result.AddedSources[component] = name + } + } + } else if p.SourceKind == SourceSnapshot { + snapshotCollection := collectionFactory.SnapshotCollection() + for component, name := range revision.Sources { + snapshot, err := snapshotCollection.ByName(name) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + err = snapshotCollection.LoadComplete(snapshot) + if err != nil { + return result, fmt.Errorf("unable to update: %s", err) + } + + sourceUUID, exists := p.Sources[component] + if exists { + if snapshot.UUID != sourceUUID { + p.UpdateSnapshot(component, snapshot) + result.UpdatedSources[component] = name + } + } else { + p.UpdateSnapshot(component, snapshot) + result.AddedSources[component] = name + } + } + } else { + return result, fmt.Errorf("unknown published repository type") + } } return result, nil diff --git a/system/t06_publish/PublishSourceAdd1Test_gold b/system/t06_publish/PublishSourceAdd1Test_gold index 345e6aef..8ac95d37 100644 --- a/system/t06_publish/PublishSourceAdd1Test_gold +++ b/system/t06_publish/PublishSourceAdd1Test_gold @@ -1 +1,3 @@ -Test +Adding component "test" with source "snap2" [snapshot]... + +You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceAdd2Test_gold b/system/t06_publish/PublishSourceAdd2Test_gold index 345e6aef..904bfc81 100644 --- a/system/t06_publish/PublishSourceAdd2Test_gold +++ b/system/t06_publish/PublishSourceAdd2Test_gold @@ -1 +1,4 @@ -Test +Adding component "test" with source "snap2" [snapshot]... +Adding component "other-test" with source "snap3" [snapshot]... + +You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceAdd3Test_gold b/system/t06_publish/PublishSourceAdd3Test_gold index 345e6aef..e5802b78 100644 --- a/system/t06_publish/PublishSourceAdd3Test_gold +++ b/system/t06_publish/PublishSourceAdd3Test_gold @@ -1 +1 @@ -Test +ERROR: unable to add: component "main" has already been added diff --git a/system/t06_publish/PublishSourceDrop1Test_gold b/system/t06_publish/PublishSourceDrop1Test_gold index 345e6aef..5f05ba78 100644 --- a/system/t06_publish/PublishSourceDrop1Test_gold +++ b/system/t06_publish/PublishSourceDrop1Test_gold @@ -1 +1 @@ -Test +Source changes have been removed successfully. diff --git a/system/t06_publish/PublishSourceList1Test_gold b/system/t06_publish/PublishSourceList1Test_gold index 345e6aef..364ef5c2 100644 --- a/system/t06_publish/PublishSourceList1Test_gold +++ b/system/t06_publish/PublishSourceList1Test_gold @@ -1 +1,3 @@ -Test +Sources: + main: snap1 [snapshot] + test: snap2 [snapshot] diff --git a/system/t06_publish/PublishSourceList2Test_gold b/system/t06_publish/PublishSourceList2Test_gold new file mode 100644 index 00000000..ba30e4af --- /dev/null +++ b/system/t06_publish/PublishSourceList2Test_gold @@ -0,0 +1,10 @@ +[ + { + "Component": "main", + "Name": "snap1" + }, + { + "Component": "test", + "Name": "snap2" + } +] diff --git a/system/t06_publish/PublishSourceList3Test_gold b/system/t06_publish/PublishSourceList3Test_gold new file mode 100644 index 00000000..03e2a99a --- /dev/null +++ b/system/t06_publish/PublishSourceList3Test_gold @@ -0,0 +1 @@ +ERROR: unable to list: no source changes exist diff --git a/system/t06_publish/PublishSourceRemove1Test_gold b/system/t06_publish/PublishSourceRemove1Test_gold index 345e6aef..761bf16b 100644 --- a/system/t06_publish/PublishSourceRemove1Test_gold +++ b/system/t06_publish/PublishSourceRemove1Test_gold @@ -1 +1,3 @@ -Test +Removing component "test" with source "snap2" [snapshot]... + +You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceRemove2Test_gold b/system/t06_publish/PublishSourceRemove2Test_gold index 345e6aef..6692ca2e 100644 --- a/system/t06_publish/PublishSourceRemove2Test_gold +++ b/system/t06_publish/PublishSourceRemove2Test_gold @@ -1 +1,4 @@ -Test +Removing component "test" with source "snap2" [snapshot]... +Removing component "other-test" with source "snap3" [snapshot]... + +You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceRemove3Test_gold b/system/t06_publish/PublishSourceRemove3Test_gold index 345e6aef..54507f25 100644 --- a/system/t06_publish/PublishSourceRemove3Test_gold +++ b/system/t06_publish/PublishSourceRemove3Test_gold @@ -1 +1 @@ -Test +ERROR: unable to remove: component "not-existent" does not exist diff --git a/system/t06_publish/PublishSourceUpdate1Test_gold b/system/t06_publish/PublishSourceUpdate1Test_gold index 345e6aef..ea66b43f 100644 --- a/system/t06_publish/PublishSourceUpdate1Test_gold +++ b/system/t06_publish/PublishSourceUpdate1Test_gold @@ -1 +1,3 @@ -Test +Updating component "main" with source "snap2" [snapshot]... + +You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceUpdate2Test_gold b/system/t06_publish/PublishSourceUpdate2Test_gold index 345e6aef..134f0d3f 100644 --- a/system/t06_publish/PublishSourceUpdate2Test_gold +++ b/system/t06_publish/PublishSourceUpdate2Test_gold @@ -1 +1,4 @@ -Test +Updating component "main" with source "snap2" [snapshot]... +Updating component "test" with source "snap3" [snapshot]... + +You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceUpdate3Test_gold b/system/t06_publish/PublishSourceUpdate3Test_gold index 345e6aef..4d2fb4c8 100644 --- a/system/t06_publish/PublishSourceUpdate3Test_gold +++ b/system/t06_publish/PublishSourceUpdate3Test_gold @@ -1 +1 @@ -Test +ERROR: unable to update: component "not-existent" does not exist diff --git a/system/t06_publish/PublishSwitch6Test_gold b/system/t06_publish/PublishSwitch6Test_gold index 9253c44d..27a20838 100644 --- a/system/t06_publish/PublishSwitch6Test_gold +++ b/system/t06_publish/PublishSwitch6Test_gold @@ -1 +1 @@ -ERROR: unable to switch: local repo with name snap1 not found +ERROR: unable to switch: not a published snapshot repository diff --git a/system/t06_publish/PublishUpdate15Test_gold b/system/t06_publish/PublishUpdate15Test_gold new file mode 100644 index 00000000..ebd8ad24 --- /dev/null +++ b/system/t06_publish/PublishUpdate15Test_gold @@ -0,0 +1,8 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components ... + +Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap3]: Created as empty}, {test: [snap2]: Created as empty} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate16Test_gold b/system/t06_publish/PublishUpdate16Test_gold new file mode 100644 index 00000000..e989f216 --- /dev/null +++ b/system/t06_publish/PublishUpdate16Test_gold @@ -0,0 +1,8 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components ... + +Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate17Test_gold b/system/t06_publish/PublishUpdate17Test_gold new file mode 100644 index 00000000..9ad84577 --- /dev/null +++ b/system/t06_publish/PublishUpdate17Test_gold @@ -0,0 +1,8 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components other-test, test... + +Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap5]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap4]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate18Test_gold b/system/t06_publish/PublishUpdate18Test_gold new file mode 100644 index 00000000..2e37b702 --- /dev/null +++ b/system/t06_publish/PublishUpdate18Test_gold @@ -0,0 +1,8 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components test... + +Published snapshot repository ./maverick [i386] publishes {other-test: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap3]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated. diff --git a/system/t06_publish/PublishUpdate8Test_gold b/system/t06_publish/PublishUpdate8Test_gold index 09aa9845..1a6ebd26 100644 --- a/system/t06_publish/PublishUpdate8Test_gold +++ b/system/t06_publish/PublishUpdate8Test_gold @@ -3,4 +3,4 @@ Generating metadata files and linking package files... Finalizing metadata files... Cleaning up prefix "." components contrib, main... -Publish for local repo ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. diff --git a/system/t06_publish/source.py b/system/t06_publish/source.py index 432f83a8..9f352e20 100644 --- a/system/t06_publish/source.py +++ b/system/t06_publish/source.py @@ -8,32 +8,13 @@ class PublishSourceAdd1Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror wheezy-contrib", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", - "aptly publish source add -component=contrib wheezy snap2" + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", ] - runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec" - + runCmd = "aptly publish source add -component=test maverick snap2" gold_processor = BaseTest.expand_environ - def check(self): - super(PublishSourceAdd1Test, self).check() - self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages') - self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.gz') - self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.bz2') - self.check_exists('public/dists/wheezy/contrib/Contents-i386.gz') - self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages') - self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.gz') - self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.bz2') - self.check_exists('public/dists/wheezy/contrib/Contents-amd64.gz') - - release = self.read_file('public/dists/wheezy/Release').split('\n') - components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) - components = sorted(components.split(' ')) - if ['contrib', 'main'] != components: - raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'contrib main')) - class PublishSourceAdd2Test(BaseTest): """ @@ -42,42 +23,14 @@ class PublishSourceAdd2Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror wheezy-contrib", - "aptly snapshot create snap3 from mirror wheezy-non-free", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", - "aptly publish source add -component=contrib,non-free wheezy snap2 snap3" + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", ] - runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec" - + runCmd = "aptly publish source add -component=test,other-test maverick snap2 snap3" gold_processor = BaseTest.expand_environ - def check(self): - super(PublishSourceAdd2Test, self).check() - self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages') - self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.gz') - self.check_exists('public/dists/wheezy/contrib/binary-i386/Packages.bz2') - self.check_exists('public/dists/wheezy/contrib/Contents-i386.gz') - self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages') - self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.gz') - self.check_exists('public/dists/wheezy/contrib/binary-amd64/Packages.bz2') - self.check_exists('public/dists/wheezy/contrib/Contents-amd64.gz') - - self.check_exists('public/dists/wheezy/non-free/binary-i386/Packages') - self.check_exists('public/dists/wheezy/non-free/binary-i386/Packages.gz') - self.check_exists('public/dists/wheezy/non-free/binary-i386/Packages.bz2') - self.check_exists('public/dists/wheezy/non-free/Contents-i386.gz') - self.check_exists('public/dists/wheezy/non-free/binary-amd64/Packages') - self.check_exists('public/dists/wheezy/non-free/binary-amd64/Packages.gz') - self.check_exists('public/dists/wheezy/non-free/binary-amd64/Packages.bz2') - self.check_exists('public/dists/wheezy/non-free/Contents-amd64.gz') - - release = self.read_file('public/dists/wheezy/Release').split('\n') - components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) - components = sorted(components.split(' ')) - if ['contrib', 'main', 'non-free'] != components: - raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'contrib main non-free')) - class PublishSourceAdd3Test(BaseTest): """ @@ -86,11 +39,11 @@ class PublishSourceAdd3Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror gnuplot-maverick", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", ] - runCmd = "aptly publish add -component=main wheezy snap2" + runCmd = "aptly publish source add -component=main maverick snap2" expectedCode = 1 gold_processor = BaseTest.expand_environ @@ -102,12 +55,44 @@ class PublishSourceList1Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror wheezy-contrib", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", + "aptly publish source add -component=test maverick snap2", ] - runCmd = "aptly publish source list" + runCmd = "aptly publish source list maverick" + gold_processor = BaseTest.expand_environ + +class PublishSourceList2Test(BaseTest): + """ + publish source list: show source changes as JSON + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", + "aptly publish source add -component=test maverick snap2", + ] + runCmd = "aptly publish source list -json maverick" + gold_processor = BaseTest.expand_environ + + +class PublishSourceList3Test(BaseTest): + """ + publish source list: show source changes (empty) + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", + ] + runCmd = "aptly publish source list maverick" + expectedCode = 1 gold_processor = BaseTest.expand_environ @@ -118,12 +103,11 @@ class PublishSourceDrop1Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror wheezy-contrib", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", ] - runCmd = "aptly publish source drop" - + runCmd = "aptly publish source drop maverick" gold_processor = BaseTest.expand_environ @@ -134,12 +118,11 @@ class PublishSourceUpdate1Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror gnuplot-maverick", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", ] - runCmd = "aptly publish source update -component=main wheezy snap2" - + runCmd = "aptly publish source update -component=main maverick snap2" gold_processor = BaseTest.expand_environ @@ -150,13 +133,12 @@ class PublishSourceUpdate2Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror wheezy-main", - "aptly snapshot create snap3 from mirror gnuplot-maverick", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main,test snap1 snap2", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main,test snap1 snap2", ] - runCmd = "aptly publish source update -component=main,test wheezy snap2 snap3" - + runCmd = "aptly publish source update -component=main,test maverick snap2 snap3" gold_processor = BaseTest.expand_environ @@ -167,11 +149,10 @@ class PublishSourceUpdate3Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", ] - runCmd = "aptly publish source update -component=not-existent wheezy snap1" - + runCmd = "aptly publish source update -component=not-existent maverick snap1" gold_processor = BaseTest.expand_environ @@ -182,12 +163,11 @@ class PublishSourceRemove1Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror wheezy-contrib", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main,contrib snap1 snap2", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main,test snap1 snap2", ] - runCmd = "aptly publish source remove -component=contrib wheezy" - + runCmd = "aptly publish source remove -component=test maverick" gold_processor = BaseTest.expand_environ @@ -198,13 +178,12 @@ class PublishSourceRemove2Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly snapshot create snap2 from mirror wheezy-contrib", - "aptly snapshot create snap3 from mirror wheezy-non-free", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main,contrib,non-free snap1 snap2 snap3", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main,test,other-test snap1 snap2 snap3", ] - runCmd = "aptly publish source remove -component=contrib,non-free wheezy" - + runCmd = "aptly publish source remove -component=test,other-test maverick" gold_processor = BaseTest.expand_environ @@ -215,9 +194,9 @@ class PublishSourceRemove3Test(BaseTest): fixtureDB = True fixturePool = True fixtureCmds = [ - "aptly snapshot create snap1 from mirror wheezy-main", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=wheezy -component=main snap1", + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main snap1", ] - runCmd = "aptly publish source remove -component=not-existent wheezy" + runCmd = "aptly publish source remove -component=not-existent maverick" expectedCode = 1 gold_processor = BaseTest.expand_environ diff --git a/system/t06_publish/update.py b/system/t06_publish/update.py index 88114d92..3a24ec16 100644 --- a/system/t06_publish/update.py +++ b/system/t06_publish/update.py @@ -175,39 +175,6 @@ class PublishUpdate3Test(BaseTest): self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') -#class PublishUpdate4Test(BaseTest): -# """ -# publish update: added some packages, but list of published archs doesn't change -# """ -# fixtureCmds = [ -# "aptly repo create local-repo", -# "aptly repo add local-repo ${files}/pyspi_0.6.1-1.3.dsc", -# "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo", -# "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb" -# ] -# runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" -# gold_processor = BaseTest.expand_environ -# -# def check(self): -# super(PublishUpdate4Test, self).check() -# -# self.check_exists('public/dists/maverick/InRelease') -# self.check_exists('public/dists/maverick/Release') -# self.check_exists('public/dists/maverick/Release.gpg') -# -# self.check_not_exists('public/dists/maverick/main/binary-i386/Packages') -# self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.gz') -# self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2') -# self.check_exists('public/dists/maverick/main/source/Sources') -# self.check_exists('public/dists/maverick/main/source/Sources.gz') -# self.check_exists('public/dists/maverick/main/source/Sources.bz2') -# -# self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') -# self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') -# self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz') -# self.check_not_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') - - class PublishUpdate5Test(BaseTest): """ publish update: no such publish @@ -216,20 +183,6 @@ class PublishUpdate5Test(BaseTest): expectedCode = 1 -class PublishUpdate6Test(BaseTest): - """ - publish update: not a local repo - """ - fixtureDB = True - fixturePool = True - fixtureCmds = [ - "aptly snapshot create snap1 from mirror gnuplot-maverick", - "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1", - ] - runCmd = "aptly publish update maverick" - expectedCode = 1 - - class PublishUpdate7Test(BaseTest): """ publish update: multiple components, add some packages @@ -487,3 +440,169 @@ class PublishUpdate14Test(BaseTest): self.check_exists('public/dists/bookworm/main/binary-i386/Packages.gz') self.check_exists('public/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb') + + +class PublishUpdate15Test(BaseTest): + """ + publish update: source added + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -architectures=i386 -component=main snap1", + "aptly publish source add -component=test,other-test maverick snap2 snap3" + ] + runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" + + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishUpdate15Test, self).check() + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/main/binary-i386/Packages') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') + + self.check_exists('public/dists/maverick/test/binary-i386/Packages') + self.check_exists('public/dists/maverick/test/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/test/binary-i386/Packages.bz2') + + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages') + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages.bz2') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + if ['main', 'other-test', 'test'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'main other-test test')) + + +class PublishUpdate16Test(BaseTest): + """ + publish update: source removed + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -architectures=i386 -component=main,test,other-test snap1 snap2 snap3", + "aptly publish source remove -component=test,other-test maverick" + ] + runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" + + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishUpdate16Test, self).check() + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/main/binary-i386/Packages') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/main/Contents-i386.gz') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + if ['main'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'main')) + + +class PublishUpdate17Test(BaseTest): + """ + publish update: source updated + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly snapshot create snap4 from mirror gnuplot-maverick", + "aptly snapshot create snap5 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -architectures=i386 -component=main,test,other-test snap1 snap2 snap3", + "aptly publish source update -component=test,other-test maverick snap4 snap5" + ] + runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" + + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishUpdate17Test, self).check() + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/main/binary-i386/Packages') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/main/Contents-i386.gz') + + self.check_exists('public/dists/maverick/test/binary-i386/Packages') + self.check_exists('public/dists/maverick/test/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/test/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/test/Contents-i386.gz') + + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages') + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/other-test/Contents-i386.gz') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + if ['main', 'other-test', 'test'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'main other-test test')) + + +class PublishUpdate18Test(BaseTest): + """ + publish update: source added, updated and removed + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 from mirror gnuplot-maverick", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -architectures=i386 -component=main,test snap1 snap2", + "aptly publish source remove -component=main maverick", + "aptly publish source update -component=test maverick snap3", + "aptly publish source add -component=other-test maverick snap1" + ] + runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" + + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishUpdate18Test, self).check() + self.check_exists('public/dists/maverick/InRelease') + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/Release.gpg') + + self.check_exists('public/dists/maverick/test/binary-i386/Packages') + self.check_exists('public/dists/maverick/test/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/test/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/test/Contents-i386.gz') + + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages') + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages.gz') + self.check_exists('public/dists/maverick/other-test/binary-i386/Packages.bz2') + self.check_exists('public/dists/maverick/other-test/Contents-i386.gz') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + if ['other-test', 'test'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'other-test test')) diff --git a/system/t12_api/publish.py b/system/t12_api/publish.py index 325802d2..4fe3f6f0 100644 --- a/system/t12_api/publish.py +++ b/system/t12_api/publish.py @@ -905,6 +905,58 @@ class PublishSwitchAPISkipCleanupTestRepo(APITest): self.check_exists("public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") +class PublishShowAPITestRepo(APITest): + """ + GET /publish/:prefix/:distribution + """ + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal( + self.upload("/api/files/" + d, + "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + self.check_equal(self.post_task("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + repo_expected = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'wheezy', + 'Label': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Origin': '', + 'Path': prefix + '/' + 'wheezy', + 'Prefix': prefix, + 'SkipContents': False, + 'MultiDist': False, + 'SourceKind': 'local', + 'Sources': [{'Component': 'main', 'Name': repo1_name}], + 'Storage': '', + 'Suite': ''} + repo = self.get("/api/publish/" + prefix + "/wheezy") + self.check_equal(repo.status_code, 200) + self.check_equal(repo_expected, repo.json()) + + class ServePublishedListTestRepo(APITest): """ GET /repos @@ -1036,3 +1088,420 @@ class ServePublishedNotFoundTestRepo(APITest): get = self.get("/repos/apiandserve/pool/main/b/boost-defaults/i-dont-exist") if get.status_code != 404: raise Exception(f"Expected status 404 != {get.status_code}") + + +class PublishSourcesAddAPITestRepo(APITest): + """ + POST /publish/:prefix/:distribution/sources + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + # Actual test + self.check_equal(self.post( + "/api/publish/" + prefix + "/wheezy/sources", + json={"Component": "test", "Name": repo2_name} + ).status_code, 201) + + sources_expected = [{"Component": "main", "Name": repo1_name}, {"Component": "test", "Name": repo2_name}] + sources = self.get("/api/publish/" + prefix + "/wheezy/sources") + self.check_equal(sources.status_code, 200) + self.check_equal(sources_expected, sources.json()) + + +class PublishSourceUpdateAPITestRepo(APITest): + """ + PUT /publish/:prefix/:distribution/sources/main + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + # Actual test + self.check_equal(self.put( + "/api/publish/" + prefix + "/wheezy/sources/main", + json={"Component": "main", "Name": repo2_name} + ).status_code, 200) + + sources_expected = [{"Component": "main", "Name": repo2_name}] + sources = self.get("/api/publish/" + prefix + "/wheezy/sources") + self.check_equal(sources.status_code, 200) + self.check_equal(sources_expected, sources.json()) + + +class PublishSourcesUpdateAPITestRepo(APITest): + """ + PUT /publish/:prefix/:distribution/sources + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + # Actual test + self.check_equal(self.put( + "/api/publish/" + prefix + "/wheezy/sources", + json=[{"Component": "test", "Name": repo1_name}, {"Component": "other-test", "Name": repo2_name}] + ).status_code, 200) + + sources_expected = [{"Component": "other-test", "Name": repo2_name}, {"Component": "test", "Name": repo1_name}] + sources = self.get("/api/publish/" + prefix + "/wheezy/sources") + self.check_equal(sources.status_code, 200) + self.check_equal(sources_expected, sources.json()) + + +class PublishSourceRemoveAPITestRepo(APITest): + """ + DELETE /publish/:prefix/:distribution/sources/test + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}, {"Component": "test", "Name": repo2_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + # Actual test + self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy/sources/test").status_code, 200) + + sources_expected = [{"Component": "main", "Name": repo1_name}] + sources = self.get("/api/publish/" + prefix + "/wheezy/sources") + self.check_equal(sources.status_code, 200) + self.check_equal(sources_expected, sources.json()) + + +class PublishSourcesDropAPITestRepo(APITest): + """ + DELETE /publish/:prefix/:distribution/sources + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}, {"Component": "test", "Name": repo2_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy/sources/test").status_code, 200) + + # Actual test + self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy/sources").status_code, 200) + + self.check_equal(self.get("/api/publish/" + prefix + "/wheezy/sources").status_code, 404) + + +class PublishSourcesListAPITestRepo(APITest): + """ + GET /publish/:prefix/:distribution/sources + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + # Actual test + self.check_equal(self.post( + "/api/publish/" + prefix + "/wheezy/sources", + json={"Component": "test", "Name": repo1_name} + ).status_code, 201) + + self.check_equal(self.put( + "/api/publish/" + prefix + "/wheezy/sources/main", + json={"Component": "main", "Name": repo2_name} + ).status_code, 200) + + self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy/sources/main").status_code, 200) + + sources_expected = [{"Component": "test", "Name": repo1_name}] + sources = self.get("/api/publish/" + prefix + "/wheezy/sources") + self.check_equal(sources.status_code, 200) + self.check_equal(sources_expected, sources.json()) + + +class PublishUpdateSourcesAPITestRepo(APITest): + """ + POST /publish/:prefix/:distribution/update + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + repo3_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo3_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo3_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "Signing": DefaultSigningOptions, + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}, {"Component": "test", "Name": repo2_name}], + } + ).status_code, 201) + + # remove 'main' component + self.check_equal(self.delete("/api/publish/" + prefix + "/wheezy/sources/main").status_code, 200) + + # update 'test' component + self.check_equal(self.put( + "/api/publish/" + prefix + "/wheezy/sources/test", + json={"Component": "test", "Name": repo1_name} + ).status_code, 200) + + # add 'other-test' component + self.check_equal(self.post( + "/api/publish/" + prefix + "/wheezy/sources", + json={"Component": "other-test", "Name": repo3_name} + ).status_code, 201) + + sources_expected = [{"Component": "other-test", "Name": repo3_name}, {"Component": "test", "Name": repo1_name}] + sources = self.get("/api/publish/" + prefix + "/wheezy/sources") + self.check_equal(sources.status_code, 200) + self.check_equal(sources_expected, sources.json()) + + # update published repository and publish new content + self.check_equal(self.post( + "/api/publish/" + prefix + "/wheezy/update", + json={ + "AcquireByHash": True, + "MultiDist": False, + "Signing": DefaultSigningOptions, + "SkipBz2": True, + "SkipContents": True, + } + ).status_code, 200) + + repo_expected = { + 'AcquireByHash': True, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'wheezy', + 'Label': '', + 'Origin': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Path': prefix + '/' + 'wheezy', + 'Prefix': prefix, + 'SkipContents': True, + 'MultiDist': False, + 'SourceKind': 'local', + 'Sources': [{"Component": "other-test", "Name": repo3_name}, {"Component": "test", "Name": repo1_name}], + 'Storage': '', + 'Suite': ''} + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) From ac5ecf946d9d4c42104f9c62a987c03fcfd15607 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Fri, 11 Oct 2024 22:26:11 +0200 Subject: [PATCH 09/25] Cleanup improved and code redundant code removed. Signed-off-by: Christoph Fiehe --- api/publish.go | 32 ++++-------------------------- cmd/publish_switch.go | 4 ++-- cmd/publish_update.go | 14 ++----------- deb/publish.go | 46 ++++++++++++++++++++++++++++++++++++------- 4 files changed, 47 insertions(+), 49 deletions(-) diff --git a/api/publish.go b/api/publish.go index bf34e651..e51ceaba 100644 --- a/api/publish.go +++ b/api/publish.go @@ -3,7 +3,6 @@ package api import ( "fmt" "net/http" - "path/filepath" "strings" "github.com/aptly-dev/aptly/aptly" @@ -484,20 +483,11 @@ func apiPublishUpdateSwitch(c *gin.Context) { } if b.SkipCleanup == nil || !*b.SkipCleanup { - publishedStorage := context.GetPublishedStorage(storage) - - err = collection.CleanupPrefixComponentFiles(published.Prefix, result.UpdatedComponents(), publishedStorage, collectionFactory, out) + err = collection.CleanupPrefixComponentFiles(context, published, result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), + collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - - // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. - for _, component := range result.RemovedComponents() { - err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) - if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) - } - } } return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil @@ -890,9 +880,6 @@ func apiPublishUpdate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - updatedComponents := result.UpdatedComponents() - removedComponents := result.RemovedComponents() - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) @@ -904,22 +891,11 @@ func apiPublishUpdate(c *gin.Context) { } if b.SkipCleanup == nil || !*b.SkipCleanup { - publishedStorage := context.GetPublishedStorage(storage) - - err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents, publishedStorage, collectionFactory, out) + err = collection.CleanupPrefixComponentFiles(context, published, + result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - - if len(removedComponents) > 0 { - // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. - for _, component := range removedComponents { - err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) - if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) - } - } - } } return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index ab11f594..eedb42d8 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -115,8 +115,8 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) if !skipCleanup { - err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, - context.GetPublishedStorage(storage), collectionFactory, context.Progress()) + err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, + []string{}, components, []string{}, collectionFactory, context.Progress()) if err != nil { return fmt.Errorf("unable to switch: %s", err) } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index da2dba57..f4337865 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "path/filepath" "github.com/aptly-dev/aptly/deb" "github.com/smira/commander" @@ -77,20 +76,11 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) if !skipCleanup { - publishedStorage := context.GetPublishedStorage(storage) - err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, result.UpdatedComponents(), - publishedStorage, collectionFactory, context.Progress()) + err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, + result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, context.Progress()) if err != nil { return fmt.Errorf("unable to update: %s", err) } - - // Cleanup files belonging to a removed component by dropping the component directory from the storage backend. - for _, component := range result.RemovedComponents() { - err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), context.Progress()) - if err != nil { - return fmt.Errorf("unable to update: %s", err) - } - } } context.Progress().Printf("\nPublished %s repository %s has been successfully updated.\n", published.SourceKind, published.String()) diff --git a/deb/publish.go b/deb/publish.go index 067f9485..0cd481a9 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1400,7 +1400,7 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix processedComponentRefs := map[string]*PackageRefList{} for _, r := range collection.list { - if r.Prefix == prefix { + if r.Prefix == prefix && !r.MultiDist { matches := false repoComponents := r.Components() @@ -1461,21 +1461,53 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix } // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair -func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string, - publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error { +func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider, + published *PublishedRepo, addedComponents, updatedComponents, removedComponents []string, + collectionFactory *CollectionFactory, progress aptly.Progress) error { collection.loadList() - if progress != nil { - progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", ")) + prefix := published.Prefix + distribution := published.Distribution + multiDist := published.MultiDist + publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage) + + components := make([]string, 0, len(addedComponents)+len(updatedComponents)+len(removedComponents)) + components = append(append(append(components, addedComponents...), updatedComponents...), removedComponents...) + sort.Strings(components) + + for _, component := range removedComponents { + if progress != nil { + progress.Printf("Removing component %q from prefix %q...\n", component, prefix) + } + + err := publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), progress) + if err != nil { + return err + } + + if multiDist { + for _, component := range removedComponents { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), progress) + if err != nil { + return err + } + } + } } + components = make([]string, 0, len(updatedComponents)+len(removedComponents)) + components = append(append(components, addedComponents...), updatedComponents...) + referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress) if err != nil { return err } for _, component := range components { + if progress != nil { + progress.Printf("Cleaning up component %q in prefix %q...\n", component, prefix) + } sort.Strings(referencedFiles[component]) rootPath := filepath.Join(prefix, "pool", component) @@ -1547,8 +1579,8 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1] if !skipCleanup && len(cleanComponents) > 0 { - err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents, - publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress) + err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, []string{}, cleanComponents, []string{}, + collectionFactory, progress) if err != nil { if !force { return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err) From f8f28e9554282eb33ec834f1be189d2ace056de9 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Fri, 11 Oct 2024 23:09:30 +0200 Subject: [PATCH 10/25] Fixing tests and fix cleanup. Signed-off-by: Christoph Fiehe --- AUTHORS | 1 + api/publish.go | 208 ++++++++++++++---- cmd/mirror_update.go | 2 +- cmd/publish_source_add.go | 4 +- cmd/publish_source_remove.go | 4 +- cmd/publish_source_update.go | 4 +- cmd/publish_switch.go | 3 +- cmd/publish_update.go | 7 +- deb/publish.go | 111 +++++++--- deb/publish_test.go | 44 ++++ system/t04_mirror/UpdateMirror10Test_gold | 2 +- system/t04_mirror/UpdateMirror11FTPTest_gold | 2 +- system/t04_mirror/UpdateMirror12Test_gold | 2 +- system/t04_mirror/UpdateMirror13Test_gold | 2 +- system/t04_mirror/UpdateMirror14Test_gold | 2 +- system/t04_mirror/UpdateMirror15Test_gold | 2 +- system/t04_mirror/UpdateMirror16Test_gold | 2 +- system/t04_mirror/UpdateMirror17Test_gold | 2 +- system/t04_mirror/UpdateMirror18Test_gold | 2 +- system/t04_mirror/UpdateMirror19Test_gold | 2 +- system/t04_mirror/UpdateMirror1Test_gold | 2 +- system/t04_mirror/UpdateMirror20Test_gold | 2 +- system/t04_mirror/UpdateMirror21Test_gold | 2 +- system/t04_mirror/UpdateMirror22Test_gold | 2 +- system/t04_mirror/UpdateMirror23Test_gold | 2 +- system/t04_mirror/UpdateMirror24Test_gold | 2 +- system/t04_mirror/UpdateMirror26Test_gold | 2 +- system/t04_mirror/UpdateMirror6Test_gold | 2 +- system/t04_mirror/UpdateMirror7Test_gold | 2 +- system/t04_mirror/UpdateMirror8Test_gold | 2 +- system/t04_mirror/UpdateMirror9Test_gold | 2 +- system/t06_publish/AzurePublish2Test_gold | 5 +- system/t06_publish/AzurePublish3Test_gold | 3 +- system/t06_publish/AzurePublish5Test_gold | 3 +- system/t06_publish/PublishDrop3Test_gold | 3 +- system/t06_publish/PublishDrop5Test_gold | 3 +- system/t06_publish/PublishDrop9Test_gold | 3 +- system/t06_publish/PublishShow5Test_gold | 5 + system/t06_publish/PublishSnapshot42Test_gold | 13 ++ system/t06_publish/PublishSourceAdd1Test_gold | 2 +- system/t06_publish/PublishSourceAdd2Test_gold | 4 +- system/t06_publish/PublishSourceAdd3Test_gold | 2 +- .../t06_publish/PublishSourceRemove1Test_gold | 2 +- .../t06_publish/PublishSourceRemove2Test_gold | 4 +- .../t06_publish/PublishSourceRemove3Test_gold | 2 +- .../t06_publish/PublishSourceUpdate1Test_gold | 2 +- .../t06_publish/PublishSourceUpdate2Test_gold | 4 +- .../t06_publish/PublishSourceUpdate3Test_gold | 2 +- system/t06_publish/PublishSwitch11Test_gold | 3 +- system/t06_publish/PublishSwitch13Test_gold | 3 +- system/t06_publish/PublishSwitch15Test_gold | 3 +- system/t06_publish/PublishSwitch16Test_gold | 3 +- system/t06_publish/PublishSwitch1Test_gold | 3 +- system/t06_publish/PublishSwitch2Test_gold | 3 +- system/t06_publish/PublishSwitch3Test_gold | 3 +- system/t06_publish/PublishSwitch4Test_gold | 3 +- system/t06_publish/PublishSwitch8Test_gold | 4 +- system/t06_publish/PublishUpdate10Test_gold | 5 +- system/t06_publish/PublishUpdate11Test_gold | 5 +- system/t06_publish/PublishUpdate12Test_gold | 2 +- system/t06_publish/PublishUpdate13Test_gold | 5 +- system/t06_publish/PublishUpdate14Test_gold | 5 +- system/t06_publish/PublishUpdate15Test_gold | 4 +- system/t06_publish/PublishUpdate16Test_gold | 10 +- system/t06_publish/PublishUpdate17Test_gold | 6 +- system/t06_publish/PublishUpdate18Test_gold | 8 +- system/t06_publish/PublishUpdate1Test_gold | 5 +- system/t06_publish/PublishUpdate2Test_gold | 5 +- system/t06_publish/PublishUpdate3Test_gold | 5 +- system/t06_publish/PublishUpdate4Test_gold | 4 +- system/t06_publish/PublishUpdate7Test_gold | 6 +- system/t06_publish/PublishUpdate8Test_gold | 6 +- system/t06_publish/S3Publish2Test_gold | 5 +- system/t06_publish/S3Publish3Test_gold | 3 +- system/t06_publish/S3Publish5Test_gold | 3 +- system/t06_publish/S3Publish6Test_gold | 5 +- system/t06_publish/SwiftPublish2Test_gold | 4 +- system/t06_publish/SwiftPublish3Test_gold | 2 +- system/t06_publish/SwiftPublish5Test_gold | 2 +- system/t06_publish/show.py | 14 ++ system/t06_publish/snapshot.py | 20 ++ system/t08_db/CleanupDB9Test_publish_drop | 3 +- system/t12_api/publish.py | 52 +++++ 83 files changed, 540 insertions(+), 174 deletions(-) create mode 100644 system/t06_publish/PublishShow5Test_gold create mode 100644 system/t06_publish/PublishSnapshot42Test_gold diff --git a/AUTHORS b/AUTHORS index 2a42fb9a..114fe886 100644 --- a/AUTHORS +++ b/AUTHORS @@ -64,3 +64,4 @@ List of contributors, in chronological order: * Golf Hu (https://github.com/hudeng-go) * Cookie Fei (https://github.com/wuhuang26) * Andrey Loukhnov (https://github.com/aol-nnov) +* Christoph Fiehe (https://github.com/cfiehe) diff --git a/api/publish.go b/api/publish.go index e51ceaba..2d71f539 100644 --- a/api/publish.go +++ b/api/publish.go @@ -30,9 +30,9 @@ type signingParams struct { type sourceParams struct { // Name of the component - Component string `binding:"required" json:"Component" example:"contrib"` + Component string `binding:"required" json:"Component"` // Name of the local repository/snapshot - Name string `binding:"required" json:"Name" example:"snap1"` + Name string `binding:"required" json:"Name"` } func getSigner(options *signingParams) (pgp.Signer, error) { @@ -77,7 +77,7 @@ func apiPublishList(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() - result := make([]*deb.PublishedRepo, 0, collection.Len()) + repos := make([]*deb.PublishedRepo, 0, collection.Len()) err := collection.ForEach(func(repo *deb.PublishedRepo) error { err := collection.LoadShallow(repo, collectionFactory) @@ -85,7 +85,7 @@ func apiPublishList(c *gin.Context) { return err } - result = append(result, repo) + repos = append(repos, repo) return nil }) @@ -95,17 +95,16 @@ func apiPublishList(c *gin.Context) { return } - c.JSON(http.StatusOK, result) + c.JSON(http.StatusOK, repos) } // @Summary Show published repository -// @Description **Get published repository by name** +// @Description **Get details of a published repository** // @Tags Publish -// @Consume json // @Produce json // @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs" // @Param distribution path string true "distribution name" -// @Success 200 {object} deb.RemoteRepo +// @Success 200 {object} deb.PublishedRepo // @Failure 404 {object} Error "Published repository not found" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution} [get] @@ -165,14 +164,15 @@ type publishedRepoCreateParams struct { } // @Summary Create published repository -// @Description Publish local repository or snapshot under specified prefix. Storage might be passed in prefix as well, e.g. `s3:packages/`. -// @Description To supply empty prefix, just remove last part (`POST /api/publish`). +// @Description **Publish local repository or snapshot under specified prefix** +// @Description +// @Description Storage might be passed in prefix as well, e.g. `s3:packages/`. To supply empty prefix, just remove last part (`POST /api/publish`). // @Tags Publish // @Param prefix path string true "publishing prefix" // @Consume json // @Param request body publishedRepoCreateParams true "Parameters" // @Produce json -// @Success 200 {object} deb.RemoteRepo +// @Success 200 {object} deb.PublishedRepo // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Source not found" // @Failure 500 {object} Error "Internal Error" @@ -337,14 +337,40 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { }) } +type publishedRepoUpdateSwitchParams struct { + // when publishing, overwrite files in pool/ directory without notice + ForceOverwrite bool ` json:"ForceOverwrite"` + // GPG options + Signing signingParams ` json:"Signing"` + // Don't generate contents indexes + SkipContents *bool ` json:"SkipContents"` + // Skip bz2 compression for index files + SkipBz2 *bool ` json:"SkipBz2"` + // Don't remove unreferenced files in prefix/component + SkipCleanup *bool ` json:"SkipCleanup"` + // only when updating published snapshots, list of objects 'Component/Name' + Snapshots []sourceParams `binding:"required" json:"Snapshots"` + // Provide index files by hash + AcquireByHash *bool ` json:"AcquireByHash"` + // Enable multiple packages with the same filename in different distributions + MultiDist *bool ` json:"MultiDist"` +} + // @Summary Update published repository -// @Description Update a published repository. +// @Description **Update a published local repository or switch published snapshot** +// @Description +// @Description API action depends on published repository contents: +// @Description * if local repository has been published, published repository would be updated to match local repository contents +// @Description * if snapshots have been been published, it is possible to switch each component to new snapshot // @Tags Publish // @Accept json // @Produce json // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" -// @Success 200 {object} deb.RemoteRepo +// @Consume json +// @Param request body publishedRepoUpdateSwitchParams true "Parameters" +// @Produce json +// @Success 200 {object} deb.PublishedRepo // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository or source not found" // @Failure 500 {object} Error "Internal Error" @@ -354,20 +380,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) - var b struct { - ForceOverwrite bool - Signing signingParams - SkipContents *bool - SkipBz2 *bool - SkipCleanup *bool - Snapshots []struct { - Component string `binding:"required"` - Name string `binding:"required"` - } - AcquireByHash *bool - MultiDist *bool - } - + var b publishedRepoUpdateSwitchParams if c.Bind(&b) != nil { return } @@ -393,7 +406,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { if published.SourceKind == deb.SourceLocalRepo { if len(b.Snapshots) > 0 { - AbortWithJSONError(c, 400, fmt.Errorf("snapshots shouldn't be given when updating local repo")) + AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo")) return } updatedComponents = published.Components() @@ -483,8 +496,9 @@ func apiPublishUpdateSwitch(c *gin.Context) { } if b.SkipCleanup == nil || !*b.SkipCleanup { - err = collection.CleanupPrefixComponentFiles(context, published, result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), - collectionFactory, out) + cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources)) + cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...) + err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } @@ -495,7 +509,9 @@ func apiPublishUpdateSwitch(c *gin.Context) { } // @Summary Delete published repository -// @Description Delete a published repository. +// @Description **Delete a published repository** +// @Description +// @Description Delete published repository, clean up files in published directory. // @Tags Publish // @Accept json // @Produce json @@ -503,7 +519,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { // @Param distribution path string true "distribution name" // @Param force query int true "force: 1 to enable" // @Param skipCleanup query int true "skipCleanup: 1 to enable" -// @Success 200 {object} task.ProcessReturnValue +// @Success 200 // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository not found" // @Failure 500 {object} Error "Internal Error" @@ -538,6 +554,16 @@ func apiPublishDrop(c *gin.Context) { }) } +// @Summary Add staged source +// @Description **Create and add a staged source** +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Produce json +// @Success 200 {object} sourceParams +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository not found" +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [post] func apiPublishSourcesCreate(c *gin.Context) { var ( @@ -576,7 +602,7 @@ func apiPublishSourcesCreate(c *gin.Context) { _, exists := sources[component] if exists { - AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component %q already exists", component)) + AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component '%s' already exists", component)) return } @@ -594,6 +620,16 @@ func apiPublishSourcesCreate(c *gin.Context) { }) } +// @Summary Get staged sources +// @Description **Get the staged source list** +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Produce json +// @Success 200 {array} sourceParams +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository not found or no source changes exist" +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [get] func apiPublishSourcesList(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) @@ -624,6 +660,18 @@ func apiPublishSourcesList(c *gin.Context) { c.JSON(http.StatusOK, revision.SourceList()) } +// @Summary Set staged sources +// @Description **Set the staged source list** +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Consume json +// @Param request body publishedRepoUpdateParams true "Parameters" +// @Produce json +// @Success 200 {array} sourceParams +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository not found" +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [put] func apiPublishSourcesUpdate(c *gin.Context) { var ( @@ -640,13 +688,13 @@ func apiPublishSourcesUpdate(c *gin.Context) { published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) return } err = collection.LoadComplete(published, collectionFactory) if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) return } @@ -672,10 +720,22 @@ func apiPublishSourcesUpdate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } - return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + return &task.ProcessReturnValue{Code: http.StatusOK, Value: revision.SourceList()}, nil }) } +// @Summary Delete staged sources +// @Description **Delete the staged source list** +// @Description +// @Description Delete staged sources and keep existing sources of published repository. +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Produce json +// @Success 200 +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository not found" +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [delete] func apiPublishSourcesDelete(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) @@ -707,10 +767,21 @@ func apiPublishSourcesDelete(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } - return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil }) } +// @Summary Update staged source +// @Description **Update the staged source of a component** +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Param component path string true "component name" +// @Produce json +// @Success 200 +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository/component not found" +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources/{component} [put] func apiPublishSourceUpdate(c *gin.Context) { var ( @@ -743,7 +814,7 @@ func apiPublishSourceUpdate(c *gin.Context) { _, exists := sources[component] if !exists { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component %q does not exist", component)) + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component '%s' does not exist", component)) return } @@ -770,10 +841,21 @@ func apiPublishSourceUpdate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } - return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil }) } +// @Summary Delete staged source +// @Description **Delete the staged source** +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Param component path string true "component name" +// @Produce json +// @Success 200 +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository not found" +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources/{component} [delete] func apiPublishSourceDelete(c *gin.Context) { var err error @@ -788,19 +870,25 @@ func apiPublishSourceDelete(c *gin.Context) { published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err)) return } err = collection.LoadComplete(published, collectionFactory) if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err)) return } revision := published.ObtainRevision() sources := revision.Sources + _, exists := sources[component] + if !exists { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: Component '%s' does not exist", component)) + return + } + delete(sources, component) resources := []string{string(published.Key())} @@ -811,10 +899,41 @@ func apiPublishSourceDelete(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } - return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil + return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil }) } +type publishedRepoUpdateParams struct { + // when publishing, overwrite files in pool/ directory without notice + ForceOverwrite bool ` json:"ForceOverwrite"` + // GPG options + Signing signingParams ` json:"Signing"` + // Don't generate contents indexes + SkipContents *bool ` json:"SkipContents"` + // Skip bz2 compression for index files + SkipBz2 *bool ` json:"SkipBz2"` + // Don't remove unreferenced files in prefix/component + SkipCleanup *bool ` json:"SkipCleanup"` + // Provide index files by hash + AcquireByHash *bool ` json:"AcquireByHash"` + // Enable multiple packages with the same filename in different distributions + MultiDist *bool ` json:"MultiDist"` +} + +// @Summary Update content of published repository +// @Description **Update the content of a published repository** +// @Description +// @Description Replace the sources of the published repository (if available) and (re-)publish new content. +// @Tags Publish +// @Param prefix path string true "publishing prefix" +// @Param distribution path string true "distribution name" +// @Consume json +// @Param request body publishedRepoUpdateParams true "Parameters" +// @Produce json +// @Success 200 {object} deb.PublishedRepo +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Published repository/component not found" +// @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/update [post] func apiPublishUpdate(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) @@ -891,8 +1010,9 @@ func apiPublishUpdate(c *gin.Context) { } if b.SkipCleanup == nil || !*b.SkipCleanup { - err = collection.CleanupPrefixComponentFiles(context, published, - result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, out) + cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources)) + cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...) + err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index 2e6df439..2aad8c53 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -267,7 +267,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to update: %s", err) } - context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name) + context.Progress().Printf("\nMirror `%s` has been updated successfully.\n", repo.Name) return err } diff --git a/cmd/publish_source_add.go b/cmd/publish_source_add.go index 2ee9e7ff..cf2b8570 100644 --- a/cmd/publish_source_add.go +++ b/cmd/publish_source_add.go @@ -44,9 +44,9 @@ func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error { name := names[i] _, exists := sources[component] if exists { - return fmt.Errorf("unable to add: component %q has already been added", component) + return fmt.Errorf("unable to add: component '%s' has already been added", component) } - context.Progress().Printf("Adding component %q with source %q [%s]...\n", component, name, published.SourceKind) + context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind) sources[component] = names[i] } diff --git a/cmd/publish_source_remove.go b/cmd/publish_source_remove.go index 1d3892c4..5a0df7a6 100644 --- a/cmd/publish_source_remove.go +++ b/cmd/publish_source_remove.go @@ -42,9 +42,9 @@ func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error { for _, component := range components { name, exists := sources[component] if !exists { - return fmt.Errorf("unable to remove: component %q does not exist", component) + return fmt.Errorf("unable to remove: component '%s' does not exist", component) } - context.Progress().Printf("Removing component %q with source %q [%s]...\n", component, name, published.SourceKind) + context.Progress().Printf("Removing component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind) delete(sources, component) } diff --git a/cmd/publish_source_update.go b/cmd/publish_source_update.go index 66c27468..0dae6439 100644 --- a/cmd/publish_source_update.go +++ b/cmd/publish_source_update.go @@ -44,9 +44,9 @@ func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error { name := names[i] _, exists := sources[component] if !exists { - return fmt.Errorf("unable to update: component %q does not exist", component) + return fmt.Errorf("unable to update: component '%s' does not exist", component) } - context.Progress().Printf("Updating component %q with source %q [%s]...\n", component, name, published.SourceKind) + context.Progress().Printf("Updating component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind) sources[component] = name } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index eedb42d8..12d73f08 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -115,8 +115,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) if !skipCleanup { - err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, - []string{}, components, []string{}, collectionFactory, context.Progress()) + err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress()) if err != nil { return fmt.Errorf("unable to switch: %s", err) } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index f4337865..589e35bf 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -76,14 +76,15 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) if !skipCleanup { - err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, - result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, context.Progress()) + cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources)) + cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...) + err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress()) if err != nil { return fmt.Errorf("unable to update: %s", err) } } - context.Progress().Printf("\nPublished %s repository %s has been successfully updated.\n", published.SourceKind, published.String()) + context.Progress().Printf("\nPublished %s repository %s has been updated successfully.\n", published.SourceKind, published.String()) return err } diff --git a/deb/publish.go b/deb/publish.go index 0cd481a9..9143e610 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1462,73 +1462,127 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider, - published *PublishedRepo, addedComponents, updatedComponents, removedComponents []string, - collectionFactory *CollectionFactory, progress aptly.Progress) error { + published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error { + + var err error collection.loadList() + storage := published.Storage prefix := published.Prefix distribution := published.Distribution - multiDist := published.MultiDist + + rootPath := filepath.Join(prefix, "dists", distribution) publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage) - components := make([]string, 0, len(addedComponents)+len(updatedComponents)+len(removedComponents)) - components = append(append(append(components, addedComponents...), updatedComponents...), removedComponents...) - sort.Strings(components) + sort.Strings(cleanComponents) + publishedComponents := published.Components() + removedComponents := utils.StrSlicesSubstract(cleanComponents, publishedComponents) + updatedComponents := utils.StrSlicesSubstract(cleanComponents, removedComponents) + + if progress != nil { + progress.Printf("Cleaning up published repository %s/%s...\n", published.StoragePrefix(), distribution) + } for _, component := range removedComponents { if progress != nil { - progress.Printf("Removing component %q from prefix %q...\n", component, prefix) + progress.Printf("Removing component '%s'...\n", component) } - err := publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), progress) + err = publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress) if err != nil { return err } - if multiDist { - for _, component := range removedComponents { - err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), progress) + // Ensure that component does not exist in multi distribution pool + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), nil) + if err != nil { + return err + } + } + + referencedFiles := map[string][]string{} + + if published.MultiDist { + rootPath = filepath.Join(prefix, "pool", distribution) + + // Get all referenced files by component for determining orphaned pool files. + for _, component := range publishedComponents { + packageList, err := NewPackageListFromRefList(published.RefList(component), collectionFactory.PackageCollection(), progress) + if err != nil { + return err + } + + packageList.ForEach(func(p *Package) error { + poolDir, err := p.PoolDirectory() + if err != nil { + return err + } + + for _, file := range p.Files() { + referencedFiles[component] = append(referencedFiles[component], filepath.Join(poolDir, file.Filename)) + } + + return nil + }) + } + } else { + rootPath = filepath.Join(prefix, "pool") + + // In case of a shared component pool directory, we must check, if a component is no longer referenced by any other + // published repository within the same prefix. + referencedComponents := map[string]struct{}{} + for _, p := range collection.list { + if p.Prefix == prefix && p.Storage == storage && !p.MultiDist { + for _, component := range p.Components() { + referencedComponents[component] = struct{}{} + } + } + } + + // Remove orphaned component pool directories in the prefix. + for _, component := range removedComponents { + _, exists := referencedComponents[component] + if !exists { + err := publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress) if err != nil { return err } } } + + // Get all referenced files by component for determining orphaned pool files. + referencedFiles, err = collection.listReferencedFilesByComponent(prefix, publishedComponents, collectionFactory, progress) + if err != nil { + return err + } } - components = make([]string, 0, len(updatedComponents)+len(removedComponents)) - components = append(append(components, addedComponents...), updatedComponents...) - - referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress) - if err != nil { - return err - } - - for _, component := range components { + for _, component := range updatedComponents { if progress != nil { - progress.Printf("Cleaning up component %q in prefix %q...\n", component, prefix) + progress.Printf("Cleaning up component '%s'...\n", component) } sort.Strings(referencedFiles[component]) - rootPath := filepath.Join(prefix, "pool", component) - existingFiles, err := publishedStorage.Filelist(rootPath) + path := filepath.Join(rootPath, component) + existingFiles, err := publishedStorage.Filelist(path) if err != nil { return err } sort.Strings(existingFiles) - filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles[component]) + orphanedFiles := utils.StrSlicesSubstract(existingFiles, referencedFiles[component]) - for _, file := range filesToDelete { - err = publishedStorage.Remove(filepath.Join(rootPath, file)) + for _, file := range orphanedFiles { + err = publishedStorage.Remove(filepath.Join(path, file)) if err != nil { return err } } } - return nil + return err } // Remove removes published repository, cleaning up directories, files @@ -1579,8 +1633,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1] if !skipCleanup && len(cleanComponents) > 0 { - err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, []string{}, cleanComponents, []string{}, - collectionFactory, progress) + err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, cleanComponents, collectionFactory, progress) if err != nil { if !force { return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err) diff --git a/deb/publish_test.go b/deb/publish_test.go index 5243a41a..eec1601f 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -2,6 +2,7 @@ package deb import ( "bytes" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -359,6 +360,25 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) { c.Check(err, ErrorMatches, "duplicate component name: main") } +func (s *PublishedRepoSuite) TestUpdate(c *C) { + revision := s.repo2.ObtainRevision() + sources := revision.Sources + sources["test"] = "local1" + + result, err := s.repo2.Update(s.factory, nil) + c.Assert(err, IsNil) + c.Assert(result, NotNil) + c.Assert(s.repo2.Revision, IsNil) + + c.Assert(result.AddedSources, DeepEquals, map[string]string{"test": "local1"}) + c.Assert(result.UpdatedSources, DeepEquals, map[string]string{"main": "local1"}) + c.Assert(result.RemovedSources, DeepEquals, map[string]string{}) + + c.Assert(result.AddedComponents(), DeepEquals, []string{"test"}) + c.Assert(result.UpdatedComponents(), DeepEquals, []string{"main"}) + c.Assert(result.RemovedComponents(), DeepEquals, []string{}) +} + func (s *PublishedRepoSuite) TestPublish(c *C) { err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false) c.Assert(err, IsNil) @@ -489,6 +509,30 @@ func (s *PublishedRepoSuite) TestEncodeDecode(c *C) { c.Assert(repo2, DeepEquals, s.repo2) } +func (s *PublishedRepoSuite) TestPublishedRepoRevision(c *C) { + revision := s.repo2.ObtainRevision() + c.Assert(revision, NotNil) + + sources := revision.Sources + c.Assert(sources, NotNil) + c.Assert(sources, DeepEquals, map[string]string{"main": "local1"}) + + sources["test1"] = "snap1" + sources["test2"] = "snap2" + + c.Assert(revision.Components(), DeepEquals, []string{"main", "test1", "test2"}) + c.Assert(revision.SourceNames(), DeepEquals, []string{"local1", "snap1", "snap2"}) + + bytes, err := json.Marshal(revision) + c.Assert(err, IsNil) + + json_expected := `{"Sources":[{"Component":"main","Name":"local1"},{"Component":"test1","Name":"snap1"},{"Component":"test2","Name":"snap2"}]}` + c.Assert(string(bytes), Equals, json_expected) + + c.Assert(s.repo2.DropRevision(), DeepEquals, revision) + c.Assert(s.repo2.Revision, IsNil) +} + type PublishedRepoCollectionSuite struct { PackageListMixinSuite db database.Storage diff --git a/system/t04_mirror/UpdateMirror10Test_gold b/system/t04_mirror/UpdateMirror10Test_gold index fa0f41f2..54251a64 100644 --- a/system/t04_mirror/UpdateMirror10Test_gold +++ b/system/t04_mirror/UpdateMirror10Test_gold @@ -18,7 +18,7 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-dbgsym_0.7.5-1~bullseyecran.0_i386.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_amd64.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_i386.deb -Mirror `flat-src` has been successfully updated. +Mirror `flat-src` has been updated successfully. Packages filtered: 110 -> 11. gpgv: using RSA key 7BA040A510E4E66ED3743EC1B8F25A8A73EACF41 gpgv: Good signature from "Johannes Ranke " diff --git a/system/t04_mirror/UpdateMirror11FTPTest_gold b/system/t04_mirror/UpdateMirror11FTPTest_gold index 24f61552..dc4bab13 100644 --- a/system/t04_mirror/UpdateMirror11FTPTest_gold +++ b/system/t04_mirror/UpdateMirror11FTPTest_gold @@ -13,7 +13,7 @@ Downloading: http://repo.aptly.info/system-tests/snapshot.debian.org/archive/deb Downloading: http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/pool/main/s/sensible-utils/sensible-utils_0.0.9+deb9u1_all.deb Downloading: http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/pool/main/s/sysvinit/sysvinit-utils_2.88dsf-59.9_i386.deb Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/dists/stretch/InRelease -Mirror `stretch-main` has been successfully updated. +Mirror `stretch-main` has been updated successfully. Packages filtered: 50604 -> 3. Retrying 0 http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/dists/stretch/InRelease... gpgv: issuer "debian-release@lists.debian.org" diff --git a/system/t04_mirror/UpdateMirror12Test_gold b/system/t04_mirror/UpdateMirror12Test_gold index eff42e76..f9ad32d5 100644 --- a/system/t04_mirror/UpdateMirror12Test_gold +++ b/system/t04_mirror/UpdateMirror12Test_gold @@ -37,7 +37,7 @@ Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archi Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/g/gnupg2/scdaemon_2.1.18-8~deb9u4_amd64.deb Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/g/gnupg2/scdaemon_2.1.18-8~deb9u4_i386.deb Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease -Mirror `stretch` has been successfully updated. +Mirror `stretch` has been updated successfully. Packages filtered: 78248 -> 20. Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease... gpgv: issuer "debian-release@lists.debian.org" diff --git a/system/t04_mirror/UpdateMirror13Test_gold b/system/t04_mirror/UpdateMirror13Test_gold index 63c166c8..b00c4ce3 100644 --- a/system/t04_mirror/UpdateMirror13Test_gold +++ b/system/t04_mirror/UpdateMirror13Test_gold @@ -58,4 +58,4 @@ Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/va Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_i386.deb -Mirror `varnish` has been successfully updated. \ No newline at end of file +Mirror `varnish` has been updated successfully. \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror14Test_gold b/system/t04_mirror/UpdateMirror14Test_gold index 0c1b5c59..edcc3695 100644 --- a/system/t04_mirror/UpdateMirror14Test_gold +++ b/system/t04_mirror/UpdateMirror14Test_gold @@ -6,4 +6,4 @@ Downloading & parsing package files... Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2 -Mirror `varnish` has been successfully updated. \ No newline at end of file +Mirror `varnish` has been updated successfully. \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror15Test_gold b/system/t04_mirror/UpdateMirror15Test_gold index 3c11f1ff..d218085c 100644 --- a/system/t04_mirror/UpdateMirror15Test_gold +++ b/system/t04_mirror/UpdateMirror15Test_gold @@ -6,4 +6,4 @@ Downloading & parsing package files... Downloading https://dl.bintray.com/smira/deb/Packages.bz2... Downloading https://dl.bintray.com/smira/deb/Release... Downloading https://dl.bintray.com/smira/deb/libboost-program-options-dev_1.49.0.1_i386.deb... -Mirror `bintray` has been successfully updated. \ No newline at end of file +Mirror `bintray` has been updated successfully. \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror16Test_gold b/system/t04_mirror/UpdateMirror16Test_gold index 3c11f1ff..d218085c 100644 --- a/system/t04_mirror/UpdateMirror16Test_gold +++ b/system/t04_mirror/UpdateMirror16Test_gold @@ -6,4 +6,4 @@ Downloading & parsing package files... Downloading https://dl.bintray.com/smira/deb/Packages.bz2... Downloading https://dl.bintray.com/smira/deb/Release... Downloading https://dl.bintray.com/smira/deb/libboost-program-options-dev_1.49.0.1_i386.deb... -Mirror `bintray` has been successfully updated. \ No newline at end of file +Mirror `bintray` has been updated successfully. \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror17Test_gold b/system/t04_mirror/UpdateMirror17Test_gold index 6341c1de..cddc0d3e 100644 --- a/system/t04_mirror/UpdateMirror17Test_gold +++ b/system/t04_mirror/UpdateMirror17Test_gold @@ -6,5 +6,5 @@ Download queue: 0 items (0 B) Downloading & parsing package files... Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz -Mirror `stretch` has been successfully updated. +Mirror `stretch` has been updated successfully. Packages filtered: 50604 -> 1. \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror18Test_gold b/system/t04_mirror/UpdateMirror18Test_gold index 0e0deb88..30dfc184 100644 --- a/system/t04_mirror/UpdateMirror18Test_gold +++ b/system/t04_mirror/UpdateMirror18Test_gold @@ -7,5 +7,5 @@ Downloading & parsing package files... Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb -Mirror `stretch` has been successfully updated. +Mirror `stretch` has been updated successfully. Packages filtered: 50604 -> 1. \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror19Test_gold b/system/t04_mirror/UpdateMirror19Test_gold index fcf7c33c..ef4a8634 100644 --- a/system/t04_mirror/UpdateMirror19Test_gold +++ b/system/t04_mirror/UpdateMirror19Test_gold @@ -12,4 +12,4 @@ Downloading: http://repo.aptly.info/system-tests/packages.pagerduty.com/pdagent/ Building download queue... Download queue: 23 items (3.46 MiB) -Mirror `pagerduty` has been successfully updated. +Mirror `pagerduty` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror1Test_gold b/system/t04_mirror/UpdateMirror1Test_gold index 63c166c8..b00c4ce3 100644 --- a/system/t04_mirror/UpdateMirror1Test_gold +++ b/system/t04_mirror/UpdateMirror1Test_gold @@ -58,4 +58,4 @@ Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/va Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_i386.deb -Mirror `varnish` has been successfully updated. \ No newline at end of file +Mirror `varnish` has been updated successfully. \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror20Test_gold b/system/t04_mirror/UpdateMirror20Test_gold index 3353fc5b..726afd93 100644 --- a/system/t04_mirror/UpdateMirror20Test_gold +++ b/system/t04_mirror/UpdateMirror20Test_gold @@ -8,7 +8,7 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/Packages.bz2 Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/r-cran-class_7.3-22-2~bullseyecran.0_amd64.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/r-cran-class_7.3-22-2~bullseyecran.0_i386.deb -Mirror `flat` has been successfully updated. +Mirror `flat` has been updated successfully. Packages filtered: 89 -> 2. openpgp: Good signature from "Johannes Ranke " openpgp: RSA key ID B8F25A8A73EACF41 \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror21Test_gold b/system/t04_mirror/UpdateMirror21Test_gold index 869ec65b..e864bf91 100644 --- a/system/t04_mirror/UpdateMirror21Test_gold +++ b/system/t04_mirror/UpdateMirror21Test_gold @@ -11,4 +11,4 @@ Downloading: http://repo.aptly.info/system-tests/packages.pagerduty.com/pdagent/ Building download queue... Download queue: 23 items (3.46 MiB) -Mirror `pagerduty` has been successfully updated. +Mirror `pagerduty` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror22Test_gold b/system/t04_mirror/UpdateMirror22Test_gold index 6724bab1..9c76ac64 100644 --- a/system/t04_mirror/UpdateMirror22Test_gold +++ b/system/t04_mirror/UpdateMirror22Test_gold @@ -8,4 +8,4 @@ Applying filter... Building download queue... Download queue: 0 items (0 B) -Mirror `libnvidia-container` has been successfully updated. +Mirror `libnvidia-container` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror23Test_gold b/system/t04_mirror/UpdateMirror23Test_gold index 5e6157c7..5b172366 100644 --- a/system/t04_mirror/UpdateMirror23Test_gold +++ b/system/t04_mirror/UpdateMirror23Test_gold @@ -23,7 +23,7 @@ Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archi Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/non-free/installer-s390x/current/images/SHA256SUMS Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/non-free/installer-s390x/current/images/SHA256SUMS -Mirror `stretch` has been successfully updated. +Mirror `stretch` has been updated successfully. Packages filtered: 49256 -> 1. Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease... Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/non-free/installer-s390x/current/images/SHA256SUMS... diff --git a/system/t04_mirror/UpdateMirror24Test_gold b/system/t04_mirror/UpdateMirror24Test_gold index e8ec96b9..dcc27631 100644 --- a/system/t04_mirror/UpdateMirror24Test_gold +++ b/system/t04_mirror/UpdateMirror24Test_gold @@ -53,7 +53,7 @@ Downloading: http://repo.aptly.info/system-tests/us.archive.ubuntu.com/ubuntu/di Downloading: http://repo.aptly.info/system-tests/us.archive.ubuntu.com/ubuntu/dists/trusty/restricted/installer-amd64/current/images/SHA256SUMS Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/us.archive.ubuntu.com/ubuntu/dists/trusty/InRelease Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/us.archive.ubuntu.com/ubuntu/dists/trusty/restricted/installer-amd64/current/images/SHA256SUMS -Mirror `trusty` has been successfully updated. +Mirror `trusty` has been updated successfully. Packages filtered: 8616 -> 1. Retrying 0 http://repo.aptly.info/system-tests/us.archive.ubuntu.com/ubuntu/dists/trusty/InRelease... Retrying 0 http://repo.aptly.info/system-tests/us.archive.ubuntu.com/ubuntu/dists/trusty/restricted/installer-amd64/current/images/SHA256SUMS... diff --git a/system/t04_mirror/UpdateMirror26Test_gold b/system/t04_mirror/UpdateMirror26Test_gold index aa6d212e..b460083f 100644 --- a/system/t04_mirror/UpdateMirror26Test_gold +++ b/system/t04_mirror/UpdateMirror26Test_gold @@ -14,7 +14,7 @@ Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archi Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb -Mirror `grab` has been successfully updated. +Mirror `grab` has been updated successfully. Packages filtered: 50604 -> 1. Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease diff --git a/system/t04_mirror/UpdateMirror6Test_gold b/system/t04_mirror/UpdateMirror6Test_gold index 84cb9694..000a38aa 100644 --- a/system/t04_mirror/UpdateMirror6Test_gold +++ b/system/t04_mirror/UpdateMirror6Test_gold @@ -18,4 +18,4 @@ Download queue: 1 items (30 B) Downloading: ${url}pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb WARNING: ${url}pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb: sha1 hash mismatch "8d3a014000038725d6daf8771b42a0784253688f" != "66b27417d37e024c46526c2f6d358a754fc552f3" -Mirror `failure` has been successfully updated. +Mirror `failure` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror7Test_gold b/system/t04_mirror/UpdateMirror7Test_gold index 14090853..bc1279d8 100644 --- a/system/t04_mirror/UpdateMirror7Test_gold +++ b/system/t04_mirror/UpdateMirror7Test_gold @@ -94,7 +94,7 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-dbgsym_0.7.5-1~bullseyecran.0_i386.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_amd64.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_i386.deb -Mirror `flat` has been successfully updated. +Mirror `flat` has been updated successfully. gpgv: using RSA key 7BA040A510E4E66ED3743EC1B8F25A8A73EACF41 gpgv: Good signature from "Johannes Ranke " gpgv: Signature made Thu Nov 2 07:43:52 2023 UTC \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror8Test_gold b/system/t04_mirror/UpdateMirror8Test_gold index 30dbccb9..382b15e2 100644 --- a/system/t04_mirror/UpdateMirror8Test_gold +++ b/system/t04_mirror/UpdateMirror8Test_gold @@ -15,4 +15,4 @@ Downloading: http://repo.aptly.info/system-tests/ppa.launchpad.net/gladky-anton/ Building download queue... Download queue: 0 items (0 B) -Mirror `gnuplot-maverick-src` has been successfully updated. +Mirror `gnuplot-maverick-src` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror9Test_gold b/system/t04_mirror/UpdateMirror9Test_gold index 1d898874..2c2dba87 100644 --- a/system/t04_mirror/UpdateMirror9Test_gold +++ b/system/t04_mirror/UpdateMirror9Test_gold @@ -157,7 +157,7 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/survival_3.5-7-1~bullseyecran.0.debian.tar.xz Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/survival_3.5-7-1~bullseyecran.0.dsc Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/survival_3.5-7.orig.tar.gz -Mirror `flat-src` has been successfully updated. +Mirror `flat-src` has been updated successfully. gpgv: using RSA key 7BA040A510E4E66ED3743EC1B8F25A8A73EACF41 gpgv: Good signature from "Johannes Ranke " gpgv: Signature made Thu Nov 2 07:43:52 2023 UTC \ No newline at end of file diff --git a/system/t06_publish/AzurePublish2Test_gold b/system/t06_publish/AzurePublish2Test_gold index ea8cafb9..0ac6b345 100644 --- a/system/t06_publish/AzurePublish2Test_gold +++ b/system/t06_publish/AzurePublish2Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository azure:test1:./maverick... +Cleaning up component 'main'... -Published local repository azure:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository azure:test1:./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/AzurePublish3Test_gold b/system/t06_publish/AzurePublish3Test_gold index c1855169..a346b469 100644 --- a/system/t06_publish/AzurePublish3Test_gold +++ b/system/t06_publish/AzurePublish3Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository azure:test1:./maverick... +Cleaning up component 'main'... Published snapshot repository azure:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/AzurePublish5Test_gold b/system/t06_publish/AzurePublish5Test_gold index cadce02c..bef652cb 100644 --- a/system/t06_publish/AzurePublish5Test_gold +++ b/system/t06_publish/AzurePublish5Test_gold @@ -1,3 +1,4 @@ -Cleaning up prefix "." components main... +Cleaning up published repository azure:test1:./sq2... +Cleaning up component 'main'... Published repository has been removed successfully. diff --git a/system/t06_publish/PublishDrop3Test_gold b/system/t06_publish/PublishDrop3Test_gold index eb8ce9c0..1da15e9a 100644 --- a/system/t06_publish/PublishDrop3Test_gold +++ b/system/t06_publish/PublishDrop3Test_gold @@ -1,4 +1,5 @@ Removing ${HOME}/.aptly/public/dists/sq1... -Cleaning up prefix "." components main... +Cleaning up published repository ./sq1... +Cleaning up component 'main'... Published repository has been removed successfully. diff --git a/system/t06_publish/PublishDrop5Test_gold b/system/t06_publish/PublishDrop5Test_gold index 79be1a16..2313ec65 100644 --- a/system/t06_publish/PublishDrop5Test_gold +++ b/system/t06_publish/PublishDrop5Test_gold @@ -1,4 +1,5 @@ Removing ${HOME}/.aptly/public/dists/sq2... -Cleaning up prefix "." components main... +Cleaning up published repository ./sq2... +Cleaning up component 'main'... Published repository has been removed successfully. diff --git a/system/t06_publish/PublishDrop9Test_gold b/system/t06_publish/PublishDrop9Test_gold index eb8ce9c0..1da15e9a 100644 --- a/system/t06_publish/PublishDrop9Test_gold +++ b/system/t06_publish/PublishDrop9Test_gold @@ -1,4 +1,5 @@ Removing ${HOME}/.aptly/public/dists/sq1... -Cleaning up prefix "." components main... +Cleaning up published repository ./sq1... +Cleaning up component 'main'... Published repository has been removed successfully. diff --git a/system/t06_publish/PublishShow5Test_gold b/system/t06_publish/PublishShow5Test_gold new file mode 100644 index 00000000..7e6f4a2c --- /dev/null +++ b/system/t06_publish/PublishShow5Test_gold @@ -0,0 +1,5 @@ +Prefix: . +Distribution: wheezy +Architectures: i386 +Sources: + main: local-repo [local] diff --git a/system/t06_publish/PublishSnapshot42Test_gold b/system/t06_publish/PublishSnapshot42Test_gold new file mode 100644 index 00000000..8cf10f19 --- /dev/null +++ b/system/t06_publish/PublishSnapshot42Test_gold @@ -0,0 +1,13 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Snapshot snap1 has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ maverick main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/PublishSourceAdd1Test_gold b/system/t06_publish/PublishSourceAdd1Test_gold index 8ac95d37..12c4aa07 100644 --- a/system/t06_publish/PublishSourceAdd1Test_gold +++ b/system/t06_publish/PublishSourceAdd1Test_gold @@ -1,3 +1,3 @@ -Adding component "test" with source "snap2" [snapshot]... +Adding component 'test' with source 'snap2' [snapshot]... You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceAdd2Test_gold b/system/t06_publish/PublishSourceAdd2Test_gold index 904bfc81..fcdf6d02 100644 --- a/system/t06_publish/PublishSourceAdd2Test_gold +++ b/system/t06_publish/PublishSourceAdd2Test_gold @@ -1,4 +1,4 @@ -Adding component "test" with source "snap2" [snapshot]... -Adding component "other-test" with source "snap3" [snapshot]... +Adding component 'test' with source 'snap2' [snapshot]... +Adding component 'other-test' with source 'snap3' [snapshot]... You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceAdd3Test_gold b/system/t06_publish/PublishSourceAdd3Test_gold index e5802b78..dd31661c 100644 --- a/system/t06_publish/PublishSourceAdd3Test_gold +++ b/system/t06_publish/PublishSourceAdd3Test_gold @@ -1 +1 @@ -ERROR: unable to add: component "main" has already been added +ERROR: unable to add: component 'main' has already been added diff --git a/system/t06_publish/PublishSourceRemove1Test_gold b/system/t06_publish/PublishSourceRemove1Test_gold index 761bf16b..40fcf4de 100644 --- a/system/t06_publish/PublishSourceRemove1Test_gold +++ b/system/t06_publish/PublishSourceRemove1Test_gold @@ -1,3 +1,3 @@ -Removing component "test" with source "snap2" [snapshot]... +Removing component 'test' with source 'snap2' [snapshot]... You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceRemove2Test_gold b/system/t06_publish/PublishSourceRemove2Test_gold index 6692ca2e..f8270658 100644 --- a/system/t06_publish/PublishSourceRemove2Test_gold +++ b/system/t06_publish/PublishSourceRemove2Test_gold @@ -1,4 +1,4 @@ -Removing component "test" with source "snap2" [snapshot]... -Removing component "other-test" with source "snap3" [snapshot]... +Removing component 'test' with source 'snap2' [snapshot]... +Removing component 'other-test' with source 'snap3' [snapshot]... You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceRemove3Test_gold b/system/t06_publish/PublishSourceRemove3Test_gold index 54507f25..d4ebcbb1 100644 --- a/system/t06_publish/PublishSourceRemove3Test_gold +++ b/system/t06_publish/PublishSourceRemove3Test_gold @@ -1 +1 @@ -ERROR: unable to remove: component "not-existent" does not exist +ERROR: unable to remove: component 'not-existent' does not exist diff --git a/system/t06_publish/PublishSourceUpdate1Test_gold b/system/t06_publish/PublishSourceUpdate1Test_gold index ea66b43f..2f218bf2 100644 --- a/system/t06_publish/PublishSourceUpdate1Test_gold +++ b/system/t06_publish/PublishSourceUpdate1Test_gold @@ -1,3 +1,3 @@ -Updating component "main" with source "snap2" [snapshot]... +Updating component 'main' with source 'snap2' [snapshot]... You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceUpdate2Test_gold b/system/t06_publish/PublishSourceUpdate2Test_gold index 134f0d3f..88178e48 100644 --- a/system/t06_publish/PublishSourceUpdate2Test_gold +++ b/system/t06_publish/PublishSourceUpdate2Test_gold @@ -1,4 +1,4 @@ -Updating component "main" with source "snap2" [snapshot]... -Updating component "test" with source "snap3" [snapshot]... +Updating component 'main' with source 'snap2' [snapshot]... +Updating component 'test' with source 'snap3' [snapshot]... You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/PublishSourceUpdate3Test_gold b/system/t06_publish/PublishSourceUpdate3Test_gold index 4d2fb4c8..c3e6eb88 100644 --- a/system/t06_publish/PublishSourceUpdate3Test_gold +++ b/system/t06_publish/PublishSourceUpdate3Test_gold @@ -1 +1 @@ -ERROR: unable to update: component "not-existent" does not exist +ERROR: unable to update: component 'not-existent' does not exist diff --git a/system/t06_publish/PublishSwitch11Test_gold b/system/t06_publish/PublishSwitch11Test_gold index 9d87127b..5f37e805 100644 --- a/system/t06_publish/PublishSwitch11Test_gold +++ b/system/t06_publish/PublishSwitch11Test_gold @@ -5,6 +5,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... Published snapshot repository ./maverick [i386, source] publishes {main: [snap2]: Snapshot from local repo [local-repo2]} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch13Test_gold b/system/t06_publish/PublishSwitch13Test_gold index 271997f3..b9e7a75a 100644 --- a/system/t06_publish/PublishSwitch13Test_gold +++ b/system/t06_publish/PublishSwitch13Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch15Test_gold b/system/t06_publish/PublishSwitch15Test_gold index 271997f3..b9e7a75a 100644 --- a/system/t06_publish/PublishSwitch15Test_gold +++ b/system/t06_publish/PublishSwitch15Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch16Test_gold b/system/t06_publish/PublishSwitch16Test_gold index d3d1cd5b..6794c13d 100644 --- a/system/t06_publish/PublishSwitch16Test_gold +++ b/system/t06_publish/PublishSwitch16Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./bookworm... +Cleaning up component 'main'... Published snapshot repository ./bookworm (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch1Test_gold b/system/t06_publish/PublishSwitch1Test_gold index 271997f3..b9e7a75a 100644 --- a/system/t06_publish/PublishSwitch1Test_gold +++ b/system/t06_publish/PublishSwitch1Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch2Test_gold b/system/t06_publish/PublishSwitch2Test_gold index 7cc00d44..7cfc4f75 100644 --- a/system/t06_publish/PublishSwitch2Test_gold +++ b/system/t06_publish/PublishSwitch2Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "ppa" components main... +Cleaning up published repository ppa/maverick... +Cleaning up component 'main'... Published snapshot repository ppa/maverick [amd64, i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch3Test_gold b/system/t06_publish/PublishSwitch3Test_gold index 271997f3..b9e7a75a 100644 --- a/system/t06_publish/PublishSwitch3Test_gold +++ b/system/t06_publish/PublishSwitch3Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch4Test_gold b/system/t06_publish/PublishSwitch4Test_gold index 899c3d53..74451164 100644 --- a/system/t06_publish/PublishSwitch4Test_gold +++ b/system/t06_publish/PublishSwitch4Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "ppa" components main... +Cleaning up published repository ppa/maverick... +Cleaning up component 'main'... Published snapshot repository ppa/maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully switched to new source. diff --git a/system/t06_publish/PublishSwitch8Test_gold b/system/t06_publish/PublishSwitch8Test_gold index a10e335f..44d1fdbe 100644 --- a/system/t06_publish/PublishSwitch8Test_gold +++ b/system/t06_publish/PublishSwitch8Test_gold @@ -3,6 +3,8 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components b, c... +Cleaning up published repository ./maverick... +Cleaning up component 'b'... +Cleaning up component 'c'... Published snapshot repository ./maverick [amd64, i386, source] publishes {a: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {b: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'}, {c: [local2]: Snapshot from local repo [local-repo]} has been successfully switched to new source. diff --git a/system/t06_publish/PublishUpdate10Test_gold b/system/t06_publish/PublishUpdate10Test_gold index 082275a3..010d5ec4 100644 --- a/system/t06_publish/PublishUpdate10Test_gold +++ b/system/t06_publish/PublishUpdate10Test_gold @@ -5,6 +5,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate11Test_gold b/system/t06_publish/PublishUpdate11Test_gold index 05b56b2a..3be101c4 100644 --- a/system/t06_publish/PublishUpdate11Test_gold +++ b/system/t06_publish/PublishUpdate11Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate12Test_gold b/system/t06_publish/PublishUpdate12Test_gold index 7488eec4..d3a202df 100644 --- a/system/t06_publish/PublishUpdate12Test_gold +++ b/system/t06_publish/PublishUpdate12Test_gold @@ -4,4 +4,4 @@ Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate13Test_gold b/system/t06_publish/PublishUpdate13Test_gold index 05b56b2a..3be101c4 100644 --- a/system/t06_publish/PublishUpdate13Test_gold +++ b/system/t06_publish/PublishUpdate13Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate14Test_gold b/system/t06_publish/PublishUpdate14Test_gold index e0d3ab8c..12ddf71c 100644 --- a/system/t06_publish/PublishUpdate14Test_gold +++ b/system/t06_publish/PublishUpdate14Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./bookworm... +Cleaning up component 'main'... -Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./bookworm [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate15Test_gold b/system/t06_publish/PublishUpdate15Test_gold index ebd8ad24..a0624c8e 100644 --- a/system/t06_publish/PublishUpdate15Test_gold +++ b/system/t06_publish/PublishUpdate15Test_gold @@ -3,6 +3,6 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components ... +Cleaning up published repository ./maverick... -Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap3]: Created as empty}, {test: [snap2]: Created as empty} has been successfully updated. +Published snapshot repository ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap3]: Created as empty}, {test: [snap2]: Created as empty} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate16Test_gold b/system/t06_publish/PublishUpdate16Test_gold index e989f216..927ac538 100644 --- a/system/t06_publish/PublishUpdate16Test_gold +++ b/system/t06_publish/PublishUpdate16Test_gold @@ -3,6 +3,12 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components ... +Cleaning up published repository ./maverick... +Removing component 'other-test'... +Removing ${HOME}/.aptly/public/dists/maverick/other-test... +Removing component 'test'... +Removing ${HOME}/.aptly/public/dists/maverick/test... +Removing ${HOME}/.aptly/public/pool/other-test... +Removing ${HOME}/.aptly/public/pool/test... -Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated. +Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate17Test_gold b/system/t06_publish/PublishUpdate17Test_gold index 9ad84577..69c0182f 100644 --- a/system/t06_publish/PublishUpdate17Test_gold +++ b/system/t06_publish/PublishUpdate17Test_gold @@ -3,6 +3,8 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components other-test, test... +Cleaning up published repository ./maverick... +Cleaning up component 'other-test'... +Cleaning up component 'test'... -Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap5]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap4]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated. +Published snapshot repository ./maverick [i386] publishes {main: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {other-test: [snap5]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap4]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate18Test_gold b/system/t06_publish/PublishUpdate18Test_gold index 2e37b702..ca805720 100644 --- a/system/t06_publish/PublishUpdate18Test_gold +++ b/system/t06_publish/PublishUpdate18Test_gold @@ -3,6 +3,10 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components test... +Cleaning up published repository ./maverick... +Removing component 'main'... +Removing ${HOME}/.aptly/public/dists/maverick/main... +Removing ${HOME}/.aptly/public/pool/main... +Cleaning up component 'test'... -Published snapshot repository ./maverick [i386] publishes {other-test: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap3]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been successfully updated. +Published snapshot repository ./maverick [i386] publishes {other-test: [snap1]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick}, {test: [snap3]: Snapshot from mirror [gnuplot-maverick]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate1Test_gold b/system/t06_publish/PublishUpdate1Test_gold index 05b56b2a..3be101c4 100644 --- a/system/t06_publish/PublishUpdate1Test_gold +++ b/system/t06_publish/PublishUpdate1Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate2Test_gold b/system/t06_publish/PublishUpdate2Test_gold index 05b56b2a..3be101c4 100644 --- a/system/t06_publish/PublishUpdate2Test_gold +++ b/system/t06_publish/PublishUpdate2Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate3Test_gold b/system/t06_publish/PublishUpdate3Test_gold index 05b56b2a..3be101c4 100644 --- a/system/t06_publish/PublishUpdate3Test_gold +++ b/system/t06_publish/PublishUpdate3Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository ./maverick... +Cleaning up component 'main'... -Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate4Test_gold b/system/t06_publish/PublishUpdate4Test_gold index 29502a02..c996227a 100644 --- a/system/t06_publish/PublishUpdate4Test_gold +++ b/system/t06_publish/PublishUpdate4Test_gold @@ -3,6 +3,6 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up component 'main' in prefix '.'... -Published local repository ./maverick [source] publishes {main: [local-repo]} has been successfully updated. +Published local repository ./maverick [source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate7Test_gold b/system/t06_publish/PublishUpdate7Test_gold index a13b260c..b0f13d5c 100644 --- a/system/t06_publish/PublishUpdate7Test_gold +++ b/system/t06_publish/PublishUpdate7Test_gold @@ -3,6 +3,8 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components contrib, main... +Cleaning up published repository ./maverick... +Cleaning up component 'contrib'... +Cleaning up component 'main'... -Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +Published local repository ./maverick [i386, source] publishes {contrib: [repo2]}, {main: [repo1]} has been updated successfully. diff --git a/system/t06_publish/PublishUpdate8Test_gold b/system/t06_publish/PublishUpdate8Test_gold index 1a6ebd26..b69df45f 100644 --- a/system/t06_publish/PublishUpdate8Test_gold +++ b/system/t06_publish/PublishUpdate8Test_gold @@ -1,6 +1,8 @@ Loading packages... Generating metadata files and linking package files... Finalizing metadata files... -Cleaning up prefix "." components contrib, main... +Cleaning up published repository ./squeeze... +Cleaning up component 'contrib'... +Cleaning up component 'main'... -Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been successfully updated. +Published local repository ./squeeze [i386] publishes {contrib: [repo2]}, {main: [repo1]} has been updated successfully. diff --git a/system/t06_publish/S3Publish2Test_gold b/system/t06_publish/S3Publish2Test_gold index d6ffd382..cd68115c 100644 --- a/system/t06_publish/S3Publish2Test_gold +++ b/system/t06_publish/S3Publish2Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository s3:test1:./maverick... +Cleaning up component 'main'... -Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/S3Publish3Test_gold b/system/t06_publish/S3Publish3Test_gold index 54c8a83e..348fbe81 100644 --- a/system/t06_publish/S3Publish3Test_gold +++ b/system/t06_publish/S3Publish3Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository s3:test1:./maverick... +Cleaning up component 'main'... Published snapshot repository s3:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new source. diff --git a/system/t06_publish/S3Publish5Test_gold b/system/t06_publish/S3Publish5Test_gold index cadce02c..0b8f635f 100644 --- a/system/t06_publish/S3Publish5Test_gold +++ b/system/t06_publish/S3Publish5Test_gold @@ -1,3 +1,4 @@ -Cleaning up prefix "." components main... +Cleaning up published repository s3:test1:./sq2... +Cleaning up component 'main'... Published repository has been removed successfully. diff --git a/system/t06_publish/S3Publish6Test_gold b/system/t06_publish/S3Publish6Test_gold index d6ffd382..cd68115c 100644 --- a/system/t06_publish/S3Publish6Test_gold +++ b/system/t06_publish/S3Publish6Test_gold @@ -3,6 +3,7 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up published repository s3:test1:./maverick... +Cleaning up component 'main'... -Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/SwiftPublish2Test_gold b/system/t06_publish/SwiftPublish2Test_gold index 1ec018b6..40ce0140 100644 --- a/system/t06_publish/SwiftPublish2Test_gold +++ b/system/t06_publish/SwiftPublish2Test_gold @@ -3,6 +3,6 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up prefix '.' components main... -Published local repository swift:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated. +Published local repository swift:test1:./maverick [i386, source] publishes {main: [local-repo]} has been updated successfully. diff --git a/system/t06_publish/SwiftPublish3Test_gold b/system/t06_publish/SwiftPublish3Test_gold index eab52653..d8334a59 100644 --- a/system/t06_publish/SwiftPublish3Test_gold +++ b/system/t06_publish/SwiftPublish3Test_gold @@ -3,6 +3,6 @@ Generating metadata files and linking package files... Finalizing metadata files... Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: -Cleaning up prefix "." components main... +Cleaning up prefix '.' components main... Published snapshot repository swift:test1:./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot. diff --git a/system/t06_publish/SwiftPublish5Test_gold b/system/t06_publish/SwiftPublish5Test_gold index cadce02c..b8a740ad 100644 --- a/system/t06_publish/SwiftPublish5Test_gold +++ b/system/t06_publish/SwiftPublish5Test_gold @@ -1,3 +1,3 @@ -Cleaning up prefix "." components main... +Cleaning up prefix '.' components main... Published repository has been removed successfully. diff --git a/system/t06_publish/show.py b/system/t06_publish/show.py index 7616b78e..a6f06824 100644 --- a/system/t06_publish/show.py +++ b/system/t06_publish/show.py @@ -51,3 +51,17 @@ class PublishShow4Test(BaseTest): "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1 ppa/smira", ] runCmd = "aptly publish show -json maverick ppa/smira" + + +class PublishShow5Test(BaseTest): + """ + publish show: existing local repo + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly repo create -distribution=wheezy local-repo", + "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -architectures=i386 local-repo" + ] + runCmd = "aptly publish show wheezy" + gold_processor = BaseTest.expand_environ diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index 4514571d..aa3fb54e 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -1344,3 +1344,23 @@ class PublishSnapshot41Test(BaseTest): self.check_exists('public/pool/main/libx/libxslt/libxslt1.1_1.1.32-2.2~deb10u2_i386.deb') self.check_exists('public/pool/main/libz/libzstd/libzstd1_1.3.8+dfsg-3+deb10u2_i386.deb') self.check_exists('public/pool/main/z/zlib/zlib1g_1.2.11.dfsg-1+deb10u2_i386.deb') + + +class PublishSnapshot42Test(BaseTest): + """ + publish snapshot: mirror with multi-dist + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + ] + runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -multi-dist snap1" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishSnapshot42Test, self).check() + self.check_not_exists( + 'public/pool/main/g/gnuplot/gnuplot-doc_4.6.1-1~maverick2_all.deb') + self.check_exists( + 'public/pool/maverick/main/g/gnuplot/gnuplot-doc_4.6.1-1~maverick2_all.deb') diff --git a/system/t08_db/CleanupDB9Test_publish_drop b/system/t08_db/CleanupDB9Test_publish_drop index c9303809..ee4bb6e3 100644 --- a/system/t08_db/CleanupDB9Test_publish_drop +++ b/system/t08_db/CleanupDB9Test_publish_drop @@ -1,4 +1,5 @@ Removing ${HOME}/.aptly/public/dists/def... -Cleaning up prefix "." components main... +Cleaning up published repository ./def... +Cleaning up component 'main'... Published repository has been removed successfully. diff --git a/system/t12_api/publish.py b/system/t12_api/publish.py index 4fe3f6f0..82751579 100644 --- a/system/t12_api/publish.py +++ b/system/t12_api/publish.py @@ -1402,6 +1402,58 @@ class PublishSourcesListAPITestRepo(APITest): self.check_equal(sources_expected, sources.json()) +class PublishSourceReplaceAPITestRepo(APITest): + """ + PUT /publish/:prefix/:distribution/sources/main + """ + fixtureGpg = True + + def check(self): + repo1_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo1_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo1_name + "/file/" + d).status_code, 200) + + repo2_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo2_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb").status_code, 200) + + self.check_equal(self.post("/api/repos/" + repo2_name + "/file/" + d).status_code, 200) + + # publishing under prefix, default distribution + prefix = self.random_name() + self.check_equal(self.post( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Component": "main", "Name": repo1_name}], + "Signing": DefaultSigningOptions, + } + ).status_code, 201) + + # Actual test + self.check_equal(self.put( + "/api/publish/" + prefix + "/wheezy/sources/main", + json={"Component": "test", "Name": repo2_name} + ).status_code, 200) + + sources_expected = [{"Component": "test", "Name": repo2_name}] + sources = self.get("/api/publish/" + prefix + "/wheezy/sources") + self.check_equal(sources.status_code, 200) + self.check_equal(sources_expected, sources.json()) + + class PublishUpdateSourcesAPITestRepo(APITest): """ POST /publish/:prefix/:distribution/update From 8cceed12f799050836b8e8777ef3456b59d698ec Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Mon, 14 Oct 2024 20:50:45 +0200 Subject: [PATCH 11/25] Fix tests. Signed-off-by: Christoph Fiehe --- api/publish.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/publish.go b/api/publish.go index 2d71f539..812972c0 100644 --- a/api/publish.go +++ b/api/publish.go @@ -349,7 +349,7 @@ type publishedRepoUpdateSwitchParams struct { // Don't remove unreferenced files in prefix/component SkipCleanup *bool ` json:"SkipCleanup"` // only when updating published snapshots, list of objects 'Component/Name' - Snapshots []sourceParams `binding:"required" json:"Snapshots"` + Snapshots []sourceParams ` json:"Snapshots"` // Provide index files by hash AcquireByHash *bool ` json:"AcquireByHash"` // Enable multiple packages with the same filename in different distributions From 7a7ff1142c954eac5f651831416de3e1bc4c4268 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Tue, 15 Oct 2024 12:33:05 +0200 Subject: [PATCH 12/25] Minor code and documentation changes. Signed-off-by: Christoph Fiehe --- api/publish.go | 92 ++++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/api/publish.go b/api/publish.go index 812972c0..a049bc0a 100644 --- a/api/publish.go +++ b/api/publish.go @@ -67,7 +67,9 @@ func slashEscape(path string) string { } // @Summary List published repositories -// @Description **Get a list of all published repositories** +// @Description **List published repositories** +// @Description +// @Description Lists repositories that have been published based on local repositories or snapshots. For each repository information about `endpoint`, `prefix` and `distribution` is listed along with `component` and architecture list. Information about snapshot or local repo being published is appended to published repository description. // @Tags Publish // @Produce json // @Success 200 {array} deb.PublishedRepo @@ -99,7 +101,9 @@ func apiPublishList(c *gin.Context) { } // @Summary Show published repository -// @Description **Get details of a published repository** +// @Description **Show published repository** +// @Description +// @Description Show detailed information of published repository. // @Tags Publish // @Produce json // @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs" @@ -121,6 +125,7 @@ func apiPublishShow(c *gin.Context) { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to show: %s", err)) return } + err = collection.LoadComplete(published, collectionFactory) if err != nil { AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to show: %s", err)) @@ -178,10 +183,17 @@ type publishedRepoCreateParams struct { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix} [post] func apiPublishRepoOrSnapshot(c *gin.Context) { + var ( + b publishedRepoCreateParams + components []string + names []string + sources []interface{} + resources []string + ) + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) - var b publishedRepoCreateParams if c.Bind(&b) != nil { return } @@ -199,10 +211,6 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return } - var components []string - var names []string - var sources []interface{} - var resources []string collectionFactory := context.NewCollectionFactory() if b.SourceKind == deb.SourceSnapshot { @@ -313,10 +321,6 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { published.AcquireByHash = *b.AcquireByHash } - if b.MultiDist != nil { - published.MultiDist = *b.MultiDist - } - duplicate := collection.CheckDuplicate(published) if duplicate != nil { collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory) @@ -376,11 +380,12 @@ type publishedRepoUpdateSwitchParams struct { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution} [put] func apiPublishUpdateSwitch(c *gin.Context) { + var b publishedRepoUpdateSwitchParams + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) - var b publishedRepoUpdateSwitchParams if c.Bind(&b) != nil { return } @@ -511,7 +516,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { // @Summary Delete published repository // @Description **Delete a published repository** // @Description -// @Description Delete published repository, clean up files in published directory. +// @Description Delete published repository and clean up files in published directory. Aptly tries to remove as many files belonging to this repository as possible. For example, if no other published repositories share the same prefix, all files inside the prefix will be removed. // @Tags Publish // @Accept json // @Produce json @@ -525,13 +530,13 @@ func apiPublishUpdateSwitch(c *gin.Context) { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution} [delete] func apiPublishDrop(c *gin.Context) { - force := c.Request.URL.Query().Get("force") == "1" - skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1" - param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) + force := c.Request.URL.Query().Get("force") == "1" + skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1" + collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() @@ -554,8 +559,10 @@ func apiPublishDrop(c *gin.Context) { }) } -// @Summary Add staged source -// @Description **Create and add a staged source** +// @Summary Add source to staged source list +// @Description **Add a source to the staged source list** +// @Description +// @Description The staged source list exists independently of the current source list of the published repository. It can be modified in multiple steps by adding, removing and updating sources. A source is a tuple of two elements comprising the name of the component and the name of the local repository or snapshot. The staged source list exists as long as it gets discarded via `drop` or applied to the published repository via `update`. // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -566,10 +573,7 @@ func apiPublishDrop(c *gin.Context) { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [post] func apiPublishSourcesCreate(c *gin.Context) { - var ( - err error - b sourceParams - ) + var b sourceParams param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) @@ -620,7 +624,7 @@ func apiPublishSourcesCreate(c *gin.Context) { }) } -// @Summary Get staged sources +// @Summary Get staged source list // @Description **Get the staged source list** // @Tags Publish // @Param prefix path string true "publishing prefix" @@ -628,7 +632,7 @@ func apiPublishSourcesCreate(c *gin.Context) { // @Produce json // @Success 200 {array} sourceParams // @Failure 400 {object} Error "Bad Request" -// @Failure 404 {object} Error "Published repository not found or no source changes exist" +// @Failure 404 {object} Error "Published repository not found or staged source list does not exist" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [get] func apiPublishSourcesList(c *gin.Context) { @@ -660,8 +664,10 @@ func apiPublishSourcesList(c *gin.Context) { c.JSON(http.StatusOK, revision.SourceList()) } -// @Summary Set staged sources +// @Summary Set staged source list // @Description **Set the staged source list** +// @Description +// @Description If the staged source list is known in advance, it can set via this method in a single call. All modifications done before are lost and the staged source list get replaced by the one given in the request body. // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -674,10 +680,7 @@ func apiPublishSourcesList(c *gin.Context) { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [put] func apiPublishSourcesUpdate(c *gin.Context) { - var ( - err error - b []sourceParams - ) + var b []sourceParams param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) @@ -703,7 +706,7 @@ func apiPublishSourcesUpdate(c *gin.Context) { } revision := published.ObtainRevision() - sources := make(map[string]string, len(b)) + sources := make(map[string]string, 0, len(b)) revision.Sources = sources for _, source := range b { @@ -724,10 +727,10 @@ func apiPublishSourcesUpdate(c *gin.Context) { }) } -// @Summary Delete staged sources +// @Summary Delete staged source list // @Description **Delete the staged source list** // @Description -// @Description Delete staged sources and keep existing sources of published repository. +// @Description Delete/Discard the staged sources and keep existing sources of published repository. // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -784,10 +787,7 @@ func apiPublishSourcesDelete(c *gin.Context) { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources/{component} [put] func apiPublishSourceUpdate(c *gin.Context) { - var ( - err error - b sourceParams - ) + var b sourceParams param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) @@ -845,8 +845,8 @@ func apiPublishSourceUpdate(c *gin.Context) { }) } -// @Summary Delete staged source -// @Description **Delete the staged source** +// @Summary Delete source from staged source list +// @Description **Delete a single source from the staged source list** // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -858,8 +858,6 @@ func apiPublishSourceUpdate(c *gin.Context) { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources/{component} [delete] func apiPublishSourceDelete(c *gin.Context) { - var err error - param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) @@ -923,7 +921,7 @@ type publishedRepoUpdateParams struct { // @Summary Update content of published repository // @Description **Update the content of a published repository** // @Description -// @Description Replace the sources of the published repository (if available) and (re-)publish new content. +// @Description Replace the current source list of the published repository by the staged one (if available) and (re-)publish the new content. // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -936,20 +934,12 @@ type publishedRepoUpdateParams struct { // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/update [post] func apiPublishUpdate(c *gin.Context) { + var b publishedRepoUpdateParams + param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) - var b struct { - AcquireByHash *bool - ForceOverwrite bool - Signing signingParams - SkipBz2 *bool - SkipCleanup *bool - SkipContents *bool - MultiDist *bool - } - if c.Bind(&b) != nil { return } From 4d6688d68ed146ccb7e1634d9e9f12f34b2afd21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 14 Oct 2024 14:20:52 +0200 Subject: [PATCH 13/25] sanitize archs --- api/publish.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/publish.go b/api/publish.go index a049bc0a..660e5e17 100644 --- a/api/publish.go +++ b/api/publish.go @@ -200,6 +200,12 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { b.Distribution = utils.SanitizePath(b.Distribution) + var archs []string + for _, arch := range b.Architectures { + archs = append(archs, utils.SanitizePath(arch)) + } + b.Architectures = archs + signer, err := getSigner(&b.Signing) if err != nil { AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err)) From a56f52ff18e690677bf08a8f44f2fb511c214375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 14 Oct 2024 14:57:39 +0200 Subject: [PATCH 14/25] update man pages --- AUTHORS | 2 +- man/aptly.1 | 530 ++++++++++++++++++++++++++++++++++-------- man/aptly.1.ronn.tmpl | 2 +- 3 files changed, 430 insertions(+), 104 deletions(-) diff --git a/AUTHORS b/AUTHORS index 114fe886..0abdc0b0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -60,7 +60,7 @@ List of contributors, in chronological order: * Nic Waller (https://github.com/sf-nwaller) * iofq (https://github.com/iofq) * Noa Resare (https://github.com/nresare) -* Ramón N.Rodriguez (https://github.com/runitonmetal) +* Ramon N.Rodriguez (https://github.com/runitonmetal) * Golf Hu (https://github.com/hudeng-go) * Cookie Fei (https://github.com/wuhuang26) * Andrey Loukhnov (https://github.com/aol-nnov) diff --git a/man/aptly.1 b/man/aptly.1 index fe986f0f..d49d2f6f 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "APTLY" "1" "January 2022" "" "" +.TH "APTLY" "1" "October 2024" "" "" . .SH "NAME" \fBaptly\fR \- Debian repository management tool @@ -25,7 +25,7 @@ aptly is a tool to create partial and full mirrors of remote repositories, manag aptly\(cqs goal is to establish repeatability and controlled changes in a package\-centric environment\. aptly allows one to fix a set of packages in a repository, so that package installation and upgrade becomes deterministic\. At the same time aptly allows one to perform controlled, fine\-grained changes in repository contents to transition your package environment to new version\. . .SH "CONFIGURATION" -aptly looks for configuration file first in \fB~/\.aptly\.conf\fR then in \fB/etc/aptly\.conf\fR and, if no config file found, new one is created in home directory\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly\fR, it will be created if missing\. +aptly looks for configuration file first in \fB~/\.aptly\.conf\fR then in \fB/usr/local/etc/aptly\.conf\fR and \fB/etc/aptly\.conf\fR\. If no config file found (or they are not readable), a new one is created in the home directory\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly/\fR, it will be created if missing\. . .P Configuration file is stored in JSON format (default values shown below): @@ -36,6 +36,10 @@ Configuration file is stored in JSON format (default values shown below): { "rootDir": "$HOME/\.aptly", + "databaseBackend": { + "type": "", + "url": "" + }, "downloadConcurrency": 4, "downloadSpeedLimit": 0, "downloadRetries": 0, @@ -51,6 +55,16 @@ Configuration file is stored in JSON format (default values shown below): "gpgDisableVerify": false, "gpgProvider": "gpg", "downloadSourcePackages": false, + "packagePoolStorage": { + "path": "$ROOTDIR/pool", + "azure": { + "accountName": "", + "accountKey": "", + "container": "repo", + "prefix": "", + "endpoint": "" + } + }, "skipLegacyPool": true, "ppaDistributorID": "ubuntu", "ppaCodename": "", @@ -84,7 +98,7 @@ Configuration file is stored in JSON format (default values shown below): "plusWorkaround": false, "disableMultiDel": false, "forceSigV2": false, - "forceVirtualHostedStyle": false, + "forceVirtualHostedStyle": true, "debug": false } }, @@ -104,8 +118,8 @@ Configuration file is stored in JSON format (default values shown below): "accountName": "", "accountKey": "", "container": "repo", - "prefix": "" - "endpoint": "blob.core.windows.net" + "prefix": "", + "endpoint": "blob\.core\.windows\.net" } } } @@ -117,85 +131,96 @@ Configuration file is stored in JSON format (default values shown below): .P Options: . -.TP -\fBrootDir\fR -is root of directory storage to store database (\fBrootDir\fR/db), downloaded packages (\fBrootDir\fR/pool) and the default for published repositories (\fBrootDir\fR/public) +.IP "\[ci]" 4 +\fBrootDir\fR: is root of directory storage to store database (\fBrootDir\fR/db), the default for downloaded packages (\fBrootDir\fR/pool) and the default for published repositories (\fBrootDir\fR/public) . -.TP -\fBdownloadConcurrency\fR -is a number of parallel download threads to use when downloading packages +.IP "\[ci]" 4 +\fBdatabaseBackend\fR: the database config; if this config is empty, use levledb backend by default . -.TP -\fBdownloadSpeedLimit\fR -limit in kbytes/sec on download speed while mirroring remote repositories +.IP "\[ci]" 4 +\fBdownloadConcurrency\fR: is a number of parallel download threads to use when downloading packages . -.TP -\fBdownloadRetries\fR -number of retries for download attempts +.IP "\[ci]" 4 +\fBdownloadSpeedLimit\fR: limit in kbytes/sec on download speed while mirroring remote repositories . -.TP -\fBdatabaseOpenAttempts\fR -number of attempts to open DB if it\(cqs locked by other instance; could be overridden with option \fB\-db\-open\-attempts\fR +.IP "\[ci]" 4 +\fBdownloadRetries\fR: number of retries for download attempts . -.TP -\fBarchitectures\fR -is a list of architectures to process; if left empty defaults to all available architectures; could be overridden with option \fB\-architectures\fR +.IP "\[ci]" 4 +\fBdatabaseOpenAttempts\fR: number of attempts to open DB if it\(cqs locked by other instance; could be overridden with option \fB\-db\-open\-attempts\fR . -.TP -\fBdependencyFollowSuggests\fR -follow contents of \fBSuggests:\fR field when processing dependencies for the package +.IP "\[ci]" 4 +\fBarchitectures\fR: is a list of architectures to process; if left empty defaults to all available architectures; could be overridden with option \fB\-architectures\fR . -.TP -\fBdependencyFollowRecommends\fR -follow contents of \fBRecommends:\fR field when processing dependencies for the package +.IP "\[ci]" 4 +\fBdependencyFollowSuggests\fR: follow contents of \fBSuggests:\fR field when processing dependencies for the package . -.TP -\fBdependencyFollowAllVariants\fR -when dependency looks like \fBpackage\-a | package\-b\fR, follow both variants always +.IP "\[ci]" 4 +\fBdependencyFollowRecommends\fR: follow contents of \fBRecommends:\fR field when processing dependencies for the package . -.TP -\fBdependencyFollowSource\fR -follow dependency from binary package to source package +.IP "\[ci]" 4 +\fBdependencyFollowAllVariants\fR: when dependency looks like \fBpackage\-a | package\-b\fR, follow both variants always . -.TP -\fBdependencyVerboseResolve\fR -print additional details while resolving dependencies (useful for debugging) +.IP "\[ci]" 4 +\fBdependencyFollowSource\fR: follow dependency from binary package to source package . -.TP -\fBgpgDisableSign\fR -don\(cqt sign published repositories with gpg(1), also can be disabled on per\-repo basis using \fB\-skip\-signing\fR flag when publishing +.IP "\[ci]" 4 +\fBdependencyVerboseResolve\fR: print additional details while resolving dependencies (useful for debugging) . -.TP -\fBgpgDisableVerify\fR -don\(cqt verify remote mirrors with gpg(1), also can be disabled on per\-mirror basis using \fB\-ignore\-signatures\fR flag when creating and updating mirrors +.IP "\[ci]" 4 +\fBgpgDisableSign\fR: don\(cqt sign published repositories with gpg(1), also can be disabled on per\-repo basis using \fB\-skip\-signing\fR flag when publishing . -.TP -\fBgpgProvider\fR -implementation of PGP signing/validation \- \fBgpg\fR for external \fBgpg\fR utility or \fBinternal\fR to use Go internal implementation; \fBgpg1\fR might be used to force use of GnuPG 1\.x, \fBgpg2\fR enables GnuPG 2\.x only; default is to use GnuPG 1\.x if available and GnuPG 2\.x otherwise +.IP "\[ci]" 4 +\fBgpgDisableVerify\fR: don\(cqt verify remote mirrors with gpg(1), also can be disabled on per\-mirror basis using \fB\-ignore\-signatures\fR flag when creating and updating mirrors . -.TP -\fBdownloadSourcePackages\fR -if enabled, all mirrors created would have flag set to download source packages; this setting could be controlled on per\-mirror basis with \fB\-with\-sources\fR flag +.IP "\[ci]" 4 +\fBgpgProvider\fR: implementation of PGP signing/validation \- \fBgpg\fR for external \fBgpg\fR utility or \fBinternal\fR to use Go internal implementation; \fBgpg1\fR might be used to force use of GnuPG 1\.x, \fBgpg2\fR enables GnuPG 2\.x only; default is to use GnuPG 1\.x if available and GnuPG 2\.x otherwise . -.TP -\fBskipLegacyPool\fR -in aptly up to version 1\.0\.0, package files were stored in internal package pool with MD5\-dervied path, since 1\.1\.0 package pool layout was changed; if option is enabled, aptly stops checking for legacy paths; by default option is enabled for new aptly installations and disabled when upgrading from older versions +.IP "\[ci]" 4 +\fBdownloadSourcePackages\fR: if enabled, all mirrors created would have flag set to download source packages; this setting could be controlled on per\-mirror basis with \fB\-with\-sources\fR flag . -.TP -\fBppaDistributorID\fR, \fBppaCodename\fR -specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command +.IP "\[ci]" 4 +\fBpackagePoolStorage\fR: configures the location to store downloaded packages (defaults to the path \fB$ROOTDIR/pool\fR), by setting the value of the \fBtype\fR: . -.TP -\fBFileSystemPublishEndpoints\fR -configuration of local filesystem publishing endpoints (see below) +.IP "\[ci]" 4 +\fBpath\fR: store the packages in the given path . -.TP -\fBS3PublishEndpoints\fR -configuration of Amazon S3 publishing endpoints (see below) +.IP "\[ci]" 4 +\fBazure\fR: store the packages in the given Azure Blob Storage container (see the section on Azure publishing below for information on the configuration) . -.TP -\fBSwiftPublishEndpoints\fR -configuration of OpenStack Swift publishing endpoints (see below) +.IP "" 0 + +. +.IP "\[ci]" 4 +\fBskipLegacyPool\fR: in aptly up to version 1\.0\.0, package files were stored in internal package pool with MD5\-dervied path, since 1\.1\.0 package pool layout was changed; if option is enabled, aptly stops checking for legacy paths; by default option is enabled for new aptly installations and disabled when upgrading from older versions +. +.IP "\[ci]" 4 +\fBppaDistributorID\fR, \fBppaCodename\fR: specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command +. +.IP "\[ci]" 4 +\fBFileSystemPublishEndpoints\fR: configuration of local filesystem publishing endpoints (see below) +. +.IP "\[ci]" 4 +\fBS3PublishEndpoints\fR: configuration of Amazon S3 publishing endpoints (see below) +. +.IP "\[ci]" 4 +\fBSwiftPublishEndpoints\fR: configuration of OpenStack Swift publishing endpoints (see below) +. +.IP "\[ci]" 4 +\fBAzurePublishEndpoints\fR: configuration of Azure publishing endpoints (see below) +. +.IP "" 0 +. +.SH "CUSTOM PACKAGE POOLS" +aptly defaults to storing downloaded packages at \fBrootDir/\fRpool\. In order to change this, you can set the \fBtype\fR key within \fBpackagePoolStorage\fR to one of two values: +. +.IP "\[ci]" 4 +\fBlocal\fR: Store the package pool locally (the default)\. In order to change the path, additionally set the \fBpath\fR key within \fBpackagePoolStorage\fR to the desired location\. +. +.IP "\[ci]" 4 +\fBazure\fR: Store the package pool in an Azure Blob Storage container\. Any keys in the below section on Azure publishing may be set on the \fBpackagePoolStorage\fR object in order to configure the Azure connection\. +. +.IP "" 0 . .SH "FILESYSTEM PUBLISHING ENDPOINTS" aptly defaults to publish to a single publish directory under \fBrootDir\fR/public\. For a more advanced publishing strategy, you can define one or more filesystem endpoints in the \fBFileSystemPublishEndpoints\fR list of the aptly configuration file\. Each endpoint has a name and the following associated settings: @@ -308,6 +333,25 @@ In order to publish to Swift, specify endpoint as \fBswift:endpoint\-name:\fR be .P \fBaptly publish snapshot jessie\-main swift:test:\fR . +.SH "AZURE PUBLISHING ENDPOINTS" +aptly can be configured to publish repositories directly to Microsoft Azure Blob Storage\. First, publishing endpoints should be described in the aptly configuration file\. Each endpoint has its name and associated settings: +. +.TP +\fBcontainer\fR +container name +. +.TP +\fBprefix\fR +(optional) do publishing under specified prefix in the container, defaults to no prefix (container root) +. +.TP +\fBaccountName\fR, \fBaccountKey\fR +Azure storage account access key to access blob storage +. +.TP +\fBendpoint\fR +endpoint URL to connect to, as described in the Azure documentation \fIhttps://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string\fR; defaults to \fBhttps://$accountName\.blob\.core\.windows\.net\fR +. .SH "PACKAGE QUERY" Some commands accept package queries to identify list of packages to process\. Package query syntax almost matches \fBreprepro\fR query language\. Query consists of the following simple terms: . @@ -431,7 +475,7 @@ list of architectures to consider during (comma\-separated), default to all avai . .TP \-\fBconfig\fR= -location of configuration file (default locations are /etc/aptly\.conf, ~/\.aptly\.conf) +location of configuration file (default locations in order: ~/\.aptly\.conf, /usr/local/etc/aptly\.conf, /etc/aptly\.conf) . .TP \-\fBdb\-open\-attempts\fR=10 @@ -507,6 +551,10 @@ disable verification of Release file signatures gpg keyring to use when verifying Release file (could be specified multiple times) . .TP +\-\fBmax\-tries\fR=1 +max download tries till process fails with download error +. +.TP \-\fBwith\-installer\fR download additional not packaged installer files . @@ -1009,13 +1057,13 @@ custom format for result printing include dependencies into search results . .SH "ADD PACKAGES TO LOCAL REPOSITORIES BASED ON \.CHANGES FILES" -\fBaptly\fR \fBrepo\fR \fBinclude\fR |\fIdirectory\fR \fB\|\.\|\.\|\.\fR +\fBaptly\fR \fBrepo\fR \fBinclude\fR . .P Command include looks for \.changes files in list of arguments or specified directories\. Each \.changes file is verified, parsed, referenced files are put into separate temporary directory and added into local repository\. Successfully imported files are removed by default\. . .P -Additionally uploads could be restricted with file\. Rules in this file control uploads based on GPG key ID of \.changes file signature and queries on \.changes file fields\. +Additionally uploads could be restricted with . .P Example: @@ -1450,6 +1498,10 @@ run GPG with detached tty set value for ButAutomaticUpgrades field . .TP +\-\fBcodename\fR= +codename to publish (defaults to distribution) +. +.TP \-\fBcomponent\fR= component name to publish (for multi\-component publishing, separate components with commas) . @@ -1474,6 +1526,10 @@ GPG keyring to use (instead of default) label to publish . .TP +\-\fBmulti\-dist\fR +enable multiple packages with the same filename in different distributions +. +.TP \-\fBnotautomatic\fR= set value for NotAutomatic field . @@ -1494,6 +1550,10 @@ GPG passphrase\-file for the key (warning: could be insecure) GPG secret keyring to use (instead of default) . .TP +\-\fBskip\-bz2\fR +don\(cqt generate bzipped indexes +. +.TP \-\fBskip\-contents\fR don\(cqt generate Contents indexes . @@ -1505,9 +1565,31 @@ don\(cqt sign Release files with GPG \-\fBsuite\fR= suite to publish (defaults to distribution) . +.SH "SHOWS DETAILS OF PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBshow\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] +. +.P +Command show displays full information of a published repository\. +. +.P +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish show wheezy +. +.fi +. +.IP "" 0 +. +.P +Options: +. .TP -\-\fBcodename\fR= -codename to publish (defaults to distribution) +\-\fBjson\fR +display record in JSON format . .SH "PUBLISH SNAPSHOT" \fBaptly\fR \fBpublish\fR \fBsnapshot\fR \fIname\fR [[\fIendpoint\fR:]\fIprefix\fR] @@ -1557,6 +1639,10 @@ run GPG with detached tty overwrite value for ButAutomaticUpgrades field . .TP +\-\fBcodename\fR= +codename to publish (defaults to distribution) +. +.TP \-\fBcomponent\fR= component name to publish (for multi\-component publishing, separate components with commas) . @@ -1581,6 +1667,10 @@ GPG keyring to use (instead of default) label to publish . .TP +\-\fBmulti\-dist\fR +enable multiple packages with the same filename in different distributions +. +.TP \-\fBnotautomatic\fR= overwrite value for NotAutomatic field . @@ -1601,6 +1691,10 @@ GPG passphrase\-file for the key (warning: could be insecure) GPG secret keyring to use (instead of default) . .TP +\-\fBskip\-bz2\fR +don\(cqt generate bzipped indexes +. +.TP \-\fBskip\-contents\fR don\(cqt generate Contents indexes . @@ -1612,18 +1706,200 @@ don\(cqt sign Release files with GPG \-\fBsuite\fR= suite to publish (defaults to distribution) . +.SH "ADD SOURCE TO STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBadd\fR \fIdistribution\fR \fIsource\fR +. +.P +The command adds sources to the staged source list of the published repository\. +. +.P +The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: +. +.IP "" 4 +. +.nf + +aptly publish add \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish add \-component=contrib wheezy ppa wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +This command assigns the snapshot wheezy\-contrib to the component contrib and adds it to published repository revision of ppa/wheezy\. +. +.P +Options: +. .TP -\-\fBcodename\fR= -codename to publish (defaults to distribution) +\-\fBcomponent\fR= +component names to add (for multi\-component publishing, separate components with commas) . -.SH "UPDATE PUBLISHED REPOSITORY BY SWITCHING TO NEW SNAPSHOT" -\fBaptly\fR \fBpublish\fR \fBswitch\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fInew\-snapshot\fR +.TP +\-\fBprefix\fR=\. +publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR +. +.SH "DROPS STAGED SOURCE CHANGES OF PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBdrop\fR \fIdistribution\fR . .P -Command switches in\-place published snapshots with new snapshot contents\. All publishing parameters are preserved (architecture list, distribution, component)\. +Command drops the staged source changes of the published repository\. . .P -For multiple component repositories, flag \-component should be given with list of components to update\. Corresponding snapshots should be given in the same order, e\.g\.: +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish source drop wheezy +. +.fi +. +.IP "" 0 +. +.P +Options: +. +.TP +\-\fBcomponent\fR= +component names to add (for multi\-component publishing, separate components with commas) +. +.TP +\-\fBprefix\fR=\. +publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR +. +.SH "LISTS REVISION OF PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBlist\fR \fIdistribution\fR +. +.P +Command lists sources of a published repository\. +. +.P +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish source list wheezy +. +.fi +. +.IP "" 0 +. +.P +Options: +. +.TP +\-\fBcomponent\fR= +component names to add (for multi\-component publishing, separate components with commas) +. +.TP +\-\fBjson\fR +display record in JSON format +. +.TP +\-\fBprefix\fR=\. +publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR +. +.SH "REMOVE SOURCE FROM STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBremove\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fIsource\fR +. +.P +The command removes sources from the staged source list of the published repository\. +. +.P +The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be removed, e\.g\.: +. +.P +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish remove \-component=contrib,non\-free wheezy filesystem:symlink:debian +. +.fi +. +.IP "" 0 +. +.P +Options: +. +.TP +\-\fBcomponent\fR= +component names to remove (for multi\-component publishing, separate components with commas) +. +.TP +\-\fBprefix\fR=\. +publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR +. +.SH "UPDATE SOURCE IN STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBupdate\fR \fIdistribution\fR \fIsource\fR +. +.P +The command updates sources in the staged source list of the published repository\. +. +.P +The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: +. +.IP "" 4 +. +.nf + +aptly publish update \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish update \-component=contrib wheezy ppa wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +Options: +. +.TP +\-\fBcomponent\fR= +component names to add (for multi\-component publishing, separate components with commas) +. +.TP +\-\fBprefix\fR=\. +publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR +. +.SH "UPDATE PUBLISHED REPOSITORY BY SWITCHING TO NEW SOURCE" +\fBaptly\fR \fBpublish\fR \fBswitch\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fInew\-source\fR +. +.P +Command switches in\-place published snapshots with new source contents\. All publishing parameters are preserved (architecture list, distribution, component)\. +. +.P +For multiple component repositories, flag \-component should be given with list of components to update\. Corresponding sources should be given in the same order, e\.g\.: . .IP "" 4 . @@ -1675,6 +1951,10 @@ GPG key ID to use when signing the release GPG keyring to use (instead of default) . .TP +\-\fBmulti\-dist\fR +enable multiple packages with the same filename in different distributions +. +.TP \-\fBpassphrase\fR= GPG passphrase for the key (warning: could be insecure) . @@ -1687,6 +1967,10 @@ GPG passphrase\-file for the key (warning: could be insecure) GPG secret keyring to use (instead of default) . .TP +\-\fBskip\-bz2\fR +don\(cqt generate bzipped indexes +. +.TP \-\fBskip\-cleanup\fR don\(cqt remove unreferenced files in prefix/component . @@ -1740,6 +2024,10 @@ GPG key ID to use when signing the release GPG keyring to use (instead of default) . .TP +\-\fBmulti\-dist\fR +enable multiple packages with the same filename in different distributions +. +.TP \-\fBpassphrase\fR= GPG passphrase for the key (warning: could be insecure) . @@ -1752,6 +2040,10 @@ GPG passphrase\-file for the key (warning: could be insecure) GPG secret keyring to use (instead of default) . .TP +\-\fBskip\-bz2\fR +don\(cqt generate bzipped indexes +. +.TP \-\fBskip\-cleanup\fR don\(cqt remove unreferenced files in prefix/component . @@ -1763,32 +2055,6 @@ don\(cqt generate Contents indexes \-\fBskip\-signing\fR don\(cqt sign Release files with GPG . -.SH "SHOWS DETAILS OF PUBLISHED REPOSITORY" -\fBaptly\fR \fBpublish\fR \fBshow\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] -. -.P -Command show displays full information of a published repository\. -. -.P -Example: -. -.IP "" 4 -. -.nf - -$ aptly publish show wheezy -. -.fi -. -.IP "" 0 -. -.P -Options: -. -.TP -\-\fBjson\fR -display record in JSON format -. .SH "SEARCH FOR PACKAGES MATCHING QUERY" \fBaptly\fR \fBpackage\fR \fBsearch\fR [\fIpackage\-query\fR] . @@ -2160,5 +2426,65 @@ Lorenzo Bolla (https://github\.com/lbolla) .IP "\[ci]" 4 Benj Fassbind (https://github\.com/randombenj) . +.IP "\[ci]" 4 +Markus Muellner (https://github\.com/mmianl) +. +.IP "\[ci]" 4 +Chuan Liu (https://github\.com/chuan) +. +.IP "\[ci]" 4 +Samuel Mutel (https://github\.com/smutel) +. +.IP "\[ci]" 4 +Russell Greene (https://github\.com/russelltg) +. +.IP "\[ci]" 4 +Wade Simmons (https://github\.com/wadey) +. +.IP "\[ci]" 4 +Steven Stone (https://github\.com/smstone) +. +.IP "\[ci]" 4 +Josh Bayfield (https://github\.com/jbayfield) +. +.IP "\[ci]" 4 +Boxjan (https://github\.com/boxjan) +. +.IP "\[ci]" 4 +Mauro Regli (https://github\.com/reglim) +. +.IP "\[ci]" 4 +Alexander Zubarev (https://github\.com/strike) +. +.IP "\[ci]" 4 +Nicolas Dostert (https://github\.com/acdn\-ndostert) +. +.IP "\[ci]" 4 +Ryan Gonzalez (https://github\.com/refi64) +. +.IP "\[ci]" 4 +Paul Cacheux (https://github\.com/paulcacheux) +. +.IP "\[ci]" 4 +Nic Waller (https://github\.com/sf\-nwaller) +. +.IP "\[ci]" 4 +iofq (https://github\.com/iofq) +. +.IP "\[ci]" 4 +Noa Resare (https://github\.com/nresare) +. +.IP "\[ci]" 4 +Ramon N\.Rodriguez (https://github\.com/runitonmetal) +. +.IP "\[ci]" 4 +Golf Hu (https://github\.com/hudeng\-go) +. +.IP "\[ci]" 4 +Cookie Fei (https://github\.com/wuhuang26) +. +.IP "\[ci]" 4 +Christoph Fiehe (https://github\.com/cfiehe) +. .IP "" 0 diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index baab5ff6..e88ba35e 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -111,7 +111,7 @@ Configuration file is stored in JSON format (default values shown below): "accountKey": "", "container": "repo", "prefix": "", - "endpoint": "" + "endpoint": "blob.core.windows.net" } } } From f4057850b97e7c858f8500d625031de32cf1f723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 15 Oct 2024 13:08:33 +0200 Subject: [PATCH 15/25] fix compile and lint errors --- api/publish.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/publish.go b/api/publish.go index 660e5e17..4c07ab1e 100644 --- a/api/publish.go +++ b/api/publish.go @@ -184,11 +184,11 @@ type publishedRepoCreateParams struct { // @Router /api/publish/{prefix} [post] func apiPublishRepoOrSnapshot(c *gin.Context) { var ( - b publishedRepoCreateParams + b publishedRepoCreateParams components []string - names []string - sources []interface{} - resources []string + names []string + sources []interface{} + resources []string ) param := slashEscape(c.Params.ByName("prefix")) @@ -712,7 +712,7 @@ func apiPublishSourcesUpdate(c *gin.Context) { } revision := published.ObtainRevision() - sources := make(map[string]string, 0, len(b)) + sources := make(map[string]string, len(b)) revision.Sources = sources for _, source := range b { From 755fdfaca24177bd463b19f78b1b2b8d8ab1af6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 15 Oct 2024 13:32:12 +0200 Subject: [PATCH 16/25] update swagger documentation - add default values - set default values --- Makefile | 2 +- api/publish.go | 129 ++++++++++++++++++++++++++----------------------- api/router.go | 12 ++--- 3 files changed, 76 insertions(+), 67 deletions(-) diff --git a/Makefile b/Makefile index 89aceb22..5e89df2b 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ serve: prepare swagger-install ## Run development server (auto recompiling) test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf ~/.aptly.conf sed -i /enableSwaggerEndpoint/s/false/true/ ~/.aptly.conf - PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,cmd,systemd -- api serve -listen 0.0.0.0:3142 + PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,cmd,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 dpkg: prepare swagger ## Build debian packages @test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1) diff --git a/api/publish.go b/api/publish.go index 4c07ab1e..b7aa3a87 100644 --- a/api/publish.go +++ b/api/publish.go @@ -15,24 +15,24 @@ import ( type signingParams struct { // Don't sign published repository - Skip bool ` json:"Skip"` + Skip bool ` json:"Skip" example:"false"` // GPG key ID to use when signing the release, if not specified default key is used - GpgKey string ` json:"GpgKey"` + GpgKey string ` json:"GpgKey" example:"A0546A43624A8331"` // GPG keyring to use (instead of default) - Keyring string ` json:"Keyring"` - // GPG secret keyring to use (instead of default) - SecretKeyring string ` json:"SecretKeyring"` + Keyring string ` json:"Keyring" example:"trustedkeys.gpg"` + // GPG secret keyring to use (instead of default) Note: depreciated with gpg2 + SecretKeyring string ` json:"SecretKeyring" example:""` // GPG passphrase to unlock private key (possibly insecure) - Passphrase string ` json:"Passphrase"` + Passphrase string ` json:"Passphrase" example:"verysecure"` // GPG passphrase file to unlock private key (possibly insecure) - PassphraseFile string ` json:"PassphraseFile"` + PassphraseFile string ` json:"PassphraseFile" example:"/etc/aptly.passphrase"` } type sourceParams struct { // Name of the component - Component string `binding:"required" json:"Component"` + Component string `binding:"required" json:"Component" example:"main"` // Name of the local repository/snapshot - Name string `binding:"required" json:"Name"` + Name string `binding:"required" json:"Name" example:"snap1"` } func getSigner(options *signingParams) (pgp.Signer, error) { @@ -67,7 +67,7 @@ func slashEscape(path string) string { } // @Summary List published repositories -// @Description **List published repositories** +// @Description **Get list of published repositories** // @Description // @Description Lists repositories that have been published based on local repositories or snapshots. For each repository information about `endpoint`, `prefix` and `distribution` is listed along with `component` and architecture list. Information about snapshot or local repo being published is appended to published repository description. // @Tags Publish @@ -101,7 +101,7 @@ func apiPublishList(c *gin.Context) { } // @Summary Show published repository -// @Description **Show published repository** +// @Description **Get published repository information** // @Description // @Description Show detailed information of published repository. // @Tags Publish @@ -137,35 +137,35 @@ func apiPublishShow(c *gin.Context) { type publishedRepoCreateParams struct { // 'local' for local repositories and 'snapshot' for snapshots - SourceKind string `binding:"required" json:"SourceKind" example:"snapshot"` + SourceKind string `binding:"required" json:"SourceKind" example:"snapshot"` // List of 'Component/Name' objects, 'Name' is either local repository or snapshot name Sources []sourceParams `binding:"required" json:"Sources"` // Distribution name, if missing Aptly would try to guess from sources - Distribution string ` json:"Distribution"` + Distribution string ` json:"Distribution" example:"bookworm"` // Value of Label: field in published repository stanza - Label string ` json:"Label"` + Label string ` json:"Label" example:""` // Value of Origin: field in published repository stanza - Origin string ` json:"Origin"` + Origin string ` json:"Origin" example:""` // when publishing, overwrite files in pool/ directory without notice - ForceOverwrite bool ` json:"ForceOverwrite"` + ForceOverwrite bool ` json:"ForceOverwrite" example:"false"` // Override list of published architectures - Architectures []string ` json:"Architectures"` + Architectures []string ` json:"Architectures" example:"amd64,armhf"` // GPG options Signing signingParams ` json:"Signing"` // Setting to yes indicates to the package manager to not install or upgrade packages from the repository without user consent - NotAutomatic string ` json:"NotAutomatic"` + NotAutomatic string ` json:"NotAutomatic" example:""` // setting to yes excludes upgrades from the NotAutomic setting - ButAutomaticUpgrades string ` json:"ButAutomaticUpgrades"` + ButAutomaticUpgrades string ` json:"ButAutomaticUpgrades" example:""` // Don't generate contents indexes - SkipContents *bool ` json:"SkipContents"` + SkipContents *bool ` json:"SkipContents" example:"false"` // Don't remove unreferenced files in prefix/component - SkipCleanup *bool ` json:"SkipCleanup"` + SkipCleanup *bool ` json:"SkipCleanup" example:"false"` // Skip bz2 compression for index files - SkipBz2 *bool ` json:"SkipBz2"` + SkipBz2 *bool ` json:"SkipBz2" example:"false"` // Provide index files by hash - AcquireByHash *bool ` json:"AcquireByHash"` + AcquireByHash *bool ` json:"AcquireByHash" example:"false"` // Enable multiple packages with the same filename in different distributions - MultiDist *bool ` json:"MultiDist"` + MultiDist *bool ` json:"MultiDist" example:"false"` } // @Summary Create published repository @@ -349,21 +349,21 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { type publishedRepoUpdateSwitchParams struct { // when publishing, overwrite files in pool/ directory without notice - ForceOverwrite bool ` json:"ForceOverwrite"` + ForceOverwrite bool ` json:"ForceOverwrite" example:"false"` // GPG options Signing signingParams ` json:"Signing"` // Don't generate contents indexes - SkipContents *bool ` json:"SkipContents"` + SkipContents *bool ` json:"SkipContents" example:"false"` // Skip bz2 compression for index files - SkipBz2 *bool ` json:"SkipBz2"` + SkipBz2 *bool ` json:"SkipBz2" example:"false"` // Don't remove unreferenced files in prefix/component - SkipCleanup *bool ` json:"SkipCleanup"` + SkipCleanup *bool ` json:"SkipCleanup" example:"false"` // only when updating published snapshots, list of objects 'Component/Name' Snapshots []sourceParams ` json:"Snapshots"` // Provide index files by hash - AcquireByHash *bool ` json:"AcquireByHash"` + AcquireByHash *bool ` json:"AcquireByHash" example:"false"` // Enable multiple packages with the same filename in different distributions - MultiDist *bool ` json:"MultiDist"` + MultiDist *bool ` json:"MultiDist" example:"false"` } // @Summary Update published repository @@ -373,7 +373,6 @@ type publishedRepoUpdateSwitchParams struct { // @Description * if local repository has been published, published repository would be updated to match local repository contents // @Description * if snapshots have been been published, it is possible to switch each component to new snapshot // @Tags Publish -// @Accept json // @Produce json // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -524,7 +523,6 @@ func apiPublishUpdateSwitch(c *gin.Context) { // @Description // @Description Delete published repository and clean up files in published directory. Aptly tries to remove as many files belonging to this repository as possible. For example, if no other published repositories share the same prefix, all files inside the prefix will be removed. // @Tags Publish -// @Accept json // @Produce json // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -565,20 +563,24 @@ func apiPublishDrop(c *gin.Context) { }) } -// @Summary Add source to staged source list -// @Description **Add a source to the staged source list** +// @Summary Add Source +// @Description **Add a source to published repo** // @Description -// @Description The staged source list exists independently of the current source list of the published repository. It can be modified in multiple steps by adding, removing and updating sources. A source is a tuple of two elements comprising the name of the component and the name of the local repository or snapshot. The staged source list exists as long as it gets discarded via `drop` or applied to the published repository via `update`. +// @Description Adds a component of a snapshot or local repository to be published. +// @Description +// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" +// @Consume json +// @Param request body sourceParams true "Parameters" // @Produce json -// @Success 200 {object} sourceParams +// @Success 200 // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository not found" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [post] -func apiPublishSourcesCreate(c *gin.Context) { +func apiPublishAddSource(c *gin.Context) { var b sourceParams param := slashEscape(c.Params.ByName("prefix")) @@ -630,18 +632,22 @@ func apiPublishSourcesCreate(c *gin.Context) { }) } -// @Summary Get staged source list -// @Description **Get the staged source list** +// @Summary List pending changes +// @Description **List changes to be applied** +// @Description +// @Description Returns added, removed or changed components of snapshots or local repository to be published. +// @Description +// @Description The changes will be applied by a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Produce json -// @Success 200 {array} sourceParams +// @Success 200 {array} []deb.PublishedRepoRevision // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository not found or staged source list does not exist" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [get] -func apiPublishSourcesList(c *gin.Context) { +func apiPublishListChanges(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) @@ -670,22 +676,23 @@ func apiPublishSourcesList(c *gin.Context) { c.JSON(http.StatusOK, revision.SourceList()) } -// @Summary Set staged source list -// @Description **Set the staged source list** +// @Summary Set Sources +// @Description **Set the sources of a published repository** // @Description -// @Description If the staged source list is known in advance, it can set via this method in a single call. All modifications done before are lost and the staged source list get replaced by the one given in the request body. -// @Tags Publish +// @Description Sets the components of snapshots or local repositories to be published. Existing Sourced will be replaced. +// @Description +// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Consume json -// @Param request body publishedRepoUpdateParams true "Parameters" +// @Param request body []sourceParams true "Parameters" // @Produce json -// @Success 200 {array} sourceParams +// @Success 200 // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository not found" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [put] -func apiPublishSourcesUpdate(c *gin.Context) { +func apiPublishSetSources(c *gin.Context) { var b []sourceParams param := slashEscape(c.Params.ByName("prefix")) @@ -733,10 +740,10 @@ func apiPublishSourcesUpdate(c *gin.Context) { }) } -// @Summary Delete staged source list -// @Description **Delete the staged source list** +// @Summary Drop Changes +// @Description **Drop pending source changes in a published repository** // @Description -// @Description Delete/Discard the staged sources and keep existing sources of published repository. +// @Description Removes all pending changes what would be applied with a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -746,7 +753,7 @@ func apiPublishSourcesUpdate(c *gin.Context) { // @Failure 404 {object} Error "Published repository not found" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [delete] -func apiPublishSourcesDelete(c *gin.Context) { +func apiPublishDropChanges(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) @@ -786,13 +793,15 @@ func apiPublishSourcesDelete(c *gin.Context) { // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Param component path string true "component name" +// @Consume json +// @Param request body sourceParams true "Parameters" // @Produce json // @Success 200 // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository/component not found" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources/{component} [put] -func apiPublishSourceUpdate(c *gin.Context) { +func apiPublishUpdateSource(c *gin.Context) { var b sourceParams param := slashEscape(c.Params.ByName("prefix")) @@ -863,7 +872,7 @@ func apiPublishSourceUpdate(c *gin.Context) { // @Failure 404 {object} Error "Published repository not found" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources/{component} [delete] -func apiPublishSourceDelete(c *gin.Context) { +func apiPublishRemoveSource(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) @@ -909,19 +918,19 @@ func apiPublishSourceDelete(c *gin.Context) { type publishedRepoUpdateParams struct { // when publishing, overwrite files in pool/ directory without notice - ForceOverwrite bool ` json:"ForceOverwrite"` + ForceOverwrite bool ` json:"ForceOverwrite" example:"false"` // GPG options Signing signingParams ` json:"Signing"` // Don't generate contents indexes - SkipContents *bool ` json:"SkipContents"` + SkipContents *bool ` json:"SkipContents" example:"false"` // Skip bz2 compression for index files - SkipBz2 *bool ` json:"SkipBz2"` + SkipBz2 *bool ` json:"SkipBz2" example:"false"` // Don't remove unreferenced files in prefix/component - SkipCleanup *bool ` json:"SkipCleanup"` + SkipCleanup *bool ` json:"SkipCleanup" example:"false"` // Provide index files by hash - AcquireByHash *bool ` json:"AcquireByHash"` + AcquireByHash *bool ` json:"AcquireByHash" example:"false"` // Enable multiple packages with the same filename in different distributions - MultiDist *bool ` json:"MultiDist"` + MultiDist *bool ` json:"MultiDist" example:"false"` } // @Summary Update content of published repository diff --git a/api/router.go b/api/router.go index 8083966b..ada14b88 100644 --- a/api/router.go +++ b/api/router.go @@ -191,12 +191,12 @@ func Router(c *ctx.AptlyContext) http.Handler { api.POST("/publish/:prefix", apiPublishRepoOrSnapshot) api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch) api.DELETE("/publish/:prefix/:distribution", apiPublishDrop) - api.POST("/publish/:prefix/:distribution/sources", apiPublishSourcesCreate) - api.GET("/publish/:prefix/:distribution/sources", apiPublishSourcesList) - api.PUT("/publish/:prefix/:distribution/sources", apiPublishSourcesUpdate) - api.DELETE("/publish/:prefix/:distribution/sources", apiPublishSourcesDelete) - api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishSourceUpdate) - api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishSourceDelete) + api.POST("/publish/:prefix/:distribution/sources", apiPublishAddSource) + api.GET("/publish/:prefix/:distribution/sources", apiPublishListChanges) + api.PUT("/publish/:prefix/:distribution/sources", apiPublishSetSources) + api.DELETE("/publish/:prefix/:distribution/sources", apiPublishDropChanges) + api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishUpdateSource) + api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishRemoveSource) api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate) } From 451de79666080f726f3b348870617e1a96c01a20 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Fri, 18 Oct 2024 14:54:44 +0200 Subject: [PATCH 17/25] Improve consistency between API and Swagger docs. Signed-off-by: Christoph Fiehe --- api/publish.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/publish.go b/api/publish.go index b7aa3a87..f5d27fb2 100644 --- a/api/publish.go +++ b/api/publish.go @@ -177,7 +177,7 @@ type publishedRepoCreateParams struct { // @Consume json // @Param request body publishedRepoCreateParams true "Parameters" // @Produce json -// @Success 200 {object} deb.PublishedRepo +// @Success 201 {object} deb.PublishedRepo // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Source not found" // @Failure 500 {object} Error "Internal Error" @@ -575,7 +575,7 @@ func apiPublishDrop(c *gin.Context) { // @Consume json // @Param request body sourceParams true "Parameters" // @Produce json -// @Success 200 +// @Success 201 // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Published repository not found" // @Failure 500 {object} Error "Internal Error" @@ -628,7 +628,7 @@ func apiPublishAddSource(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } - return &task.ProcessReturnValue{Code: http.StatusCreated, Value: published}, nil + return &task.ProcessReturnValue{Code: http.StatusCreated, Value: gin.H{}}, nil }) } From 21013a83171142c2db462deefa088a64f2f69b82 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Fri, 18 Oct 2024 21:22:12 +0200 Subject: [PATCH 18/25] Command descriptions fixed. Signed-off-by: Christoph Fiehe --- cmd/publish_source_add.go | 6 +++--- cmd/publish_source_drop.go | 2 +- cmd/publish_source_remove.go | 4 ++-- cmd/publish_source_update.go | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/publish_source_add.go b/cmd/publish_source_add.go index cf2b8570..a5633460 100644 --- a/cmd/publish_source_add.go +++ b/cmd/publish_source_add.go @@ -74,16 +74,16 @@ The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be modified. The number of given components must be equal to the number of given sources, e.g.: - aptly publish add -component=main,contrib wheezy wheezy-main wheezy-contrib + aptly publish source add -component=main,contrib wheezy wheezy-main wheezy-contrib Example: - $ aptly publish add -component=contrib wheezy ppa wheezy-contrib + $ aptly publish source add -component=contrib wheezy ppa wheezy-contrib This command assigns the snapshot wheezy-contrib to the component contrib and adds it to published repository revision of ppa/wheezy. `, - Flag: *flag.NewFlagSet("aptly-publish-add", flag.ExitOnError), + Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError), } cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") diff --git a/cmd/publish_source_drop.go b/cmd/publish_source_drop.go index d038e0e4..d954bbf3 100644 --- a/cmd/publish_source_drop.go +++ b/cmd/publish_source_drop.go @@ -53,7 +53,7 @@ Example: $ aptly publish source drop wheezy `, - Flag: *flag.NewFlagSet("aptly-publish-revision-create", flag.ExitOnError), + Flag: *flag.NewFlagSet("aptly-publish-source-drop", flag.ExitOnError), } cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") diff --git a/cmd/publish_source_remove.go b/cmd/publish_source_remove.go index 5a0df7a6..a15676bb 100644 --- a/cmd/publish_source_remove.go +++ b/cmd/publish_source_remove.go @@ -73,9 +73,9 @@ multiple components should be removed, e.g.: Example: - $ aptly publish remove -component=contrib,non-free wheezy filesystem:symlink:debian + $ aptly publish source remove -component=contrib,non-free wheezy filesystem:symlink:debian `, - Flag: *flag.NewFlagSet("aptly-publish-remove", flag.ExitOnError), + Flag: *flag.NewFlagSet("aptly-publish-source-remove", flag.ExitOnError), } cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)") diff --git a/cmd/publish_source_update.go b/cmd/publish_source_update.go index 0dae6439..71b8b5f7 100644 --- a/cmd/publish_source_update.go +++ b/cmd/publish_source_update.go @@ -74,13 +74,13 @@ The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be modified. The number of given components must be equal to the number of given sources, e.g.: - aptly publish update -component=main,contrib wheezy wheezy-main wheezy-contrib + aptly publish source update -component=main,contrib wheezy wheezy-main wheezy-contrib Example: - $ aptly publish update -component=contrib wheezy ppa wheezy-contrib + $ aptly publish source update -component=contrib wheezy ppa wheezy-contrib `, - Flag: *flag.NewFlagSet("aptly-publish-revision-source-update", flag.ExitOnError), + Flag: *flag.NewFlagSet("aptly-publish-source-update", flag.ExitOnError), } cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") From bd01cd40335250e1d0fcd3542b298f172dcecad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sat, 19 Oct 2024 00:42:34 +0200 Subject: [PATCH 19/25] update swagger documentation --- api/publish.go | 77 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/api/publish.go b/api/publish.go index f5d27fb2..9b20ebf6 100644 --- a/api/publish.go +++ b/api/publish.go @@ -66,10 +66,12 @@ func slashEscape(path string) string { return result } -// @Summary List published repositories +// @Summary List Published Repositories // @Description **Get list of published repositories** // @Description // @Description Lists repositories that have been published based on local repositories or snapshots. For each repository information about `endpoint`, `prefix` and `distribution` is listed along with `component` and architecture list. Information about snapshot or local repo being published is appended to published repository description. +// @Description +// @Description See also: `aptly publish list` // @Tags Publish // @Produce json // @Success 200 {array} deb.PublishedRepo @@ -100,10 +102,12 @@ func apiPublishList(c *gin.Context) { c.JSON(http.StatusOK, repos) } -// @Summary Show published repository +// @Summary Show Published Repository // @Description **Get published repository information** // @Description -// @Description Show detailed information of published repository. +// @Description Show detailed information of a published repository. +// @Description +// @Description See also: `aptly publish show` // @Tags Publish // @Produce json // @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs" @@ -168,10 +172,12 @@ type publishedRepoCreateParams struct { MultiDist *bool ` json:"MultiDist" example:"false"` } -// @Summary Create published repository -// @Description **Publish local repository or snapshot under specified prefix** +// @Summary Create Published Repository +// @Description **Publish a local repository or snapshot** // @Description // @Description Storage might be passed in prefix as well, e.g. `s3:packages/`. To supply empty prefix, just remove last part (`POST /api/publish`). +// @Description +// @Description See also: `aptly publish create` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Consume json @@ -366,12 +372,14 @@ type publishedRepoUpdateSwitchParams struct { MultiDist *bool ` json:"MultiDist" example:"false"` } -// @Summary Update published repository -// @Description **Update a published local repository or switch published snapshot** +// @Summary Update Published Repository +// @Description **Update a published local repository or switch snapshot** // @Description // @Description API action depends on published repository contents: // @Description * if local repository has been published, published repository would be updated to match local repository contents // @Description * if snapshots have been been published, it is possible to switch each component to new snapshot +// @Description +// @Description See also: `aptly publish update` / `aptly publish switch` // @Tags Publish // @Produce json // @Param prefix path string true "publishing prefix" @@ -518,10 +526,12 @@ func apiPublishUpdateSwitch(c *gin.Context) { }) } -// @Summary Delete published repository +// @Summary Delete Published Repository // @Description **Delete a published repository** // @Description // @Description Delete published repository and clean up files in published directory. Aptly tries to remove as many files belonging to this repository as possible. For example, if no other published repositories share the same prefix, all files inside the prefix will be removed. +// @Description +// @Description See also: `aptly publish drop` // @Tags Publish // @Produce json // @Param prefix path string true "publishing prefix" @@ -563,12 +573,14 @@ func apiPublishDrop(c *gin.Context) { }) } -// @Summary Add Source -// @Description **Add a source to published repo** +// @Summary Add Source Component +// @Description **Add a source component to a published repo** // @Description // @Description Adds a component of a snapshot or local repository to be published. // @Description // @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description +// @Description See also: `aptly publish source add` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -633,18 +645,20 @@ func apiPublishAddSource(c *gin.Context) { } // @Summary List pending changes -// @Description **List changes to be applied** +// @Description **List source component changes to be applied** // @Description // @Description Returns added, removed or changed components of snapshots or local repository to be published. // @Description // @Description The changes will be applied by a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description +// @Description See also: `aptly publish source list` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Produce json -// @Success 200 {array} []deb.PublishedRepoRevision +// @Success 200 {array} []deb.SourceEntry // @Failure 400 {object} Error "Bad Request" -// @Failure 404 {object} Error "Published repository not found or staged source list does not exist" +// @Failure 404 {object} Error "Published repository pending changes not found" // @Failure 500 {object} Error "Internal Error" // @Router /api/publish/{prefix}/{distribution}/sources [get] func apiPublishListChanges(c *gin.Context) { @@ -676,12 +690,15 @@ func apiPublishListChanges(c *gin.Context) { c.JSON(http.StatusOK, revision.SourceList()) } -// @Summary Set Sources -// @Description **Set the sources of a published repository** +// @Summary Replace Source Components +// @Description **Replace the source components of a published repository** // @Description // @Description Sets the components of snapshots or local repositories to be published. Existing Sourced will be replaced. // @Description // @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description +// @Description See also: `aptly publish source replace` +// @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Consume json @@ -740,10 +757,12 @@ func apiPublishSetSources(c *gin.Context) { }) } -// @Summary Drop Changes -// @Description **Drop pending source changes in a published repository** +// @Summary Drop pending changes +// @Description **Drop pending source component changes of a published repository** // @Description // @Description Removes all pending changes what would be applied with a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description +// @Description See also: `aptly publish source drop` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -787,8 +806,12 @@ func apiPublishDropChanges(c *gin.Context) { }) } -// @Summary Update staged source -// @Description **Update the staged source of a component** +// @Summary Update Source Component +// @Description **Update the source component of a published repository** +// @Description +// @Description Publish pending source component changes which were added with `Add/Remove/Replace Source Components` +// @Description +// @Description See also: `aptly publish update` / `aptly publish switch` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -860,8 +883,12 @@ func apiPublishUpdateSource(c *gin.Context) { }) } -// @Summary Delete source from staged source list -// @Description **Delete a single source from the staged source list** +// @Summary Remove Source Component +// @Description **Remove a source component from a published repo** +// @Description +// @Description Remove a source (snapshot / local repo) component from a published repository. +// @Description +// @Description See also: `aptly publish source remove` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -933,10 +960,12 @@ type publishedRepoUpdateParams struct { MultiDist *bool ` json:"MultiDist" example:"false"` } -// @Summary Update content of published repository -// @Description **Update the content of a published repository** +// @Summary Update Published Repository +// @Description **Update a published repository** // @Description -// @Description Replace the current source list of the published repository by the staged one (if available) and (re-)publish the new content. +// @Description Apply pending changes and republish +// @Description +// @Description See also: `aptly publish update` / `aptly publish switch` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" From eb9421105314fd3b8e1a0b25bbdf48214b51f3df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 22 Oct 2024 15:33:14 +0200 Subject: [PATCH 20/25] fix race conditions --- api/publish.go | 49 +++++++++++-------------------------------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/api/publish.go b/api/publish.go index 9b20ebf6..1dadf36b 100644 --- a/api/publish.go +++ b/api/publish.go @@ -273,9 +273,8 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { collection := collectionFactory.PublishedRepoCollection() - resources = append(resources, string(published.Key())) taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"", - b.SourceKind, published.StoragePrefix(), published.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`)) + b.SourceKind, param, b.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`)) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { taskDetail := task.PublishDetail{ Detail: detail, @@ -419,27 +418,21 @@ func apiPublishUpdateSwitch(c *gin.Context) { return } - var updatedComponents []string - var updatedSnapshots []string - if published.SourceKind == deb.SourceLocalRepo { if len(b.Snapshots) > 0 { AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo")) return } - updatedComponents = published.Components() - } else if published.SourceKind == "snapshot" { + } else if published.SourceKind == deb.SourceSnapshot { for _, snapshotInfo := range b.Snapshots { - snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name) + _, err2 := snapshotCollection.ByName(snapshotInfo.Name) if err2 != nil { AbortWithJSONError(c, http.StatusNotFound, err2) return } - updatedComponents = append(updatedComponents, snapshotInfo.Component) - updatedSnapshots = append(updatedSnapshots, snapshot.Name) } } else { - AbortWithJSONError(c, 500, fmt.Errorf("unknown published repository type")) + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type")) return } @@ -459,17 +452,6 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.MultiDist = *b.MultiDist } - revision := published.ObtainRevision() - sources := revision.Sources - - if published.SourceKind == deb.SourceSnapshot { - for _, snapshotInfo := range b.Snapshots { - component := snapshotInfo.Component - name := snapshotInfo.Name - sources[component] = name - } - } - resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { @@ -478,23 +460,14 @@ func apiPublishUpdateSwitch(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err) } - if published.SourceKind == deb.SourceLocalRepo { - for _, component := range updatedComponents { - published.UpdateLocalRepo(component) - } - } else if published.SourceKind == "snapshot" { + revision := published.ObtainRevision() + sources := revision.Sources + + if published.SourceKind == deb.SourceSnapshot { for _, snapshotInfo := range b.Snapshots { - snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name) - if err2 != nil { - return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err2 - } - - err2 = snapshotCollection.LoadComplete(snapshot) - if err2 != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err2 - } - - published.UpdateSnapshot(snapshotInfo.Component, snapshot) + component := snapshotInfo.Component + name := snapshotInfo.Name + sources[component] = name } } From ee3124cfc65b9404612a4beae0e75ad9637b1b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 22 Oct 2024 16:54:49 +0200 Subject: [PATCH 21/25] update bash completion --- completion.d/aptly | 28 +++++++++++++++++++++++++++- system/docker-wrapper | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/completion.d/aptly b/completion.d/aptly index 8caf47f8..b7966bc2 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -54,12 +54,14 @@ _aptly() { cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" + prevprev="${COMP_WORDS[COMP_CWORD-2]}" commands="api config db graph mirror package publish repo serve snapshot task version" options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider=" db_subcommands="cleanup recover" mirror_subcommands="create drop edit show list rename search update" - publish_subcommands="drop list repo snapshot switch update" + publish_subcommands="drop list repo snapshot switch update source" + publish_source_subcommands="drop list add remove update" snapshot_subcommands="create diff drop filter list merge pull rename search show verify" repo_subcommands="add copy create drop edit import include list move remove rename search show" package_subcommands="search show" @@ -148,6 +150,17 @@ _aptly() esac fi + case "$prevprev" in + "publish") + case "$prev" in + "source") + COMPREPLY=($(compgen -W "${publish_source_subcommands}" -- ${cur})) + return 0 + ;; + esac + ;; + esac + case "$cmd" in "mirror") case "$subcmd" in @@ -575,6 +588,19 @@ _aptly() ;; esac ;; + "source") + case "$subcmd" in + "add") + return 0 + ;; + "list") + if [[ $numargs -eq 0 ]]; then + COMPREPLY=($(compgen -W "-raw" -- ${cur})) + return 0 + fi + ;; + esac + ;; "package") case "$subcmd" in "search") diff --git a/system/docker-wrapper b/system/docker-wrapper index 3bee17e7..0375ddaa 100755 --- a/system/docker-wrapper +++ b/system/docker-wrapper @@ -6,6 +6,7 @@ chown -R `stat -c %u /work/src` /var/lib/aptly args="$@" if [ -z "$args" ]; then + cp /work/src/completion.d/aptly /usr/share/bash-completion/completions/ cmd="bash" else cmd="make $@" From c9309c926cd2562d813c21e741b624239ded0652 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Wed, 23 Oct 2024 15:26:31 +0200 Subject: [PATCH 22/25] Command to replace the whole staged source list added. Signed-off-by: Christoph Fiehe --- cmd/publish.go | 1 + cmd/publish_source_add.go | 2 +- cmd/publish_source_replace.go | 88 +++++++++++++++++++ completion.d/aptly | 2 +- .../PublishSourceReplace1Test_gold | 5 ++ system/t06_publish/source.py | 16 ++++ 6 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 cmd/publish_source_replace.go create mode 100644 system/t06_publish/PublishSourceReplace1Test_gold diff --git a/cmd/publish.go b/cmd/publish.go index 887929d6..4217ff87 100644 --- a/cmd/publish.go +++ b/cmd/publish.go @@ -52,6 +52,7 @@ func makeCmdPublishSource() *commander.Command { makeCmdPublishSourceDrop(), makeCmdPublishSourceList(), makeCmdPublishSourceRemove(), + makeCmdPublishSourceReplace(), makeCmdPublishSourceUpdate(), }, } diff --git a/cmd/publish_source_add.go b/cmd/publish_source_add.go index a5633460..4c1e9989 100644 --- a/cmd/publish_source_add.go +++ b/cmd/publish_source_add.go @@ -48,7 +48,7 @@ func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error { } context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind) - sources[component] = names[i] + sources[component] = name } err = collectionFactory.PublishedRepoCollection().Update(published) diff --git a/cmd/publish_source_replace.go b/cmd/publish_source_replace.go new file mode 100644 index 00000000..10d5b588 --- /dev/null +++ b/cmd/publish_source_replace.go @@ -0,0 +1,88 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishSourceReplace(cmd *commander.Command, args []string) error { + if len(args) < 2 { + cmd.Usage() + return commander.ErrCommandError + } + + distribution := args[0] + names := args[1:] + components := strings.Split(context.Flags().Lookup("component").Value.String(), ",") + + if len(names) != len(components) { + return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names)) + } + + prefix := context.Flags().Lookup("prefix").Value.String() + storage, prefix := deb.ParsePrefix(prefix) + + collectionFactory := context.NewCollectionFactory() + published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to add: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) + if err != nil { + return fmt.Errorf("unable to add: %s", err) + } + + revision := published.ObtainRevision() + sources := revision.Sources + context.Progress().Printf("Clearing staged source list...\n") + clear(sources) + + for i, component := range components { + name := names[i] + context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind) + + sources[component] = name + } + + err = collectionFactory.PublishedRepoCollection().Update(published) + if err != nil { + return fmt.Errorf("unable to save to DB: %s", err) + } + + context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n", + distribution, published.StoragePrefix()) + + return err +} + +func makeCmdPublishSourceReplace() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishSourceReplace, + UsageLine: "replace ", + Short: "replace staged source list of published repository", + Long: ` +The command replaces the staged source list of the published repository. + +The flag -component is mandatory. Use a comma-separated list of components, if +multiple components should be modified. The number of given components must be +equal to the number of given sources, e.g.: + + aptly publish source replace -component=main,contrib wheezy wheezy-main wheezy-contrib + +Example: + + $ aptly publish source replace -component=contrib wheezy ppa wheezy-contrib + +`, + Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError), + } + cmd.Flag.String("prefix", ".", "publishing prefix in the form of [:]") + cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)") + + return cmd +} diff --git a/completion.d/aptly b/completion.d/aptly index b7966bc2..3cb54ac5 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -61,7 +61,7 @@ _aptly() db_subcommands="cleanup recover" mirror_subcommands="create drop edit show list rename search update" publish_subcommands="drop list repo snapshot switch update source" - publish_source_subcommands="drop list add remove update" + publish_source_subcommands="drop list add remove update replace" snapshot_subcommands="create diff drop filter list merge pull rename search show verify" repo_subcommands="add copy create drop edit import include list move remove rename search show" package_subcommands="search show" diff --git a/system/t06_publish/PublishSourceReplace1Test_gold b/system/t06_publish/PublishSourceReplace1Test_gold new file mode 100644 index 00000000..61d88254 --- /dev/null +++ b/system/t06_publish/PublishSourceReplace1Test_gold @@ -0,0 +1,5 @@ +Clearing staged source list... +Adding component 'main-new' with source 'snap2' [snapshot]... +Adding component 'test-new' with source 'snap3' [snapshot]... + +You can run 'aptly publish update maverick .' to update the content of the published repository. diff --git a/system/t06_publish/source.py b/system/t06_publish/source.py index 9f352e20..78c81c60 100644 --- a/system/t06_publish/source.py +++ b/system/t06_publish/source.py @@ -156,6 +156,22 @@ class PublishSourceUpdate3Test(BaseTest): gold_processor = BaseTest.expand_environ +class PublishSourceReplace1Test(BaseTest): + """ + publish source replace: Replace existing sources + """ + fixtureDB = True + fixturePool = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror gnuplot-maverick", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=main,test snap1 snap2", + ] + runCmd = "aptly publish source replace -component=main-new,test-new maverick snap2 snap3" + gold_processor = BaseTest.expand_environ + + class PublishSourceRemove1Test(BaseTest): """ publish source remove: Remove single source From f79423a4eeabcbe9cd9772807b4f1dea0255d333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Thu, 31 Oct 2024 13:57:37 +0100 Subject: [PATCH 23/25] update swagger documentation --- api/publish.go | 48 ++++++++++++++++++++++------------- cmd/publish_source_add.go | 6 +++-- cmd/publish_source_drop.go | 4 +-- cmd/publish_source_remove.go | 6 +++-- cmd/publish_source_replace.go | 7 ++--- cmd/publish_source_update.go | 6 +++-- cmd/publish_update.go | 20 ++++++++++----- 7 files changed, 61 insertions(+), 36 deletions(-) diff --git a/api/publish.go b/api/publish.go index 1dadf36b..b2f4ffad 100644 --- a/api/publish.go +++ b/api/publish.go @@ -69,7 +69,7 @@ func slashEscape(path string) string { // @Summary List Published Repositories // @Description **Get list of published repositories** // @Description -// @Description Lists repositories that have been published based on local repositories or snapshots. For each repository information about `endpoint`, `prefix` and `distribution` is listed along with `component` and architecture list. Information about snapshot or local repo being published is appended to published repository description. +// @Description Return list of published repositories including detailed information. // @Description // @Description See also: `aptly publish list` // @Tags Publish @@ -175,7 +175,9 @@ type publishedRepoCreateParams struct { // @Summary Create Published Repository // @Description **Publish a local repository or snapshot** // @Description -// @Description Storage might be passed in prefix as well, e.g. `s3:packages/`. To supply empty prefix, just remove last part (`POST /api/publish`). +// @Description Create a published repository. +// @Description +// @Description The prefix may contain a storage specifier, e.g. `s3:packages/`, or it may also be empty to publish to the root directory. // @Description // @Description See also: `aptly publish create` // @Tags Publish @@ -372,11 +374,15 @@ type publishedRepoUpdateSwitchParams struct { } // @Summary Update Published Repository -// @Description **Update a published local repository or switch snapshot** +// @Description **Update a published repository** // @Description -// @Description API action depends on published repository contents: -// @Description * if local repository has been published, published repository would be updated to match local repository contents -// @Description * if snapshots have been been published, it is possible to switch each component to new snapshot +// @Description Update a published local repository or switch snapshot. +// @Description +// @Description For published local repositories: +// @Description * update to match local repository contents +// @Description +// @Description For published snapshots: +// @Description * switch components to new snapshot // @Description // @Description See also: `aptly publish update` / `aptly publish switch` // @Tags Publish @@ -502,7 +508,9 @@ func apiPublishUpdateSwitch(c *gin.Context) { // @Summary Delete Published Repository // @Description **Delete a published repository** // @Description -// @Description Delete published repository and clean up files in published directory. Aptly tries to remove as many files belonging to this repository as possible. For example, if no other published repositories share the same prefix, all files inside the prefix will be removed. +// @Description Delete a distribution of a published repository and remove associated files. +// @Description +// @Description If no other published repositories share the same prefix, all files inside the prefix will be removed. // @Description // @Description See also: `aptly publish drop` // @Tags Publish @@ -549,9 +557,9 @@ func apiPublishDrop(c *gin.Context) { // @Summary Add Source Component // @Description **Add a source component to a published repo** // @Description -// @Description Adds a component of a snapshot or local repository to be published. +// @Description Add a component of a snapshot or local repository to be published. // @Description -// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (i.e `PUT /api/publish/{prefix}/{distribution}` / `POST /api/publish/{prefix}/{distribution}/update`). // @Description // @Description See also: `aptly publish source add` // @Tags Publish @@ -620,9 +628,9 @@ func apiPublishAddSource(c *gin.Context) { // @Summary List pending changes // @Description **List source component changes to be applied** // @Description -// @Description Returns added, removed or changed components of snapshots or local repository to be published. +// @Description Return added, removed or changed components of snapshots or local repository to be published. // @Description -// @Description The changes will be applied by a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description The changes will be applied by a subsequent publish update call (i.e. `PUT /api/publish/{prefix}/{distribution}` / `POST /api/publish/{prefix}/{distribution}/update`). // @Description // @Description See also: `aptly publish source list` // @Tags Publish @@ -668,7 +676,7 @@ func apiPublishListChanges(c *gin.Context) { // @Description // @Description Sets the components of snapshots or local repositories to be published. Existing Sourced will be replaced. // @Description -// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (i.e `PUT /api/publish/{prefix}/{distribution}` / `POST /api/publish/{prefix}/{distribution}/update`). // @Description // @Description See also: `aptly publish source replace` // @Tags Publish @@ -733,7 +741,7 @@ func apiPublishSetSources(c *gin.Context) { // @Summary Drop pending changes // @Description **Drop pending source component changes of a published repository** // @Description -// @Description Removes all pending changes what would be applied with a subsequent publish update call (See `PUT /api/publish/{prefix}/{distribution}`). +// @Description Remove all pending changes what would be applied with a subsequent publish update call (i.e. `PUT /api/publish/{prefix}/{distribution}` / `POST /api/publish/{prefix}/{distribution}/update`). // @Description // @Description See also: `aptly publish source drop` // @Tags Publish @@ -782,9 +790,11 @@ func apiPublishDropChanges(c *gin.Context) { // @Summary Update Source Component // @Description **Update the source component of a published repository** // @Description -// @Description Publish pending source component changes which were added with `Add/Remove/Replace Source Components` +// @Description Update a component of a snapshot or local repository to be published. // @Description -// @Description See also: `aptly publish update` / `aptly publish switch` +// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (i.e `PUT /api/publish/{prefix}/{distribution}` / `POST /api/publish/{prefix}/{distribution}/update`). +// @Description +// @Description See also: `aptly publish source update` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" @@ -859,7 +869,9 @@ func apiPublishUpdateSource(c *gin.Context) { // @Summary Remove Source Component // @Description **Remove a source component from a published repo** // @Description -// @Description Remove a source (snapshot / local repo) component from a published repository. +// @Description Remove a source component (snapshot / local repo) from a published repository. +// @Description +// @Description This call does not publish the changes, but rather schedules them for a subsequent publish update call (i.e `PUT /api/publish/{prefix}/{distribution}` / `POST /api/publish/{prefix}/{distribution}/update`). // @Description // @Description See also: `aptly publish source remove` // @Tags Publish @@ -936,9 +948,9 @@ type publishedRepoUpdateParams struct { // @Summary Update Published Repository // @Description **Update a published repository** // @Description -// @Description Apply pending changes and republish +// @Description Publish pending source component changes which were added with `Add/Remove/Replace Source Components` // @Description -// @Description See also: `aptly publish update` / `aptly publish switch` +// @Description See also: `aptly publish update` // @Tags Publish // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" diff --git a/cmd/publish_source_add.go b/cmd/publish_source_add.go index 4c1e9989..b052c668 100644 --- a/cmd/publish_source_add.go +++ b/cmd/publish_source_add.go @@ -66,9 +66,11 @@ func makeCmdPublishSourceAdd() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceAdd, UsageLine: "add ", - Short: "add source to staged source list of published repository", + Short: "add source components to a published repo", Long: ` -The command adds sources to the staged source list of the published repository. +The command adds components of a snapshot or local repository to be published. + +This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'. The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be modified. The number of given components must be diff --git a/cmd/publish_source_drop.go b/cmd/publish_source_drop.go index d954bbf3..704e36e7 100644 --- a/cmd/publish_source_drop.go +++ b/cmd/publish_source_drop.go @@ -45,9 +45,9 @@ func makeCmdPublishSourceDrop() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceDrop, UsageLine: "drop ", - Short: "drops staged source changes of published repository", + Short: "drop pending source component changes of a published repository", Long: ` -Command drops the staged source changes of the published repository. +Remove all pending changes what would be applied with a subsequent 'aptly publish update'. Example: diff --git a/cmd/publish_source_remove.go b/cmd/publish_source_remove.go index a15676bb..d291bcf0 100644 --- a/cmd/publish_source_remove.go +++ b/cmd/publish_source_remove.go @@ -64,9 +64,11 @@ func makeCmdPublishSourceRemove() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceRemove, UsageLine: "remove [[:]] ", - Short: "remove source from staged source list of published repository", + Short: "remove source components from a published repo", Long: ` -The command removes sources from the staged source list of the published repository. +The command removes source components (snapshot / local repo) from a published repository. + +This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'. The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be removed, e.g.: diff --git a/cmd/publish_source_replace.go b/cmd/publish_source_replace.go index 10d5b588..c4031b2e 100644 --- a/cmd/publish_source_replace.go +++ b/cmd/publish_source_replace.go @@ -64,9 +64,11 @@ func makeCmdPublishSourceReplace() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceReplace, UsageLine: "replace ", - Short: "replace staged source list of published repository", + Short: "replace the source components of a published repository", Long: ` -The command replaces the staged source list of the published repository. +The command replaces the source components of a snapshot or local repository to be published. + +This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'. The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be modified. The number of given components must be @@ -77,7 +79,6 @@ equal to the number of given sources, e.g.: Example: $ aptly publish source replace -component=contrib wheezy ppa wheezy-contrib - `, Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError), } diff --git a/cmd/publish_source_update.go b/cmd/publish_source_update.go index 71b8b5f7..af207fb3 100644 --- a/cmd/publish_source_update.go +++ b/cmd/publish_source_update.go @@ -66,9 +66,11 @@ func makeCmdPublishSourceUpdate() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishSourceUpdate, UsageLine: "update ", - Short: "update source in staged source list of published repository", + Short: "update the source components of a published repository", Long: ` -The command updates sources in the staged source list of the published repository. +The command updates the source components of a snapshot or local repository to be published. + +This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'. The flag -component is mandatory. Use a comma-separated list of components, if multiple components should be modified. The number of given components must be diff --git a/cmd/publish_update.go b/cmd/publish_update.go index 589e35bf..4b38282f 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -93,15 +93,21 @@ func makeCmdPublishUpdate() *commander.Command { cmd := &commander.Command{ Run: aptlyPublishUpdate, UsageLine: "update [[:]]", - Short: "update published local repository", + Short: "update published repository", Long: ` -Command re-publishes (updates) published local repository. -and should be occupied with local repository published -using command aptly publish repo. Update happens in-place with -minimum possible downtime for published repository. +The command updates updates a published repository after applying pending changes to the sources. -For multiple component published repositories, all local repositories -are updated. +For published local repositories: + + * update to match local repository contents + +For published snapshots: + + * switch components to new snapshot + +The update happens in-place with minimum possible downtime for published repository. + +For multiple component published repositories, all local repositories are updated. Example: From 0ceff44421c0eb9c76e772ac5968607cc9b216ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 1 Nov 2024 20:01:45 +0100 Subject: [PATCH 24/25] improve log --- cmd/publish_source_replace.go | 2 +- system/t06_publish/PublishSourceReplace1Test_gold | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/publish_source_replace.go b/cmd/publish_source_replace.go index c4031b2e..17801e5a 100644 --- a/cmd/publish_source_replace.go +++ b/cmd/publish_source_replace.go @@ -39,7 +39,7 @@ func aptlyPublishSourceReplace(cmd *commander.Command, args []string) error { revision := published.ObtainRevision() sources := revision.Sources - context.Progress().Printf("Clearing staged source list...\n") + context.Progress().Printf("Replacing source list...\n") clear(sources) for i, component := range components { diff --git a/system/t06_publish/PublishSourceReplace1Test_gold b/system/t06_publish/PublishSourceReplace1Test_gold index 61d88254..c3b03ff9 100644 --- a/system/t06_publish/PublishSourceReplace1Test_gold +++ b/system/t06_publish/PublishSourceReplace1Test_gold @@ -1,4 +1,4 @@ -Clearing staged source list... +Replacing source list... Adding component 'main-new' with source 'snap2' [snapshot]... Adding component 'test-new' with source 'snap3' [snapshot]... From a4c53689caf033149980c090217ed304a594d6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 1 Nov 2024 20:17:28 +0100 Subject: [PATCH 25/25] docker-wrapper: ignore root user some systems (MacOS) might have root permissions on the volume directories. --- system/docker-wrapper | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/system/docker-wrapper b/system/docker-wrapper index 0375ddaa..1d92f392 100755 --- a/system/docker-wrapper +++ b/system/docker-wrapper @@ -1,8 +1,10 @@ #!/bin/sh -e # make sure files are written with correct user ownership -usermod -u `stat -c %u /work/src` aptly >/dev/null -chown -R `stat -c %u /work/src` /var/lib/aptly +if [ `stat -c %u /work/src` -ne 0 ]; then + usermod -u `stat -c %u /work/src` aptly >/dev/null + chown -R `stat -c %u /work/src` /var/lib/aptly +fi args="$@" if [ -z "$args" ]; then