Allow management of components

This commit allows to add, remove and update components of published repositories without the need to recreate them.

Signed-off-by: Christoph Fiehe <c.fiehe@eurodata.de>

cmd/publish.go

@@ -34,10 +34,25 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishDrop(),
 			makeCmdPublishList(),
 			makeCmdPublishRepo(),
+			makeCmdPublishShow(),
 			makeCmdPublishSnapshot(),
+			makeCmdPublishSource(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
-			makeCmdPublishShow(),
+		},
+	}
+}
+
+func makeCmdPublishSource() *commander.Command {
+	return &commander.Command{
+		UsageLine: "source",
+		Short:     "manage sources of published repository",
+		Subcommands: []*commander.Command{
+			makeCmdPublishSourceAdd(),
+			makeCmdPublishSourceDrop(),
+			makeCmdPublishSourceList(),
+			makeCmdPublishSourceRemove(),
+			makeCmdPublishSourceUpdate(),
 		},
 	}
 }

cmd/publish_show.go

@@ -52,7 +52,8 @@ func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
 
 	fmt.Printf("Sources:\n")
-	for component, sourceID := range repo.Sources {
+	for _, component := range repo.Components() {
+		sourceID := repo.Sources[component]
 		var name string
 		if repo.SourceKind == deb.SourceSnapshot {
 			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)

cmd/publish_snapshot.go

@@ -150,6 +150,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
 	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
 		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)

cmd/publish_source_add.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if exists {
+			return fmt.Errorf("unable to add: Component %q has already been added", component)
+		}
+		context.Progress().Printf("Adding component %q with source %q [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = names[i]
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPublishSourceAdd() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceAdd,
+		UsageLine: "add <distribution> <source>",
+		Short:     "add package source to published repository",
+		Long: `
+The command adds (in place) one or multiple package sources to a published repository.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish add -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish add -component=contrib wheezy ppa wheezy-contrib
+
+This command assigns the snapshot wheezy-contrib to the component contrib and
+adds it to published repository revision of ppa/wheezy.
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_drop.go

@@ -0,0 +1,62 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceDrop(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	published.DropRevision()
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("Source changes have been removed successfully.\n")
+
+	return err
+}
+
+func makeCmdPublishSourceDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceDrop,
+		UsageLine: "drop <distribution>",
+		Short:     "drops revision of published repository",
+		Long: `
+Command drops revision of a published repository.
+
+Example:
+
+    $ aptly publish revision drop wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-revision-create", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_list.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceList(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	published, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(published, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	if published.Revision == nil {
+		return fmt.Errorf("unable to list: No source changes exist")
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishSourceListJSON(published)
+	}
+
+	return aptlyPublishSourceListTxt(published)
+}
+
+func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	fmt.Printf("Sources:\n")
+	for _, component := range revision.Components() {
+		name := revision.Sources[component]
+		fmt.Printf("  %s: %s [%s]\n", component, name, published.SourceKind)
+	}
+
+	return nil
+}
+
+func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	output, err := json.MarshalIndent(revision.ToJSON()["Sources"], "", "  ")
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	fmt.Println(string(output))
+
+	return nil
+}
+
+func makeCmdPublishSourceList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceList,
+		UsageLine: "list <distribution>",
+		Short:     "lists revision of published repository",
+		Long: `
+Command lists sources of a published repository.
+
+Example:
+
+    $ aptly publish source list wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-list", flag.ExitOnError),
+	}
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_remove.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error {
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(components) == 0 {
+		return fmt.Errorf("unable to remove: Missing components, specify at least one component")
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for _, component := range components {
+		name, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to remove: Component %q is not part of revision", component)
+		}
+		context.Progress().Printf("Removing component %q with source %q [%s]...\n", component, name, published.SourceKind)
+
+		delete(sources, component)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPublishSourceRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceRemove,
+		UsageLine: "remove <distribution> [[<endpoint>:]<prefix>] <source>",
+		Short:     "remove package source to published repository",
+		Long: `
+The command removes one or multiple components from a published repository.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be removed, e.g.:
+
+Example:
+
+	$ aptly publish remove -component=contrib,non-free wheezy filesystem:symlink:debian
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-remove", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_update.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to update: Component %q does not exist", component)
+		}
+		context.Progress().Printf("Updating component %q with source %q [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPublishSourceUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceUpdate,
+		UsageLine: "update <distribution> <source>",
+		Short:     "update package source to published repository",
+		Long: `
+The command updates one or multiple components in a published repository.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish update -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish update -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-revision-source-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_switch.go

@@ -5,12 +5,16 @@ import (
 	"strings"
 
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
-	var err error
+	var (
+		err   error
+		names []string
+	)
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
@@ -22,11 +26,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	distribution := args[0]
 	param := "."
 
-	var (
-		names    []string
-		snapshot *deb.Snapshot
-	)
-
 	if len(args) == len(components)+2 {
 		param = args[1]
 		names = args[2:]
@@ -41,16 +40,12 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	collectionFactory := context.NewCollectionFactory()
 	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
-	}
-
-	if published.SourceKind != deb.SourceSnapshot {
-		return fmt.Errorf("unable to update: not a snapshot publish")
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	publishedComponents := published.Components()
@@ -62,18 +57,46 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
 	}
 
-	for i, component := range components {
-		snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
-		if err != nil {
-			return fmt.Errorf("unable to switch: %s", err)
-		}
+	if published.SourceKind == deb.SourceLocalRepo {
+		localRepoCollection := collectionFactory.LocalRepoCollection()
+		for i, component := range components {
+			if !utils.StrSliceHasItem(publishedComponents, component) {
+				return fmt.Errorf("unable to switch: component %s does not exist in published repository", component)
+			}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return fmt.Errorf("unable to switch: %s", err)
-		}
+			localRepo, err := localRepoCollection.ByName(names[i])
+			if err != nil {
+				return fmt.Errorf("unable to switch: %s", err)
+			}
 
-		published.UpdateSnapshot(component, snapshot)
+			err = localRepoCollection.LoadComplete(localRepo)
+			if err != nil {
+				return fmt.Errorf("unable to switch: %s", err)
+			}
+
+			published.UpdateLocalRepo(component, localRepo)
+		}
+	} else if published.SourceKind == deb.SourceSnapshot {
+		snapshotCollection := collectionFactory.SnapshotCollection()
+		for i, component := range components {
+			if !utils.StrSliceHasItem(publishedComponents, component) {
+				return fmt.Errorf("unable to switch: component %s does not exist in published repository", component)
+			}
+
+			snapshot, err := snapshotCollection.ByName(names[i])
+			if err != nil {
+				return fmt.Errorf("unable to switch: %s", err)
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				return fmt.Errorf("unable to switch: %s", err)
+			}
+
+			published.UpdateSnapshot(component, snapshot)
+		}
+	} else {
+		return fmt.Errorf("unknown published repository type")
 	}
 
 	signer, err := getSigner(context.Flags())
@@ -114,11 +137,11 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
 			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
 		if err != nil {
-			return fmt.Errorf("unable to update: %s", err)
+			return fmt.Errorf("unable to switch: %s", err)
 		}
 	}
 
-	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been successfully switched to new source.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -126,15 +149,15 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
-		Short:     "update published repository by switching to new snapshot",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-source>",
+		Short:     "update published repository by switching to new source",
 		Long: `
-Command switches in-place published snapshots with new snapshot contents. All
+Command switches in-place published snapshots with new source contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
 For multiple component repositories, flag -component should be given with
-list of components to update. Corresponding snapshots should be given in the
+list of components to update. Corresponding sources should be given in the
 same order, e.g.:
 
 	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib

cmd/publish_update.go

@@ -31,18 +31,14 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != deb.SourceLocalRepo {
-		return fmt.Errorf("unable to update: not a local repository publish")
-	}
-
 	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	components := published.Components()
-	for _, component := range components {
-		published.UpdateLocalRepo(component)
+	result, err := published.Update(collectionFactory, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	signer, err := getSigner(context.Flags())
@@ -80,14 +76,14 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, result.UpdatedComponents(),
 			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
 	}
 
-	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been successfully updated.\n", published.SourceKind, published.String())
 
 	return err
 }