Integrate PublishedRepos with storages & context. #15

cmd/context.go

@@ -8,6 +8,7 @@ import (
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -28,7 +29,7 @@ type AptlyContext struct {
 	downloader        aptly.Downloader
 	database          database.Storage
 	packagePool       aptly.PackagePool
-	publishedStorage  aptly.PublishedStorage
+	publishedStorages map[string]aptly.PublishedStorage
 	collectionFactory *deb.CollectionFactory
 	dependencyOptions int
 	architecturesList []string
@@ -40,6 +41,9 @@ type AptlyContext struct {
 
 var context *AptlyContext
 
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
 // FatalError is type for panicking to abort execution with non-zero
 // exit code and print meaningful explanation
 type FatalError struct {
@@ -200,12 +204,30 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 }
 
 // PublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) PublishedStorage() aptly.PublishedStorage {
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
-	if context.publishedStorage == nil {
+	publishedStorage, ok := context.publishedStorages[name]
-		context.publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.Config().S3PublishRoots[name[4:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[4:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
+				params.Region, params.Bucket, params.ACL, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
 	}
 
-	return context.publishedStorage
+	return publishedStorage
 }
 
 // ShutdownContext shuts context down
@@ -241,7 +263,11 @@ func ShutdownContext() {
 func InitContext(flags *flag.FlagSet) error {
 	var err error
 
-	context = &AptlyContext{flags: flags, dependencyOptions: -1}
+	context = &AptlyContext{
+		flags:             flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
 
 	if aptly.EnableDebug {
 		cpuprofile := flags.Lookup("cpuprofile").Value.String()

cmd/publish.go

@@ -4,6 +4,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"strings"
 )
 
 func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
@@ -24,6 +25,17 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 
 }
 
+func parsePrefix(param string) (storage, prefix string) {
+	i := strings.LastIndex(param, ":")
+	if i != -1 {
+		storage = param[:i+1]
+		prefix = param[i+1:]
+	} else {
+		prefix = param
+	}
+	return
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_drop.go

@@ -13,13 +13,15 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution,
+	storage, prefix := parsePrefix(param)
+
+	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
 		context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)

cmd/publish_snapshot.go

@@ -20,13 +20,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	var prefix string
+	var param string
 	if len(args) == len(components)+1 {
-		prefix = args[len(components)]
+		param = args[len(components)]
 		args = args[0 : len(args)-1]
 	} else {
-		prefix = ""
+		param = ""
 	}
+	storage, prefix := parsePrefix(param)
 
 	var (
 		sources = []interface{}{}
@@ -111,7 +112,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	distribution := context.flags.Lookup("distribution").Value.String()
 
-	published, err := deb.NewPublishedRepo(prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -129,7 +130,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -149,7 +150,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.PublishedStorage().(aptly.LocalPublishedStorage); ok {
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
 		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
 			localStorage.PublicPath())
 	}

cmd/publish_switch.go

@@ -19,7 +19,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	var (
 		names    []string
@@ -27,15 +27,17 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	)
 
 	if len(args) == len(components)+2 {
-		prefix = args[1]
+		param = args[1]
 		names = args[2:]
 	} else {
 		names = args[1:]
 	}
 
+	storage, prefix := parsePrefix(param)
+
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -77,7 +79,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -88,7 +90,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}

cmd/publish_update.go

@@ -15,15 +15,16 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
+	storage, prefix := parsePrefix(param)
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -47,7 +48,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -58,7 +59,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}

cmd/serve.go

@@ -84,7 +84,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	publicPath := context.PublishedStorage().(aptly.LocalPublishedStorage).PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

deb/publish.go

@@ -865,14 +865,17 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 	return
 }
 
-// ByPrefixDistribution looks up repository by storage, prefix & distribution
+// ByStoragePrefixDistribution looks up repository by storage, prefix & distribution
 func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage, prefix, distribution string) (*PublishedRepo, error) {
 	for _, r := range collection.list {
 		if r.Prefix == prefix && r.Distribution == distribution && r.Storage == storage {
 			return r, nil
 		}
 	}
-	return nil, fmt.Errorf("published repo with storage/prefix/distribution %s/%s/%s not found", storage, prefix, distribution)
+	if storage != "" {
+		storage += ":"
+	}
+	return nil, fmt.Errorf("published repo with storage:prefix/distribution %s%s/%s not found", storage, prefix, distribution)
 }
 
 // ByUUID looks up repository by uuid

system/t06_publish/PublishDrop6Test_gold

@@ -1 +1 @@
-ERROR: unable to remove: published repo with prefix/distribution ./sq1 not found
+ERROR: unable to remove: published repo with storage:prefix/distribution ./sq1 not found

system/t06_publish/PublishSwitch5Test_gold

@@ -1 +1 @@
-ERROR: unable to update: published repo with prefix/distribution ppa/maverick not found
+ERROR: unable to update: published repo with storage:prefix/distribution ppa/maverick not found