Add swift backend for repository publishing

Gomfile

@@ -10,6 +10,7 @@ gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb5706
 gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
 gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
+gom 'github.com/ncw/swift', :commit => '71da36c561d45ba5420b99262689f91f2d9b8db4'
 gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
 gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
 gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=context database deb files http query s3 utils
+PACKAGES=context database deb files http query swift s3 utils
-ALL_PACKAGES=api aptly context cmd console database deb files http query s3 utils
+ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python

context/context.go

@@ -10,6 +10,7 @@ import (
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
 	"github.com/smira/aptly/s3"
+	"github.com/smira/aptly/swift"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -263,6 +264,18 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 			if err != nil {
 				Fatal(err)
 			}
+		} else if strings.HasPrefix(name, "swift:") {
+			params, ok := context.Config().SwiftPublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
+				params.AuthUrl, params.Tenant, params.TenantId, params.Container, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
 		} else {
 			Fatal(fmt.Errorf("unknown published storage format: %v", name))
 		}

swift/public.go

@@ -0,0 +1,193 @@
+package swift
+
+import (
+	"fmt"
+        "github.com/ncw/swift"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/files"
+        "time"
+	"os"
+	"path/filepath"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on Swift)
+type PublishedStorage struct {
+        conn      swift.Connection
+        container string
+        prefix    string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+// NewPublishedStorage creates new instance of PublishedStorage with specified Swift access
+// keys, tenant and tenantId
+func NewPublishedStorage(username string, password string, authUrl string, tenant string, tenantId string, container string, prefix string) (*PublishedStorage, error) {
+        if username == "" {
+                username = os.Getenv("OS_USERNAME")
+        }
+        if password == "" {
+                password = os.Getenv("OS_PASSWORD")
+        }
+        if authUrl == "" {
+                authUrl = os.Getenv("OS_AUTH_URL")
+        }
+        if tenant == "" {
+                tenant = os.Getenv("OS_TENANT_NAME")
+        }
+        if tenantId == "" {
+                tenantId = os.Getenv("OS_TENANT_ID")
+        }
+
+        ct := swift.Connection{
+                UserName:       username,
+                ApiKey:         password,
+                AuthUrl:        authUrl,
+                UserAgent:      "aptly/" + aptly.Version,
+                Tenant:         tenant,
+                TenantId:       tenantId,
+                ConnectTimeout: 60 * time.Second,
+                Timeout:        60 * time.Second,
+        }
+        err := ct.Authenticate()
+        if err != nil {
+                return nil, fmt.Errorf("Swift authentication failed: %s", err)
+        }
+
+        result := &PublishedStorage{
+                conn:      ct,
+                container: container,
+                prefix:    prefix,
+        }
+
+        return result, nil
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return fmt.Sprintf("Swift: %s:%s/%s", storage.conn.StorageUrl, storage.container, storage.prefix)
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+        // no op for Swift
+        return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+        _, err = storage.conn.ObjectPut(storage.container, filepath.Join(storage.prefix, path), source, false, "", "", nil)
+
+	if err != nil {
+		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
+	}
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	err := storage.conn.ObjectDelete(storage.container, filepath.Join(storage.prefix, path))
+
+	if err != nil {
+		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
+	}
+	return nil
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+        path = filepath.Join(storage.prefix, path)
+        opts := swift.ObjectsOpts{
+                Prefix: storage.prefix,
+        }
+        if objects, err := storage.conn.ObjectNamesAll(storage.container, &opts); err != nil {
+		return fmt.Errorf("error removing dir %s from %s: %s", path, storage, err)
+        } else {
+                if _, err := storage.conn.BulkDelete(storage.container, objects); err != nil {
+                        if err == swift.Forbidden {
+                                for _, name := range objects {
+                                        if storage.conn.ObjectDelete(storage.container, name) != nil {
+                                                return nil
+                                        }
+                                }
+                        }
+                }
+        }
+
+        return nil
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+	sourcePath, sourceMD5 string, force bool) error {
+	// verify that package pool is local pool in filesystem
+	_ = sourcePool.(*files.PackagePool)
+
+	baseName := filepath.Base(sourcePath)
+	relPath := filepath.Join(publishedDirectory, baseName)
+	poolPath := filepath.Join(storage.prefix, relPath)
+
+	var (
+		info swift.Object
+		err  error
+	)
+
+	info, _, err = storage.conn.Object(storage.container, poolPath)
+	if err != nil {
+		if err != swift.ObjectNotFound {
+			return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err)
+		}
+	} else {
+		if !force && info.Hash != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+
+		}
+	}
+
+	return storage.PutFile(relPath, sourcePath)
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	prefix = filepath.Join(storage.prefix, prefix)
+	if prefix != "" {
+		prefix += "/"
+	}
+        opts := swift.ObjectsOpts{
+                Prefix: prefix,
+        }
+	contents, err := storage.conn.ObjectNamesAll(storage.container, &opts)
+	if err != nil {
+		return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+	}
+
+	return contents, nil
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	err := storage.conn.ObjectMove(storage.container, filepath.Join(storage.prefix, oldName), storage.container, filepath.Join(storage.prefix, newName))
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err)
+	}
+
+        return nil
+}

swift/swift.go

@@ -0,0 +1,2 @@
+// Package swift handles publishing to OpenStack Swift
+package swift

swift/swift_test.go

@@ -0,0 +1,12 @@
+package swift
+
+import (
+	"testing"
+
+  . "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

utils/config.go

@@ -8,20 +8,21 @@ import (
 
 // ConfigStructure is structure of main configuration
 type ConfigStructure struct {
-	RootDir                string                   `json:"rootDir"`
+	RootDir                string                      `json:"rootDir"`
-	DownloadConcurrency    int                      `json:"downloadConcurrency"`
+	DownloadConcurrency    int                         `json:"downloadConcurrency"`
-	DownloadLimit          int64                    `json:"downloadSpeedLimit"`
+	DownloadLimit          int64                       `json:"downloadSpeedLimit"`
-	Architectures          []string                 `json:"architectures"`
+	Architectures          []string                    `json:"architectures"`
-	DepFollowSuggests      bool                     `json:"dependencyFollowSuggests"`
+	DepFollowSuggests      bool                        `json:"dependencyFollowSuggests"`
-	DepFollowRecommends    bool                     `json:"dependencyFollowRecommends"`
+	DepFollowRecommends    bool                        `json:"dependencyFollowRecommends"`
-	DepFollowAllVariants   bool                     `json:"dependencyFollowAllVariants"`
+	DepFollowAllVariants   bool                        `json:"dependencyFollowAllVariants"`
-	DepFollowSource        bool                     `json:"dependencyFollowSource"`
+	DepFollowSource        bool                        `json:"dependencyFollowSource"`
-	GpgDisableSign         bool                     `json:"gpgDisableSign"`
+	GpgDisableSign         bool                        `json:"gpgDisableSign"`
-	GpgDisableVerify       bool                     `json:"gpgDisableVerify"`
+	GpgDisableVerify       bool                        `json:"gpgDisableVerify"`
-	DownloadSourcePackages bool                     `json:"downloadSourcePackages"`
+	DownloadSourcePackages bool                        `json:"downloadSourcePackages"`
-	PpaDistributorID       string                   `json:"ppaDistributorID"`
+	PpaDistributorID       string                      `json:"ppaDistributorID"`
-	PpaCodename            string                   `json:"ppaCodename"`
+	PpaCodename            string                      `json:"ppaCodename"`
-	S3PublishRoots         map[string]S3PublishRoot `json:"S3PublishEndpoints"`
+	S3PublishRoots         map[string]S3PublishRoot    `json:"S3PublishEndpoints"`
+	SwiftPublishRoots      map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints"`
 }
 
 // S3PublishRoot describes single S3 publishing entry point
@@ -37,6 +38,17 @@ type S3PublishRoot struct {
 	PlusWorkaround   bool   `json:"plusWorkaround"`
 }
 
+// SwiftPublishRoot describes single OpenStack Swift publishing entry point
+type SwiftPublishRoot struct {
+        UserName  string `json:"osname"`
+        Password  string `json:"password"`
+        AuthUrl   string `json:"authurl"`
+        Tenant    string `json:"tenant"`
+        TenantId  string `json:"tenantid"`
+        Prefix    string `json:"prefix"`
+        Container string `json:"container"`
+}
+
 // Config is configuration for aptly, shared by all modules
 var Config = ConfigStructure{
 	RootDir:                filepath.Join(os.Getenv("HOME"), ".aptly"),
@@ -53,6 +65,7 @@ var Config = ConfigStructure{
 	PpaDistributorID:       "ubuntu",
 	PpaCodename:            "",
 	S3PublishRoots:         map[string]S3PublishRoot{},
+	SwiftPublishRoots:      map[string]SwiftPublishRoot{},
 }
 
 // LoadConfig loads configuration from json file