From 7f038be1cbcb43885a4bb5a5db0d5b521490600f Mon Sep 17 00:00:00 2001 From: Sebastien Badia Date: Wed, 7 Jan 2015 17:26:21 +0100 Subject: [PATCH] Add swift backend for repository publishing --- Gomfile | 1 + Makefile | 4 +- context/context.go | 13 +++ swift/public.go | 193 ++++++++++++++++++++++++++++++++++++++++++++ swift/swift.go | 2 + swift/swift_test.go | 12 +++ utils/config.go | 41 ++++++---- 7 files changed, 250 insertions(+), 16 deletions(-) create mode 100644 swift/public.go create mode 100644 swift/swift.go create mode 100644 swift/swift_test.go diff --git a/Gomfile b/Gomfile index 4a4ce593..75a60f40 100644 --- a/Gomfile +++ b/Gomfile @@ -10,6 +10,7 @@ gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb5706 gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4' gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405' gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b' +gom 'github.com/ncw/swift', :commit => '71da36c561d45ba5420b99262689f91f2d9b8db4' gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47' gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107' gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa' diff --git a/Makefile b/Makefile index af873d9a..d2bef587 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GOVERSION=$(shell go version | awk '{print $$3;}') -PACKAGES=context database deb files http query s3 utils -ALL_PACKAGES=api aptly context cmd console database deb files http query s3 utils +PACKAGES=context database deb files http query swift s3 utils +ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils BINPATH=$(abspath ./_vendor/bin) GOM_ENVIRONMENT=-test PYTHON?=python diff --git a/context/context.go b/context/context.go index 1f464aa1..c471f01e 100644 --- a/context/context.go +++ b/context/context.go @@ -10,6 +10,7 @@ import ( "github.com/smira/aptly/files" "github.com/smira/aptly/http" "github.com/smira/aptly/s3" + "github.com/smira/aptly/swift" "github.com/smira/aptly/utils" "github.com/smira/commander" "github.com/smira/flag" @@ -263,6 +264,18 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto if err != nil { Fatal(err) } + } else if strings.HasPrefix(name, "swift:") { + params, ok := context.Config().SwiftPublishRoots[name[6:]] + if !ok { + Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:])) + } + + var err error + publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password, + params.AuthUrl, params.Tenant, params.TenantId, params.Container, params.Prefix) + if err != nil { + Fatal(err) + } } else { Fatal(fmt.Errorf("unknown published storage format: %v", name)) } diff --git a/swift/public.go b/swift/public.go new file mode 100644 index 00000000..cbb91bf2 --- /dev/null +++ b/swift/public.go @@ -0,0 +1,193 @@ +package swift + +import ( + "fmt" + "github.com/ncw/swift" + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/files" + "time" + "os" + "path/filepath" +) + +// PublishedStorage abstract file system with published files (actually hosted on Swift) +type PublishedStorage struct { + conn swift.Connection + container string + prefix string +} + +// Check interface +var ( + _ aptly.PublishedStorage = (*PublishedStorage)(nil) +) + +// NewPublishedStorage creates new instance of PublishedStorage with specified Swift access +// keys, tenant and tenantId +func NewPublishedStorage(username string, password string, authUrl string, tenant string, tenantId string, container string, prefix string) (*PublishedStorage, error) { + if username == "" { + username = os.Getenv("OS_USERNAME") + } + if password == "" { + password = os.Getenv("OS_PASSWORD") + } + if authUrl == "" { + authUrl = os.Getenv("OS_AUTH_URL") + } + if tenant == "" { + tenant = os.Getenv("OS_TENANT_NAME") + } + if tenantId == "" { + tenantId = os.Getenv("OS_TENANT_ID") + } + + ct := swift.Connection{ + UserName: username, + ApiKey: password, + AuthUrl: authUrl, + UserAgent: "aptly/" + aptly.Version, + Tenant: tenant, + TenantId: tenantId, + ConnectTimeout: 60 * time.Second, + Timeout: 60 * time.Second, + } + err := ct.Authenticate() + if err != nil { + return nil, fmt.Errorf("Swift authentication failed: %s", err) + } + + result := &PublishedStorage{ + conn: ct, + container: container, + prefix: prefix, + } + + return result, nil +} + +// String +func (storage *PublishedStorage) String() string { + return fmt.Sprintf("Swift: %s:%s/%s", storage.conn.StorageUrl, storage.container, storage.prefix) +} + +// MkDir creates directory recursively under public path +func (storage *PublishedStorage) MkDir(path string) error { + // no op for Swift + return nil +} + +// PutFile puts file into published storage at specified path +func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error { + var ( + source *os.File + err error + ) + source, err = os.Open(sourceFilename) + if err != nil { + return err + } + defer source.Close() + + _, err = storage.conn.ObjectPut(storage.container, filepath.Join(storage.prefix, path), source, false, "", "", nil) + + if err != nil { + return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err) + } + return nil +} + +// Remove removes single file under public path +func (storage *PublishedStorage) Remove(path string) error { + err := storage.conn.ObjectDelete(storage.container, filepath.Join(storage.prefix, path)) + + if err != nil { + return fmt.Errorf("error deleting %s from %s: %s", path, storage, err) + } + return nil +} + +// RemoveDirs removes directory structure under public path +func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error { + path = filepath.Join(storage.prefix, path) + opts := swift.ObjectsOpts{ + Prefix: storage.prefix, + } + if objects, err := storage.conn.ObjectNamesAll(storage.container, &opts); err != nil { + return fmt.Errorf("error removing dir %s from %s: %s", path, storage, err) + } else { + if _, err := storage.conn.BulkDelete(storage.container, objects); err != nil { + if err == swift.Forbidden { + for _, name := range objects { + if storage.conn.ObjectDelete(storage.container, name) != nil { + return nil + } + } + } + } + } + + return nil +} + +// LinkFromPool links package file from pool to dist's pool location +// +// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/) +// sourcePool is instance of aptly.PackagePool +// sourcePath is filepath to package file in package pool +// +// LinkFromPool returns relative path for the published file to be included in package index +func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool, + sourcePath, sourceMD5 string, force bool) error { + // verify that package pool is local pool in filesystem + _ = sourcePool.(*files.PackagePool) + + baseName := filepath.Base(sourcePath) + relPath := filepath.Join(publishedDirectory, baseName) + poolPath := filepath.Join(storage.prefix, relPath) + + var ( + info swift.Object + err error + ) + + info, _, err = storage.conn.Object(storage.container, poolPath) + if err != nil { + if err != swift.ObjectNotFound { + return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err) + } + } else { + if !force && info.Hash != sourceMD5 { + return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage) + + } + } + + return storage.PutFile(relPath, sourcePath) +} + +// Filelist returns list of files under prefix +func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { + prefix = filepath.Join(storage.prefix, prefix) + if prefix != "" { + prefix += "/" + } + opts := swift.ObjectsOpts{ + Prefix: prefix, + } + contents, err := storage.conn.ObjectNamesAll(storage.container, &opts) + if err != nil { + return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err) + } + + return contents, nil +} + +// RenameFile renames (moves) file +func (storage *PublishedStorage) RenameFile(oldName, newName string) error { + err := storage.conn.ObjectMove(storage.container, filepath.Join(storage.prefix, oldName), storage.container, filepath.Join(storage.prefix, newName)) + if err != nil { + return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err) + } + + return nil +} diff --git a/swift/swift.go b/swift/swift.go new file mode 100644 index 00000000..dc4be30a --- /dev/null +++ b/swift/swift.go @@ -0,0 +1,2 @@ +// Package swift handles publishing to OpenStack Swift +package swift diff --git a/swift/swift_test.go b/swift/swift_test.go new file mode 100644 index 00000000..1ba91de9 --- /dev/null +++ b/swift/swift_test.go @@ -0,0 +1,12 @@ +package swift + +import ( + "testing" + + . "gopkg.in/check.v1" +) + +// Launch gocheck tests +func Test(t *testing.T) { + TestingT(t) +} diff --git a/utils/config.go b/utils/config.go index 96755a76..60a85ecc 100644 --- a/utils/config.go +++ b/utils/config.go @@ -8,20 +8,21 @@ import ( // ConfigStructure is structure of main configuration type ConfigStructure struct { - RootDir string `json:"rootDir"` - DownloadConcurrency int `json:"downloadConcurrency"` - DownloadLimit int64 `json:"downloadSpeedLimit"` - Architectures []string `json:"architectures"` - DepFollowSuggests bool `json:"dependencyFollowSuggests"` - DepFollowRecommends bool `json:"dependencyFollowRecommends"` - DepFollowAllVariants bool `json:"dependencyFollowAllVariants"` - DepFollowSource bool `json:"dependencyFollowSource"` - GpgDisableSign bool `json:"gpgDisableSign"` - GpgDisableVerify bool `json:"gpgDisableVerify"` - DownloadSourcePackages bool `json:"downloadSourcePackages"` - PpaDistributorID string `json:"ppaDistributorID"` - PpaCodename string `json:"ppaCodename"` - S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints"` + RootDir string `json:"rootDir"` + DownloadConcurrency int `json:"downloadConcurrency"` + DownloadLimit int64 `json:"downloadSpeedLimit"` + Architectures []string `json:"architectures"` + DepFollowSuggests bool `json:"dependencyFollowSuggests"` + DepFollowRecommends bool `json:"dependencyFollowRecommends"` + DepFollowAllVariants bool `json:"dependencyFollowAllVariants"` + DepFollowSource bool `json:"dependencyFollowSource"` + GpgDisableSign bool `json:"gpgDisableSign"` + GpgDisableVerify bool `json:"gpgDisableVerify"` + DownloadSourcePackages bool `json:"downloadSourcePackages"` + PpaDistributorID string `json:"ppaDistributorID"` + PpaCodename string `json:"ppaCodename"` + S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints"` + SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints"` } // S3PublishRoot describes single S3 publishing entry point @@ -37,6 +38,17 @@ type S3PublishRoot struct { PlusWorkaround bool `json:"plusWorkaround"` } +// SwiftPublishRoot describes single OpenStack Swift publishing entry point +type SwiftPublishRoot struct { + UserName string `json:"osname"` + Password string `json:"password"` + AuthUrl string `json:"authurl"` + Tenant string `json:"tenant"` + TenantId string `json:"tenantid"` + Prefix string `json:"prefix"` + Container string `json:"container"` +} + // Config is configuration for aptly, shared by all modules var Config = ConfigStructure{ RootDir: filepath.Join(os.Getenv("HOME"), ".aptly"), @@ -53,6 +65,7 @@ var Config = ConfigStructure{ PpaDistributorID: "ubuntu", PpaCodename: "", S3PublishRoots: map[string]S3PublishRoot{}, + SwiftPublishRoots: map[string]SwiftPublishRoot{}, } // LoadConfig loads configuration from json file