Merge branch 'sbadia-swift'

Conflicts:
	AUTHORS

.travis.yml

@@ -8,12 +8,13 @@ env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+    - secure: "Nah4qrYI/uVhOlrL5xwgvMtK4TZPqw3hh+ApQTq5V6/EsSYLBGBoWEgR8LaKas8aaemeOEJJx6rwNBCpsF+UT7Hmf4eIsrV7dGSRRFZtd1VA8wI40iOWbsAHFWdeCYdXzqyrRWjXqm0ZCcKrnQeh/4N9i5vCLrAMjtCaUQuJKWY="
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -y python-virtualenv graphviz
   - virtualenv env
   - . env/bin/activate
-  - pip install boto requests
+  - pip install boto requests python-swiftclient
 install:
   - make prepare
 

AUTHORS

@@ -14,3 +14,4 @@ List of contributors, in chronological order:
 * Andrea Bernardo Ciddio (https://github.com/bcandrea)
 * Michael Koval (https://github.com/mkoval)
 * Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)

Gomfile

@@ -11,6 +11,7 @@ gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb5706
 gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
 gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
+gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
 gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
 gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
 gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=context database deb files http query s3 utils
+PACKAGES=context database deb files http query swift s3 utils
-ALL_PACKAGES=api aptly context cmd console database deb files http query s3 utils
+ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python

api/publish.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 )
 
+// SigningOptions is a shared between publish API GPG options structure
 type SigningOptions struct {
 	Skip           bool
 	Batch          bool

cmd/task_run.go

@@ -18,13 +18,15 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 		var text string
 		cmdArgs := []string{}
 
-		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+		var finfo os.FileInfo
+		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
 			return fmt.Errorf("no such file, %s\n", filename)
 		}
 
 		fmt.Println("Reading file...\n")
 
-		file, err := os.Open(filename)
+		var file *os.File
+		file, err = os.Open(filename)
 
 		if err != nil {
 			return err

context/context.go

@@ -10,6 +10,7 @@ import (
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
 	"github.com/smira/aptly/s3"
+	"github.com/smira/aptly/swift"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -326,6 +327,18 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 			if err != nil {
 				Fatal(err)
 			}
+		} else if strings.HasPrefix(name, "swift:") {
+			params, ok := context.Config().SwiftPublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
+				params.AuthURL, params.Tenant, params.TenantID, params.Container, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
 		} else {
 			Fatal(fmt.Errorf("unknown published storage format: %v", name))
 		}

deb/graph.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 )
 
+// BuildGraph generates graph contents from aptly object database
 func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, error) {
 	var err error
 

deb/snapshot.go

@@ -297,6 +297,7 @@ func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) err
 	return err
 }
 
+// ForEachSorted runs method for each snapshot following some sort order
 func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler func(*Snapshot) error) error {
 	sorter, err := newSnapshotSorter(sortMethod, collection)
 	if err != nil {

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "December 2014" "" ""
+.TH "APTLY" "1" "February 2015" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -60,6 +60,18 @@ Configuration file is stored in JSON format (default values shown below):
       "encryptionMethod": "",
       "plusWorkaround": false
     }
+  },
+  "SwiftPublishEndpoints": {
+    "test": {
+      "container": "repo",
+      "osname": "",
+      "password": "",
+      "prefix": "",
+      "authurl": "",
+      "tenant": "",
+      "tenantid": ""
+    }
+  }
 }
 .
 .fi
@@ -162,6 +174,35 @@ In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before p
 .P
 \fBaptly publish snapshot wheezy\-main s3:test:\fR
 .
+.SH "OPENSTACK SWIFT PUBLISHING ENDPOINTS"
+aptly could be configured to publish repository directly to OpenStack Swift\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
+.
+.TP
+\fBcontainer\fR
+container name
+.
+.TP
+\fBprefix\fR
+(optional) do publishing under specified prefix in the container, defaults to no prefix (container root)
+.
+.TP
+\fBosname\fR, \fBpassword\fR
+(optional) OpenStack credentials to access Keystone\. If not supplied, environment variables \fBOS_USERNAME\fR and \fBOS_PASSWORD\fR are used\.
+.
+.TP
+\fBtenant\fR, \fBtenantid\fR
+(optional) OpenStack tenant name and id (in order to use v2 authentication)\.
+.
+.TP
+\fBauthurl\fR
+(optional) the full url of Keystone server (including port, and version)\. example \fBhttp://identity\.example\.com:5000/v2\.0\fR
+.
+.P
+In order to publish to Swift, specify endpoint as \fBswift:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
+.
+.P
+\fBaptly publish snapshot jessie\-main swift:test:\fR
+.
 .SH "PACKAGE QUERY"
 Some commands accept package queries to identify list of packages to process\. Package query syntax almost matches \fBreprepro\fR query language\. Query consists of the following simple terms:
 .

man/aptly.1.ronn.tmpl

@@ -52,6 +52,18 @@ Configuration file is stored in JSON format (default values shown below):
           "encryptionMethod": "",
           "plusWorkaround": false
         }
+      },
+      "SwiftPublishEndpoints": {
+        "test": {
+          "container": "repo",
+          "osname": "",
+          "password": "",
+          "prefix": "",
+          "authurl": "",
+          "tenant": "",
+          "tenantid": ""
+        }
+      }
     }
 
 Options:
@@ -144,6 +156,31 @@ publishing prefix on the command line, e.g.:
 
   `aptly publish snapshot wheezy-main s3:test:`
 
+## OPENSTACK SWIFT PUBLISHING ENDPOINTS
+
+aptly could be configured to publish repository directly to OpenStack Swift. First,
+publishing endpoints should be described in aptly configuration file. Each endpoint
+has name and associated settings:
+
+   * `container`:
+     container name
+   * `prefix`:
+     (optional) do publishing under specified prefix in the container, defaults to
+     no prefix (container root)
+   * `osname`, `password`:
+     (optional) OpenStack credentials to access Keystone. If not supplied,
+     environment variables `OS_USERNAME` and `OS_PASSWORD` are used.
+   * `tenant`, `tenantid`:
+     (optional) OpenStack tenant name and id (in order to use v2 authentication).
+   * `authurl`:
+     (optional) the full url of Keystone server (including port, and version).
+     example `http://identity.example.com:5000/v2.0`
+
+In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before
+publishing prefix on the command line, e.g.:
+
+  `aptly publish snapshot jessie-main swift:test:`
+
 ## PACKAGE QUERY
 
 Some commands accept package queries to identify list of packages to process.

swift/public.go

@@ -0,0 +1,229 @@
+package swift
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/ncw/swift"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/files"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on Swift)
+type PublishedStorage struct {
+	conn              swift.Connection
+	container         string
+	prefix            string
+	supportBulkDelete bool
+}
+
+type swiftInfo map[string]interface{}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+// NewPublishedStorage creates new instance of PublishedStorage with specified Swift access
+// keys, tenant and tenantId
+func NewPublishedStorage(username string, password string, authURL string, tenant string, tenantID string, container string, prefix string) (*PublishedStorage, error) {
+	if username == "" {
+		if username = os.Getenv("OS_USERNAME"); username == "" {
+			username = os.Getenv("ST_USER")
+		}
+	}
+	if password == "" {
+		if password = os.Getenv("OS_PASSWORD"); password == "" {
+			password = os.Getenv("ST_KEY")
+		}
+	}
+	if authURL == "" {
+		if authURL = os.Getenv("OS_AUTH_URL"); authURL == "" {
+			authURL = os.Getenv("ST_AUTH")
+		}
+	}
+	if tenant == "" {
+		tenant = os.Getenv("OS_TENANT_NAME")
+	}
+	if tenantID == "" {
+		tenantID = os.Getenv("OS_TENANT_ID")
+	}
+
+	ct := swift.Connection{
+		UserName:       username,
+		ApiKey:         password,
+		AuthUrl:        authURL,
+		UserAgent:      "aptly/" + aptly.Version,
+		Tenant:         tenant,
+		TenantId:       tenantID,
+		ConnectTimeout: 60 * time.Second,
+		Timeout:        60 * time.Second,
+	}
+	err := ct.Authenticate()
+	if err != nil {
+		return nil, fmt.Errorf("swift authentication failed: %s", err)
+	}
+
+	var bulkDelete bool
+	resp, err := http.Get(filepath.Join(ct.StorageUrl, "..", "..") + "/info")
+	if err == nil {
+		defer resp.Body.Close()
+		decoder := json.NewDecoder(resp.Body)
+		var infos swiftInfo
+		if decoder.Decode(&infos) == nil {
+			_, bulkDelete = infos["bulk_delete"]
+		}
+	}
+
+	result := &PublishedStorage{
+		conn:              ct,
+		container:         container,
+		prefix:            prefix,
+		supportBulkDelete: bulkDelete,
+	}
+
+	return result, nil
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return fmt.Sprintf("Swift: %s:%s/%s", storage.conn.StorageUrl, storage.container, storage.prefix)
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+	// no op for Swift
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	_, err = storage.conn.ObjectPut(storage.container, filepath.Join(storage.prefix, path), source, false, "", "", nil)
+
+	if err != nil {
+		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
+	}
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	err := storage.conn.ObjectDelete(storage.container, filepath.Join(storage.prefix, path))
+
+	if err != nil {
+		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
+	}
+	return nil
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+	path = filepath.Join(storage.prefix, path)
+	opts := swift.ObjectsOpts{
+		Prefix: path,
+	}
+
+	objects, err := storage.conn.ObjectNamesAll(storage.container, &opts)
+	if err != nil {
+		return fmt.Errorf("error removing dir %s from %s: %s", path, storage, err)
+	}
+
+	for index, name := range objects {
+		objects[index] = name[len(storage.prefix):]
+	}
+
+	multiDelete := true
+	if storage.supportBulkDelete {
+		_, err := storage.conn.BulkDelete(storage.container, objects)
+		multiDelete = err != nil
+	}
+	if multiDelete {
+		for _, name := range objects {
+			if err := storage.conn.ObjectDelete(storage.container, name); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+	sourcePath, sourceMD5 string, force bool) error {
+	// verify that package pool is local pool in filesystem
+	_ = sourcePool.(*files.PackagePool)
+
+	baseName := filepath.Base(sourcePath)
+	relPath := filepath.Join(publishedDirectory, baseName)
+	poolPath := filepath.Join(storage.prefix, relPath)
+
+	var (
+		info swift.Object
+		err  error
+	)
+
+	info, _, err = storage.conn.Object(storage.container, poolPath)
+	if err != nil {
+		if err != swift.ObjectNotFound {
+			return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err)
+		}
+	} else {
+		if !force && info.Hash != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+
+		}
+	}
+
+	return storage.PutFile(relPath, sourcePath)
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	prefix = filepath.Join(storage.prefix, prefix)
+	if prefix != "" {
+		prefix += "/"
+	}
+	opts := swift.ObjectsOpts{
+		Prefix: prefix,
+	}
+	contents, err := storage.conn.ObjectNamesAll(storage.container, &opts)
+	if err != nil {
+		return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+	}
+
+	for index, name := range contents {
+		contents[index] = name[len(prefix):]
+	}
+
+	return contents, nil
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	err := storage.conn.ObjectMove(storage.container, filepath.Join(storage.prefix, oldName), storage.container, filepath.Join(storage.prefix, newName))
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err)
+	}
+
+	return nil
+}

swift/public_test.go

@@ -0,0 +1,187 @@
+package swift
+
+import (
+	"github.com/ncw/swift/swifttest"
+	"github.com/smira/aptly/files"
+
+	. "gopkg.in/check.v1"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const (
+	TestAddress = "localhost:5324"
+	AuthURL     = "http://" + TestAddress + "/v1.0"
+)
+
+type PublishedStorageSuite struct {
+	srv                      *swifttest.SwiftServer
+	storage, prefixedStorage *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	var err error
+	s.srv, err = swifttest.NewSwiftServer(TestAddress)
+	c.Assert(err, IsNil)
+	c.Assert(s.srv, NotNil)
+
+	s.storage, err = NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "test", "")
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "test", "lala")
+	c.Assert(err, IsNil)
+
+	s.storage.conn.ContainerCreate("test", nil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+	s.srv.Close()
+}
+
+func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
+	stor, err := NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "", "")
+	c.Check(stor, NotNil)
+	c.Check(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err := s.storage.conn.ObjectGetBytes("test", "a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to swift!"))
+
+	err = s.prefixedStorage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err = s.storage.conn.ObjectGetBytes("test", "lala/a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to swift!"))
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		err = s.storage.PutFile(path, filepath.Join(dir, "a"))
+		c.Check(err, IsNil)
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.Remove("a/b.txt")
+	c.Check(err, IsNil)
+
+	_, err = s.storage.conn.ObjectGetBytes("test", "a/b.txt")
+	c.Check(err, ErrorMatches, "Object Not Found")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	c.Skip("bulk-delete not available in s3test")
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		err = s.storage.PutFile(path, filepath.Join(dir, "a"))
+		c.Check(err, IsNil)
+	}
+
+	err = s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	c.Skip("copy not available in s3test")
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root)
+
+	sourcePath := filepath.Join(root, "pool/c1/df/mars-invaders_1.03.deb")
+	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+
+	sourcePath2 := filepath.Join(root, "pool/e9/df/mars-invaders_1.03.deb")
+	err = os.MkdirAll(filepath.Dir(sourcePath2), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	c.Check(err, IsNil)
+
+	data, err := s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	c.Check(err, IsNil)
+
+	data, err = s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	data, err = s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", true)
+	c.Check(err, IsNil)
+
+	data, err = s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Spam"))
+}

swift/swift.go

@@ -0,0 +1,2 @@
+// Package swift handles publishing to OpenStack Swift
+package swift

swift/swift_test.go

@@ -0,0 +1,12 @@
+package swift
+
+import (
+	"testing"
+
+  . "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

system/run.py

@@ -10,6 +10,7 @@ import traceback
 
 from lib import BaseTest
 from s3_lib import S3Test
+from swift_lib import SwiftTest
 from api_lib import APITest
 
 try:
@@ -37,7 +38,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
             o = getattr(testModule, name)
 
             if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and
-                    o is not S3Test and o is not APITest):
+                    o is not SwiftTest and o is not S3Test and o is not APITest):
                 continue
 
             newBase = o.__bases__[0]

system/swift_lib.py

@@ -0,0 +1,87 @@
+from lib import BaseTest
+import uuid
+import os
+
+try:
+    import swiftclient
+
+    if 'OS_USERNAME' in os.environ and 'OS_PASSWORD' in os.environ:
+        auth_username = os.environ.get('OS_USERNAME')
+        auth_password = os.environ.get('OS_PASSWORD')
+        # Using auth version 2 /v2.0/
+        auth_url = os.environ.get('OS_AUTH_URL')
+        auth_tenant = os.environ.get('OS_TENANT_NAME')
+
+        account_username = "%s:%s" % (auth_tenant, auth_username)
+        swift_conn = swiftclient.Connection(auth_url, account_username,
+                                            auth_password, auth_version=2)
+    elif 'ST_USER' in os.environ and 'ST_KEY' in os.environ:
+        auth_username = os.environ.get('ST_USER')
+        auth_password = os.environ.get('ST_KEY')
+        auth_url = os.environ.get('ST_AUTH')
+        # Using auth version 1 (/auth/v1.0)
+        swift_conn = swiftclient.Connection(auth_url, auth_username,
+                                            auth_password, auth_version=1)
+    else:
+        swift_conn = None
+except ImportError:
+    swift_conn = None
+
+
+class SwiftTest(BaseTest):
+    """
+    BaseTest + support for Swift
+    """
+
+    def fixture_available(self):
+        return super(SwiftTest, self).fixture_available() and swift_conn is not None
+
+    def prepare(self):
+        self.container_name = "aptly-sys-test-" + str(uuid.uuid4())
+        swift_conn.put_container(self.container_name)
+
+        self.configOverride = {"SwiftPublishEndpoints": {
+            "test1": {
+                "container": self.container_name,
+            }
+        }}
+
+        super(SwiftTest, self).prepare()
+
+    def shutdown(self):
+        if hasattr(self, "container_name"):
+            for obj in swift_conn.get_container(self.container_name,
+                                                full_listing=True)[1]:
+                swift_conn.delete_object(self.container_name, obj.get("name"))
+
+            swift_conn.delete_container(self.container_name)
+        super(SwiftTest, self).shutdown()
+
+    def check_path(self, path):
+        if not hasattr(self, "container_contents"):
+            self.container_contents = [obj.get('name') for obj in
+                                       swift_conn.get_container(self.container_name)[1]]
+
+        if path in self.container_contents:
+            return True
+
+        if not path.endswith("/"):
+            path = path + "/"
+
+        for item in self.container_contents:
+            if item.startswith(path):
+                return True
+
+        return False
+
+    def check_exists(self, path):
+        if not self.check_path(path):
+            raise Exception("path %s doesn't exist" % (path, ))
+
+    def check_not_exists(self, path):
+        if self.check_path(path):
+            raise Exception("path %s exists" % (path, ))
+
+    def read_file(self, path):
+        hdrs, body = swift_conn.get_object(self.container_name, path)
+        return body

system/t02_config/ConfigShowTest_gold

@@ -12,5 +12,6 @@
     "downloadSourcePackages": false,
     "ppaDistributorID": "ubuntu",
     "ppaCodename": "",
-    "S3PublishEndpoints": {}
+    "S3PublishEndpoints": {},
+    "SwiftPublishEndpoints": {}
 }

system/t02_config/CreateConfigTest_gold

@@ -12,5 +12,6 @@
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
   "ppaCodename": "",
-  "S3PublishEndpoints": {}
+  "S3PublishEndpoints": {},
+  "SwiftPublishEndpoints": {}
 }

system/t06_publish/SwiftPublish1Test_binary

@@ -0,0 +1,25 @@
+Package: libboost-program-options-dev
+Version: 1.49.0.1
+Installed-Size: 26
+Priority: optional
+Section: libdevel
+Maintainer: Debian Boost Team <pkg-boost-devel@lists.alioth.debian.org>
+Architecture: i386
+Description: program options library for C++ (default version)
+ This package forms part of the Boost C++ Libraries collection.
+ .
+ Library to let program developers obtain program options, that is
+ (name, value) pairs from the user, via conventional methods such as
+ command line and config file.
+ .
+ This package is a dependency package, which depends on Debian's default
+ Boost version (currently 1.49).
+MD5sum: 0035d7822b2f8f0ec4013f270fd650c2
+SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b
+SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12
+Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb
+Size: 2738
+Homepage: http://www.boost.org/libs/program_options/
+Source: boost-defaults
+Depends: libboost-program-options1.49-dev
+

system/t06_publish/SwiftPublish1Test_gold

@@ -0,0 +1,13 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Local repo local-repo has been successfully published.
+Now you can add following line to apt sources:
+  deb http://your-server/ maverick main
+  deb-src http://your-server/ maverick main
+Don't forget to add your GPG key to apt with apt-key.
+
+You can also use `aptly serve` to publish your repositories over HTTP quickly.

system/t06_publish/SwiftPublish1Test_release

@@ -0,0 +1,34 @@
+Origin: . maverick
+Label: . maverick
+Codename: maverick
+Date: Wed, 1 Oct 2014 08:48:48 UTC
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+ b844530d1336e9a3c431f0d36cfc01b0      602 main/binary-i386/Packages.gz
+ 1d7829dac8923aafe499f313abfaadd7      652 main/binary-i386/Packages.bz2
+ 307b6495eab59c221e2ff8962896631b     2300 main/source/Sources
+ 65dd7338cfac70762457b586629e87e4      839 main/source/Sources.gz
+ 5cc219da21fdb8a96b265bca1c4c0808     1009 main/source/Sources.bz2
+ 60b30b7b0c62ae04bb3bc457abadaced       90 main/binary-i386/Release
+ 945211dc923a8d1b97835232648c0aa7       92 main/source/Release
+ d419bd11e2b7fe9669bccdf67a18ca17      984 main/binary-i386/Packages
+SHA1:
+ 1b314cedcf18a6d08d4aabbd8b9b5605ba293d04      602 main/binary-i386/Packages.gz
+ 5406a984c100b20fbebacdbac24ae3378885f73b      652 main/binary-i386/Packages.bz2
+ e30d7bc51cd042ee987316967bf3043ab95c8ce9     2300 main/source/Sources
+ d60a7032080848eb48bcf68962698ba642dcc383      839 main/source/Sources.gz
+ fb194b90e0e0efd456a7346c4224294018b6677d     1009 main/source/Sources.bz2
+ 2bfef2580deadf6863ee6f893e8b9a2c7522e1ed       90 main/binary-i386/Release
+ 8b98a2148d157bf87cc1955ef00ba1ba31275f94       92 main/source/Release
+ be80e1c588c6052f30865e44e3f1429f730d5bc8      984 main/binary-i386/Packages
+SHA256:
+ a079102fdc72e6228229aaa8e5e6ad59b582026419737e81e11a8af2addd125e      602 main/binary-i386/Packages.gz
+ 25d101a333e85d952afc74f684cef3716d69e3c33d8a4b1544faec683c1b5d96      652 main/binary-i386/Packages.bz2
+ bcf1fcf1ca2d1bb5565da8b4c39052d906832ad4885c21682d605b830e55a506     2300 main/source/Sources
+ 3e6cf6dc079333cdf01905957c611702f4ee10f654c84895ac7bf166bbbbd3bc      839 main/source/Sources.gz
+ 47b9d37fa81d23d227dd26e85821dd4f74db8f17ddefbe6ca686f62ddfedd8ad     1009 main/source/Sources.bz2
+ 1d91164164e6310a5e5fc93390995028956f657490a9ce7aa136dc94291828a8       90 main/binary-i386/Release
+ 2d75333511325affcefe66c6cfbaa6ab21e6aa0e85a6b4fa39a4191146b81460       92 main/source/Release
+ 59643cc2d105694d6876dc328290a1c949b4e91e62ee8db396abac83a7034f9f      984 main/binary-i386/Packages

system/t06_publish/SwiftPublish1Test_sources

@@ -0,0 +1,42 @@
+Package: pyspi
+Version: 0.6.1-1.3
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Architecture: any
+Binary: python-at-spi
+Standards-Version: 3.7.3
+Format: 1.0
+Files: 22ff26db69b73d3438fdde21ab5ba2f1 3456 pyspi_0.6.1-1.3.diff.gz
+ b72cb94699298a117b7c82641c68b6fd 1782 pyspi_0.6.1-1.3.dsc
+ def336bd566ea688a06ec03db7ccf1f4 29063 pyspi_0.6.1.orig.tar.gz
+Checksums-Sha1: 95a2468e4bbce730ba286f2211fa41861b9f1d90 3456 pyspi_0.6.1-1.3.diff.gz
+ 56c8a9b1f4ab636052be8966690998cbe865cd6c 1782 pyspi_0.6.1-1.3.dsc
+ 9694b80acc171c0a5bc99f707933864edfce555e 29063 pyspi_0.6.1.orig.tar.gz
+Vcs-Svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Homepage: http://people.redhat.com/zcerza/dogtail
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+Directory: pool/main/p/pyspi
+Checksums-Sha256: 2e770b28df948f3197ed0b679bdea99f3f2bf745e9ddb440c677df9c3aeaee3c 3456 pyspi_0.6.1-1.3.diff.gz
+ d494aaf526f1ec6b02f14c2f81e060a5722d6532ddc760ec16972e45c2625989 1782 pyspi_0.6.1-1.3.dsc
+ 64069ee828c50b1c597d10a3fefbba279f093a4723965388cdd0ac02f029bfb9 29063 pyspi_0.6.1.orig.tar.gz
+
+Package: pyspi
+Version: 0.6.1-1.4
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Architecture: any
+Vcs-Svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Standards-Version: 3.7.3
+Homepage: http://people.redhat.com/zcerza/dogtail
+Directory: pool/main/p/pyspi
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+Checksums-Sha256: 289d3aefa970876e9c43686ce2b02f478d7f3ed35a713928464a98d54ae4fca3 893 pyspi-0.6.1-1.3.stripped.dsc
+ 2e770b28df948f3197ed0b679bdea99f3f2bf745e9ddb440c677df9c3aeaee3c 3456 pyspi_0.6.1-1.3.diff.gz
+ 64069ee828c50b1c597d10a3fefbba279f093a4723965388cdd0ac02f029bfb9 29063 pyspi_0.6.1.orig.tar.gz
+Format: 1.0
+Checksums-Sha1: 5005fbd1f30637edc1d380b30f45db9b79100d07 893 pyspi-0.6.1-1.3.stripped.dsc
+ 95a2468e4bbce730ba286f2211fa41861b9f1d90 3456 pyspi_0.6.1-1.3.diff.gz
+ 9694b80acc171c0a5bc99f707933864edfce555e 29063 pyspi_0.6.1.orig.tar.gz
+Binary: python-at-spi
+Files: 2f5bd47cf38852b6fc927a50f98c1448 893 pyspi-0.6.1-1.3.stripped.dsc
+ 22ff26db69b73d3438fdde21ab5ba2f1 3456 pyspi_0.6.1-1.3.diff.gz
+ def336bd566ea688a06ec03db7ccf1f4 29063 pyspi_0.6.1.orig.tar.gz
+

system/t06_publish/SwiftPublish2Test_binary

@@ -0,0 +1,25 @@
+Package: libboost-program-options-dev
+Version: 1.49.0.1
+Installed-Size: 26
+Priority: optional
+Section: libdevel
+Maintainer: Debian Boost Team <pkg-boost-devel@lists.alioth.debian.org>
+Architecture: i386
+Description: program options library for C++ (default version)
+ This package forms part of the Boost C++ Libraries collection.
+ .
+ Library to let program developers obtain program options, that is
+ (name, value) pairs from the user, via conventional methods such as
+ command line and config file.
+ .
+ This package is a dependency package, which depends on Debian's default
+ Boost version (currently 1.49).
+MD5sum: 0035d7822b2f8f0ec4013f270fd650c2
+SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b
+SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12
+Source: boost-defaults
+Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb
+Depends: libboost-program-options1.49-dev
+Homepage: http://www.boost.org/libs/program_options/
+Size: 2738
+

system/t06_publish/SwiftPublish2Test_gold

@@ -0,0 +1,8 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." components main...
+
+Publish for local repo swift:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/SwiftPublish2Test_release

@@ -0,0 +1,34 @@
+Origin: . maverick
+Label: . maverick
+Codename: maverick
+Date: Wed, 1 Oct 2014 09:13:14 UTC
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+ d41d8cd98f00b204e9800998ecf8427e        0 main/source/Sources
+ f41c10a4b35cd3e1ec8abb9c2ab676ed       23 main/source/Sources.gz
+ 4059d198768f9f8dc9372dc1c54bc3c3       14 main/source/Sources.bz2
+ 60b30b7b0c62ae04bb3bc457abadaced       90 main/binary-i386/Release
+ 945211dc923a8d1b97835232648c0aa7       92 main/source/Release
+ db76ccafa3c9e4c1dba620259df78f87      984 main/binary-i386/Packages
+ d666eb8b2fc8a0ef525d37aff33c7b2f      603 main/binary-i386/Packages.gz
+ ca2b3a9fc60f4a0a1091b9f0357b11eb      651 main/binary-i386/Packages.bz2
+SHA1:
+ da39a3ee5e6b4b0d3255bfef95601890afd80709        0 main/source/Sources
+ 92c6cff562771f64540523a54baaa0b2afe54b3f       23 main/source/Sources.gz
+ 64a543afbb5f4bf728636bdcbbe7a2ed0804adc2       14 main/source/Sources.bz2
+ 2bfef2580deadf6863ee6f893e8b9a2c7522e1ed       90 main/binary-i386/Release
+ 8b98a2148d157bf87cc1955ef00ba1ba31275f94       92 main/source/Release
+ 7dcfa6945771369da0a22c2f90f2300b5d238662      984 main/binary-i386/Packages
+ ba6efb87b17aa8d08476b3f181702e4d3199794e      603 main/binary-i386/Packages.gz
+ 0b36a014d1a5ccbf3d73de0035970737659e3c0f      651 main/binary-i386/Packages.bz2
+SHA256:
+ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/source/Sources
+ 1775fca35fb6a4d31c541746eaea63c5cb3c00280c8b5a351d4e944cdca7489d       23 main/source/Sources.gz
+ d3dda84eb03b9738d118eb2be78e246106900493c0ae07819ad60815134a8058       14 main/source/Sources.bz2
+ 1d91164164e6310a5e5fc93390995028956f657490a9ce7aa136dc94291828a8       90 main/binary-i386/Release
+ 2d75333511325affcefe66c6cfbaa6ab21e6aa0e85a6b4fa39a4191146b81460       92 main/source/Release
+ 0e2e7586903004efb49dd419be8a98260dab502352c4b1bf6074f658220aef4e      984 main/binary-i386/Packages
+ e2bd1d551b4983253cc26004504ead7b6987e609db8cb7185ab3dde69d346acd      603 main/binary-i386/Packages.gz
+ 81bcd3d47fc3e9dbe1e201d7ec1b356dd2ae3bc5c171f76247243a64755c25d6      651 main/binary-i386/Packages.bz2

system/t06_publish/SwiftPublish3Test_binary

@@ -0,0 +1,29 @@
+Package: gnuplot-x11
+Version: 4.6.1-1~maverick2
+Installed-Size: 1604
+Priority: optional
+Section: math
+Maintainer: Debian Science Team <debian-science-maintainers@lists.alioth.debian.org>
+Architecture: i386
+Description: Command-line driven interactive plotting program
+ Gnuplot is a portable command-line driven interactive data and function
+ plotting utility that supports lots of output formats, including drivers
+ for many printers, (La)TeX, (x)fig, Postscript, and so on. The X11-output
+ is packaged in gnuplot-x11.
+ .
+ Data files and self-defined functions can be manipulated by the internal
+ C-like language. Can perform smoothing, spline-fitting, or nonlinear fits,
+ and can work with complex numbers.
+ .
+ This package contains the terminal driver that enables gnuplot to plot
+ images interactively under X11. Most users will want this, it is however
+ packaged separately so that low-end systems don't need X installed to use
+ gnuplot.
+MD5sum: fcad938905d0ace50a6ce0c73b2c6583
+SHA1: 02f9a93097a8f798a054e26154dbe5789088c069
+Replaces: gnuplot (<< 4.0.0)
+Filename: pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb
+Depends: gnuplot-nox (>= 4.6.1-1~maverick2), libc6 (>= 2.11), libcairo2 (>= 1.6.0), libedit2 (>= 2.5.cvs.20010821-1), libgcc1 (>= 1:4.1.1), libgd2-noxpm (>= 2.0.36~rc1~dfsg) | libgd2-xpm (>= 2.0.36~rc1~dfsg), libglib2.0-0 (>= 2.12.0), liblua5.1-0, libpango1.0-0 (>= 1.14.0), libstdc++6 (>= 4.1.1), libwxbase2.8-0 (>= 2.8.11.0), libwxgtk2.8-0 (>= 2.8.11.0), libx11-6
+Size: 724388
+Source: gnuplot
+

system/t06_publish/SwiftPublish3Test_gold

@@ -0,0 +1,8 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." components main...
+
+Publish for snapshot swift:test1:./maverick [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot.

system/t06_publish/SwiftPublish3Test_release

@@ -0,0 +1,34 @@
+Origin: . maverick
+Label: . maverick
+Codename: maverick
+Date: Wed, 1 Oct 2014 09:16:49 UTC
+Architectures: amd64 i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+ 4717e26fc4a8703cd8886feb8ff9532d       91 main/binary-amd64/Release
+ 60b30b7b0c62ae04bb3bc457abadaced       90 main/binary-i386/Release
+ 2b810443a56c38746aba877b84fc74a1     1526 main/binary-amd64/Packages
+ 28bced4c89869001d9fe6b7c553dd1df      862 main/binary-amd64/Packages.gz
+ aaa2ee36bda75a9c66e31881ae128016      931 main/binary-amd64/Packages.bz2
+ aac26f9e4705d03000094f76d475aea2     1524 main/binary-i386/Packages
+ 158aec0342fc4ca52178b4512c5ee1b5      862 main/binary-i386/Packages.gz
+ 34859d0bf49cb66045de43d01b1de311      939 main/binary-i386/Packages.bz2
+SHA1:
+ 93c9982ebbb6a74a118d07e500b596097c8c4780       91 main/binary-amd64/Release
+ 2bfef2580deadf6863ee6f893e8b9a2c7522e1ed       90 main/binary-i386/Release
+ 876cafdad8672c4b0b66baec5b12213d2bcb4cf3     1526 main/binary-amd64/Packages
+ b3e2e9ad945a190e2ce4aeb36d1946d9ad04a075      862 main/binary-amd64/Packages.gz
+ bc8a7022261b79f5aeacdca551c51aeb7530b969      931 main/binary-amd64/Packages.bz2
+ 7eca65cdb4a4a6bcb51747f2c8d4829f4457f22b     1524 main/binary-i386/Packages
+ e1f5ab02bdd1fcaa0ab93c5680919f612692992c      862 main/binary-i386/Packages.gz
+ 8a7f311f39316dcedc8a199421116ba92a941028      939 main/binary-i386/Packages.bz2
+SHA256:
+ 73aa8d6aaf47a1bf3c546869ceb09a882a8c2d840f81878e552fe2d1260ac4e2       91 main/binary-amd64/Release
+ 1d91164164e6310a5e5fc93390995028956f657490a9ce7aa136dc94291828a8       90 main/binary-i386/Release
+ f47ca8ea0dc02b4423b1291b302e5594c0ac5c01da72c6f9de1ae17d3eddef2f     1526 main/binary-amd64/Packages
+ 0a939f23e1ed98ec3cf2033eb5665d4c40e7494d6331f453ac2043be3e234897      862 main/binary-amd64/Packages.gz
+ abdb8e2537c11272fc9f70ccbcbd2ee867ae797666d3bf11a51972fa2f4d0325      931 main/binary-amd64/Packages.bz2
+ 7b1e711ab4647a3e200af742690ffee76bcf7244f597fda699495e29177b1c71     1524 main/binary-i386/Packages
+ 5723a156f299c657b2eebd1c17ff1a0ca3f50036fc9a1b6c7d9f985a1841c171      862 main/binary-i386/Packages.gz
+ 41f396a3b5c7f78d743971a1011706c6782c8abac3168ff862fa301255baa040      939 main/binary-i386/Packages.bz2

system/t06_publish/SwiftPublish4Test_gold

@@ -0,0 +1,4 @@
+Published repositories:
+  * swift:test1:./maverick [amd64, i386] publishes {main: [local-repo]}
+  * swift:test1:./xyz [amd64, i386] publishes {main: [local-repo]}
+  * swift:test1:prefix/maverick [amd64, i386] publishes {main: [local-repo]}

system/t06_publish/SwiftPublish5Test_gold

@@ -0,0 +1,3 @@
+Cleaning up prefix "." components main...
+
+Published repository has been removed successfully.

system/t06_publish/__init__.py

@@ -9,3 +9,4 @@ from .snapshot import *
 from .switch import *
 from .update import *
 from .s3 import *
+from .swift import *

system/t06_publish/swift.py

@@ -0,0 +1,158 @@
+from swift_lib import SwiftTest
+
+
+def strip_processor(output):
+    return "\n".join([l for l in output.split("\n") if not l.startswith(' ') and not l.startswith('Date:')])
+
+
+class SwiftPublish1Test(SwiftTest):
+    """
+    publish to Swift: from repo
+    """
+    fixtureCmds = [
+        "aptly repo create -distribution=maverick local-repo",
+        "aptly repo add local-repo ${files}",
+    ]
+    runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo swift:test1:"
+
+    def check(self):
+        super(SwiftPublish1Test, self).check()
+
+        self.check_exists('dists/maverick/InRelease')
+        self.check_exists('dists/maverick/Release')
+        self.check_exists('dists/maverick/Release.gpg')
+
+        self.check_exists('dists/maverick/main/binary-i386/Packages')
+        self.check_exists('dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('dists/maverick/main/source/Sources')
+        self.check_exists('dists/maverick/main/source/Sources.gz')
+        self.check_exists('dists/maverick/main/source/Sources.bz2')
+
+        self.check_exists('pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+        # # verify contents except of sums
+        self.check_file_contents('dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+        self.check_file_contents('dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+
+class SwiftPublish2Test(SwiftTest):
+    """
+    publish to Swift: publish update removed some packages
+    """
+    fixtureCmds = [
+        "aptly repo create -distribution=maverick local-repo",
+        "aptly repo add local-repo ${files}/",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo swift:test1:",
+        "aptly repo remove local-repo pyspi"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick swift:test1:"
+
+    def check(self):
+        super(SwiftPublish2Test, self).check()
+
+        self.check_exists('dists/maverick/InRelease')
+        self.check_exists('dists/maverick/Release')
+        self.check_exists('dists/maverick/Release.gpg')
+
+        self.check_exists('dists/maverick/main/binary-i386/Packages')
+        self.check_exists('dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('dists/maverick/main/source/Sources')
+        self.check_exists('dists/maverick/main/source/Sources.gz')
+        self.check_exists('dists/maverick/main/source/Sources.bz2')
+
+        self.check_not_exists('pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+        self.check_file_contents('dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+
+class SwiftPublish3Test(SwiftTest):
+    """
+    publish to Swift: publish switch - removed some packages
+    """
+    fixtureDB = True
+    fixturePool = True
+    fixtureCmds = [
+        "aptly snapshot create snap1 from mirror gnuplot-maverick",
+        "aptly snapshot create snap2 empty",
+        "aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11",
+        "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1 swift:test1:",
+    ]
+    runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick swift:test1: snap3"
+
+    def check(self):
+        super(SwiftPublish3Test, self).check()
+
+        self.check_exists('dists/maverick/InRelease')
+        self.check_exists('dists/maverick/Release')
+        self.check_exists('dists/maverick/Release.gpg')
+
+        self.check_exists('dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('dists/maverick/main/binary-amd64/Packages')
+        self.check_exists('dists/maverick/main/binary-amd64/Packages.gz')
+        self.check_exists('dists/maverick/main/binary-amd64/Packages.bz2')
+
+        self.check_exists('pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb')
+        self.check_exists('pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_amd64.deb')
+        self.check_not_exists('pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_i386.deb')
+        self.check_not_exists('pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_amd64.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+
+class SwiftPublish4Test(SwiftTest):
+    """
+    publish to Swift: multiple repos, list
+    """
+    fixtureCmds = [
+        "aptly repo create -distribution=maverick local-repo",
+        "aptly repo add local-repo ${udebs}",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo swift:test1:",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=xyz local-repo swift:test1:",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo swift:test1:prefix",
+    ]
+    runCmd = "aptly publish list"
+
+
+class SwiftPublish5Test(SwiftTest):
+    """
+    publish to Swift: publish drop - component cleanup
+    """
+    fixtureCmds = [
+        "aptly repo create local1",
+        "aptly repo create local2",
+        "aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb",
+        "aptly repo add local2 ${files}",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 swift:test1:",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 swift:test1:",
+    ]
+    runCmd = "aptly publish drop sq2 swift:test1:"
+
+    def check(self):
+        super(SwiftPublish5Test, self).check()
+
+        self.check_exists('dists/sq1')
+        self.check_not_exists('dists/sq2')
+        self.check_exists('pool/main/')
+
+        self.check_not_exists('pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')

utils/config.go

@@ -8,20 +8,21 @@ import (
 
 // ConfigStructure is structure of main configuration
 type ConfigStructure struct {
-	RootDir                string                   `json:"rootDir"`
+	RootDir                string                      `json:"rootDir"`
-	DownloadConcurrency    int                      `json:"downloadConcurrency"`
+	DownloadConcurrency    int                         `json:"downloadConcurrency"`
-	DownloadLimit          int64                    `json:"downloadSpeedLimit"`
+	DownloadLimit          int64                       `json:"downloadSpeedLimit"`
-	Architectures          []string                 `json:"architectures"`
+	Architectures          []string                    `json:"architectures"`
-	DepFollowSuggests      bool                     `json:"dependencyFollowSuggests"`
+	DepFollowSuggests      bool                        `json:"dependencyFollowSuggests"`
-	DepFollowRecommends    bool                     `json:"dependencyFollowRecommends"`
+	DepFollowRecommends    bool                        `json:"dependencyFollowRecommends"`
-	DepFollowAllVariants   bool                     `json:"dependencyFollowAllVariants"`
+	DepFollowAllVariants   bool                        `json:"dependencyFollowAllVariants"`
-	DepFollowSource        bool                     `json:"dependencyFollowSource"`
+	DepFollowSource        bool                        `json:"dependencyFollowSource"`
-	GpgDisableSign         bool                     `json:"gpgDisableSign"`
+	GpgDisableSign         bool                        `json:"gpgDisableSign"`
-	GpgDisableVerify       bool                     `json:"gpgDisableVerify"`
+	GpgDisableVerify       bool                        `json:"gpgDisableVerify"`
-	DownloadSourcePackages bool                     `json:"downloadSourcePackages"`
+	DownloadSourcePackages bool                        `json:"downloadSourcePackages"`
-	PpaDistributorID       string                   `json:"ppaDistributorID"`
+	PpaDistributorID       string                      `json:"ppaDistributorID"`
-	PpaCodename            string                   `json:"ppaCodename"`
+	PpaCodename            string                      `json:"ppaCodename"`
-	S3PublishRoots         map[string]S3PublishRoot `json:"S3PublishEndpoints"`
+	S3PublishRoots         map[string]S3PublishRoot    `json:"S3PublishEndpoints"`
+	SwiftPublishRoots      map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints"`
 }
 
 // S3PublishRoot describes single S3 publishing entry point
@@ -37,6 +38,17 @@ type S3PublishRoot struct {
 	PlusWorkaround   bool   `json:"plusWorkaround"`
 }
 
+// SwiftPublishRoot describes single OpenStack Swift publishing entry point
+type SwiftPublishRoot struct {
+	UserName  string `json:"osname"`
+	Password  string `json:"password"`
+	AuthURL   string `json:"authurl"`
+	Tenant    string `json:"tenant"`
+	TenantID  string `json:"tenantid"`
+	Prefix    string `json:"prefix"`
+	Container string `json:"container"`
+}
+
 // Config is configuration for aptly, shared by all modules
 var Config = ConfigStructure{
 	RootDir:                filepath.Join(os.Getenv("HOME"), ".aptly"),
@@ -53,6 +65,7 @@ var Config = ConfigStructure{
 	PpaDistributorID:       "ubuntu",
 	PpaCodename:            "",
 	S3PublishRoots:         map[string]S3PublishRoot{},
+	SwiftPublishRoots:      map[string]SwiftPublishRoot{},
 }
 
 // LoadConfig loads configuration from json file

utils/config_test.go

@@ -34,6 +34,9 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		Region: "us-east-1",
 		Bucket: "repo"}}
 
+	s.config.SwiftPublishRoots = map[string]SwiftPublishRoot{"test": SwiftPublishRoot{
+		Container: "repo"}}
+
 	err := SaveConfig(configname, &s.config)
 	c.Assert(err, IsNil)
 
@@ -71,6 +74,17 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		"      \"encryptionMethod\": \"\",\n"+
 		"      \"plusWorkaround\": false\n"+
 		"    }\n"+
+		"  },\n"+
+                "  \"SwiftPublishEndpoints\": {\n" +
+		"    \"test\": {\n"+
+		"      \"osname\": \"\",\n"+
+		"      \"password\": \"\",\n"+
+		"      \"authurl\": \"\",\n"+
+		"      \"tenant\": \"\",\n"+
+		"      \"tenantid\": \"\",\n"+
+		"      \"prefix\": \"\",\n"+
+		"      \"container\": \"repo\"\n"+
+		"    }\n"+
 		"  }\n"+
 		"}")
 }