Merge pull request #664 from sliverc/acquire-by-hash

Support Acquire-By-Hash for index files

api/publish.go

@@ -238,6 +238,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
 		}
+		AcquireByHash *bool
 	}
 
 	if c.Bind(&b) != nil {
@@ -317,6 +318,10 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.SkipContents = *b.SkipContents
 	}
 
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
 		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))

aptly/interfaces.go

@@ -73,6 +73,14 @@ type PublishedStorage interface {
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
 // FileSystemPublishedStorage is published storage on filesystem

bash_completion.d/aptly

@@ -499,7 +499,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))

cmd/publish_repo.go

@@ -47,6 +47,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_snapshot.go

@@ -137,6 +137,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
@@ -227,6 +231,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

deb/index_files.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -20,18 +21,20 @@ type indexFiles struct {
 	tempDir          string
 	suffix           string
 	indexes          map[string]*indexFile
+	acquireByHash    bool
 }
 
 type indexFile struct {
-	parent       *indexFiles
+	parent        *indexFiles
-	discardable  bool
+	discardable   bool
-	compressable bool
+	compressable  bool
-	onlyGzip     bool
+	onlyGzip      bool
-	signable     bool
+	signable      bool
-	relativePath string
+	acquireByHash bool
-	tempFilename string
+	relativePath  string
-	tempFile     *os.File
+	tempFilename  string
-	w            *bufio.Writer
+	tempFile      *os.File
+	w             *bufio.Writer
 }
 
 func (file *indexFile) BufWriter() (*bufio.Writer, error) {
@@ -91,11 +94,22 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
 	}
 
-	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+
+	err = file.parent.publishedStorage.MkDir(filedir)
 	if err != nil {
 		return fmt.Errorf("unable to create dir: %s", err)
 	}
 
+	if file.acquireByHash {
+		for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} {
+			err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
+			if err != nil {
+				return fmt.Errorf("unable to create dir: %s", err)
+			}
+		}
+	}
+
 	for _, ext := range exts {
 		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
 			file.tempFilename+ext)
@@ -107,6 +121,16 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
 				filepath.Join(file.parent.basePath, file.relativePath+ext)
 		}
+
+		if file.acquireByHash {
+			sums := file.parent.generatedFiles[file.relativePath+ext]
+			for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} {
+				err = packageIndexByHash(file, ext, hash, sum)
+				if err != nil {
+					return fmt.Errorf("unable to build hash file: %s", err)
+				}
+			}
+		}
 	}
 
 	if file.signable && signer != nil {
@@ -143,7 +167,53 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	return nil
 }
 
-func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
+	src := filepath.Join(file.parent.basePath, file.relativePath)
+	indexfile := path.Base(src + ext)
+	src = src + file.parent.suffix + ext
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+	dst := filepath.Join(filedir, "by-hash", hash)
+	sumfilePath := filepath.Join(dst, sum)
+
+	// link already exists? do nothing
+	exists, err := file.parent.publishedStorage.FileExists(sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err)
+	}
+	if exists {
+		return nil
+	}
+
+	// create the link
+	err = file.parent.publishedStorage.HardLink(src, sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
+	}
+
+	// if a previous index file already exists exists, backup symlink
+	if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)); exists {
+		// if exists, remove old symlink
+		if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")); exists {
+			var link string
+			link, err = file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old"))
+			if err != nil {
+				file.parent.publishedStorage.Remove(link)
+			}
+			file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old"))
+		}
+		file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile),
+			filepath.Join(dst, indexfile+".old"))
+	}
+
+	// create symlink
+	err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile))
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err)
+	}
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
 	return &indexFiles{
 		publishedStorage: publishedStorage,
 		basePath:         basePath,
@@ -152,6 +222,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 		tempDir:          tempDir,
 		suffix:           suffix,
 		indexes:          make(map[string]*indexFile),
+		acquireByHash:    acquireByHash,
 	}
 }
 
@@ -175,11 +246,12 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  false,
+			discardable:   false,
-			compressable: true,
+			compressable:  true,
-			signable:     false,
+			signable:      false,
-			relativePath: relativePath,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -208,11 +280,12 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  udeb,
+			discardable:   udeb,
-			compressable: false,
+			compressable:  false,
-			signable:     false,
+			signable:      false,
-			relativePath: relativePath,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -237,12 +310,13 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  true,
+			discardable:   true,
-			compressable: true,
+			compressable:  true,
-			onlyGzip:     true,
+			onlyGzip:      true,
-			signable:     false,
+			signable:      false,
-			relativePath: relativePath,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file

deb/publish.go

@@ -64,6 +64,9 @@ type PublishedRepo struct {
 
 	// True if repo is being re-published
 	rePublishing bool
+
+	// Provide index files per hash also
+	AcquireByHash bool
 }
 
 // ParsePrefix splits [storage:]prefix into components
@@ -556,7 +559,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	defer os.RemoveAll(tempDir)
 
-	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix)
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
 
 	for component, list := range lists {
 		hadUdebs := false
@@ -683,6 +686,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Component"] = component
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
+				if p.AcquireByHash {
+					release["Acquire-By-Hash"] = "yes"
+				}
 
 				var bufWriter *bufio.Writer
 				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
@@ -720,6 +726,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
+	if p.AcquireByHash {
+		release["Acquire-By-Hash"] = "yes"
+	}
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""

files/public.go

@@ -247,3 +247,27 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
 }
+
+// SymLink creates a symbolic link, which can be read with ReadLink
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	return os.Symlink(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// HardLink creates a hardlink of a file
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return os.Link(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	if _, err := os.Lstat(filepath.Join(storage.rootPath, path)); os.IsNotExist(err) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	return os.Readlink(path)
+}

files/public_test.go

@@ -103,6 +103,48 @@ func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, false)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	exists, _ = s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.SymLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestHardLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.HardLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)

man/aptly.1

@@ -1373,6 +1373,10 @@ $ aptly publish repo testing
 Options:
 .
 .TP
+\-\fBacquire\-by\-hash\fR
+provide index files by hash
+.
+.TP
 \-\fBbatch\fR
 run GPG with detached tty
 .
@@ -1468,6 +1472,10 @@ $ aptly publish snapshot wheezy\-main
 Options:
 .
 .TP
+\-\fBacquire\-by\-hash\fR
+provide index files by hash
+.
+.TP
 \-\fBbatch\fR
 run GPG with detached tty
 .

s3/public.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/request"
@@ -384,3 +385,71 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 	return storage.Remove(oldName)
 }
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+
+	params := &s3.CopyObjectInput{
+		Bucket:     aws.String(storage.bucket),
+		CopySource: aws.String(filepath.Join(storage.prefix, src)),
+		Key:        aws.String(filepath.Join(storage.prefix, dst)),
+		ACL:        aws.String(storage.acl),
+		Metadata: map[string]*string{
+			"SymLink": aws.String(src),
+		},
+		MetadataDirective: aws.String("REPLACE"),
+	}
+
+	if storage.storageClass != "" {
+		params.StorageClass = aws.String(storage.storageClass)
+	}
+	if storage.encryptionMethod != "" {
+		params.ServerSideEncryption = aws.String(storage.encryptionMethod)
+	}
+
+	_, err := storage.s3.CopyObject(params)
+	if err != nil {
+		return fmt.Errorf("error symlinking %s -> %s in %s: %s", src, dst, storage, err)
+	}
+
+	return err
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(storage.bucket),
+		Key:    aws.String(filepath.Join(storage.prefix, path)),
+	}
+	_, err := storage.s3.HeadObject(params)
+	if err != nil {
+		aerr, ok := err.(awserr.Error)
+		if ok && aerr.Code() == s3.ErrCodeNoSuchKey {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path.
+// This simply reads text file created with SymLink
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(storage.bucket),
+		Key:    aws.String(filepath.Join(storage.prefix, path)),
+	}
+	output, err := storage.s3.HeadObject(params)
+	if err != nil {
+		return "", err
+	}
+
+	return aws.StringValue(output.Metadata["SymLink"]), nil
+}

s3/public_test.go

@@ -280,3 +280,30 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 }
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.SymLink("a/b", "a/b.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not available in s3test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	exists, err := s.storage.FileExists("a/b")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, _ = s.storage.FileExists("a/b.invalid")
+	// Comment out as there is an error in s3test implementation
+	// c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

swift/public.go

@@ -264,3 +264,49 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 	return nil
 }
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	srcObjectName := filepath.Join(storage.prefix, src)
+	dstObjectName := filepath.Join(storage.prefix, dst)
+
+	headers := map[string]string{
+		"SymLink": srcObjectName,
+	}
+	_, err := storage.conn.ObjectCopy(storage.container, srcObjectName, storage.container, dstObjectName, headers)
+	if err != nil {
+		return fmt.Errorf("error symlinking %s -> %s in %s: %s", srcObjectName, dstObjectName, storage, err)
+	}
+	return err
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	_, _, err := storage.conn.Object(storage.container, filepath.Join(storage.prefix, path))
+
+	if err != nil {
+		if err == swift.ObjectNotFound {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	srcObjectName := filepath.Join(storage.prefix, path)
+	_, headers, err := storage.conn.Object(storage.container, srcObjectName)
+	if err != nil {
+		return "", fmt.Errorf("error reading symlink %s in %s: %s", srcObjectName, storage, err)
+	}
+
+	return headers["SymLink"], nil
+}

swift/public_test.go

@@ -194,3 +194,39 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Check(err, IsNil)
 	c.Check(data, DeepEquals, []byte("Spam"))
 }
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.SymLink("a/b.txt", "a/b.txt.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.txt.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not availbale in s3test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	exists, err := s.storage.FileExists("a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, err = s.storage.FileExists("a/b.invalid")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

system/run.py

@@ -99,7 +99,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
         print "\nFAILURES (%d):" % (len(fails), )
 
         for (test, t, typ, val, tb, testModule) in fails:
-            print "%s:%s %s" % (test, t.__class__.__name__, testModule.__name__ + ": " + t.__doc__.strip())
+            doc = t.__doc__ or ''
+            print "%s:%s %s" % (test, t.__class__.__name__, testModule.__name__ + ": " + doc.strip())
             traceback.print_exception(typ, val, tb)
             print "=" * 60
 

system/t06_publish/snapshot.py

@@ -219,7 +219,7 @@ class PublishSnapshot5Test(BaseTest):
     fixtureCmds = [
         "aptly snapshot create snap5 from mirror gnuplot-maverick",
     ]
-    runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira"
+    runCmd = "aptly publish snapshot -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira"
 
     gold_processor = BaseTest.expand_environ
 
@@ -231,12 +231,15 @@ class PublishSnapshot5Test(BaseTest):
         self.check_exists('public/ppa/smira/dists/squeeze/Release.gpg')
 
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages')
+        self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/e98cd30fc76fbe7fa3ea25717efa1c92')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.bz2')
-        self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-i386.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages')
+        self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/ab073d1f73bed52e7356c91161e8667e')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.bz2')
+
+        self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-i386.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-amd64.gz')
 
         self.check_exists('public/ppa/smira/pool/main/g/gnuplot/gnuplot-doc_4.6.1-1~maverick2_all.deb')

system/t06_publish/update.py

@@ -95,7 +95,7 @@ class PublishUpdate2Test(BaseTest):
     fixtureCmds = [
         "aptly repo create local-repo",
         "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb ${files}/pyspi_0.6.1-1.3.dsc",
-        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly publish repo -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
         "aptly repo add local-repo ${files}/pyspi-0.6.1-1.3.stripped.dsc"
     ]
     runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick"
@@ -113,8 +113,14 @@ class PublishUpdate2Test(BaseTest):
         self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
         self.check_exists('public/dists/maverick/main/Contents-i386.gz')
         self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.old')
         self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.gz.old')
         self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.bz2')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.bz2.old')
 
         self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
         self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')