New upstream version 1.1.1

s3/public.go

@@ -2,6 +2,7 @@ package s3
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -12,8 +13,9 @@ import (
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/pkg/errors"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/go-aws-auth"
 )
 
@@ -50,7 +52,10 @@ func NewPublishedStorageRaw(
 		storageClass = ""
 	}
 
-	sess := session.New(config)
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, err
+	}
 
 	result := &PublishedStorage{
 		s3:               s3.New(sess),
@@ -134,6 +139,17 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}
 	defer source.Close()
 
+	err = storage.putFile(path, source)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
+	}
+
+	return err
+}
+
+// putFile uploads file-like object to
+func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) error {
+
 	params := &s3.PutObjectInput{
 		Bucket: aws.String(storage.bucket),
 		Key:    aws.String(filepath.Join(storage.prefix, path)),
@@ -147,13 +163,18 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 		params.ServerSideEncryption = aws.String(storage.encryptionMethod)
 	}
 
-	_, err = storage.s3.PutObject(params)
+	_, err := storage.s3.PutObject(params)
 	if err != nil {
-		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
+		return err
 	}
 
-	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
-		return storage.PutFile(strings.Replace(path, "+", " ", -1), sourceFilename)
+	if storage.plusWorkaround && strings.Contains(path, "+") {
+		_, err = source.Seek(0, 0)
+		if err != nil {
+			return err
+		}
+
+		return storage.putFile(strings.Replace(path, "+", " ", -1), source)
 	}
 	return nil
 }
@@ -166,10 +187,10 @@ func (storage *PublishedStorage) Remove(path string) error {
 	}
 	_, err := storage.s3.DeleteObject(params)
 	if err != nil {
-		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
+		return errors.Wrap(err, fmt.Sprintf("error deleting %s from %s", path, storage))
 	}
 
-	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
+	if storage.plusWorkaround && strings.Contains(path, "+") {
 		// try to remove workaround version, but don't care about result
 		_ = storage.Remove(strings.Replace(path, "+", " ", -1))
 	}
@@ -239,23 +260,16 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
-	sourcePath, sourceMD5 string, force bool) error {
-	// verify that package pool is local pool in filesystem
-	_ = sourcePool.(*files.PackagePool)
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool,
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	baseName := filepath.Base(sourcePath)
 	relPath := filepath.Join(publishedDirectory, baseName)
 	poolPath := filepath.Join(storage.prefix, relPath)
 
-	var (
-		err error
-	)
-
 	if storage.pathCache == nil {
 		paths, md5s, err := storage.internalFilelist(storage.prefix, true)
 		if err != nil {
-			return fmt.Errorf("error caching paths under prefix: %s", err)
+			return errors.Wrap(err, "error caching paths under prefix")
 		}
 
 		storage.pathCache = make(map[string]string, len(paths))
@@ -266,8 +280,13 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 	}
 
 	destinationMD5, exists := storage.pathCache[relPath]
+	sourceMD5 := sourceChecksums.MD5
 
 	if exists {
+		if sourceMD5 == "" {
+			return fmt.Errorf("unable to compare object, MD5 checksum missing")
+		}
+
 		if destinationMD5 == sourceMD5 {
 			return nil
 		}
@@ -278,9 +297,17 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 		}
 	}
 
-	err = storage.PutFile(relPath, sourcePath)
+	source, err := sourcePool.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.putFile(relPath, source)
 	if err == nil {
 		storage.pathCache[relPath] = sourceMD5
+	} else {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
 	}
 
 	return err
@@ -308,7 +335,7 @@ func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaro
 
 	err = storage.s3.ListObjectsPages(params, func(contents *s3.ListObjectsOutput, lastPage bool) bool {
 		for _, key := range contents.Contents {
-			if storage.plusWorkaround && hidePlusWorkaround && strings.Index(*key.Key, " ") != -1 {
+			if storage.plusWorkaround && hidePlusWorkaround && strings.Contains(*key.Key, " ") {
 				// if we use plusWorkaround, we want to hide those duplicates
 				/// from listing
 				continue

s3/public_test.go

@@ -3,7 +3,6 @@ package s3
 import (
 	"bytes"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 
 	. "gopkg.in/check.v1"
@@ -12,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 
 	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/utils"
 )
 
 type PublishedStorageSuite struct {
@@ -216,42 +216,44 @@ func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 
 func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	root := c.MkDir()
-	pool := files.NewPackagePool(root)
+	pool := files.NewPackagePool(root, false)
+	cs := files.NewMockChecksumStorage()
 
-	sourcePath := filepath.Join(root, "pool/c1/df/mars-invaders_1.03.deb")
-	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
 	c.Assert(err, IsNil)
+	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
 
-	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
 	c.Assert(err, IsNil)
+	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
 
-	sourcePath2 := filepath.Join(root, "pool/e9/df/mars-invaders_1.03.deb")
-	err = os.MkdirAll(filepath.Dir(sourcePath2), 0755)
+	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
 	c.Assert(err, IsNil)
-
-	err = ioutil.WriteFile(sourcePath2, []byte("Spam"), 0644)
+	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
 	c.Assert(err, IsNil)
 
 	// first link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// duplicate link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", false)
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict and force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", true)
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))

s3/server_test.go

@@ -646,7 +646,7 @@ func (objr objectResource) put(a *action) interface{} {
 		fatalError(400, "TODO", "read error")
 	}
 	gotHash := sum.Sum(nil)
-	if expectHash != nil && bytes.Compare(gotHash, expectHash) != 0 {
+	if expectHash != nil && !bytes.Equal(gotHash, expectHash) {
 		fatalError(400, "BadDigest", "The Content-MD5 you specified did not match what we received")
 	}
 	if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength {