s3: fix test

s3/public_test.go

@@ -6,6 +6,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 
 	. "gopkg.in/check.v1"
 
@@ -50,6 +52,17 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 	s.srv.Quit()
 }
 
+func (s *PublishedStorageSuite) checkGetRequestsEqual(c *C, prefix string, expectedGetRequestUris []string) {
+	getRequests := make([]string, 0, len(s.srv.Requests))
+	for _, r := range s.srv.Requests {
+		if r.Method == "GET" && strings.HasPrefix(r.RequestURI, prefix) {
+			getRequests = append(getRequests, r.RequestURI)
+		}
+	}
+	sort.Strings(getRequests)
+	c.Check(getRequests, DeepEquals, expectedGetRequestUris)
+}
+
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
 	resp, err := s.storage.s3.GetObject(context.TODO(), &s3.GetObjectInput{
 		Bucket: aws.String(s.storage.bucket),
@@ -301,7 +314,6 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	// 2nd link from pool, providing wrong path for source file
 	//
 	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
-	s.prefixedStorage.pathCache = nil
 	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)
 
@@ -335,7 +347,7 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) {
 	c.Check(err, IsNil)
 
 	// Check only one listing request was done to the server
-	s.checkGetRequestsEqual(c, "/test?", []string{"/test?max-keys=1000&prefix="})
+	s.checkGetRequestsEqual(c, "/test?", []string{"/test?list-type=2&max-keys=1000&prefix=pool%2F"})
 
 	s.srv.Requests = nil
 	// Publish two packages at a different prefix
@@ -345,10 +357,8 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) {
 	err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
-	// Check only one listing request was done to the server
-	s.checkGetRequestsEqual(c, "/test?", []string{
-		"/test?max-keys=1000&prefix=publish-prefix%2F",
-	})
+	// Check no listing request was done to the server (pathCache is used)
+	s.checkGetRequestsEqual(c, "/test?", []string{})
 
 	s.srv.Requests = nil
 	// Publish two packages at a prefixed storage
@@ -360,24 +370,21 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) {
 
 	// Check only one listing request was done to the server
 	s.checkGetRequestsEqual(c, "/test?", []string{
-		"/test?max-keys=1000&prefix=lala%2F",
+		"/test?list-type=2&max-keys=1000&prefix=lala%2Flala%2Fpool%2F",
 	})
 
-	s.srv.Requests = nil
 	// Publish two packages at a prefixed storage plus a publish prefix.
+	s.srv.Requests = nil
 	err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
-	// Check only one listing request was done to the server
-	s.checkGetRequestsEqual(c, "/test?", []string{
-		"/test?max-keys=1000&prefix=lala%2Fpublish-prefix%2F",
-	})
+	// Check no listing request was done to the server (pathCache is used)
+	s.checkGetRequestsEqual(c, "/test?", []string{})
 
 	// This step checks that files already exists in S3 and skip upload (which would fail if not skipped).
-	s.prefixedStorage.pathCache = nil
 	err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)
 	c.Check(err, IsNil)
 	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)

s3/server_test.go

@@ -59,6 +59,12 @@ func (c *Config) send409Conflict() bool {
 	return false
 }
 
+// Request stores the method and URI of an HTTP request.
+type Request struct {
+	Method     string
+	RequestURI string
+}
+
 // Server is a fake S3 server for testing purposes.
 // All of the data for the server is kept in memory.
 type Server struct {
@@ -68,6 +74,8 @@ type Server struct {
 	mu       sync.Mutex
 	buckets  map[string]*bucket
 	config   *Config
+	// Requests holds a log of all requests received by the server.
+	Requests []Request
 }
 
 type bucket struct {
@@ -140,6 +148,7 @@ func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) {
 	if debug {
 		log.Printf("s3test %q %q", req.Method, req.URL)
 	}
+	srv.Requests = append(srv.Requests, Request{req.Method, req.RequestURI})
 	a := &action{
 		srv:   srv,
 		w:     w,
@@ -330,6 +339,10 @@ type Owner struct {
 	DisplayName string
 }
 
+type CommonPrefix struct {
+	Prefix string
+}
+
 // The ListResp type holds the results of a List bucket operation.
 type ListResp struct {
 	Name       string
@@ -344,7 +357,7 @@ type ListResp struct {
 	// http://goo.gl/YjQTc
 	IsTruncated    bool
 	Contents       []Key
-	CommonPrefixes []string `xml:">Prefix"`
+	CommonPrefixes []CommonPrefix
 }
 
 // The Key type represents an item stored in an S3 bucket.
@@ -403,7 +416,7 @@ func (r bucketResource) get(a *action) interface{} {
 		MaxKeys:   maxKeys,
 	}
 
-	var prefixes []string
+	var prefixes []CommonPrefix
 	for _, obj := range objs {
 		if !strings.HasPrefix(obj.name, prefix) {
 			continue
@@ -413,7 +426,7 @@ func (r bucketResource) get(a *action) interface{} {
 		if delimiter != "" {
 			if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 {
 				name = obj.name[:len(prefix)+i+len(delimiter)]
-				if prefixes != nil && prefixes[len(prefixes)-1] == name {
+				if prefixes != nil && prefixes[len(prefixes)-1].Prefix == name {
 					continue
 				}
 				isPrefix = true
@@ -427,7 +440,7 @@ func (r bucketResource) get(a *action) interface{} {
 			break
 		}
 		if isPrefix {
-			prefixes = append(prefixes, name)
+			prefixes = append(prefixes, CommonPrefix{name})
 		} else {
 			// Contents contains only keys not found in CommonPrefixes
 			resp.Contents = append(resp.Contents, obj.s3Key())