Files
aptly/s3/public_test.go
T
Martin Simon 563e4eab2e s3: match the SymLink metadata key case-insensitively in ReadLink
S3 lowercases user metadata keys. SymLink writes "SymLink" and ReadLink
looks up output.Metadata["SymLink"], but the value comes back under
"symlink", so the lookup misses and ReadLink returns an empty string with
a nil error. To the caller that reads as "this object is not a link"
rather than as a failure.

deb.packageIndexByHash uses ReadLink to find the physical file its .old
pointer refers to, and removes it before rotating:

    linkTarget, err = publishedStorage.ReadLink(oldIndexPath)
    if err == nil {
        _ = publishedStorage.Remove(linkTarget)
    }

With "" returned the Remove is a no-op and its error is discarded, so the
superseded by-hash index file is never deleted. The pointers rotate
correctly and the physical files accumulate: one per index file per hash
algorithm per publish, indefinitely. Filesystem backends are unaffected
because they have real symlinks.

Measured on a live R2-backed archive publishing three suites a few times
a day: dists/ held 711 objects, of which 672 were by-hash and only 39
were current indexes, growing by about 48 objects per publish since
Acquire-By-Hash was enabled.

The existing TestSymLink cannot catch this, because it ends in
c.Skip("copy not available in s3test") and never runs. The regression
test added here writes the metadata with PutObject instead, so it needs
no CopyObject support, and fails before this change with
obtained string = "".
2026-08-31 11:17:35 +02:00

452 lines
16 KiB
Go

package s3
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"sort"
"strings"
. "gopkg.in/check.v1"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
)
type PublishedStorageSuite struct {
srv *Server
storage, prefixedStorage *PublishedStorage
noSuchBucketStorage *PublishedStorage
}
var _ = Suite(&PublishedStorageSuite{})
func (s *PublishedStorageSuite) SetUpTest(c *C) {
var err error
s.srv, err = NewServer(&Config{})
c.Assert(err, IsNil)
c.Assert(s.srv, NotNil)
s.storage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "test", "", "", "", "", false, true, false, false, false)
c.Assert(err, IsNil)
s.prefixedStorage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "test", "", "lala", "", "", false, true, false, false, false)
c.Assert(err, IsNil)
s.noSuchBucketStorage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "no-bucket", "", "", "", "", false, true, false, false, false)
c.Assert(err, IsNil)
_, err = s.storage.s3.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String("test"),
CreateBucketConfiguration: &types.CreateBucketConfiguration{
LocationConstraint: "test-1",
}})
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) TearDownTest(c *C) {
s.srv.Quit()
}
func (s *PublishedStorageSuite) checkGetRequestsEqual(c *C, prefix string, expectedGetRequestUris []string) {
getRequests := make([]string, 0, len(s.srv.Requests))
for _, r := range s.srv.Requests {
if r.Method == "GET" && strings.HasPrefix(r.RequestURI, prefix) {
getRequests = append(getRequests, r.RequestURI)
}
}
sort.Strings(getRequests)
c.Check(getRequests, DeepEquals, expectedGetRequestUris)
}
func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
resp, err := s.storage.s3.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(s.storage.bucket),
Key: aws.String(path),
})
c.Assert(err, IsNil)
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
c.Assert(err, IsNil)
return body
}
func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
_, err := s.storage.s3.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(s.storage.bucket),
Key: aws.String(path),
})
c.Assert(err, ErrorMatches, ".*StatusCode: 404.*")
}
func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
_, err := s.storage.s3.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(s.storage.bucket),
Key: aws.String(path),
Body: bytes.NewReader(data),
ContentType: aws.String("binary/octet-stream"),
ACL: types.ObjectCannedACLPrivate,
})
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) TestPutFile(c *C) {
dir := c.MkDir()
err := os.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644)
c.Assert(err, IsNil)
err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
c.Check(err, IsNil)
c.Check(s.GetFile(c, "a/b.txt"), DeepEquals, []byte("welcome to s3!"))
err = s.prefixedStorage.PutFile("a/b.txt", filepath.Join(dir, "a"))
c.Check(err, IsNil)
c.Check(s.GetFile(c, "lala/a/b.txt"), DeepEquals, []byte("welcome to s3!"))
}
func (s *PublishedStorageSuite) TestPutFilePlusWorkaround(c *C) {
s.storage.plusWorkaround = true
dir := c.MkDir()
err := os.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644)
c.Assert(err, IsNil)
err = s.storage.PutFile("a/b+c.txt", filepath.Join(dir, "a"))
c.Check(err, IsNil)
c.Check(s.GetFile(c, "a/b+c.txt"), DeepEquals, []byte("welcome to s3!"))
c.Check(s.GetFile(c, "a/b c.txt"), DeepEquals, []byte("welcome to s3!"))
}
func (s *PublishedStorageSuite) TestFilelist(c *C) {
paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
for _, path := range paths {
s.PutFile(c, path, []byte("test"))
}
list, err := s.storage.Filelist("")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
list, err = s.storage.Filelist("test")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a", "b"})
list, err = s.storage.Filelist("test2")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
list, err = s.prefixedStorage.Filelist("")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a", "b", "c"})
}
func (s *PublishedStorageSuite) TestFilelistPlusWorkaround(c *C) {
s.storage.plusWorkaround = true
s.prefixedStorage.plusWorkaround = true
paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
for _, path := range paths {
s.PutFile(c, path, []byte("test"))
}
list, err := s.storage.Filelist("")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "test/a+1", "testa"})
list, err = s.storage.Filelist("test")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a+1"})
list, err = s.storage.Filelist("test2")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
list, err = s.prefixedStorage.Filelist("")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a+b", "c"})
}
func (s *PublishedStorageSuite) TestRemove(c *C) {
s.PutFile(c, "a/b", []byte("test"))
err := s.storage.Remove("a/b")
c.Check(err, IsNil)
s.AssertNoFile(c, "a/b")
s.PutFile(c, "lala/xyz", []byte("test"))
errp := s.prefixedStorage.Remove("xyz")
c.Check(errp, IsNil)
s.AssertNoFile(c, "lala/xyz")
}
func (s *PublishedStorageSuite) TestRemoveNoSuchBucket(c *C) {
err := s.noSuchBucketStorage.Remove("a/b")
c.Check(err, IsNil)
}
func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) {
s.storage.plusWorkaround = true
s.PutFile(c, "a/b+c", []byte("test"))
s.PutFile(c, "a/b", []byte("test"))
err := s.storage.Remove("a/b+c")
c.Check(err, IsNil)
s.AssertNoFile(c, "a/b+c")
s.AssertNoFile(c, "a/b c")
err = s.storage.Remove("a/b")
c.Check(err, IsNil)
s.AssertNoFile(c, "a/b")
}
func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
s.storage.plusWorkaround = true
paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
for _, path := range paths {
s.PutFile(c, path, []byte("test"))
}
err := s.storage.RemoveDirs("test", nil)
c.Check(err, IsNil)
list, err := s.storage.Filelist("")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "testa"})
}
func (s *PublishedStorageSuite) TestRemoveDirsPlusWorkaround(c *C) {
paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
for _, path := range paths {
s.PutFile(c, path, []byte("test"))
}
err := s.storage.RemoveDirs("test", nil)
c.Check(err, IsNil)
list, err := s.storage.Filelist("")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
}
func (s *PublishedStorageSuite) TestRemoveDirsNoSuchBucket(c *C) {
err := s.noSuchBucketStorage.RemoveDirs("a/b", nil)
c.Check(err, ErrorMatches, ".*StatusCode: 404.*")
}
func (s *PublishedStorageSuite) TestRenameFile(c *C) {
c.Skip("copy not available in s3test")
}
func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
root := c.MkDir()
pool := files.NewPackagePool(root, false)
cs := files.NewMockChecksumStorage()
tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
err := os.WriteFile(tmpFile1, []byte("Contents"), 0644)
c.Assert(err, IsNil)
cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
err = os.WriteFile(tmpFile2, []byte("Spam"), 0644)
c.Assert(err, IsNil)
cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
_ = os.MkdirAll(filepath.Dir(tmpFile3), 0777)
err = os.WriteFile(tmpFile3, []byte("Contents"), 0644)
c.Assert(err, IsNil)
cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
c.Assert(err, IsNil)
src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
c.Assert(err, IsNil)
src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
c.Assert(err, IsNil)
// first link from pool
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// duplicate link from pool
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with conflict
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
c.Check(err, ErrorMatches, ".*file already exists and is different.*")
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with conflict and force
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
// for prefixed storage:
// first link from pool
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
// 2nd link from pool, providing wrong path for source file
//
// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with nested file name
err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
}
func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) {
root := c.MkDir()
pool := files.NewPackagePool(root, false)
cs := files.NewMockChecksumStorage()
tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
err := os.WriteFile(tmpFile1, []byte("Contents"), 0644)
c.Assert(err, IsNil)
cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
c.Assert(err, IsNil)
// Publish two packages at the same publish prefix
err = s.storage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
err = s.storage.LinkFromPool("", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
// Check only one listing request was done to the server
s.checkGetRequestsEqual(c, "/test?", []string{"/test?encryption=", "/test?encryption=", "/test?list-type=2&max-keys=1000&prefix=pool%2F"})
s.srv.Requests = nil
// Publish two packages at a different prefix
err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
// Check no listing request was done to the server (pathCache is used)
s.checkGetRequestsEqual(c, "/test?", []string{})
s.srv.Requests = nil
// Publish two packages at a prefixed storage
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
// Check only one listing request was done to the server
s.checkGetRequestsEqual(c, "/test?", []string{
"/test?list-type=2&max-keys=1000&prefix=lala%2Fpool%2F",
})
// Publish two packages at a prefixed storage plus a publish prefix.
s.srv.Requests = nil
err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
// Check no listing request was done to the server (pathCache is used)
s.checkGetRequestsEqual(c, "/test?", []string{})
// This step checks that files already exists in S3 and skip upload (which would fail if not skipped).
err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)
c.Check(err, IsNil)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)
c.Check(err, IsNil)
err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)
c.Check(err, IsNil)
err = s.storage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)
c.Check(err, IsNil)
}
func (s *PublishedStorageSuite) TestSymLink(c *C) {
s.PutFile(c, "a/b", []byte("test"))
err := s.storage.SymLink("a/b", "a/b.link")
c.Check(err, IsNil)
var link string
link, err = s.storage.ReadLink("a/b.link")
c.Check(err, IsNil)
c.Check(link, Equals, "a/b")
c.Skip("copy not available in s3test")
}
func (s *PublishedStorageSuite) TestReadLinkMetadataKeyIsCaseInsensitive(c *C) {
// S3 lowercases user metadata keys, so a value written as "SymLink" comes
// back as "symlink". A case-sensitive map lookup therefore returns "" with
// a nil error, which reads as "this is not a link" rather than as failure.
//
// deb.packageIndexByHash relies on ReadLink to find the physical file its
// .old pointer refers to, and removes it before rotating. With "" returned
// the Remove is a no-op whose error is discarded, so every superseded
// by-hash index file is left behind. On a live S3-backed archive that is
// one leaked object per index per hash algorithm per publish, forever;
// filesystem backends are unaffected because they have real symlinks.
//
// Written with PutObject rather than SymLink because the test server has
// no CopyObject, which is why TestSymLink above is skipped and why this
// path was never covered.
_, err := s.storage.s3.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String("test"),
Key: aws.String("a/b.link"),
Body: bytes.NewReader([]byte("test")),
Metadata: map[string]string{"symlink": "a/b"},
})
c.Assert(err, IsNil)
link, err := s.storage.ReadLink("a/b.link")
c.Check(err, IsNil)
c.Check(link, Equals, "a/b")
}
func (s *PublishedStorageSuite) TestFileExists(c *C) {
s.PutFile(c, "a/b", []byte("test"))
exists, err := s.storage.FileExists("a/b")
c.Check(err, IsNil)
c.Check(exists, Equals, true)
exists, _ = s.storage.FileExists("a/b.invalid")
// Comment out as there is an error in s3test implementation
// c.Check(err, IsNil)
c.Check(exists, Equals, false)
}