Fix s3 etag issue (Closes: #983877)

2026-06-15 07:00:52 +00:00 · 2021-03-03 10:46:05 +01:00
parent b61875fa9c
commit ea9657dae0
2 changed files with 107 additions and 0 deletions
@@ -0,0 +1,106 @@
+Description: fix etag issue with s3 backend
+ The S3 backend relies on ETag S3 returns being equal to the MD5 of
+ the object, but it’s not necessarily true. For that purpose we store
+ the MD5 object in a separate metadata field as well to make sure it
+ isn’t lost. When the value returned clearly doesn’t look like a valid
+ MD5 hash (length isn’t exactly 32 characters), attempt to retrieve
+ the MD5 hash possibly stored in the metadata.
+ .
+  We cannot always do this since user-defined metadata isn’t returned
+  by the ListObjects call, so verifying it for each object is
+  expensive as it requires one HEAD request per each object.
+Author:  Andrej Shadura <andrew.shadura@collabora.co.uk>
+Origin: upstream
+Bug: https://github.com/aptly-dev/aptly/issues/923
+Bug-Debian: 983877
+Applied-Upstream: commit:2422d3a
+Reviewed-by: Sebastien Delafond <sdelafond@gmail.com>
+Last-Update: 2021-03-02
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+---
+diff --git a/s3/public.go b/s3/public.go
+index a3e85a99..d6c161ad 100644
+--- a/s3/public.go
+++ b/s3/public.go
+@@ -142,7 +142,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
+ 	}
+ 	defer source.Close()
+ 
+-	err = storage.putFile(path, source)
+	err = storage.putFile(path, source, "")
+ 	if err != nil {
+ 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
+ 	}
+@@ -150,8 +150,22 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
+ 	return err
+ }
+ 
+// getMD5 retrieves MD5 stored in the metadata, if any
+func (storage *PublishedStorage) getMD5(path string) (string, error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(storage.bucket),
+		Key:    aws.String(filepath.Join(storage.prefix, path)),
+	}
+	output, err := storage.s3.HeadObject(params)
+	if err != nil {
+		return "", err
+	}
+
+	return aws.StringValue(output.Metadata["Md5"]), nil
+}
+
+ // putFile uploads file-like object to
+-func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) error {
+func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker, sourceMD5 string) error {
+ 
+ 	params := &s3.PutObjectInput{
+ 		Bucket: aws.String(storage.bucket),
+@@ -165,6 +179,11 @@ func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) erro
+ 	if storage.encryptionMethod != "" {
+ 		params.ServerSideEncryption = aws.String(storage.encryptionMethod)
+ 	}
+	if sourceMD5 != "" {
+		params.Metadata = map[string]*string{
+			"Md5": aws.String(sourceMD5),
+		}
+	}
+ 
+ 	_, err := storage.s3.PutObject(params)
+ 	if err != nil {
+@@ -177,7 +196,7 @@ func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) erro
+ 			return err
+ 		}
+ 
+-		return storage.putFile(strings.Replace(path, "+", " ", -1), source)
+		return storage.putFile(strings.Replace(path, "+", " ", -1), source, sourceMD5)
+ 	}
+ 	return nil
+ }
+@@ -298,6 +317,17 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
+ 	sourceMD5 := sourceChecksums.MD5
+ 
+ 	if exists {
+		if len(destinationMD5) != 32 {
+			// doesn’t look like a valid MD5,
+			// attempt to fetch one from the metadata
+			var err error
+			destinationMD5, err = storage.getMD5(relPath)
+			if err != nil {
+				err = errors.Wrap(err, fmt.Sprintf("error verifying MD5 for %s: %s", storage, poolPath))
+				return err
+			}
+			storage.pathCache[relPath] = destinationMD5
+		}
+ 		if sourceMD5 == "" {
+ 			return fmt.Errorf("unable to compare object, MD5 checksum missing")
+ 		}
+@@ -318,7 +348,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
+ 	}
+ 	defer source.Close()
+ 
+-	err = storage.putFile(relPath, source)
+	err = storage.putFile(relPath, source, sourceMD5)
+ 	if err == nil {
+ 		storage.pathCache[relPath] = sourceMD5
+ 	} else {
@@ -1,2 +1,3 @@
 Fix-UUID-struct-field-not-encoded-in-msgpack.patch
 Fix-time.Time-msgpack-decoding-backwards-compatibili.patch
+s3-etag.patch