vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-01-11 03:11:50 +00:00
Code Issues Projects Releases Wiki Activity

Store MD5 in a separate metadata field as well

Browse Source 
The S3 backend relies on ETag S3 returns being equal to the MD5 of the
object, but it’s not necessarily true. For that purpose we store the MD5
object in a separate metadata field as well to make sure it isn’t lost.

From https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html:

> The entity tag is a hash of the object. The ETag reflects changes only
> to the contents of an object, not its metadata. The ETag may or may not
> be an MD5 digest of the object data. Whether or not it depends on how
> the object was created and how it is encrypted as described below:
>
> Objects created by the PUT Object, POST Object, or Copy operation,
> or through the AWS Management Console, and are encrypted by SSE-S3 or
> plaintext, have ETags that are an MD5 digest of their object data.
>
> Objects created by the PUT Object, POST Object, or Copy operation,
> or through the AWS Management Console, and are encrypted by SSE-C or
> SSE-KMS, have ETags that are not an MD5 digest of their object data.
>
> If an object is created by either the Multipart Upload or Part Copy
> operation, the ETag is not an MD5 digest, regardless of the method
> of encryption.

Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
This commit is contained in:
Andrej Shadura
2020-07-22 19:36:36 +02:00
committed by Lorenzo Bolla
parent af2564c580
commit 960cf76c42
1 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
s3/public.go
View File

@@ -142,7 +142,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
}
defer source.Close()
err = storage.putFile(path, source)
err = storage.putFile(path, source, "")
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
}
@@ -151,7 +151,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
}
// putFile uploads file-like object to
func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) error {
func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker, sourceMD5 string) error {
params := &s3.PutObjectInput{
Bucket: aws.String(storage.bucket),
@@ -165,6 +165,11 @@ func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) erro
if storage.encryptionMethod != "" {
params.ServerSideEncryption = aws.String(storage.encryptionMethod)
}
if sourceMD5 != "" {
params.Metadata = map[string]*string{
"Md5": aws.String(sourceMD5),
}
}
_, err := storage.s3.PutObject(params)
if err != nil {
@@ -177,7 +182,7 @@ func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) erro
return err
}
return storage.putFile(strings.Replace(path, "+", " ", -1), source)
return storage.putFile(strings.Replace(path, "+", " ", -1), source, sourceMD5)
}
return nil
}
@@ -318,7 +323,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
}
defer source.Close()
err = storage.putFile(relPath, source)
err = storage.putFile(relPath, source, sourceMD5)
if err == nil {
storage.pathCache[relPath] = sourceMD5
} else {