Update Go AWS SDK to the latest version

This commit is contained in:
Andrey Smirnov
2019-07-13 00:03:55 +03:00
committed by Andrey Smirnov
parent d08be990ef
commit 94a72b23ff
2183 changed files with 885887 additions and 228114 deletions
@@ -0,0 +1,22 @@
# Example
This is an example using the AWS SDK for Go to download an S3 object with a
progress bar.
# Usage
The example uses the bucket name provided, one key for object, and output the
progress to stdout.
```prompt
AWS_PROFILE=my-profile AWS_REGION=us-west-2 go run -tags example getObjectWithProgress.go cool-bucket my/object/prefix/cool_thing.zip
2019/02/22 13:04:52 File size is: 35.9 MB
2019/02/22 13:04:53 File size:35943530 downloaded:8580 percentage:0%
2019/02/22 13:04:53 File size:35943530 downloaded:17580 percentage:0%
2019/02/22 13:04:53 File size:35943530 downloaded:33940 percentage:0%
2019/02/22 13:04:53 File size:35943530 downloaded:34988 percentage:0%
2019/02/22 13:04:53 File size:35943530 downloaded:51348 percentage:0%
2019/02/22 13:04:53 File size:35943530 downloaded:52396 percentage:0%
...
```
@@ -0,0 +1,133 @@
// +build example
package main
import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strings"
"sync/atomic"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
// progressWriter tracks the download progress of a file from S3 to a file
// as the writeAt method is called, the byte size is added to the written total,
// and then a log is printed of the written percentage from the total size
// it looks like this on the command line:
// 2019/02/22 12:59:15 File size:35943530 downloaded:16360 percentage:0%
// 2019/02/22 12:59:15 File size:35943530 downloaded:16988 percentage:0%
// 2019/02/22 12:59:15 File size:35943530 downloaded:33348 percentage:0%
type progressWriter struct {
written int64
writer io.WriterAt
size int64
}
func (pw *progressWriter) WriteAt(p []byte, off int64) (int, error) {
atomic.AddInt64(&pw.written, int64(len(p)))
percentageDownloaded := float32(pw.written*100) / float32(pw.size)
fmt.Printf("File size:%d downloaded:%d percentage:%.2f%%\r", pw.size, pw.written, percentageDownloaded)
return pw.writer.WriteAt(p, off)
}
func byteCountDecimal(b int64) string {
const unit = 1000
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
}
func getFileSize(svc *s3.S3, bucket string, prefix string) (filesize int64, error error) {
params := &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(prefix),
}
resp, err := svc.HeadObject(params)
if err != nil {
return 0, err
}
return *resp.ContentLength, nil
}
func parseFilename(keyString string) (filename string) {
ss := strings.Split(keyString, "/")
s := ss[len(ss)-1]
return s
}
func main() {
if len(os.Args) < 2 {
log.Println("USAGE ERROR: AWS_REGION=us-east-1 go run getObjWithProgress.go bucket-name object-key")
return
}
bucket := os.Args[1]
key := os.Args[2]
filename := parseFilename(key)
sess, err := session.NewSession()
if err != nil {
panic(err)
}
s3Client := s3.New(sess)
downloader := s3manager.NewDownloader(sess)
size, err := getFileSize(s3Client, bucket, key)
if err != nil {
panic(err)
}
log.Println("Starting download, size:", byteCountDecimal(size))
cwd, err := os.Getwd()
if err != nil {
panic(err)
}
temp, err := ioutil.TempFile(cwd, "getObjWithProgress-tmp-")
if err != nil {
panic(err)
}
tempfileName := temp.Name()
writer := &progressWriter{writer: temp, size: size, written: 0}
params := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
if _, err := downloader.Download(writer, params); err != nil {
log.Printf("Download failed! Deleting tempfile: %s", tempfileName)
os.Remove(tempfileName)
panic(err)
}
if err := temp.Close(); err != nil {
panic(err)
}
if err := os.Rename(temp.Name(), filename); err != nil {
panic(err)
}
fmt.Println()
log.Println("File downloaded! Avaliable at:", filename)
}