Update Go AWS SDK to the latest version

2026-05-31 04:30:44 +00:00 · 2019-07-13 00:03:55 +03:00
parent d08be990ef
commit 94a72b23ff
2183 changed files with 885887 additions and 228114 deletions
@@ -0,0 +1,22 @@
+# Example
+
+This is an example using the AWS SDK for Go to download an S3 object with a
+progress bar.
+
+# Usage
+
+The example uses the bucket name provided, one key for object, and output the
+progress to stdout.
+
+```prompt
+AWS_PROFILE=my-profile AWS_REGION=us-west-2 go run -tags example getObjectWithProgress.go cool-bucket my/object/prefix/cool_thing.zip
+
+2019/02/22 13:04:52 File size is: 35.9 MB
+2019/02/22 13:04:53 File size:35943530 downloaded:8580 percentage:0%
+2019/02/22 13:04:53 File size:35943530 downloaded:17580 percentage:0%
+2019/02/22 13:04:53 File size:35943530 downloaded:33940 percentage:0%
+2019/02/22 13:04:53 File size:35943530 downloaded:34988 percentage:0%
+2019/02/22 13:04:53 File size:35943530 downloaded:51348 percentage:0%
+2019/02/22 13:04:53 File size:35943530 downloaded:52396 percentage:0%
+...
+```
@@ -0,0 +1,133 @@
+// +build example
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"sync/atomic"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+// progressWriter tracks the download progress of a file from S3 to a file
+// as the writeAt method is called, the byte size is added to the written total,
+// and then a log is printed of the written percentage from the total size
+// it looks like this on the command line:
+//  2019/02/22 12:59:15 File size:35943530 downloaded:16360 percentage:0%
+//  2019/02/22 12:59:15 File size:35943530 downloaded:16988 percentage:0%
+//  2019/02/22 12:59:15 File size:35943530 downloaded:33348 percentage:0%
+type progressWriter struct {
+	written int64
+	writer  io.WriterAt
+	size    int64
+}
+
+func (pw *progressWriter) WriteAt(p []byte, off int64) (int, error) {
+	atomic.AddInt64(&pw.written, int64(len(p)))
+
+	percentageDownloaded := float32(pw.written*100) / float32(pw.size)
+
+	fmt.Printf("File size:%d downloaded:%d percentage:%.2f%%\r", pw.size, pw.written, percentageDownloaded)
+
+	return pw.writer.WriteAt(p, off)
+}
+
+func byteCountDecimal(b int64) string {
+	const unit = 1000
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
+}
+
+func getFileSize(svc *s3.S3, bucket string, prefix string) (filesize int64, error error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(prefix),
+	}
+
+	resp, err := svc.HeadObject(params)
+	if err != nil {
+		return 0, err
+	}
+
+	return *resp.ContentLength, nil
+}
+
+func parseFilename(keyString string) (filename string) {
+	ss := strings.Split(keyString, "/")
+	s := ss[len(ss)-1]
+	return s
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		log.Println("USAGE ERROR: AWS_REGION=us-east-1 go run getObjWithProgress.go bucket-name object-key")
+		return
+	}
+
+	bucket := os.Args[1]
+	key := os.Args[2]
+
+	filename := parseFilename(key)
+
+	sess, err := session.NewSession()
+	if err != nil {
+		panic(err)
+	}
+
+	s3Client := s3.New(sess)
+	downloader := s3manager.NewDownloader(sess)
+	size, err := getFileSize(s3Client, bucket, key)
+	if err != nil {
+		panic(err)
+	}
+
+	log.Println("Starting download, size:", byteCountDecimal(size))
+	cwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+
+	temp, err := ioutil.TempFile(cwd, "getObjWithProgress-tmp-")
+	if err != nil {
+		panic(err)
+	}
+	tempfileName := temp.Name()
+
+	writer := &progressWriter{writer: temp, size: size, written: 0}
+	params := &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	}
+
+	if _, err := downloader.Download(writer, params); err != nil {
+		log.Printf("Download failed! Deleting tempfile: %s", tempfileName)
+		os.Remove(tempfileName)
+		panic(err)
+	}
+
+	if err := temp.Close(); err != nil {
+		panic(err)
+	}
+
+	if err := os.Rename(temp.Name(), filename); err != nil {
+		panic(err)
+	}
+
+	fmt.Println()
+	log.Println("File downloaded! Avaliable at:", filename)
+}
@@ -24,11 +24,11 @@ AWS credentials. See the [`Configuring Credentials`](http://docs.aws.amazon.com/
 section of the SDK's API Reference guide on how the SDK loads your AWS credentials.

 The server requires the S3 `-b bucket` the presigned URLs will be generated for. A
-`-r region` is only needed if the bucket is in AWS China or AWS Gov Cloud. For 
+`-r region` is only needed if the bucket is in AWS China or AWS Gov Cloud. For
 buckets in AWS the server will use the [`s3manager.GetBucketRegion`](http://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#GetBucketRegion) utility to lookup the bucket's region.

 You should run the service in the background or in a separate terminal tab before
-moving onto the client. 
+moving onto the client.


 ```sh
@@ -43,7 +43,7 @@ defaults.

 Use the client application to request a presigned URL from the server and use
 that presigned URL to download the object from S3. Calling the client with the
-`-get key` flag will do this. An optional `-f filename` flag can be provided as 
+`-get key` flag will do this. An optional `-f filename` flag can be provided as
 well to write the object to. If no flag is provided the object will be written
 to `stdout`

@@ -63,7 +63,7 @@ URL. The `method` value can be `GET` or `PUT` for the `GetObject` or `PutObject`
 curl -v "http://127.0.0.1:8080/presign/my-object/key?method=GET"
 ```

-The server will respond with a JSON value. The value contains three pieces of 
+The server will respond with a JSON value. The value contains three pieces of
 information that the client will need to correctly make the request. First is
 the presigned URL. This is the URL the client will make the request to. Second
 is the HTTP method the request should be sent as. This is included to simplify
@@ -97,7 +97,7 @@ service
 go run -tags example client/client.go -put "my-object/key" -f filename
 ```

-Like the download case this will make a HTTP request to the server for the 
+Like the download case this will make a HTTP request to the server for the
 presigned URL. The Server will respond with a presigned URL for S3's `PutObject`
 API operation. In addition the `method` query parameter the client will also
 include a `contentLength` this value instructs the server to generate the presigned
@@ -118,7 +118,7 @@ such as additional constraints the server puts on the presigned URLs like
 `Content-Type`.

 In addition to adding constraints to the presigned URLs the service could be
-updated to obfuscate S3 object's key. Instead of the client knowing the object's 
+updated to obfuscate S3 object's key. Instead of the client knowing the object's
 key, a lookup system could be used instead. This could be substitution based,
 or lookup into an external data store such as DynamoDB.

@@ -19,10 +19,10 @@ putBucketAcl <params>
 ```

 ```sh
-go run -tags example putObjectAcl.go 
-	-bucket <bucket> 
-	-key <key> 
-	-owner-name <name> 
+go run -tags example putObjectAcl.go
+	-bucket <bucket>
+	-key <key>
+	-owner-name <name>
 	-owner-id <id>
 	-grantee-type <some type>
 	-user-id <user-id>
@@ -0,0 +1,14 @@
+# Example
+
+This is an example using the AWS SDK for Go to upload object with progress.
+We use CustomReader to implement it
+
+
+# Usage
+
+The example uses the bucket name provided, one key for object, and output the progress to stdout.
+The Object size should larger than 5M or your will not see the progress
+
+```sh
+AWS_REGION=<region> go run putObjWithProcess.go <credential> <bucket> <key for object> <local file name>
+```
@@ -0,0 +1,102 @@
+// +build example
+
+package main
+
+import (
+	"log"
+	"os"
+	"sync/atomic"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+type CustomReader struct {
+	fp   *os.File
+	size int64
+	read int64
+}
+
+func (r *CustomReader) Read(p []byte) (int, error) {
+	return r.fp.Read(p)
+}
+
+func (r *CustomReader) ReadAt(p []byte, off int64) (int, error) {
+	n, err := r.fp.ReadAt(p, off)
+	if err != nil {
+		return n, err
+	}
+
+	// Got the length have read( or means has uploaded), and you can construct your message
+	atomic.AddInt64(&r.read, int64(n))
+
+	// I have no idea why the read length need to be div 2,
+	// maybe the request read once when Sign and actually send call ReadAt again
+	// It works for me
+	log.Printf("total read:%d    progress:%d%%\n", r.read/2, int(float32(r.read*100/2)/float32(r.size)))
+
+	return n, err
+}
+
+func (r *CustomReader) Seek(offset int64, whence int) (int64, error) {
+	return r.fp.Seek(offset, whence)
+}
+
+func main() {
+	if len(os.Args) < 4 {
+		log.Println("USAGE ERROR: AWS_REGION=us-east-1 go run putObjWithProcess.go <credential> <bucket> <key for object> <local file name>")
+		return
+	}
+
+	credential := os.Args[1]
+	bucket := os.Args[2]
+	key := os.Args[3]
+	fileName := os.Args[4]
+
+	creds := credentials.NewSharedCredentials(credential, "default")
+	if _, err := creds.Get(); err != nil {
+		log.Println("ERROR:", err)
+		return
+	}
+
+	sess := session.New(&aws.Config{
+		Credentials: creds,
+	})
+
+	file, err := os.Open(fileName)
+	if err != nil {
+		log.Println("ERROR:", err)
+		return
+	}
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		log.Println("ERROR:", err)
+		return
+	}
+
+	reader := &CustomReader{
+		fp:   file,
+		size: fileInfo.Size(),
+	}
+
+	uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
+		u.PartSize = 5 * 1024 * 1024
+		u.LeavePartsOnError = true
+	})
+
+	output, err := uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   reader,
+	})
+
+	if err != nil {
+		log.Println("ERROR:", err)
+		return
+	}
+
+	log.Println(output.Location)
+}
@@ -2,7 +2,7 @@

 sync will upload a given directory to Amazon S3 using the upload iterator interface defined in the
 s3manager package. This example uses a path that is specified during runtime to walk and build keys
-to upload to Amazon S3. It will use the keys to upload the files/folders to Amazon S3. 
+to upload to Amazon S3. It will use the keys to upload the files/folders to Amazon S3.

 # Usage

@@ -14,7 +14,7 @@ sync <params>
 ```

 ```sh
-go run -tags example sync.go 
+go run -tags example sync.go
 	-region <region> // required
 	-bucket <bucket> // required
 	-path  <path> // required
@@ -5,6 +5,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"mime"
 	"os"
 	"path/filepath"
 	"strings"
@@ -69,15 +70,22 @@ func (iter *SyncFolderIterator) UploadObject() s3manager.BatchUploadObject {
 		iter.err = err
 	}

+	extension := filepath.Ext(fi.key)
+	mimeType := mime.TypeByExtension(extension)
+
+	if mimeType == "" {
+		mimeType = "binary/octet-stream"
+	}
+
 	input := s3manager.UploadInput{
-		Bucket: &iter.bucket,
-		Key:    &fi.key,
-		Body:   body,
+		Bucket:      &iter.bucket,
+		Key:         &fi.key,
+		Body:        body,
+		ContentType: &mimeType,
 	}

 	return s3manager.BatchUploadObject{
-		&input,
-		nil,
+		Object: &input,
 	}
 }

@@ -101,11 +109,11 @@ func main() {

 	iter := NewSyncFolderIterator(*pathPtr, *bucketPtr)
 	if err := uploader.UploadWithIterator(aws.BackgroundContext(), iter); err != nil {
-		fmt.Fprintf(os.Stderr, "unexpected error has occured: %v", err)
+		fmt.Fprintf(os.Stderr, "unexpected error has occurred: %v", err)
 	}

 	if err := iter.Err(); err != nil {
-		fmt.Fprintf(os.Stderr, "unexpected error occured during file walking: %v", err)
+		fmt.Fprintf(os.Stderr, "unexpected error occurred during file walking: %v", err)
 	}

 	fmt.Println("Success")