Update Go AWS SDK to the latest version

2026-05-08 22:30:41 +00:00 · 2019-07-13 00:03:55 +03:00
parent d08be990ef
commit 94a72b23ff
2183 changed files with 885887 additions and 228114 deletions
@@ -22,7 +22,7 @@ Compile the plugin with:
 JSON Credentials File
 ---

-This example plugin will read the credentials from a JSON file pointed to by 
+This example plugin will read the credentials from a JSON file pointed to by
 the `PLUGIN_CREDS_FILE` environment variable. The contents of the file are
 the credentials, Key, Secret, and Token. The `Token` filed does not need to be
 set if your credentials do not have one.
@@ -38,7 +38,7 @@ set if your credentials do not have one.
 Example Application
 ---

-The `main.go` file in this folder demonstrates how you can configure the SDK to 
+The `main.go` file in this folder demonstrates how you can configure the SDK to
 use a plugin to retrieve credentials with.

 Compile and run application:
@@ -27,7 +27,7 @@ CLI parameters
    -p=id partition id, e.g: aws
    -r=id region id, e.g: us-west-2
    -s=id service id, e.g: s3
-    
+
    -partitions Lists all partitions.
    -regions Lists all regions in a partition. Requires partition ID.
             If service ID is also provided will show endpoints for a service.
@@ -0,0 +1,23 @@
+# Example
+
+Example of using the AWS SDK for Go with an HTTPS_PROXY that requires client
+TLS certificates. The example will use the proxy configured via the environment
+variable `HTTPS_PROXY` proxy a request for the Amazon S3 `ListBuckets` API
+operation call.
+
+The example assumes credentials are provided in the environment, or shared
+credentials file `~/.aws/credentials`. The `certificate` and `key` files paths
+are required to be specified when the example is run. An certificate authority
+(CA) file path can also be optionally specified.
+
+Refer to [httpproxy.FromEnvironment](https://godoc.org/golang.org/x/net/http/httpproxy#FromEnvironment)
+for details using `HTTPS_PROXY` with the Go HTTP client. 
+
+## Usage:
+
+```sh
+export HTTPS_PROXY=https://127.0.0.1:8443
+export AWS_REGION=us-west-2
+go run -cert <certfile> -key <keyfile> [-ca <cafile>]
+```
+
@@ -0,0 +1,123 @@
+// +build example
+
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"golang.org/x/net/http2"
+)
+
+// Example of creating an HTTP Client configured with a client TLS
+// certificates. Can be used with endpoints such as HTTPS_PROXY that require
+// client certificates.
+//
+// Requires a cert and key flags, and optionally takes a CA file.
+//
+// To run:
+//   go run -cert <certfile> -key <keyfile> [-ca <cafile>]
+//
+// You can generate self signed cert and key pem files
+// go run $(go env GOROOT)/src/crypto/tls/generate_cert.go -host <hostname>
+func main() {
+	var clientCertFile, clientKeyFile, caFile string
+	flag.StringVar(&clientCertFile, "cert", "cert.pem", "The `certificate file` to load.")
+	flag.StringVar(&clientKeyFile, "key", "key.pem", "The `key file` to load.")
+	flag.StringVar(&caFile, "ca", "ca.pem", "The `root CA` to load.")
+	flag.Parse()
+
+	if len(clientCertFile) == 0 || len(clientKeyFile) == 0 {
+		flag.PrintDefaults()
+		log.Fatalf("client certificate and key required")
+	}
+
+	tlsCfg, err := tlsConfigWithClientCert(clientCertFile, clientKeyFile, caFile)
+	if err != nil {
+		log.Fatalf("failed to load client cert, %v", err)
+	}
+
+	// Copy of net/http.DefaultTransport with TLS config loaded
+	tr := defaultHTTPTransport()
+	tr.TLSClientConfig = tlsCfg
+
+	// re-enable HTTP/2 because modifing TLS config will prevent auto support
+	// for HTTP/2.
+	http2.ConfigureTransport(tr)
+
+	// Configure the SDK's session with the HTTP client with TLS client
+	// certificate support enabled. This session will be used to create all
+	// SDK's API clients.
+	sess, err := session.NewSession(&aws.Config{
+		HTTPClient: &http.Client{
+			Transport: tr,
+		},
+	})
+
+	// Create each API client will the session configured with the client TLS
+	// certificate.
+	svc := s3.New(sess)
+
+	resp, err := svc.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		log.Fatalf("failed to list buckets, %v", err)
+	}
+
+	fmt.Println("Buckets")
+	fmt.Println(resp)
+}
+
+func tlsConfigWithClientCert(clientCertFile, clientKeyFile, caFile string) (*tls.Config, error) {
+	clientCert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load certificat files, %s, %s, %v",
+			clientCertFile, clientKeyFile, err)
+	}
+
+	tlsCfg := &tls.Config{
+		Certificates: []tls.Certificate{
+			clientCert,
+		},
+	}
+
+	if len(caFile) != 0 {
+		cert, err := ioutil.ReadFile(caFile)
+		if err != nil {
+			return nil, fmt.Errorf("unable to load root CA file, %s, %v",
+				caFile, err)
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(cert)
+		tlsCfg.RootCAs = caCertPool
+	}
+
+	tlsCfg.BuildNameToCertificate()
+
+	return tlsCfg, nil
+}
+
+func defaultHTTPTransport() *http.Transport {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:          100,
+		MaxIdleConnsPerHost:   10, // Increased idle connections per host
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+}
@@ -0,0 +1,40 @@
+# Example
+
+Demonstrates how the Go standard library `httptrace` can be used with the SDK
+to collect HTTP request tracing timing using the SDK's API operation methods
+like SNS's `PublishWithContext`.
+
+The `trace.go` file demonstrates how the `httptrace` package's `ClientTrace`
+can be created to gather timing information from HTTP requests made.
+
+The `logger.go` file demonstrates how the trace information can be combined to
+retrieve timing data for the different stages of the request.
+
+The `config.go` file provides additional configuration settings to control how
+the HTTP client and its transport is configured. Such as, timeouts, and
+keepalive.
+
+## Usage
+
+Run the example providing your SNS topic's ARN as the `-topic` parameter. This
+example assumes that the region is provided via the environment variable and
+the AWS shared credentials file (~/.aws/credentials)'s `default` provide
+provides credentials.
+
+```sh
+AWS_REGION=us-west-2 go run -tags example . -topic arn:aws:sns:us-west-2:0123456789:mytopicname
+```
+
+Once the example starts you'll be prompted with a `Message:` statement. Input
+the message that you'd like to send to the topic on a single line and hit
+`enter` to send it.
+
+```
+Message: My Really cool Message
+```
+
+The example will output the http trace timing information for how long the request took.
+
+```
+Latency: 79.863505ms ConnectionReused: false DNSStartAt: 280.452µs DNSDoneAt: 1.526342ms DNSDur: 1.24589ms ConnectStartAt: 1.533484ms ConnectDoneAt: 11.290792ms ConnectDur: 9.757308ms TLSStatAt: 11.331066ms TLSDoneAt: 33.912968ms TLSDur: 22.581902ms RequestWritten 34.951272ms RespFirstByte: 79.534808ms WaitRespFirstByte: 44.583536ms
+```
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"net"
+	"net/http"
+	"time"
+)
+
+// NewClient creates a new HTTP Client using the ClientConfig values.
+func NewClient(cfg ClientConfig) *http.Client {
+	tr := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   cfg.Timeouts.Connect,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:    100,
+		IdleConnTimeout: cfg.Timeouts.IdleConnection,
+
+		DisableKeepAlives:     !cfg.KeepAlive,
+		TLSHandshakeTimeout:   cfg.Timeouts.TLSHandshake,
+		ExpectContinueTimeout: cfg.Timeouts.ExpectContinue,
+		ResponseHeaderTimeout: cfg.Timeouts.ResponseHeader,
+	}
+
+	return &http.Client{
+		Transport: tr,
+	}
+}
@@ -0,0 +1,54 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+	"time"
+)
+
+// ClientConfig provides the timeouts from CLI flags and default values for the
+// example apps's configuration.
+type ClientConfig struct {
+	KeepAlive bool
+	Timeouts  Timeouts
+}
+
+// SetupFlags initializes the CLI flags.
+func (c *ClientConfig) SetupFlags(prefix string, flagset *flag.FlagSet) {
+	prefix += "client."
+
+	flagset.BoolVar(&c.KeepAlive, prefix+"http-keep-alive", true,
+		"Specifies if HTTP keep alive is enabled.")
+
+	c.Timeouts.SetupFlags(prefix, flagset)
+}
+
+// Timeouts collection of HTTP client timeout values.
+type Timeouts struct {
+	IdleConnection time.Duration
+	Connect        time.Duration
+	TLSHandshake   time.Duration
+	ExpectContinue time.Duration
+	ResponseHeader time.Duration
+}
+
+// SetupFlags initializes the CLI flags.
+func (c *Timeouts) SetupFlags(prefix string, flagset *flag.FlagSet) {
+	prefix += "timeout."
+
+	flagset.DurationVar(&c.IdleConnection, prefix+"idle-conn", 90*time.Second,
+		"The `timeout` of idle connects to the remote host.")
+
+	flagset.DurationVar(&c.Connect, prefix+"connect", 30*time.Second,
+		"The `timeout` connecting to the remote host.")
+
+	defTR := http.DefaultTransport.(*http.Transport)
+
+	flagset.DurationVar(&c.TLSHandshake, prefix+"tls", defTR.TLSHandshakeTimeout,
+		"The `timeout` waiting for the TLS handshake to complete.")
+
+	c.ExpectContinue = defTR.ExpectContinueTimeout
+
+	flagset.DurationVar(&c.ResponseHeader, prefix+"response-header", defTR.ResponseHeaderTimeout,
+		"The `timeout` waiting for the TLS handshake to complete.")
+}
@@ -0,0 +1,100 @@
+// build example
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+// RecordTrace outputs the request trace as text.
+func RecordTrace(w io.Writer, trace *RequestTrace) {
+	attempt := AttemptReport{
+		Reused:     trace.Reused,
+		Latency:    trace.Finish.Sub(trace.Start),
+		ReqWritten: trace.RequestWritten.Sub(trace.Start),
+	}
+
+	if !trace.FirstResponseByte.IsZero() {
+		attempt.RespFirstByte = trace.FirstResponseByte.Sub(trace.Start)
+		attempt.WaitRespFirstByte = trace.FirstResponseByte.Sub(trace.RequestWritten)
+	}
+
+	if !trace.Reused {
+		attempt.DNSStart = trace.DNSStart.Sub(trace.Start)
+		attempt.DNSDone = trace.DNSDone.Sub(trace.Start)
+		attempt.DNS = trace.DNSDone.Sub(trace.DNSStart)
+
+		attempt.ConnectStart = trace.ConnectStart.Sub(trace.Start)
+		attempt.ConnectDone = trace.ConnectDone.Sub(trace.Start)
+		attempt.Connect = trace.ConnectDone.Sub(trace.ConnectStart)
+
+		attempt.TLSHandshakeStart = trace.TLSHandshakeStart.Sub(trace.Start)
+		attempt.TLSHandshakeDone = trace.TLSHandshakeDone.Sub(trace.Start)
+		attempt.TLSHandshake = trace.TLSHandshakeDone.Sub(trace.TLSHandshakeStart)
+	}
+
+	_, err := fmt.Fprintln(w,
+		"Latency:",
+		attempt.Latency,
+
+		"ConnectionReused:",
+		fmt.Sprintf("%t", attempt.Reused),
+
+		"DNSStartAt:",
+		fmt.Sprintf("%s", attempt.DNSStart),
+		"DNSDoneAt:",
+		fmt.Sprintf("%s", attempt.DNSDone),
+		"DNSDur:",
+		fmt.Sprintf("%s", attempt.DNS),
+
+		"ConnectStartAt:",
+		fmt.Sprintf("%s", attempt.ConnectStart),
+		"ConnectDoneAt:",
+		fmt.Sprintf("%s", attempt.ConnectDone),
+		"ConnectDur:",
+		fmt.Sprintf("%s", attempt.Connect),
+
+		"TLSStatAt:",
+		fmt.Sprintf("%s", attempt.TLSHandshakeStart),
+		"TLSDoneAt:",
+		fmt.Sprintf("%s", attempt.TLSHandshakeDone),
+		"TLSDur:",
+		fmt.Sprintf("%s", attempt.TLSHandshake),
+
+		"RequestWritten",
+		fmt.Sprintf("%s", attempt.ReqWritten),
+		"RespFirstByte:",
+		fmt.Sprintf("%s", attempt.RespFirstByte),
+		"WaitRespFirstByte:",
+		fmt.Sprintf("%s", attempt.WaitRespFirstByte),
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to write request trace, %v\n", err)
+	}
+}
+
+// AttemptReport proviedes the structured timing information.
+type AttemptReport struct {
+	Latency time.Duration
+	Reused  bool
+	Err     error
+
+	DNSStart time.Duration
+	DNSDone  time.Duration
+	DNS      time.Duration
+
+	ConnectStart time.Duration
+	ConnectDone  time.Duration
+	Connect      time.Duration
+
+	TLSHandshakeStart time.Duration
+	TLSHandshakeDone  time.Duration
+	TLSHandshake      time.Duration
+
+	ReqWritten        time.Duration
+	RespFirstByte     time.Duration
+	WaitRespFirstByte time.Duration
+}
@@ -0,0 +1,93 @@
+// build example
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sns"
+)
+
+var clientCfg ClientConfig
+var topicARN string
+
+func init() {
+	clientCfg.SetupFlags("", flag.CommandLine)
+
+	flag.CommandLine.StringVar(&topicARN, "topic", "",
+		"The topic `ARN` to send messages to")
+}
+
+func main() {
+	if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
+		flag.CommandLine.PrintDefaults()
+		exitErrorf(err, "failed to parse CLI commands")
+	}
+	if len(topicARN) == 0 {
+		flag.CommandLine.PrintDefaults()
+		exitErrorf(errors.New("topic ARN required"), "")
+	}
+
+	httpClient := NewClient(clientCfg)
+	sess, err := session.NewSession(&aws.Config{
+		HTTPClient: httpClient,
+
+		// Disable Retries to prevent the httptrace's getting mixed up on
+		// retries.
+		MaxRetries: aws.Int(0),
+	})
+	if err != nil {
+		exitErrorf(err, "failed to load config")
+	}
+
+	// Start making the requests.
+	svc := sns.New(sess)
+	ctx := context.Background()
+
+	fmt.Printf("Message: ")
+
+	// Scan messages from the input with newline separators.
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		trace, err := publishMessage(ctx, svc, topicARN, scanner.Text())
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to publish message, %v", err)
+		}
+		RecordTrace(os.Stdout, trace)
+
+		fmt.Println()
+		fmt.Printf("Message: ")
+	}
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "failed to read input, %v", err)
+	}
+}
+
+// publishMessage will send the message to the SNS topic returning an request
+// trace for metrics.
+func publishMessage(ctx context.Context, svc *sns.SNS, topic, msg string) (*RequestTrace, error) {
+	traceCtx := NewRequestTrace(ctx)
+	defer traceCtx.RequestDone()
+
+	_, err := svc.PublishWithContext(traceCtx, &sns.PublishInput{
+		TopicArn: &topic,
+		Message:  &msg,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return traceCtx, nil
+}
+
+func exitErrorf(err error, msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, "FAILED: %v\n"+msg+"\n", append([]interface{}{err}, args...)...)
+	os.Exit(1)
+}
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http/httptrace"
+	"time"
+)
+
+// RequestTrace provides the trace time stamps of the HTTP request's segments.
+type RequestTrace struct {
+	context.Context
+
+	Start, Finish time.Time
+
+	Reused bool
+
+	DNSStart, DNSDone                   time.Time
+	ConnectStart, ConnectDone           time.Time
+	TLSHandshakeStart, TLSHandshakeDone time.Time
+	RequestWritten                      time.Time
+	FirstResponseByte                   time.Time
+}
+
+// NewRequestTrace returns a initialized RequestTrace for an
+// httptrace.ClientTrace, based on the context passed.
+func NewRequestTrace(ctx context.Context) *RequestTrace {
+	rt := &RequestTrace{
+		Start: time.Now(),
+	}
+
+	trace := &httptrace.ClientTrace{
+		GetConn:              rt.getConn,
+		GotConn:              rt.gotConn,
+		PutIdleConn:          rt.putIdleConn,
+		GotFirstResponseByte: rt.gotFirstResponseByte,
+		Got100Continue:       rt.got100Continue,
+		DNSStart:             rt.dnsStart,
+		DNSDone:              rt.dnsDone,
+		ConnectStart:         rt.connectStart,
+		ConnectDone:          rt.connectDone,
+		TLSHandshakeStart:    rt.tlsHandshakeStart,
+		TLSHandshakeDone:     rt.tlsHandshakeDone,
+		WroteHeaders:         rt.wroteHeaders,
+		Wait100Continue:      rt.wait100Continue,
+		WroteRequest:         rt.wroteRequest,
+	}
+
+	rt.Context = httptrace.WithClientTrace(ctx, trace)
+
+	return rt
+}
+
+// TotalLatency returns the total time the request took.
+func (rt *RequestTrace) TotalLatency() time.Duration {
+	return rt.Finish.Sub(rt.Start)
+}
+
+// RequestDone completes the request trace.
+func (rt *RequestTrace) RequestDone() {
+	rt.Finish = time.Now()
+}
+
+func (rt *RequestTrace) getConn(hostPort string) {}
+func (rt *RequestTrace) gotConn(info httptrace.GotConnInfo) {
+	rt.Reused = info.Reused
+}
+func (rt *RequestTrace) putIdleConn(err error) {}
+func (rt *RequestTrace) gotFirstResponseByte() {
+	rt.FirstResponseByte = time.Now()
+}
+func (rt *RequestTrace) got100Continue() {}
+func (rt *RequestTrace) dnsStart(info httptrace.DNSStartInfo) {
+	rt.DNSStart = time.Now()
+}
+func (rt *RequestTrace) dnsDone(info httptrace.DNSDoneInfo) {
+	rt.DNSDone = time.Now()
+}
+func (rt *RequestTrace) connectStart(network, addr string) {
+	rt.ConnectStart = time.Now()
+}
+func (rt *RequestTrace) connectDone(network, addr string, err error) {
+	rt.ConnectDone = time.Now()
+}
+func (rt *RequestTrace) tlsHandshakeStart() {
+	rt.TLSHandshakeStart = time.Now()
+}
+func (rt *RequestTrace) tlsHandshakeDone(state tls.ConnectionState, err error) {
+	rt.TLSHandshakeDone = time.Now()
+}
+func (rt *RequestTrace) wroteHeaders()    {}
+func (rt *RequestTrace) wait100Continue() {}
+func (rt *RequestTrace) wroteRequest(info httptrace.WroteRequestInfo) {
+	rt.RequestWritten = time.Now()
+}