mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-05-06 22:18:28 +00:00
Upgrade AWS SDK to the latest version
This commit is contained in:
Andrey Smirnov
+49
@@ -0,0 +1,49 @@
|
||||
Retrieve Credentials with Go Plugin
|
||||
===
|
||||
|
||||
This example demonstrates how you can take advantage of Go 1.8's new Plugin
|
||||
functionality to retrieve AWS credentials dynamically from a plugin compiled
|
||||
separate from your application.
|
||||
|
||||
Usage
|
||||
---
|
||||
|
||||
Example Plugin
|
||||
---
|
||||
|
||||
You can find the plugin at `plugin/plugin.go` nested within this example. The plugin
|
||||
demonstrates what symbol the SDK will use when lookup up the credential provider
|
||||
and the type signature that needs to be implemented.
|
||||
|
||||
Compile the plugin with:
|
||||
|
||||
go build -tags example -o myPlugin.so -buildmode=plugin plugin/plugin.go
|
||||
|
||||
JSON Credentials File
|
||||
---
|
||||
|
||||
This example plugin will read the credentials from a JSON file pointed to by
|
||||
the `PLUGIN_CREDS_FILE` environment variable. The contents of the file are
|
||||
the credentials, Key, Secret, and Token. The `Token` filed does not need to be
|
||||
set if your credentials do not have one.
|
||||
|
||||
```json
|
||||
{
|
||||
"Key": "MyAWSCredAccessKeyID",
|
||||
"Secret": "MyAWSCredSecretKey",
|
||||
"Token": "MyAWSCredToken"
|
||||
}
|
||||
```
|
||||
|
||||
Example Application
|
||||
---
|
||||
|
||||
The `main.go` file in this folder demonstrates how you can configure the SDK to
|
||||
use a plugin to retrieve credentials with.
|
||||
|
||||
Compile and run application:
|
||||
|
||||
go build -tags example -o myApp main.go
|
||||
|
||||
PLUGIN_CREDS_FILE=pathToCreds.json ./myApp myPlugin.so myBucket myObjectKey
|
||||
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
// +build example,go18
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"plugin"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials/plugincreds"
|
||||
"github.com/aws/aws-sdk-go/aws/endpoints"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
// Example application which loads a Go Plugin file, and uses the credential
|
||||
// provider defined within the plugin to get credentials for making a S3
|
||||
// request.
|
||||
//
|
||||
// The example will derive the bucket's region automatically if a AWS_REGION
|
||||
// environment variable is not defined.
|
||||
//
|
||||
// Build:
|
||||
// go build -tags example -o myApp main.go
|
||||
//
|
||||
// Usage:
|
||||
// ./myApp <compiled plugin> <bucket> <object key>
|
||||
func main() {
|
||||
if len(os.Args) < 4 {
|
||||
exitErrorf("Usage: myApp <compiled plugin>, <bucket> <object key>")
|
||||
}
|
||||
|
||||
pluginFilename := os.Args[1]
|
||||
bucket := os.Args[2]
|
||||
key := os.Args[3]
|
||||
|
||||
// Open plugin, and load it into the process.
|
||||
p, err := plugin.Open(pluginFilename)
|
||||
if err != nil {
|
||||
exitErrorf("failed to open plugin, %s, %v", pluginFilename, err)
|
||||
}
|
||||
|
||||
// Create a new Credentials value which will source the provider's Retrieve
|
||||
// and IsExpired functions from the plugin.
|
||||
creds, err := plugincreds.NewCredentials(p)
|
||||
if err != nil {
|
||||
exitErrorf("failed to load plugin provider, %v", err)
|
||||
}
|
||||
|
||||
// Example to configure a Session with the newly created credentials that
|
||||
// will be sourced using the plugin's functionality.
|
||||
sess := session.Must(session.NewSession(&aws.Config{
|
||||
Credentials: creds,
|
||||
}))
|
||||
|
||||
// If the region is not available attempt to derive the bucket's region
|
||||
// from a query to S3 for the bucket's metadata
|
||||
region := aws.StringValue(sess.Config.Region)
|
||||
if len(region) == 0 {
|
||||
region, err = s3manager.GetBucketRegion(context.Background(), sess, bucket, endpoints.UsEast1RegionID)
|
||||
if err != nil {
|
||||
exitErrorf("failed to get bucket region, %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create the S3 service client for the target region
|
||||
svc := s3.New(sess, aws.NewConfig().WithRegion(region))
|
||||
|
||||
// Get the object's details
|
||||
result, err := svc.HeadObject(&s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
fmt.Println(result, err)
|
||||
}
|
||||
|
||||
func exitErrorf(format string, args ...interface{}) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
Generated
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
// +build example,go18
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Example plugin that will retrieve credentials from a JSON file that the
|
||||
// "PLUGIN_CREDS_FILE" environment variable points to
|
||||
//
|
||||
// Build with:
|
||||
// go build -tags example -o plugin.so -buildmode=plugin plugin.go
|
||||
func main() {}
|
||||
|
||||
var myCredProvider provider
|
||||
|
||||
func init() {
|
||||
// Initialize a mock credential provider with stubs
|
||||
myCredProvider = provider{Filename: os.Getenv("PLUGIN_CREDS_FILE")}
|
||||
}
|
||||
|
||||
// GetAWSSDKCredentialProvider is the symbol SDK will lookup and use to
|
||||
// get the credential provider's retrieve and isExpired functions.
|
||||
func GetAWSSDKCredentialProvider() (func() (key, secret, token string, err error), func() bool) {
|
||||
return myCredProvider.Retrieve, myCredProvider.IsExpired
|
||||
}
|
||||
|
||||
// mock implementation of a type that returns retrieves credentials and
|
||||
// returns if they have expired.
|
||||
type provider struct {
|
||||
Filename string
|
||||
|
||||
loaded bool
|
||||
}
|
||||
|
||||
func (p *provider) Retrieve() (key, secret, token string, err error) {
|
||||
f, err := os.Open(p.Filename)
|
||||
if err != nil {
|
||||
return "", "", "", errors.Wrapf(err, "failed to open credentials file, %q", p.Filename)
|
||||
}
|
||||
decoder := json.NewDecoder(f)
|
||||
|
||||
creds := struct {
|
||||
Key, Secret, Token string
|
||||
}{}
|
||||
|
||||
if err := decoder.Decode(&creds); err != nil {
|
||||
return "", "", "", errors.Wrap(err, "failed to decode credentials file")
|
||||
}
|
||||
|
||||
p.loaded = true
|
||||
return creds.Key, creds.Secret, creds.Token, nil
|
||||
}
|
||||
|
||||
func (p *provider) IsExpired() bool {
|
||||
return !p.loaded
|
||||
}
|
||||
+9
-1
@@ -3,4 +3,12 @@ Custom Endpoint Example
|
||||
|
||||
This example provides examples on how you can provide custom endpoints, and logic to how endpoints are resolved by the SDK.
|
||||
|
||||
The example creates multiple clients with different endpoint configuraiton. From a custom endpoint resolver that wraps the defeault resolver so that any S3 service client created uses the custom endpoint, to how you can provide your own logic to a single service's endpoint resolving.
|
||||
The example creates multiple clients with different endpoint configuration. From a custom endpoint resolver that wraps the default resolver so that any Amazon S3 service client created uses the custom endpoint, to how you can provide your own logic to a single service's endpoint resolving.
|
||||
|
||||
|
||||
Usage
|
||||
---
|
||||
|
||||
```sh
|
||||
go run -tags example customeEndpoint.go
|
||||
```
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
Enumerate Regions and Endpoints Example
|
||||
===
|
||||
|
||||
Demostrates how the SDK's endpoints can be enumerated over to discover regions, services, and endpoints defined by the SDK's Regions and Endpoints metadata.
|
||||
Demonstrates how the SDK's endpoints can be enumerated over to discover regions, services, and endpoints defined by the SDK's Regions and Endpoints metadata.
|
||||
|
||||
Usage
|
||||
---
|
||||
@@ -10,7 +10,7 @@ The following parameters can be used to enumerate the SDK's partition metadata.
|
||||
|
||||
Example:
|
||||
|
||||
go run enumEndpoints.go -p aws -services -r us-west-2
|
||||
go run -tags example enumEndpoints.go -p aws -services -r us-west-2
|
||||
|
||||
Output:
|
||||
|
||||
|
||||
+1
-1
@@ -10,4 +10,4 @@ Will default to shared config file, but can load from environment if provided.
|
||||
## Usage:
|
||||
|
||||
# Upload myfile.txt to myBucket/myKey. Must complete within 10 minutes or will fail
|
||||
go run withContext.go -b mybucket -k myKey -d 10m < myfile.txt
|
||||
go run -tags example withContext.go -b mybucket -k myKey -d 10m < myfile.txt
|
||||
|
||||
+21
-6
@@ -10,6 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
@@ -36,21 +38,34 @@ func main() {
|
||||
sess := session.Must(session.NewSession())
|
||||
svc := s3.New(sess)
|
||||
|
||||
// Create a context with a timeout that will abort the upload if it takes
|
||||
// more than the passed in timeout.
|
||||
ctx := context.Background()
|
||||
var cancelFn func()
|
||||
if timeout > 0 {
|
||||
ctx, cancelFn = context.WithTimeout(ctx, timeout)
|
||||
}
|
||||
// Ensure the context is canceled to prevent leaking.
|
||||
// See context package for more information, https://golang.org/pkg/context/
|
||||
defer cancelFn()
|
||||
|
||||
// Uploads the object to S3. The Context will interrupt the request
|
||||
resp, err := svc.PutObjectWithContext(ctx, &s3.PutObjectInput{
|
||||
// Uploads the object to S3. The Context will interrupt the request if the
|
||||
// timeout expires.
|
||||
_, err := svc.PutObjectWithContext(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
Body: os.Stdin,
|
||||
})
|
||||
if err != nil {
|
||||
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode {
|
||||
// If the SDK can determine the request or retry delay was canceled
|
||||
// by a context the CanceledErrorCode error code will be returned.
|
||||
fmt.Fprintf(os.Stderr, "upload canceled due to timeout, %v\n", err)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "failed to upload object, %v\n", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Println(resp, err)
|
||||
|
||||
// Cleanup context
|
||||
cancelFn()
|
||||
fmt.Printf("successfully uploaded file to %s/%s\n", bucket, key)
|
||||
}
|
||||
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
# Example
|
||||
|
||||
`scan` is an example how to use Amazon DynamoDB's `expression` package to fill
|
||||
the member fields of Amazon DynamoDB's Operation input types.
|
||||
|
||||
## Representing DynamoDB Expressions
|
||||
|
||||
In the example, the variable `filt` represents a `FilterExpression`. Note that
|
||||
DynamoDB item attributes are represented using the function `Name()` and
|
||||
DynamoDB item values are similarly represented using the function `Value()`. In
|
||||
this context, the string `"Artist"` represents the name of the item attribute
|
||||
that we want to evaluate and the string `"No One You Know"` represents the value
|
||||
we want to evaluate the item attribute against. The relationship between the two
|
||||
[operands](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html#Expressions.OperatorsAndFunctions.Syntax)
|
||||
are specified using the method `Equal()`.
|
||||
|
||||
Similarly, the variable `proj` represents a `ProjectionExpression`. The list of
|
||||
item attribute names comprising the `ProjectionExpression` are specified as
|
||||
arguments to the function `NamesList()`. The `expression` package utilizes the
|
||||
type safety of Go and if an item value were to be used as an argument to the
|
||||
function `NamesList()`, a compile time error is returned. The pattern of
|
||||
representing DynamoDB Expressions by indicating relationships between `operands`
|
||||
with functions is consistent throughout the whole `expression` package.
|
||||
|
||||
```go
|
||||
filt := expression.Name("Artist").Equal(expression.Value("No One You Know"))
|
||||
// let :a be an ExpressionAttributeValue representing the string "No One You Know"
|
||||
// equivalent FilterExpression: "Artist = :a"
|
||||
|
||||
proj := expression.NamesList(expression.Name("SongTitle"), expression.Name("AlbumTitle"))
|
||||
// equivalent ProjectionExpression: "SongTitle, AlbumTitle"
|
||||
```
|
||||
|
||||
## Creating an `Expression`
|
||||
|
||||
In the example, the variable `expr` is an instance of an `Expression` type. An
|
||||
`Expression` is built using a builder pattern. First, a new `Builder` is
|
||||
initialized by the `NewBuilder()` function. Then, types representing DynamoDB
|
||||
Expressions are added to the `Builder` by methods `WithFilter()` and
|
||||
`WithProjection()`. The `Build()` method returns an instance of an `Expression`
|
||||
and an error. The error will be either an `InvalidParameterError` or an
|
||||
`UnsetParameterError`.
|
||||
|
||||
```go
|
||||
filt := expression.Name("Artist").Equal(expression.Value("No One You Know"))
|
||||
proj := expression.NamesList(expression.Name("SongTitle"), expression.Name("AlbumTitle"))
|
||||
|
||||
expr, err := expression.NewBuilder().WithFilter(filt).WithProjection(proj).Build()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
```
|
||||
|
||||
## Filling in the fields of a DynamoDB `Scan` API
|
||||
|
||||
In the example, the getter methods of the `Expression` type is used to get the
|
||||
formatted DynamoDB Expression strings. The `ExpressionAttributeNames` and
|
||||
`ExpressionAttributeValues` member field of the DynamoDB API must always be
|
||||
assigned when using an `Expression` since all item attribute names and values
|
||||
are aliased. That means that if the `ExpressionAttributeNames` and
|
||||
`ExpressionAttributeValues` member is not assigned with the corresponding
|
||||
`Names()` and `Values()` methods, the DynamoDB operation will run into a logic
|
||||
error.
|
||||
|
||||
```go
|
||||
filt := expression.Name("Artist").Equal(expression.Value("No One You Know"))
|
||||
proj := expression.NamesList(expression.Name("SongTitle"), expression.Name("AlbumTitle"))
|
||||
expr, err := expression.NewBuilder().WithFilter(filt).WithProjection(proj).Build()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
input := &dynamodb.ScanInput{
|
||||
ExpressionAttributeNames: expr.Names(),
|
||||
ExpressionAttributeValues: expr.Values(),
|
||||
FilterExpression: expr.Filter(),
|
||||
ProjectionExpression: expr.Projection(),
|
||||
TableName: aws.String("Music"),
|
||||
}
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
`go run -tags example scan.go -table "<table_name>" -region "<optional_region>"`
|
||||
|
||||
## Output
|
||||
|
||||
```
|
||||
{
|
||||
Count: #SomeNumber,
|
||||
Items: [{
|
||||
AlbumTitle: {
|
||||
#SomeAlbumTitle
|
||||
},
|
||||
SongTitle: {
|
||||
#SomeSongTitle
|
||||
}
|
||||
}],
|
||||
...
|
||||
ScannedCount: #SomeNumber,
|
||||
}
|
||||
```
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
// +build example
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/dynamodb"
|
||||
"github.com/aws/aws-sdk-go/service/dynamodb/expression"
|
||||
)
|
||||
|
||||
func exitWithError(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := Config{}
|
||||
if err := cfg.Load(); err != nil {
|
||||
exitWithError(fmt.Errorf("failed to load config, %v", err))
|
||||
}
|
||||
|
||||
// Create the config specifying the Region for the DynamoDB table.
|
||||
// If Config.Region is not set the region must come from the shared
|
||||
// config or AWS_REGION environment variable.
|
||||
awscfg := &aws.Config{}
|
||||
if len(cfg.Region) > 0 {
|
||||
awscfg.WithRegion(cfg.Region)
|
||||
}
|
||||
|
||||
// Create the session that the DynamoDB service will use.
|
||||
sess := session.Must(session.NewSession(awscfg))
|
||||
|
||||
// Create the DynamoDB service client to make the query request with.
|
||||
svc := dynamodb.New(sess)
|
||||
|
||||
// Create the Expression to fill the input struct with.
|
||||
filt := expression.Name("Artist").Equal(expression.Value("No One You Know"))
|
||||
proj := expression.NamesList(expression.Name("SongTitle"), expression.Name("AlbumTitle"))
|
||||
expr, err := expression.NewBuilder().WithFilter(filt).WithProjection(proj).Build()
|
||||
if err != nil {
|
||||
exitWithError(fmt.Errorf("failed to create the Expression, %v", err))
|
||||
}
|
||||
|
||||
// Build the query input parameters
|
||||
params := &dynamodb.ScanInput{
|
||||
ExpressionAttributeNames: expr.Names(),
|
||||
ExpressionAttributeValues: expr.Values(),
|
||||
FilterExpression: expr.Filter(),
|
||||
ProjectionExpression: expr.Projection(),
|
||||
TableName: aws.String(cfg.Table),
|
||||
}
|
||||
if cfg.Limit > 0 {
|
||||
params.Limit = aws.Int64(cfg.Limit)
|
||||
}
|
||||
|
||||
// Make the DynamoDB Query API call
|
||||
result, err := svc.Scan(params)
|
||||
if err != nil {
|
||||
exitWithError(fmt.Errorf("failed to make Query API call, %v", err))
|
||||
}
|
||||
|
||||
fmt.Println(result)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Table string // required
|
||||
Region string // optional
|
||||
Limit int64 // optional
|
||||
}
|
||||
|
||||
func (c *Config) Load() error {
|
||||
flag.Int64Var(&c.Limit, "limit", 0, "Limit is the max items to be returned, 0 is no limit")
|
||||
flag.StringVar(&c.Table, "table", "", "Table to Query on")
|
||||
flag.StringVar(&c.Region, "region", "", "AWS Region the table is in")
|
||||
flag.Parse()
|
||||
|
||||
if len(c.Table) == 0 {
|
||||
flag.PrintDefaults()
|
||||
return fmt.Errorf("table name is required.")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+3
-2
@@ -13,6 +13,7 @@ type Item struct {
|
||||
Data map[string]interface{}
|
||||
}
|
||||
```
|
||||
Use Go tags to define what the name is of the attribute in your DynamoDB table. See [AWS SDK for Go API Reference: Marshal](https://docs.aws.amazon.com/sdk-for-go/api/service/dynamodb/dynamodbattribute/#Marshal) for more information.
|
||||
|
||||
In DynamoDB the structure of the item to be returned will be:
|
||||
```json
|
||||
@@ -28,12 +29,12 @@ In DynamoDB the structure of the item to be returned will be:
|
||||
|
||||
## Usage
|
||||
|
||||
`scanItems.go -table "<table_name>" -region "<optional_region>"`
|
||||
`go run -tags example scanItems.go -table "<table_name>" -region "<optional_region>"`
|
||||
|
||||
## Output
|
||||
|
||||
```
|
||||
0: Key: 123, Desc: An item in the DynamoDB table
|
||||
0: Key: 123, Desc: An item in the DynamoDB table
|
||||
Num Data Values: 0
|
||||
1: Key: 2, Desc: Second ddb item
|
||||
Num Data Values: 2
|
||||
|
||||
+4
-1
@@ -6,6 +6,7 @@ methods connecting to DynamoDB, or as `unitTest` demonstrates, create your own
|
||||
## Test-compatible DynamoDB field
|
||||
If you use `*dynamodb.DynamoDB` as a field, you will be unable to unit test it,
|
||||
as documented in #88. Cast it instead as `dynamodbiface.DynamoDBAPI`:
|
||||
|
||||
```go
|
||||
type ItemGetter struct {
|
||||
DynamoDB dynamodbiface.DynamoDBAPI
|
||||
@@ -14,6 +15,7 @@ type ItemGetter struct {
|
||||
|
||||
## Querying actual DynamoDB
|
||||
You'll need an `*aws.Config` and `*session.Session` for these to work correctly:
|
||||
|
||||
```go
|
||||
// Setup
|
||||
var getter = new(ItemGetter)
|
||||
@@ -30,6 +32,7 @@ Construct a `fakeDynamoDB` and add the necessary methods for each of those
|
||||
structs (custom ones for `ItemGetter` and [whatever methods you're using for
|
||||
DynamoDB](https://github.com/aws/aws-sdk-go/blob/master/service/dynamodb/dynamodbiface/interface.go)),
|
||||
and you're good to go!
|
||||
|
||||
```go
|
||||
type fakeDynamoDB struct {
|
||||
dynamodbiface.DynamoDBAPI
|
||||
@@ -42,7 +45,7 @@ getter.DynamoDB.GetItem(/* ... */)
|
||||
|
||||
## Output
|
||||
```
|
||||
$ go test -cover
|
||||
$ go test -tags example -cover
|
||||
PASS
|
||||
coverage: 100.0% of statements
|
||||
ok _/Users/shatil/workspace/aws-sdk-go/example/service/dynamodb/unitTest 0.008s
|
||||
|
||||
+2
-2
@@ -1,11 +1,11 @@
|
||||
# Example
|
||||
|
||||
filter_ec2_by_tag is an example using the AWS SDK for Go to list ec2 instaces that match provided tag name filter.
|
||||
This is an example using the AWS SDK for Go to list ec2 instances that match provided tag name filter.
|
||||
|
||||
|
||||
# Usage
|
||||
|
||||
The example uses the the bucket name provided, and lists all object keys in a bucket.
|
||||
The example uses the bucket name provided, and lists all object keys in a bucket.
|
||||
|
||||
```sh
|
||||
go run -tags example filter_ec2_by_tag.go <name_filter>
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
|
||||
# Example Fetch By region
|
||||
|
||||
This is an example using the AWS SDK for Go to list ec2 instances instance state By different region . By default it fetch all running and stopped instance
|
||||
|
||||
|
||||
# Usage
|
||||
|
||||
|
||||
```sh
|
||||
# To fetch the stopped instance of all region use below:
|
||||
./filter_ec2_by_region --state running --state stopped
|
||||
|
||||
# To fetch the stopped and running instance for region us-west-1 and eu-west-1 use below:
|
||||
./filter_ec2_by_region --state running --state stopped --region us-west-1 --region=eu-west-1
|
||||
```
|
||||
|
||||
## Sample Output
|
||||
|
||||
```
|
||||
Fetching instace details for region: ap-south-1 with criteria: [running][stopped]**
|
||||
printing instance details.....
|
||||
instance id i-************
|
||||
current State stopped
|
||||
done for region ap-south-1 ****
|
||||
|
||||
|
||||
|
||||
Fetching instace details for region: eu-west-2 with criteria: [running][stopped]**
|
||||
There is no instance for the for region eu-west-2 with the matching Criteria: [running][stopped]
|
||||
done for region eu-west-2 ****
|
||||
```
|
||||
Generated
Vendored
+137
@@ -0,0 +1,137 @@
|
||||
// +build example
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
)
|
||||
|
||||
// Prints a list of instances for each region. If no regions are provided
|
||||
// all regions will be searched. The state is required.
|
||||
//
|
||||
// Will use the AWS SDK for Go's default credential chain and region. You can
|
||||
// specify the region with the AWS_REGION environment variable.
|
||||
//
|
||||
// Usage: instancesByRegion -state <value> [-state val...] [-region region...]
|
||||
func main() {
|
||||
states, regions := parseArguments()
|
||||
|
||||
if len(states) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", usage())
|
||||
os.Exit(1)
|
||||
}
|
||||
instanceCriteria := " "
|
||||
for _, state := range states {
|
||||
instanceCriteria += "[" + state + "]"
|
||||
}
|
||||
|
||||
if len(regions) == 0 {
|
||||
var err error
|
||||
regions, err = fetchRegion()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
for _, region := range regions {
|
||||
sess := session.Must(session.NewSession(&aws.Config{
|
||||
Region: aws.String(region),
|
||||
}))
|
||||
|
||||
ec2Svc := ec2.New(sess)
|
||||
params := &ec2.DescribeInstancesInput{
|
||||
Filters: []*ec2.Filter{
|
||||
&ec2.Filter{
|
||||
Name: aws.String("instance-state-name"),
|
||||
Values: aws.StringSlice(states),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := ec2Svc.DescribeInstances(params)
|
||||
if err != nil {
|
||||
fmt.Println("Error", err)
|
||||
} else {
|
||||
fmt.Printf("\n\n\nFetching instace details for region: %s with criteria: %s**\n ", region, instanceCriteria)
|
||||
if len(result.Reservations) == 0 {
|
||||
fmt.Printf("There is no instance for the for region %s with the matching Criteria:%s \n", region, instanceCriteria)
|
||||
}
|
||||
for _, reservation := range result.Reservations {
|
||||
|
||||
fmt.Println("printing instance details.....")
|
||||
for _, instance := range reservation.Instances {
|
||||
fmt.Println("instance id " + *instance.InstanceId)
|
||||
fmt.Println("current State " + *instance.State.Name)
|
||||
}
|
||||
}
|
||||
fmt.Printf("done for region %s **** \n", region)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func fetchRegion() ([]string, error) {
|
||||
awsSession := session.Must(session.NewSession(&aws.Config{}))
|
||||
|
||||
svc := ec2.New(awsSession)
|
||||
awsRegions, err := svc.DescribeRegions(&ec2.DescribeRegionsInput{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
regions := make([]string, 0, len(awsRegions.Regions))
|
||||
for _, region := range awsRegions.Regions {
|
||||
regions = append(regions, *region.RegionName)
|
||||
}
|
||||
|
||||
return regions, nil
|
||||
}
|
||||
|
||||
type flagArgs []string
|
||||
|
||||
func (a flagArgs) String() string {
|
||||
return strings.Join(a.Args(), ",")
|
||||
}
|
||||
|
||||
func (a *flagArgs) Set(value string) error {
|
||||
*a = append(*a, value)
|
||||
return nil
|
||||
}
|
||||
func (a flagArgs) Args() []string {
|
||||
return []string(a)
|
||||
}
|
||||
|
||||
func parseArguments() (states []string, regions []string) {
|
||||
var stateArgs, regionArgs flagArgs
|
||||
|
||||
flag.Var(&stateArgs, "state", "state list")
|
||||
flag.Var(®ionArgs, "region", "region list")
|
||||
flag.Parse()
|
||||
|
||||
if flag.NFlag() != 0 {
|
||||
states = append([]string{}, stateArgs.Args()...)
|
||||
regions = append([]string{}, regionArgs.Args()...)
|
||||
}
|
||||
|
||||
return states, regions
|
||||
}
|
||||
|
||||
func usage() string {
|
||||
return `
|
||||
|
||||
Missing mandatory flag 'state'. Please use like below Example:
|
||||
|
||||
To fetch the stopped instance of all region use below:
|
||||
./filter_ec2_by_region -state running -state stopped
|
||||
|
||||
To fetch the stopped and running instance for region us-west-1 and eu-west-1 use below:
|
||||
./filter_ec2_by_region -state running -state stopped -region us-west-1 -region=eu-west-1
|
||||
`
|
||||
}
|
||||
Generated
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
# Example
|
||||
|
||||
This is an example using the AWS SDK for Go to create an Amazon RDS DB token using the
|
||||
rdsutils package.
|
||||
|
||||
# Usage
|
||||
|
||||
```sh
|
||||
go run -tags example iam_authetnication.go <region> <db user> <db name> <endpoint to database> <iam arn>
|
||||
```
|
||||
|
||||
Output:
|
||||
```
|
||||
Successfully opened connection to database
|
||||
```
|
||||
Generated
Vendored
+47
@@ -0,0 +1,47 @@
|
||||
// +build example,skip
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/rds/rdsutils"
|
||||
)
|
||||
|
||||
// Usage ./iam_authentication <region> <db user> <db name> <endpoint to database> <iam arn>
|
||||
func main() {
|
||||
if len(os.Args) < 5 {
|
||||
log.Println("USAGE ERROR: go run concatenateObjects.go <region> <endpoint to database> <iam arn>")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
awsRegion := os.Args[1]
|
||||
dbUser := os.Args[2]
|
||||
dbName := os.Args[3]
|
||||
dbEndpoint := os.Args[4]
|
||||
awsCreds := stscreds.NewCredentials(session.New(&aws.Config{Region: &awsRegion}), os.Args[5])
|
||||
authToken, err := rdsutils.BuildAuthToken(dbEndpoint, awsRegion, dbUser, awsCreds)
|
||||
|
||||
// Create the MySQL DNS string for the DB connection
|
||||
// user:password@protocol(endpoint)/dbname?<params>
|
||||
dnsStr := fmt.Sprintf("%s:%s@tcp(%s)/%s?tls=true",
|
||||
dbUser, authToken, dbEndpoint, dbName,
|
||||
)
|
||||
|
||||
driver := mysql.MySQLDriver{}
|
||||
_ = driver
|
||||
// Use db to perform SQL operations on database
|
||||
if _, err = sql.Open("mysql", dnsStr); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
fmt.Println("Successfully opened connection to database")
|
||||
}
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
# Example
|
||||
|
||||
concatObjects is an example using the AWS SDK for Go to concatenate two objects together.
|
||||
This is an example using the AWS SDK for Go to concatenate two objects together.
|
||||
We use `UploadPartCopy` which uses an object for a part. Here in this example we have two parts, or in other words
|
||||
two objects that we want to concatenate together.
|
||||
|
||||
|
||||
# Usage
|
||||
|
||||
The example uses the the bucket name provided, two keys for each object, and lastly the output key.
|
||||
The example uses the bucket name provided, two keys for each object, and lastly the output key.
|
||||
|
||||
```sh
|
||||
AWS_REGION=<region> go run -tags example concatenateObjects.go <bucket> <key for object 1> <key for object 2> <key for output>
|
||||
|
||||
+2
-2
@@ -1,11 +1,11 @@
|
||||
# Example
|
||||
|
||||
listObjects is an example using the AWS SDK for Go to list objects' key in a S3 bucket.
|
||||
This is an example using the AWS SDK for Go to list objects' key in a S3 bucket.
|
||||
|
||||
|
||||
# Usage
|
||||
|
||||
The example uses the the bucket name provided, and lists all object keys in a bucket.
|
||||
The example uses the bucket name provided, and lists all object keys in a bucket.
|
||||
|
||||
```sh
|
||||
go run -tags example listObjects.go <bucket>
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -1,6 +1,6 @@
|
||||
## Example
|
||||
|
||||
listObjectsConcurrentlv is an example using the AWS SDK for Go concurrently to list the encrypted objects in the S3 buckets owned by an account.
|
||||
This is an example using the AWS SDK for Go concurrently to list the encrypted objects in the S3 buckets owned by an account.
|
||||
|
||||
## Usage
|
||||
|
||||
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
# Presigned Amazon S3 API Operation Example
|
||||
|
||||
This example demonstrates how you can build a client application to retrieve and
|
||||
upload object data from Amazon S3 without needing to know anything about Amazon
|
||||
S3 or have access to any AWS credentials. Only the service would have knowledge
|
||||
of how and where the objects are stored in Amazon S3.
|
||||
|
||||
The example is split into two parts `server.go` and `client.go`. These two parts
|
||||
simulate the client/server architecture. In this example the client will represent
|
||||
a third part user that will request resource URLs from the service. The service
|
||||
will generate presigned S3 URLs which the client can use to download and
|
||||
upload S3 object content.
|
||||
|
||||
The service supports generating presigned URLs for two S3 APIs; `GetObject` and
|
||||
`PutObject`. The client will request a presigned URL from the service with an
|
||||
object Key. In this example the value is the S3 object's `key`. Alternatively,
|
||||
you could use your own pattern with no visible relation to the S3 object's key.
|
||||
The server would then perform a cross reference with client provided value to
|
||||
one that maps to the S3 object's key.
|
||||
|
||||
Before using the client to upload and download S3 objects you'll need to start the
|
||||
service. The service will use the SDK's default credential chain to source your
|
||||
AWS credentials. See the [`Configuring Credentials`](http://docs.aws.amazon.com/sdk-for-go/api/)
|
||||
section of the SDK's API Reference guide on how the SDK loads your AWS credentials.
|
||||
|
||||
The server requires the S3 `-b bucket` the presigned URLs will be generated for. A
|
||||
`-r region` is only needed if the bucket is in AWS China or AWS Gov Cloud. For
|
||||
buckets in AWS the server will use the [`s3manager.GetBucketRegion`](http://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#GetBucketRegion) utility to lookup the bucket's region.
|
||||
|
||||
You should run the service in the background or in a separate terminal tab before
|
||||
moving onto the client.
|
||||
|
||||
|
||||
```sh
|
||||
go run -tags example server/server.go -b mybucket
|
||||
> Starting Server On: 127.0.0.1:8080
|
||||
```
|
||||
|
||||
Use the `--help` flag to see a list of additional configuration flags, and their
|
||||
defaults.
|
||||
|
||||
## Downloading an Amazon S3 Object
|
||||
|
||||
Use the client application to request a presigned URL from the server and use
|
||||
that presigned URL to download the object from S3. Calling the client with the
|
||||
`-get key` flag will do this. An optional `-f filename` flag can be provided as
|
||||
well to write the object to. If no flag is provided the object will be written
|
||||
to `stdout`
|
||||
|
||||
```sh
|
||||
go run -tags example client/client.go -get "my-object/key" -f outputfilename
|
||||
```
|
||||
|
||||
Use the `--help` flag to see a list of additional configuration flags, and their
|
||||
defaults.
|
||||
|
||||
The following curl request demonstrates the request the client makes to the server
|
||||
for the presigned URL for the `my-object/key` S3 object. The `method` query
|
||||
parameter lets the server know that we are requesting the `GetObject`'s presigned
|
||||
URL. The `method` value can be `GET` or `PUT` for the `GetObject` or `PutObject` APIs.
|
||||
|
||||
```sh
|
||||
curl -v "http://127.0.0.1:8080/presign/my-object/key?method=GET"
|
||||
```
|
||||
|
||||
The server will respond with a JSON value. The value contains three pieces of
|
||||
information that the client will need to correctly make the request. First is
|
||||
the presigned URL. This is the URL the client will make the request to. Second
|
||||
is the HTTP method the request should be sent as. This is included to simplify
|
||||
the client's request building. Finally the response will include a list of
|
||||
additional headers that the client should include that the presigned request
|
||||
was signed with.
|
||||
|
||||
```json
|
||||
{
|
||||
"URL": "https://mybucket.s3-us-west-2.amazonaws.com/my-object/key?<signature>",
|
||||
"Method": "GET",
|
||||
"Header": {
|
||||
"x-amz-content-sha256":["UNSIGNED-PAYLOAD"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
With this URL our client will build a HTTP request for the S3 object's data. The
|
||||
`client.go` will then write the object's data to the `filename` if one is provided,
|
||||
or to `stdout` of a filename is not set in the command line arguments.
|
||||
|
||||
## Uploading a File to Amazon S3
|
||||
|
||||
Just like the download, uploading a file to S3 will use a presigned URL requested
|
||||
from the server. The resigned URL will be built into an HTTP request using the
|
||||
URL, Method, and Headers. The `-put key` flag will upload the content of `-f filename`
|
||||
or stdin if no filename is provided to S3 using a presigned URL provided by the
|
||||
service
|
||||
|
||||
```sh
|
||||
go run -tags example client/client.go -put "my-object/key" -f filename
|
||||
```
|
||||
|
||||
Like the download case this will make a HTTP request to the server for the
|
||||
presigned URL. The Server will respond with a presigned URL for S3's `PutObject`
|
||||
API operation. In addition the `method` query parameter the client will also
|
||||
include a `contentLength` this value instructs the server to generate the presigned
|
||||
PutObject request with a `Content-Length` header value included in the signature.
|
||||
This is done so the content that is uploaded by the client can only be the size
|
||||
the presigned request was generated for.
|
||||
|
||||
```sh
|
||||
curl -v "http://127.0.0.1:8080/presign/my-object/key?method=PUT&contentLength=1024"
|
||||
```
|
||||
|
||||
## Expanding the Example
|
||||
|
||||
This example provides a spring board you can use to vend presigned URLs to your
|
||||
clients instead of streaming the object's content through your service. This
|
||||
client and server example can be expanded and customized. Adding new functionality
|
||||
such as additional constraints the server puts on the presigned URLs like
|
||||
`Content-Type`.
|
||||
|
||||
In addition to adding constraints to the presigned URLs the service could be
|
||||
updated to obfuscate S3 object's key. Instead of the client knowing the object's
|
||||
key, a lookup system could be used instead. This could be substitution based,
|
||||
or lookup into an external data store such as DynamoDB.
|
||||
|
||||
+266
@@ -0,0 +1,266 @@
|
||||
// +build example
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// client.go is an example of a client that will request URLs from a service that
|
||||
// the client will use to upload and download content with.
|
||||
//
|
||||
// The server must be started before the client is run.
|
||||
//
|
||||
// Use "--help" command line argument flag to see all options and defaults. If
|
||||
// filename is not provided the client will read from stdin for uploads and
|
||||
// write to stdout for downloads.
|
||||
//
|
||||
// Usage:
|
||||
// go run -tags example client.go -get myObjectKey -f filename
|
||||
func main() {
|
||||
method, filename, key, serverURL := loadConfig()
|
||||
|
||||
var err error
|
||||
|
||||
switch method {
|
||||
case GetMethod:
|
||||
// Requests the URL from the server that the client will use to download
|
||||
// the content from. The content will be written to the file pointed to
|
||||
// by filename. Creating it if the file does not exist. If filename is
|
||||
// not set the contents will be written to stdout.
|
||||
err = downloadFile(serverURL, key, filename)
|
||||
case PutMethod:
|
||||
// Requests the URL from the service that the client will use to upload
|
||||
// content to. The content will be read from the file pointed to by the
|
||||
// filename. If the filename is not set, content will be read from stdin.
|
||||
err = uploadFile(serverURL, key, filename)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
exitError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// loadConfig configures the client based on the command line arguments used.
|
||||
func loadConfig() (method Method, serverURL, key, filename string) {
|
||||
var getKey, putKey string
|
||||
flag.StringVar(&getKey, "get", "",
|
||||
"Downloads the object from S3 by the `key`. Writes the object to a file the filename is provided, otherwise writes to stdout.")
|
||||
flag.StringVar(&putKey, "put", "",
|
||||
"Uploads data to S3 at the `key` provided. Uploads the file if filename is provided, otherwise reads from stdin.")
|
||||
flag.StringVar(&serverURL, "s", "http://127.0.0.1:8080", "Required `URL` the client will request presigned S3 operation from.")
|
||||
flag.StringVar(&filename, "f", "", "The `filename` of the file to upload and get from S3.")
|
||||
flag.Parse()
|
||||
|
||||
var errs Errors
|
||||
|
||||
if len(serverURL) == 0 {
|
||||
errs = append(errs, fmt.Errorf("server URL required"))
|
||||
}
|
||||
|
||||
if !((len(getKey) != 0) != (len(putKey) != 0)) {
|
||||
errs = append(errs, fmt.Errorf("either `get` or `put` can be provided, and one of the two is required."))
|
||||
}
|
||||
|
||||
if len(getKey) > 0 {
|
||||
method = GetMethod
|
||||
key = getKey
|
||||
} else {
|
||||
method = PutMethod
|
||||
key = putKey
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "Failed to load configuration:%v\n", errs)
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
return method, filename, key, serverURL
|
||||
}
|
||||
|
||||
// downloadFile will request a URL from the server that the client can download
|
||||
// the content pointed to by "key". The content will be written to the file
|
||||
// pointed to by filename, creating the file if it doesn't exist. If filename
|
||||
// is not set the content will be written to stdout.
|
||||
func downloadFile(serverURL, key, filename string) error {
|
||||
var w *os.File
|
||||
if len(filename) > 0 {
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create download file %s, %v", filename, err)
|
||||
}
|
||||
w = f
|
||||
} else {
|
||||
w = os.Stdout
|
||||
}
|
||||
defer w.Close()
|
||||
|
||||
// Get the presigned URL from the remote service.
|
||||
req, err := getPresignedRequest(serverURL, "GET", key, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get get presigned request, %v", err)
|
||||
}
|
||||
|
||||
// Gets the file contents with the URL provided by the service.
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to do GET request, %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("failed to get S3 object, %d:%s",
|
||||
resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
if _, err = io.Copy(w, resp.Body); err != nil {
|
||||
return fmt.Errorf("failed to write S3 object, %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFile will request a URL from the service that the client can use to
|
||||
// upload content to. The content will be read from the file pointed to by filename.
|
||||
// If filename is not set the content will be read from stdin.
|
||||
func uploadFile(serverURL, key, filename string) error {
|
||||
var r io.ReadCloser
|
||||
var size int64
|
||||
if len(filename) > 0 {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open upload file %s, %v", filename, err)
|
||||
}
|
||||
|
||||
// Get the size of the file so that the constraint of Content-Length
|
||||
// can be included with the presigned URL. This can be used by the
|
||||
// server or client to ensure the content uploaded is of a certain size.
|
||||
//
|
||||
// These constraints can further be expanded to include things like
|
||||
// Content-Type. Additionally constraints such as X-Amz-Content-Sha256
|
||||
// header set restricting the content of the file to only the content
|
||||
// the client initially made the request with. This prevents the object
|
||||
// from being overwritten or used to upload other unintended content.
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat file, %s, %v", filename, err)
|
||||
}
|
||||
|
||||
size = stat.Size()
|
||||
r = f
|
||||
} else {
|
||||
buf := &bytes.Buffer{}
|
||||
io.Copy(buf, os.Stdin)
|
||||
size = int64(buf.Len())
|
||||
|
||||
r = ioutil.NopCloser(buf)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Get the Presigned URL from the remote service. Pass in the file's
|
||||
// size if it is known so that the presigned URL returned will be required
|
||||
// to be used with the size of content requested.
|
||||
req, err := getPresignedRequest(serverURL, "PUT", key, size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get get presigned request, %v", err)
|
||||
}
|
||||
req.Body = r
|
||||
|
||||
// Upload the file contents to S3.
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to do GET request, %v", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("failed to put S3 object, %d:%s",
|
||||
resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getPresignRequest will request a URL from the service for the content specified
|
||||
// by the key and method. Returns a constructed Request that can be used to
|
||||
// upload or download content with based on the method used.
|
||||
//
|
||||
// If the PUT method is used the request's Body will need to be set on the returned
|
||||
// request value.
|
||||
func getPresignedRequest(serverURL, method, key string, contentLen int64) (*http.Request, error) {
|
||||
u := fmt.Sprintf("%s/presign/%s?method=%s&contentLength=%d",
|
||||
serverURL, key, method, contentLen,
|
||||
)
|
||||
|
||||
resp, err := http.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to make request for presigned URL, %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed to get valid presign response, %s", resp.Status)
|
||||
}
|
||||
|
||||
p := PresignResp{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&p); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode response body, %v", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(p.Method, p.URL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build presigned request, %v", err)
|
||||
}
|
||||
|
||||
for k, vs := range p.Header {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
// Need to ensure that the content length member is set of the HTTP Request
|
||||
// or the request will not be transmitted correctly with a content length
|
||||
// value across the wire.
|
||||
if contLen := req.Header.Get("Content-Length"); len(contLen) > 0 {
|
||||
req.ContentLength, _ = strconv.ParseInt(contLen, 10, 64)
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
type Method int
|
||||
|
||||
const (
|
||||
PutMethod Method = iota
|
||||
GetMethod
|
||||
)
|
||||
|
||||
type Errors []error
|
||||
|
||||
func (es Errors) Error() string {
|
||||
out := make([]string, len(es))
|
||||
for _, e := range es {
|
||||
out = append(out, e.Error())
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
type PresignResp struct {
|
||||
Method, URL string
|
||||
Header http.Header
|
||||
}
|
||||
|
||||
func exitError(err error) {
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
// +build example
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/endpoints"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3iface"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
// server.go is an example of a service that vends lists for requests for presigned
|
||||
// URLs for S3 objects. The service supports two S3 operations, "GetObject" and
|
||||
// "PutObject".
|
||||
//
|
||||
// Example GetObject request to the service for the object with the key "MyObjectKey":
|
||||
//
|
||||
// curl -v "http://127.0.0.1:8080/presign/my-object/key?method=GET"
|
||||
//
|
||||
// Example PutObject request to the service for the object with the key "MyObjectKey":
|
||||
//
|
||||
// curl -v "http://127.0.0.1:8080/presign/my-object/key?method=PUT&contentLength=1024"
|
||||
//
|
||||
// Use "--help" command line argument flag to see all options and defaults.
|
||||
//
|
||||
// Usage:
|
||||
// go run -tags example service.go -b myBucket
|
||||
func main() {
|
||||
addr, bucket, region := loadConfig()
|
||||
|
||||
// Create a AWS SDK for Go Session that will load credentials using the SDK's
|
||||
// default credential change.
|
||||
sess := session.Must(session.NewSession())
|
||||
|
||||
// Use the GetBucketRegion utility to lookup the bucket's region automatically.
|
||||
// The service.go will only do this correctly for AWS regions. For AWS China
|
||||
// and AWS Gov Cloud the region needs to be specified to let the service know
|
||||
// to look in those partitions instead of AWS.
|
||||
if len(region) == 0 {
|
||||
var err error
|
||||
region, err = s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, endpoints.UsWest2RegionID)
|
||||
if err != nil {
|
||||
exitError(fmt.Errorf("failed to get bucket region, %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new S3 service client that will be use by the service to generate
|
||||
// presigned URLs with. Not actual API requests will be made with this client.
|
||||
// The credentials loaded when the Session was created above will be used
|
||||
// to sign the requests with.
|
||||
s3Svc := s3.New(sess, &aws.Config{
|
||||
Region: aws.String(region),
|
||||
})
|
||||
|
||||
// Start the server listening and serve presigned URLs for GetObject and
|
||||
// PutObject requests.
|
||||
if err := listenAndServe(addr, bucket, s3Svc); err != nil {
|
||||
exitError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig() (addr, bucket, region string) {
|
||||
flag.StringVar(&bucket, "b", "", "S3 `bucket` object should be uploaded to.")
|
||||
flag.StringVar(®ion, "r", "", "AWS `region` the bucket exists in, If not set region will be looked up, only valid for AWS Regions, not AWS China or Gov Cloud.")
|
||||
flag.StringVar(&addr, "a", "127.0.0.1:8080", "The TCP `address` the server will be started on.")
|
||||
flag.Parse()
|
||||
|
||||
if len(bucket) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "bucket is required")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
return addr, bucket, region
|
||||
}
|
||||
|
||||
func listenAndServe(addr, bucket string, svc s3iface.S3API) error {
|
||||
l, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start service listener, %v", err)
|
||||
}
|
||||
|
||||
const presignPath = "/presign/"
|
||||
|
||||
// Create the HTTP handler for the "/presign/" path prefix. This will handle
|
||||
// all requests on this path, extracting the object's key from the path.
|
||||
http.HandleFunc(presignPath, func(w http.ResponseWriter, r *http.Request) {
|
||||
var u string
|
||||
var err error
|
||||
var signedHeaders http.Header
|
||||
|
||||
query := r.URL.Query()
|
||||
|
||||
var contentLen int64
|
||||
// Optionally the Content-Length header can be included with the signature
|
||||
// of the request. This is helpful to ensure the content uploaded is the
|
||||
// size that is expected. Constraints like these can be further expanded
|
||||
// with headers such as `Content-Type`. These can be enforced by the service
|
||||
// requiring the client to satisfying those constraints when uploading
|
||||
//
|
||||
// In addition the client could provide the service with a SHA256 of the
|
||||
// content to be uploaded. This prevents any other third party from uploading
|
||||
// anything else with the presigned URL
|
||||
if contLenStr := query.Get("contentLength"); len(contLenStr) > 0 {
|
||||
contentLen, err = strconv.ParseInt(contLenStr, 10, 64)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "unable to parse request content length, %v", err)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Extract the object key from the path
|
||||
key := strings.Replace(r.URL.Path, presignPath, "", 1)
|
||||
method := query.Get("method")
|
||||
|
||||
switch method {
|
||||
case "PUT":
|
||||
// For creating PutObject presigned URLs
|
||||
fmt.Println("Received request to presign PutObject for,", key)
|
||||
sdkReq, _ := svc.PutObjectRequest(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
|
||||
// If ContentLength is 0 the header will not be included in the signature.
|
||||
ContentLength: aws.Int64(contentLen),
|
||||
})
|
||||
u, signedHeaders, err = sdkReq.PresignRequest(15 * time.Minute)
|
||||
case "GET":
|
||||
// For creating GetObject presigned URLs
|
||||
fmt.Println("Received request to presign GetObject for,", key)
|
||||
sdkReq, _ := svc.GetObjectRequest(&s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
u, signedHeaders, err = sdkReq.PresignRequest(15 * time.Minute)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "invalid method provided, %s, %v\n", method, err)
|
||||
err = fmt.Errorf("invalid request")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Create the response back to the client with the information on the
|
||||
// presigned request and additional headers to include.
|
||||
if err := json.NewEncoder(w).Encode(PresignResp{
|
||||
Method: method,
|
||||
URL: u,
|
||||
Header: signedHeaders,
|
||||
}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to encode presign response, %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
fmt.Println("Starting Server On:", "http://"+l.Addr().String())
|
||||
|
||||
s := &http.Server{}
|
||||
return s.Serve(l)
|
||||
}
|
||||
|
||||
// PresignResp provides the Go representation of the JSON value that will be
|
||||
// sent to the client.
|
||||
type PresignResp struct {
|
||||
Method, URL string
|
||||
Header http.Header
|
||||
}
|
||||
|
||||
func exitError(err error) {
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
+2
@@ -1,3 +1,5 @@
|
||||
// +build example
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
Generated
Vendored
+2
-2
@@ -1,9 +1,9 @@
|
||||
# Example
|
||||
|
||||
This example shows how the SDK's API interfaces can be used by your code intead of the concrete service client type directly. Using this pattern allows you to mock out your code's usage of the SDK's service client for testing.
|
||||
This example shows how the SDK's API interfaces can be used by your code instead of the concrete service client type directly. Using this pattern allows you to mock out your code's usage of the SDK's service client for testing.
|
||||
|
||||
# Usage
|
||||
|
||||
Use the `go test` tool to verify the `Queue` type's `GetMessages` function correctly unmarshals the SQS message responses.
|
||||
|
||||
`go test ./`
|
||||
`go test -tags example ifaceExample.go`
|
||||
|
||||
Generated
Vendored
+2
@@ -1,3 +1,5 @@
|
||||
// +build example
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
Generated
Vendored
+2
@@ -1,3 +1,5 @@
|
||||
// +build example
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
Reference in New Issue
Block a user