deps: update aws-sdk-go for ec2metadata bugfix

Newer versions of aws-sdk-go contain a bugfix for retrieving
region vs availability-zone information.
This commit is contained in:
Seth Hoenig 2020-10-06 13:05:29 -05:00
parent fac7fb5d5d
commit 3f5ecd6f42
105 changed files with 33301 additions and 3017 deletions

2
go.mod
View File

@ -25,7 +25,7 @@ require (
github.com/NYTimes/gziphandler v1.0.1
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e
github.com/armon/go-metrics v0.3.4
github.com/aws/aws-sdk-go v1.25.41
github.com/aws/aws-sdk-go v1.35.3
github.com/boltdb/bolt v1.3.1
github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f
github.com/containerd/go-cni v0.0.0-20190904155053-d20b7eebc7ee

8
go.sum
View File

@ -112,6 +112,8 @@ github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
github.com/aws/aws-sdk-go v1.25.41 h1:/hj7nZ0586wFqpwjNpzWiUTwtaMgxAZNZKHay80MdXw=
github.com/aws/aws-sdk-go v1.25.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54=
github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@ -274,6 +276,7 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
@ -488,6 +491,10 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht
github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joyent/triton-go v0.0.0-20180628001255-830d2b111e62/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA=
github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 h1:BvV6PYcRz0yGnWXNZrd5wginNT1GfFfPvvWpPbjfFL8=
github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA=
@ -863,6 +870,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=

93
vendor/github.com/aws/aws-sdk-go/aws/arn/arn.go generated vendored Normal file
View File

@ -0,0 +1,93 @@
// Package arn provides a parser for interacting with Amazon Resource Names.
package arn
import (
"errors"
"strings"
)
const (
arnDelimiter = ":"
arnSections = 6
arnPrefix = "arn:"
// zero-indexed
sectionPartition = 1
sectionService = 2
sectionRegion = 3
sectionAccountID = 4
sectionResource = 5
// errors
invalidPrefix = "arn: invalid prefix"
invalidSections = "arn: not enough sections"
)
// ARN captures the individual fields of an Amazon Resource Name.
// See http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html for more information.
type ARN struct {
// The partition that the resource is in. For standard AWS regions, the partition is "aws". If you have resources in
// other partitions, the partition is "aws-partitionname". For example, the partition for resources in the China
// (Beijing) region is "aws-cn".
Partition string
// The service namespace that identifies the AWS product (for example, Amazon S3, IAM, or Amazon RDS). For a list of
// namespaces, see
// http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#genref-aws-service-namespaces.
Service string
// The region the resource resides in. Note that the ARNs for some resources do not require a region, so this
// component might be omitted.
Region string
// The ID of the AWS account that owns the resource, without the hyphens. For example, 123456789012. Note that the
// ARNs for some resources don't require an account number, so this component might be omitted.
AccountID string
// The content of this part of the ARN varies by service. It often includes an indicator of the type of resource —
// for example, an IAM user or Amazon RDS database - followed by a slash (/) or a colon (:), followed by the
// resource name itself. Some services allows paths for resource names, as described in
// http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arns-paths.
Resource string
}
// Parse parses an ARN into its constituent parts.
//
// Some example ARNs:
// arn:aws:elasticbeanstalk:us-east-1:123456789012:environment/My App/MyEnvironment
// arn:aws:iam::123456789012:user/David
// arn:aws:rds:eu-west-1:123456789012:db:mysql-db
// arn:aws:s3:::my_corporate_bucket/exampleobject.png
func Parse(arn string) (ARN, error) {
if !strings.HasPrefix(arn, arnPrefix) {
return ARN{}, errors.New(invalidPrefix)
}
sections := strings.SplitN(arn, arnDelimiter, arnSections)
if len(sections) != arnSections {
return ARN{}, errors.New(invalidSections)
}
return ARN{
Partition: sections[sectionPartition],
Service: sections[sectionService],
Region: sections[sectionRegion],
AccountID: sections[sectionAccountID],
Resource: sections[sectionResource],
}, nil
}
// IsARN returns whether the given string is an ARN by looking for
// whether the string starts with "arn:" and contains the correct number
// of sections delimited by colons(:).
func IsARN(arn string) bool {
return strings.HasPrefix(arn, arnPrefix) && strings.Count(arn, ":") >= arnSections-1
}
// String returns the canonical representation of the ARN
func (arn ARN) String() string {
return arnPrefix +
arn.Partition + arnDelimiter +
arn.Service + arnDelimiter +
arn.Region + arnDelimiter +
arn.AccountID + arnDelimiter +
arn.Resource
}

View File

@ -43,7 +43,7 @@ type Config struct {
// An optional endpoint URL (hostname only or fully qualified URI)
// that overrides the default generated endpoint for a client. Set this
// to `""` to use the default generated endpoint.
// to `nil` or the value to `""` to use the default generated endpoint.
//
// Note: You must still provide a `Region` value when specifying an
// endpoint for a client.
@ -138,7 +138,7 @@ type Config struct {
// `ExpectContinueTimeout` for information on adjusting the continue wait
// timeout. https://golang.org/pkg/net/http/#Transport
//
// You should use this flag to disble 100-Continue if you experience issues
// You should use this flag to disable 100-Continue if you experience issues
// with proxies or third party S3 compatible services.
S3Disable100Continue *bool
@ -161,6 +161,17 @@ type Config struct {
// on GetObject API calls.
S3DisableContentMD5Validation *bool
// Set this to `true` to have the S3 service client to use the region specified
// in the ARN, when an ARN is provided as an argument to a bucket parameter.
S3UseARNRegion *bool
// Set this to `true` to enable the SDK to unmarshal API response header maps to
// normalized lower case map keys.
//
// For example S3's X-Amz-Meta prefixed header will be unmarshaled to lower case
// Metadata member's map keys. The value of the header in the map is unaffected.
LowerCaseHeaderMaps *bool
// Set this to `true` to disable the EC2Metadata client from overriding the
// default http.Client's Timeout. This is helpful if you do not want the
// EC2Metadata client to create a new http.Client. This options is only
@ -172,7 +183,7 @@ type Config struct {
//
// Example:
// sess := session.Must(session.NewSession(aws.NewConfig()
// .WithEC2MetadataDiableTimeoutOverride(true)))
// .WithEC2MetadataDisableTimeoutOverride(true)))
//
// svc := s3.New(sess)
//
@ -183,7 +194,7 @@ type Config struct {
// both IPv4 and IPv6 addressing.
//
// Setting this for a service which does not support dual stack will fail
// to make requets. It is not recommended to set this value on the session
// to make requests. It is not recommended to set this value on the session
// as it will apply to all service clients created with the session. Even
// services which don't support dual stack endpoints.
//
@ -227,6 +238,7 @@ type Config struct {
// EnableEndpointDiscovery will allow for endpoint discovery on operations that
// have the definition in its model. By default, endpoint discovery is off.
// To use EndpointDiscovery, Endpoint should be unset or set to an empty string.
//
// Example:
// sess := session.Must(session.NewSession(&aws.Config{
@ -385,6 +397,13 @@ func (c *Config) WithS3DisableContentMD5Validation(enable bool) *Config {
}
// WithS3UseARNRegion sets a config S3UseARNRegion value and
// returning a Config pointer for chaining
func (c *Config) WithS3UseARNRegion(enable bool) *Config {
c.S3UseARNRegion = &enable
return c
}
// WithUseDualStack sets a config UseDualStack value returning a Config
// pointer for chaining.
func (c *Config) WithUseDualStack(enable bool) *Config {
@ -513,6 +532,10 @@ func mergeInConfig(dst *Config, other *Config) {
dst.S3DisableContentMD5Validation = other.S3DisableContentMD5Validation
}
if other.S3UseARNRegion != nil {
dst.S3UseARNRegion = other.S3UseARNRegion
}
if other.UseDualStack != nil {
dst.UseDualStack = other.UseDualStack
}

View File

@ -2,42 +2,8 @@
package aws
import "time"
// An emptyCtx is a copy of the Go 1.7 context.emptyCtx type. This is copied to
// provide a 1.6 and 1.5 safe version of context that is compatible with Go
// 1.7's Context.
//
// An emptyCtx is never canceled, has no values, and has no deadline. It is not
// struct{}, since vars of this type must have distinct addresses.
type emptyCtx int
func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
return
}
func (*emptyCtx) Done() <-chan struct{} {
return nil
}
func (*emptyCtx) Err() error {
return nil
}
func (*emptyCtx) Value(key interface{}) interface{} {
return nil
}
func (e *emptyCtx) String() string {
switch e {
case backgroundCtx:
return "aws.BackgroundContext"
}
return "unknown empty Context"
}
var (
backgroundCtx = new(emptyCtx)
import (
"github.com/aws/aws-sdk-go/internal/context"
)
// BackgroundContext returns a context that will never be canceled, has no
@ -52,5 +18,5 @@ var (
//
// See https://golang.org/pkg/context for more information on Contexts.
func BackgroundContext() Context {
return backgroundCtx
return context.BackgroundCtx
}

View File

@ -161,7 +161,7 @@ func handleSendError(r *request.Request, err error) {
}
// Catch all request errors, and let the default retrier determine
// if the error is retryable.
r.Error = awserr.New("RequestError", "send request failed", err)
r.Error = awserr.New(request.ErrCodeRequestError, "send request failed", err)
// Override the error with a context canceled error, if that was canceled.
ctx := r.Context()
@ -225,6 +225,8 @@ var ValidateEndpointHandler = request.NamedHandler{Name: "core.ValidateEndpointH
if r.ClientInfo.SigningRegion == "" && aws.StringValue(r.Config.Region) == "" {
r.Error = aws.ErrMissingRegion
} else if r.ClientInfo.Endpoint == "" {
// Was any endpoint provided by the user, or one was derived by the
// SDK's endpoint resolver?
r.Error = aws.ErrMissingEndpoint
}
}}

View File

@ -0,0 +1,22 @@
// +build !go1.7
package credentials
import (
"github.com/aws/aws-sdk-go/internal/context"
)
// backgroundContext returns a context that will never be canceled, has no
// values, and no deadline. This context is used by the SDK to provide
// backwards compatibility with non-context API operations and functionality.
//
// Go 1.6 and before:
// This context function is equivalent to context.Background in the Go stdlib.
//
// Go 1.7 and later:
// The context returned will be the value returned by context.Background()
//
// See https://golang.org/pkg/context for more information on Contexts.
func backgroundContext() Context {
return context.BackgroundCtx
}

View File

@ -0,0 +1,20 @@
// +build go1.7
package credentials
import "context"
// backgroundContext returns a context that will never be canceled, has no
// values, and no deadline. This context is used by the SDK to provide
// backwards compatibility with non-context API operations and functionality.
//
// Go 1.6 and before:
// This context function is equivalent to context.Background in the Go stdlib.
//
// Go 1.7 and later:
// The context returned will be the value returned by context.Background()
//
// See https://golang.org/pkg/context for more information on Contexts.
func backgroundContext() Context {
return context.Background()
}

View File

@ -0,0 +1,39 @@
// +build !go1.9
package credentials
import "time"
// Context is an copy of the Go v1.7 stdlib's context.Context interface.
// It is represented as a SDK interface to enable you to use the "WithContext"
// API methods with Go v1.6 and a Context type such as golang.org/x/net/context.
//
// This type, aws.Context, and context.Context are equivalent.
//
// See https://golang.org/pkg/context on how to use contexts.
type Context interface {
// Deadline returns the time when work done on behalf of this context
// should be canceled. Deadline returns ok==false when no deadline is
// set. Successive calls to Deadline return the same results.
Deadline() (deadline time.Time, ok bool)
// Done returns a channel that's closed when work done on behalf of this
// context should be canceled. Done may return nil if this context can
// never be canceled. Successive calls to Done return the same value.
Done() <-chan struct{}
// Err returns a non-nil error value after Done is closed. Err returns
// Canceled if the context was canceled or DeadlineExceeded if the
// context's deadline passed. No other values for Err are defined.
// After Done is closed, successive calls to Err return the same value.
Err() error
// Value returns the value associated with this context for key, or nil
// if no value is associated with key. Successive calls to Value with
// the same key returns the same result.
//
// Use context values only for request-scoped data that transits
// processes and API boundaries, not for passing optional parameters to
// functions.
Value(key interface{}) interface{}
}

View File

@ -0,0 +1,13 @@
// +build go1.9
package credentials
import "context"
// Context is an alias of the Go stdlib's context.Context interface.
// It can be used within the SDK's API operation "WithContext" methods.
//
// This type, aws.Context, and context.Context are equivalent.
//
// See https://golang.org/pkg/context on how to use contexts.
type Context = context.Context

View File

@ -50,10 +50,11 @@ package credentials
import (
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/internal/sync/singleflight"
)
// AnonymousCredentials is an empty Credential object that can be used as
@ -106,6 +107,13 @@ type Provider interface {
IsExpired() bool
}
// ProviderWithContext is a Provider that can retrieve credentials with a Context
type ProviderWithContext interface {
Provider
RetrieveWithContext(Context) (Value, error)
}
// An Expirer is an interface that Providers can implement to expose the expiration
// time, if known. If the Provider cannot accurately provide this info,
// it should not implement this interface.
@ -197,20 +205,68 @@ func (e *Expiry) ExpiresAt() time.Time {
// first instance of the credentials Value. All calls to Get() after that
// will return the cached credentials Value until IsExpired() returns true.
type Credentials struct {
creds Value
forceRefresh bool
m sync.RWMutex
creds atomic.Value
sf singleflight.Group
provider Provider
}
// NewCredentials returns a pointer to a new Credentials with the provider set.
func NewCredentials(provider Provider) *Credentials {
return &Credentials{
provider: provider,
forceRefresh: true,
c := &Credentials{
provider: provider,
}
c.creds.Store(Value{})
return c
}
// GetWithContext returns the credentials value, or error if the credentials
// Value failed to be retrieved. Will return early if the passed in context is
// canceled.
//
// Will return the cached credentials Value if it has not expired. If the
// credentials Value has expired the Provider's Retrieve() will be called
// to refresh the credentials.
//
// If Credentials.Expire() was called the credentials Value will be force
// expired, and the next call to Get() will cause them to be refreshed.
//
// Passed in Context is equivalent to aws.Context, and context.Context.
func (c *Credentials) GetWithContext(ctx Context) (Value, error) {
if curCreds := c.creds.Load(); !c.isExpired(curCreds) {
return curCreds.(Value), nil
}
// Cannot pass context down to the actual retrieve, because the first
// context would cancel the whole group when there is not direct
// association of items in the group.
resCh := c.sf.DoChan("", func() (interface{}, error) {
return c.singleRetrieve(&suppressedContext{ctx})
})
select {
case res := <-resCh:
return res.Val.(Value), res.Err
case <-ctx.Done():
return Value{}, awserr.New("RequestCanceled",
"request context canceled", ctx.Err())
}
}
func (c *Credentials) singleRetrieve(ctx Context) (creds interface{}, err error) {
if curCreds := c.creds.Load(); !c.isExpired(curCreds) {
return curCreds.(Value), nil
}
if p, ok := c.provider.(ProviderWithContext); ok {
creds, err = p.RetrieveWithContext(ctx)
} else {
creds, err = c.provider.Retrieve()
}
if err == nil {
c.creds.Store(creds)
}
return creds, err
}
// Get returns the credentials value, or error if the credentials Value failed
@ -223,30 +279,7 @@ func NewCredentials(provider Provider) *Credentials {
// If Credentials.Expire() was called the credentials Value will be force
// expired, and the next call to Get() will cause them to be refreshed.
func (c *Credentials) Get() (Value, error) {
// Check the cached credentials first with just the read lock.
c.m.RLock()
if !c.isExpired() {
creds := c.creds
c.m.RUnlock()
return creds, nil
}
c.m.RUnlock()
// Credentials are expired need to retrieve the credentials taking the full
// lock.
c.m.Lock()
defer c.m.Unlock()
if c.isExpired() {
creds, err := c.provider.Retrieve()
if err != nil {
return Value{}, err
}
c.creds = creds
c.forceRefresh = false
}
return c.creds, nil
return c.GetWithContext(backgroundContext())
}
// Expire expires the credentials and forces them to be retrieved on the
@ -255,10 +288,7 @@ func (c *Credentials) Get() (Value, error) {
// This will override the Provider's expired state, and force Credentials
// to call the Provider's Retrieve().
func (c *Credentials) Expire() {
c.m.Lock()
defer c.m.Unlock()
c.forceRefresh = true
c.creds.Store(Value{})
}
// IsExpired returns if the credentials are no longer valid, and need
@ -267,33 +297,43 @@ func (c *Credentials) Expire() {
// If the Credentials were forced to be expired with Expire() this will
// reflect that override.
func (c *Credentials) IsExpired() bool {
c.m.RLock()
defer c.m.RUnlock()
return c.isExpired()
return c.isExpired(c.creds.Load())
}
// isExpired helper method wrapping the definition of expired credentials.
func (c *Credentials) isExpired() bool {
return c.forceRefresh || c.provider.IsExpired()
func (c *Credentials) isExpired(creds interface{}) bool {
return creds == nil || creds.(Value) == Value{} || c.provider.IsExpired()
}
// ExpiresAt provides access to the functionality of the Expirer interface of
// the underlying Provider, if it supports that interface. Otherwise, it returns
// an error.
func (c *Credentials) ExpiresAt() (time.Time, error) {
c.m.RLock()
defer c.m.RUnlock()
expirer, ok := c.provider.(Expirer)
if !ok {
return time.Time{}, awserr.New("ProviderNotExpirer",
fmt.Sprintf("provider %s does not support ExpiresAt()", c.creds.ProviderName),
fmt.Sprintf("provider %s does not support ExpiresAt()", c.creds.Load().(Value).ProviderName),
nil)
}
if c.forceRefresh {
if c.creds.Load().(Value) == (Value{}) {
// set expiration time to the distant past
return time.Time{}, nil
}
return expirer.ExpiresAt(), nil
}
type suppressedContext struct {
Context
}
func (s *suppressedContext) Deadline() (deadline time.Time, ok bool) {
return time.Time{}, false
}
func (s *suppressedContext) Done() <-chan struct{} {
return nil
}
func (s *suppressedContext) Err() error {
return nil
}

View File

@ -7,6 +7,7 @@ import (
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/credentials"
@ -87,7 +88,14 @@ func NewCredentialsWithClient(client *ec2metadata.EC2Metadata, options ...func(*
// Error will be returned if the request fails, or unable to extract
// the desired credentials.
func (m *EC2RoleProvider) Retrieve() (credentials.Value, error) {
credsList, err := requestCredList(m.Client)
return m.RetrieveWithContext(aws.BackgroundContext())
}
// RetrieveWithContext retrieves credentials from the EC2 service.
// Error will be returned if the request fails, or unable to extract
// the desired credentials.
func (m *EC2RoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
credsList, err := requestCredList(ctx, m.Client)
if err != nil {
return credentials.Value{ProviderName: ProviderName}, err
}
@ -97,7 +105,7 @@ func (m *EC2RoleProvider) Retrieve() (credentials.Value, error) {
}
credsName := credsList[0]
roleCreds, err := requestCred(m.Client, credsName)
roleCreds, err := requestCred(ctx, m.Client, credsName)
if err != nil {
return credentials.Value{ProviderName: ProviderName}, err
}
@ -130,8 +138,8 @@ const iamSecurityCredsPath = "iam/security-credentials/"
// requestCredList requests a list of credentials from the EC2 service.
// If there are no credentials, or there is an error making or receiving the request
func requestCredList(client *ec2metadata.EC2Metadata) ([]string, error) {
resp, err := client.GetMetadata(iamSecurityCredsPath)
func requestCredList(ctx aws.Context, client *ec2metadata.EC2Metadata) ([]string, error) {
resp, err := client.GetMetadataWithContext(ctx, iamSecurityCredsPath)
if err != nil {
return nil, awserr.New("EC2RoleRequestError", "no EC2 instance role found", err)
}
@ -154,8 +162,8 @@ func requestCredList(client *ec2metadata.EC2Metadata) ([]string, error) {
//
// If the credentials cannot be found, or there is an error reading the response
// and error will be returned.
func requestCred(client *ec2metadata.EC2Metadata, credsName string) (ec2RoleCredRespBody, error) {
resp, err := client.GetMetadata(sdkuri.PathJoin(iamSecurityCredsPath, credsName))
func requestCred(ctx aws.Context, client *ec2metadata.EC2Metadata, credsName string) (ec2RoleCredRespBody, error) {
resp, err := client.GetMetadataWithContext(ctx, sdkuri.PathJoin(iamSecurityCredsPath, credsName))
if err != nil {
return ec2RoleCredRespBody{},
awserr.New("EC2RoleRequestError",

View File

@ -116,7 +116,13 @@ func (p *Provider) IsExpired() bool {
// Retrieve will attempt to request the credentials from the endpoint the Provider
// was configured for. And error will be returned if the retrieval fails.
func (p *Provider) Retrieve() (credentials.Value, error) {
resp, err := p.getCredentials()
return p.RetrieveWithContext(aws.BackgroundContext())
}
// RetrieveWithContext will attempt to request the credentials from the endpoint the Provider
// was configured for. And error will be returned if the retrieval fails.
func (p *Provider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
resp, err := p.getCredentials(ctx)
if err != nil {
return credentials.Value{ProviderName: ProviderName},
awserr.New("CredentialsEndpointError", "failed to load credentials", err)
@ -148,7 +154,7 @@ type errorOutput struct {
Message string `json:"message"`
}
func (p *Provider) getCredentials() (*getCredentialsOutput, error) {
func (p *Provider) getCredentials(ctx aws.Context) (*getCredentialsOutput, error) {
op := &request.Operation{
Name: "GetCredentials",
HTTPMethod: "GET",
@ -156,6 +162,7 @@ func (p *Provider) getCredentials() (*getCredentialsOutput, error) {
out := &getCredentialsOutput{}
req := p.Client.NewRequest(op, nil, out)
req.SetContext(ctx)
req.HTTPRequest.Header.Set("Accept", "application/json")
if authToken := p.AuthorizationToken; len(authToken) != 0 {
req.HTTPRequest.Header.Set("Authorization", authToken)

View File

@ -90,6 +90,7 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/internal/sdkio"
)
const (
@ -142,7 +143,7 @@ const (
// DefaultBufSize limits buffer size from growing to an enormous
// amount due to a faulty process.
DefaultBufSize = 1024
DefaultBufSize = int(8 * sdkio.KibiByte)
// DefaultTimeout default limit on time a process can run.
DefaultTimeout = time.Duration(1) * time.Minute

View File

@ -17,8 +17,9 @@ var (
ErrSharedCredentialsHomeNotFound = awserr.New("UserHomeNotFound", "user home directory not found.", nil)
)
// A SharedCredentialsProvider retrieves credentials from the current user's home
// directory, and keeps track if those credentials are expired.
// A SharedCredentialsProvider retrieves access key pair (access key ID,
// secret access key, and session token if present) credentials from the current
// user's home directory, and keeps track if those credentials are expired.
//
// Profile ini file example: $HOME/.aws/credentials
type SharedCredentialsProvider struct {

View File

@ -19,7 +19,9 @@ type StaticProvider struct {
}
// NewStaticCredentials returns a pointer to a new Credentials object
// wrapping a static credentials value provider.
// wrapping a static credentials value provider. Token is only required
// for temporary security credentials retrieved via STS, otherwise an empty
// string can be passed for this parameter.
func NewStaticCredentials(id, secret, token string) *Credentials {
return NewCredentials(&StaticProvider{Value: Value{
AccessKeyID: id,

View File

@ -87,6 +87,7 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/sdkrand"
"github.com/aws/aws-sdk-go/service/sts"
)
@ -118,6 +119,10 @@ type AssumeRoler interface {
AssumeRole(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error)
}
type assumeRolerWithContext interface {
AssumeRoleWithContext(aws.Context, *sts.AssumeRoleInput, ...request.Option) (*sts.AssumeRoleOutput, error)
}
// DefaultDuration is the default amount of time in minutes that the credentials
// will be valid for.
var DefaultDuration = time.Duration(15) * time.Minute
@ -144,6 +149,13 @@ type AssumeRoleProvider struct {
// Session name, if you wish to reuse the credentials elsewhere.
RoleSessionName string
// Optional, you can pass tag key-value pairs to your session. These tags are called session tags.
Tags []*sts.Tag
// A list of keys for session tags that you want to set as transitive.
// If you set a tag key as transitive, the corresponding key and value passes to subsequent sessions in a role chain.
TransitiveTagKeys []*string
// Expiry duration of the STS credentials. Defaults to 15 minutes if not set.
Duration time.Duration
@ -157,6 +169,29 @@ type AssumeRoleProvider struct {
// size.
Policy *string
// The ARNs of IAM managed policies you want to use as managed session policies.
// The policies must exist in the same account as the role.
//
// This parameter is optional. You can provide up to 10 managed policy ARNs.
// However, the plain text that you use for both inline and managed session
// policies can't exceed 2,048 characters.
//
// An AWS conversion compresses the passed session policies and session tags
// into a packed binary format that has a separate limit. Your request can fail
// for this limit even if your plain text meets the other requirements. The
// PackedPolicySize response element indicates by percentage how close the policies
// and tags for your request are to the upper size limit.
//
// Passing policies to this operation returns new temporary credentials. The
// resulting session's permissions are the intersection of the role's identity-based
// policy and the session policies. You can use the role's temporary credentials
// in subsequent AWS API calls to access resources in the account that owns
// the role. You cannot use session policies to grant more permissions than
// those allowed by the identity-based policy of the role that is being assumed.
// For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
// in the IAM User Guide.
PolicyArns []*sts.PolicyDescriptorType
// The identification number of the MFA device that is associated with the user
// who is making the AssumeRole call. Specify this value if the trust policy
// of the role being assumed includes a condition that requires MFA authentication.
@ -258,6 +293,11 @@ func NewCredentialsWithClient(svc AssumeRoler, roleARN string, options ...func(*
// Retrieve generates a new set of temporary credentials using STS.
func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {
return p.RetrieveWithContext(aws.BackgroundContext())
}
// RetrieveWithContext generates a new set of temporary credentials using STS.
func (p *AssumeRoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
// Apply defaults where parameters are not set.
if p.RoleSessionName == "" {
// Try to work out a role name that will hopefully end up unique.
@ -269,10 +309,13 @@ func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {
}
jitter := time.Duration(sdkrand.SeededRand.Float64() * p.MaxJitterFrac * float64(p.Duration))
input := &sts.AssumeRoleInput{
DurationSeconds: aws.Int64(int64((p.Duration - jitter) / time.Second)),
RoleArn: aws.String(p.RoleARN),
RoleSessionName: aws.String(p.RoleSessionName),
ExternalId: p.ExternalID,
DurationSeconds: aws.Int64(int64((p.Duration - jitter) / time.Second)),
RoleArn: aws.String(p.RoleARN),
RoleSessionName: aws.String(p.RoleSessionName),
ExternalId: p.ExternalID,
Tags: p.Tags,
PolicyArns: p.PolicyArns,
TransitiveTagKeys: p.TransitiveTagKeys,
}
if p.Policy != nil {
input.Policy = p.Policy
@ -295,7 +338,15 @@ func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {
}
}
roleOutput, err := p.Client.AssumeRole(input)
var roleOutput *sts.AssumeRoleOutput
var err error
if c, ok := p.Client.(assumeRolerWithContext); ok {
roleOutput, err = c.AssumeRoleWithContext(ctx, input)
} else {
roleOutput, err = p.Client.AssumeRole(input)
}
if err != nil {
return credentials.Value{ProviderName: ProviderName}, err
}

View File

@ -28,15 +28,46 @@ const (
// compare test values.
var now = time.Now
// TokenFetcher shuold return WebIdentity token bytes or an error
type TokenFetcher interface {
FetchToken(credentials.Context) ([]byte, error)
}
// FetchTokenPath is a path to a WebIdentity token file
type FetchTokenPath string
// FetchToken returns a token by reading from the filesystem
func (f FetchTokenPath) FetchToken(ctx credentials.Context) ([]byte, error) {
data, err := ioutil.ReadFile(string(f))
if err != nil {
errMsg := fmt.Sprintf("unable to read file at %s", f)
return nil, awserr.New(ErrCodeWebIdentity, errMsg, err)
}
return data, nil
}
// WebIdentityRoleProvider is used to retrieve credentials using
// an OIDC token.
type WebIdentityRoleProvider struct {
credentials.Expiry
PolicyArns []*sts.PolicyDescriptorType
client stsiface.STSAPI
// Duration the STS credentials will be valid for. Truncated to seconds.
// If unset, the assumed role will use AssumeRoleWithWebIdentity's default
// expiry duration. See
// https://docs.aws.amazon.com/sdk-for-go/api/service/sts/#STS.AssumeRoleWithWebIdentity
// for more information.
Duration time.Duration
// The amount of time the credentials will be refreshed before they expire.
// This is useful refresh credentials before they expire to reduce risk of
// using credentials as they expire. If unset, will default to no expiry
// window.
ExpiryWindow time.Duration
tokenFilePath string
client stsiface.STSAPI
tokenFetcher TokenFetcher
roleARN string
roleSessionName string
}
@ -52,9 +83,15 @@ func NewWebIdentityCredentials(c client.ConfigProvider, roleARN, roleSessionName
// NewWebIdentityRoleProvider will return a new WebIdentityRoleProvider with the
// provided stsiface.STSAPI
func NewWebIdentityRoleProvider(svc stsiface.STSAPI, roleARN, roleSessionName, path string) *WebIdentityRoleProvider {
return NewWebIdentityRoleProviderWithToken(svc, roleARN, roleSessionName, FetchTokenPath(path))
}
// NewWebIdentityRoleProviderWithToken will return a new WebIdentityRoleProvider with the
// provided stsiface.STSAPI and a TokenFetcher
func NewWebIdentityRoleProviderWithToken(svc stsiface.STSAPI, roleARN, roleSessionName string, tokenFetcher TokenFetcher) *WebIdentityRoleProvider {
return &WebIdentityRoleProvider{
client: svc,
tokenFilePath: path,
tokenFetcher: tokenFetcher,
roleARN: roleARN,
roleSessionName: roleSessionName,
}
@ -64,10 +101,16 @@ func NewWebIdentityRoleProvider(svc stsiface.STSAPI, roleARN, roleSessionName, p
// 'WebIdentityTokenFilePath' specified destination and if that is empty an
// error will be returned.
func (p *WebIdentityRoleProvider) Retrieve() (credentials.Value, error) {
b, err := ioutil.ReadFile(p.tokenFilePath)
return p.RetrieveWithContext(aws.BackgroundContext())
}
// RetrieveWithContext will attempt to assume a role from a token which is located at
// 'WebIdentityTokenFilePath' specified destination and if that is empty an
// error will be returned.
func (p *WebIdentityRoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
b, err := p.tokenFetcher.FetchToken(ctx)
if err != nil {
errMsg := fmt.Sprintf("unable to read file at %s", p.tokenFilePath)
return credentials.Value{}, awserr.New(ErrCodeWebIdentity, errMsg, err)
return credentials.Value{}, awserr.New(ErrCodeWebIdentity, "failed fetching WebIdentity token: ", err)
}
sessionName := p.roleSessionName
@ -76,11 +119,22 @@ func (p *WebIdentityRoleProvider) Retrieve() (credentials.Value, error) {
// uses unix time in nanoseconds to uniquely identify sessions.
sessionName = strconv.FormatInt(now().UnixNano(), 10)
}
var duration *int64
if p.Duration != 0 {
duration = aws.Int64(int64(p.Duration / time.Second))
}
req, resp := p.client.AssumeRoleWithWebIdentityRequest(&sts.AssumeRoleWithWebIdentityInput{
PolicyArns: p.PolicyArns,
RoleArn: &p.roleARN,
RoleSessionName: &sessionName,
WebIdentityToken: aws.String(string(b)),
DurationSeconds: duration,
})
req.SetContext(ctx)
// InvalidIdentityToken error is a temporary error that can occur
// when assuming an Role with a JWT web identity token.
req.RetryErrorCodes = append(req.RetryErrorCodes, sts.ErrCodeInvalidIdentityTokenException)

View File

@ -89,7 +89,7 @@ func getMetricException(err awserr.Error) metricException {
code := err.Code()
switch code {
case "RequestError",
case request.ErrCodeRequestError,
request.ErrCodeSerialization,
request.CanceledErrorCode:
return sdkException{

View File

@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/sdkuri"
@ -15,15 +16,16 @@ import (
// getToken uses the duration to return a token for EC2 metadata service,
// or an error if the request failed.
func (c *EC2Metadata) getToken(duration time.Duration) (tokenOutput, error) {
func (c *EC2Metadata) getToken(ctx aws.Context, duration time.Duration) (tokenOutput, error) {
op := &request.Operation{
Name: "GetToken",
HTTPMethod: "PUT",
HTTPPath: "/api/token",
HTTPPath: "/latest/api/token",
}
var output tokenOutput
req := c.NewRequest(op, nil, &output)
req.SetContext(ctx)
// remove the fetch token handler from the request handlers to avoid infinite recursion
req.Handlers.Sign.RemoveByName(fetchTokenHandlerName)
@ -31,7 +33,7 @@ func (c *EC2Metadata) getToken(duration time.Duration) (tokenOutput, error) {
// Swap the unmarshalMetadataHandler with unmarshalTokenHandler on this request.
req.Handlers.Unmarshal.Swap(unmarshalMetadataHandlerName, unmarshalTokenHandler)
ttl := strconv.FormatInt(int64(duration / time.Second),10)
ttl := strconv.FormatInt(int64(duration/time.Second), 10)
req.HTTPRequest.Header.Set(ttlHeader, ttl)
err := req.Send()
@ -50,15 +52,24 @@ func (c *EC2Metadata) getToken(duration time.Duration) (tokenOutput, error) {
// instance metadata service. The content will be returned as a string, or
// error if the request failed.
func (c *EC2Metadata) GetMetadata(p string) (string, error) {
return c.GetMetadataWithContext(aws.BackgroundContext(), p)
}
// GetMetadataWithContext uses the path provided to request information from the EC2
// instance metadata service. The content will be returned as a string, or
// error if the request failed.
func (c *EC2Metadata) GetMetadataWithContext(ctx aws.Context, p string) (string, error) {
op := &request.Operation{
Name: "GetMetadata",
HTTPMethod: "GET",
HTTPPath: sdkuri.PathJoin("/meta-data", p),
HTTPPath: sdkuri.PathJoin("/latest/meta-data", p),
}
output := &metadataOutput{}
req := c.NewRequest(op, nil, output)
req.SetContext(ctx)
err := req.Send()
return output.Content, err
}
@ -67,14 +78,22 @@ func (c *EC2Metadata) GetMetadata(p string) (string, error) {
// there is no user-data setup for the EC2 instance a "NotFoundError" error
// code will be returned.
func (c *EC2Metadata) GetUserData() (string, error) {
return c.GetUserDataWithContext(aws.BackgroundContext())
}
// GetUserDataWithContext returns the userdata that was configured for the service. If
// there is no user-data setup for the EC2 instance a "NotFoundError" error
// code will be returned.
func (c *EC2Metadata) GetUserDataWithContext(ctx aws.Context) (string, error) {
op := &request.Operation{
Name: "GetUserData",
HTTPMethod: "GET",
HTTPPath: "/user-data",
HTTPPath: "/latest/user-data",
}
output := &metadataOutput{}
req := c.NewRequest(op, nil, output)
req.SetContext(ctx)
err := req.Send()
return output.Content, err
@ -84,14 +103,22 @@ func (c *EC2Metadata) GetUserData() (string, error) {
// instance metadata service for dynamic data. The content will be returned
// as a string, or error if the request failed.
func (c *EC2Metadata) GetDynamicData(p string) (string, error) {
return c.GetDynamicDataWithContext(aws.BackgroundContext(), p)
}
// GetDynamicDataWithContext uses the path provided to request information from the EC2
// instance metadata service for dynamic data. The content will be returned
// as a string, or error if the request failed.
func (c *EC2Metadata) GetDynamicDataWithContext(ctx aws.Context, p string) (string, error) {
op := &request.Operation{
Name: "GetDynamicData",
HTTPMethod: "GET",
HTTPPath: sdkuri.PathJoin("/dynamic", p),
HTTPPath: sdkuri.PathJoin("/latest/dynamic", p),
}
output := &metadataOutput{}
req := c.NewRequest(op, nil, output)
req.SetContext(ctx)
err := req.Send()
return output.Content, err
@ -101,7 +128,14 @@ func (c *EC2Metadata) GetDynamicData(p string) (string, error) {
// instance. Error is returned if the request fails or is unable to parse
// the response.
func (c *EC2Metadata) GetInstanceIdentityDocument() (EC2InstanceIdentityDocument, error) {
resp, err := c.GetDynamicData("instance-identity/document")
return c.GetInstanceIdentityDocumentWithContext(aws.BackgroundContext())
}
// GetInstanceIdentityDocumentWithContext retrieves an identity document describing an
// instance. Error is returned if the request fails or is unable to parse
// the response.
func (c *EC2Metadata) GetInstanceIdentityDocumentWithContext(ctx aws.Context) (EC2InstanceIdentityDocument, error) {
resp, err := c.GetDynamicDataWithContext(ctx, "instance-identity/document")
if err != nil {
return EC2InstanceIdentityDocument{},
awserr.New("EC2MetadataRequestError",
@ -120,7 +154,12 @@ func (c *EC2Metadata) GetInstanceIdentityDocument() (EC2InstanceIdentityDocument
// IAMInfo retrieves IAM info from the metadata API
func (c *EC2Metadata) IAMInfo() (EC2IAMInfo, error) {
resp, err := c.GetMetadata("iam/info")
return c.IAMInfoWithContext(aws.BackgroundContext())
}
// IAMInfoWithContext retrieves IAM info from the metadata API
func (c *EC2Metadata) IAMInfoWithContext(ctx aws.Context) (EC2IAMInfo, error) {
resp, err := c.GetMetadataWithContext(ctx, "iam/info")
if err != nil {
return EC2IAMInfo{},
awserr.New("EC2MetadataRequestError",
@ -145,24 +184,36 @@ func (c *EC2Metadata) IAMInfo() (EC2IAMInfo, error) {
// Region returns the region the instance is running in.
func (c *EC2Metadata) Region() (string, error) {
resp, err := c.GetMetadata("placement/availability-zone")
return c.RegionWithContext(aws.BackgroundContext())
}
// RegionWithContext returns the region the instance is running in.
func (c *EC2Metadata) RegionWithContext(ctx aws.Context) (string, error) {
ec2InstanceIdentityDocument, err := c.GetInstanceIdentityDocumentWithContext(ctx)
if err != nil {
return "", err
}
if len(resp) == 0 {
return "", awserr.New("EC2MetadataError", "invalid Region response", nil)
// extract region from the ec2InstanceIdentityDocument
region := ec2InstanceIdentityDocument.Region
if len(region) == 0 {
return "", awserr.New("EC2MetadataError", "invalid region received for ec2metadata instance", nil)
}
// returns region without the suffix. Eg: us-west-2a becomes us-west-2
return resp[:len(resp)-1], nil
// returns region
return region, nil
}
// Available returns if the application has access to the EC2 Metadata service.
// Can be used to determine if application is running within an EC2 Instance and
// the metadata service is available.
func (c *EC2Metadata) Available() bool {
if _, err := c.GetMetadata("instance-id"); err != nil {
return c.AvailableWithContext(aws.BackgroundContext())
}
// AvailableWithContext returns if the application has access to the EC2 Metadata service.
// Can be used to determine if application is running within an EC2 Instance and
// the metadata service is available.
func (c *EC2Metadata) AvailableWithContext(ctx aws.Context) bool {
if _, err := c.GetMetadataWithContext(ctx, "instance-id"); err != nil {
return false
}

View File

@ -5,6 +5,10 @@
// variable "AWS_EC2_METADATA_DISABLED=true". This environment variable set to
// true instructs the SDK to disable the EC2 Metadata client. The client cannot
// be used while the environment variable is set to true, (case insensitive).
//
// The endpoint of the EC2 IMDS client can be configured via the environment
// variable, AWS_EC2_METADATA_SERVICE_ENDPOINT when creating the client with a
// Session. See aws/session#Options.EC2IMDSEndpoint for more details.
package ec2metadata
import (
@ -12,6 +16,7 @@ import (
"errors"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
@ -41,7 +46,7 @@ const (
enableTokenProviderHandlerName = "enableTokenProviderHandler"
// TTL constants
defaultTTL = 21600 * time.Second
defaultTTL = 21600 * time.Second
ttlExpirationWindow = 30 * time.Second
)
@ -69,6 +74,9 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *EC2Metadata {
// a client when not using a session. Generally using just New with a session
// is preferred.
//
// Will remove the URL path from the endpoint provided to ensure the EC2 IMDS
// client is able to communicate with the EC2 IMDS API.
//
// If an unmodified HTTP client is provided from the stdlib default, or no client
// the EC2RoleProvider's EC2Metadata HTTP client's timeout will be shortened.
// To disable this set Config.EC2MetadataDisableTimeoutOverride to false. Enabled by default.
@ -80,8 +88,19 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
// use a shorter timeout than default because the metadata
// service is local if it is running, and to fail faster
// if not running on an ec2 instance.
Timeout: 5 * time.Second,
Timeout: 1 * time.Second,
}
// max number of retries on the client operation
cfg.MaxRetries = aws.Int(2)
}
if u, err := url.Parse(endpoint); err == nil {
// Remove path from the endpoint since it will be added by requests.
// This is an artifact of the SDK adding `/latest` to the endpoint for
// EC2 IMDS, but this is now moved to the operation definition.
u.Path = ""
u.RawPath = ""
endpoint = u.String()
}
svc := &EC2Metadata{

View File

@ -46,7 +46,7 @@ func (t *tokenProvider) fetchTokenHandler(r *request.Request) {
return
}
output, err := t.client.getToken(t.configuredTTL)
output, err := t.client.getToken(r.Context(), t.configuredTTL)
if err != nil {
@ -62,7 +62,7 @@ func (t *tokenProvider) fetchTokenHandler(r *request.Request) {
// Check if request timed out while waiting for response
if e, ok := requestFailureError.OrigErr().(awserr.Error); ok {
if e.Code() == "RequestError" {
if e.Code() == request.ErrCodeRequestError {
atomic.StoreUint32(&t.disabled, 1)
}
}

View File

@ -93,7 +93,7 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
}
func custAddS3DualStack(p *partition) {
if p.ID != "aws" {
if !(p.ID == "aws" || p.ID == "aws-cn" || p.ID == "aws-us-gov") {
return
}

File diff suppressed because it is too large Load Diff

View File

@ -317,7 +317,7 @@ func (p Partition) EndpointFor(service, region string, opts ...func(*Options)) (
// Regions returns a map of Regions indexed by their ID. This is useful for
// enumerating over the regions in a partition.
func (p Partition) Regions() map[string]Region {
rs := map[string]Region{}
rs := make(map[string]Region, len(p.p.Regions))
for id, r := range p.p.Regions {
rs[id] = Region{
id: id,
@ -332,7 +332,7 @@ func (p Partition) Regions() map[string]Region {
// Services returns a map of Service indexed by their ID. This is useful for
// enumerating over the services in a partition.
func (p Partition) Services() map[string]Service {
ss := map[string]Service{}
ss := make(map[string]Service, len(p.p.Services))
for id := range p.p.Services {
ss[id] = Service{
id: id,
@ -419,7 +419,7 @@ func (s Service) Regions() map[string]Region {
// A region is the AWS region the service exists in. Whereas a Endpoint is
// an URL that can be resolved to a instance of a service.
func (s Service) Endpoints() map[string]Endpoint {
es := map[string]Endpoint{}
es := make(map[string]Endpoint, len(s.p.Services[s.id].Endpoints))
for id := range s.p.Services[s.id].Endpoints {
es[id] = Endpoint{
id: id,

View File

@ -7,6 +7,8 @@ import (
"strings"
)
var regionValidationRegex = regexp.MustCompile(`^[[:alnum:]]([[:alnum:]\-]*[[:alnum:]])?$`)
type partitions []partition
func (ps partitions) EndpointFor(service, region string, opts ...func(*Options)) (ResolvedEndpoint, error) {
@ -124,7 +126,7 @@ func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (
defs := []endpoint{p.Defaults, s.Defaults}
return e.resolve(service, p.ID, region, p.DNSSuffix, defs, opt), nil
return e.resolve(service, p.ID, region, p.DNSSuffix, defs, opt)
}
func serviceList(ss services) []string {
@ -233,7 +235,7 @@ func getByPriority(s []string, p []string, def string) string {
return s[0]
}
func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs []endpoint, opts Options) ResolvedEndpoint {
func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs []endpoint, opts Options) (ResolvedEndpoint, error) {
var merged endpoint
for _, def := range defs {
merged.mergeIn(def)
@ -260,6 +262,10 @@ func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs [
region = signingRegion
}
if !validateInputRegion(region) {
return ResolvedEndpoint{}, fmt.Errorf("invalid region identifier format provided")
}
u := strings.Replace(hostname, "{service}", service, 1)
u = strings.Replace(u, "{region}", region, 1)
u = strings.Replace(u, "{dnsSuffix}", dnsSuffix, 1)
@ -274,7 +280,7 @@ func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs [
SigningName: signingName,
SigningNameDerived: signingNameDerived,
SigningMethod: getByPriority(e.SignatureVersions, signerPriority, defaultSigner),
}
}, nil
}
func getEndpointScheme(protocols []string, disableSSL bool) string {
@ -339,3 +345,7 @@ const (
boxedFalse
boxedTrue
)
func validateInputRegion(region string) bool {
return regionValidationRegex.MatchString(region)
}

View File

@ -9,7 +9,8 @@ func isErrConnectionReset(err error) bool {
return false
}
if strings.Contains(err.Error(), "connection reset") ||
if strings.Contains(err.Error(), "use of closed network connection") ||
strings.Contains(err.Error(), "connection reset") ||
strings.Contains(err.Error(), "broken pipe") {
return true
}

View File

@ -10,6 +10,7 @@ import (
type Handlers struct {
Validate HandlerList
Build HandlerList
BuildStream HandlerList
Sign HandlerList
Send HandlerList
ValidateResponse HandlerList
@ -28,6 +29,7 @@ func (h *Handlers) Copy() Handlers {
return Handlers{
Validate: h.Validate.copy(),
Build: h.Build.copy(),
BuildStream: h.BuildStream.copy(),
Sign: h.Sign.copy(),
Send: h.Send.copy(),
ValidateResponse: h.ValidateResponse.copy(),
@ -46,6 +48,7 @@ func (h *Handlers) Copy() Handlers {
func (h *Handlers) Clear() {
h.Validate.Clear()
h.Build.Clear()
h.BuildStream.Clear()
h.Send.Clear()
h.Sign.Clear()
h.Unmarshal.Clear()
@ -67,6 +70,9 @@ func (h *Handlers) IsEmpty() bool {
if h.Build.Len() != 0 {
return false
}
if h.BuildStream.Len() != 0 {
return false
}
if h.Send.Len() != 0 {
return false
}
@ -320,3 +326,18 @@ func MakeAddToUserAgentFreeFormHandler(s string) func(*Request) {
AddToUserAgent(r, s)
}
}
// WithSetRequestHeaders updates the operation request's HTTP header to contain
// the header key value pairs provided. If the header key already exists in the
// request's HTTP header set, the existing value(s) will be replaced.
func WithSetRequestHeaders(h map[string]string) Option {
return withRequestHeader(h).SetRequestHeaders
}
type withRequestHeader map[string]string
func (h withRequestHeader) SetRequestHeaders(r *Request) {
for k, v := range h {
r.HTTPRequest.Header[k] = []string{v}
}
}

View File

@ -36,6 +36,10 @@ const (
// API request that was canceled. Requests given a aws.Context may
// return this error when canceled.
CanceledErrorCode = "RequestCanceled"
// ErrCodeRequestError is an error preventing the SDK from continuing to
// process the request.
ErrCodeRequestError = "RequestError"
)
// A Request is the service request to be made.
@ -51,6 +55,7 @@ type Request struct {
HTTPRequest *http.Request
HTTPResponse *http.Response
Body io.ReadSeeker
streamingBody io.ReadCloser
BodyStart int64 // offset from beginning of Body that the request body starts
Params interface{}
Error error
@ -130,8 +135,6 @@ func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers,
err = awserr.New("InvalidEndpointURL", "invalid endpoint uri", err)
}
SanitizeHostForHeader(httpReq)
r := &Request{
Config: cfg,
ClientInfo: clientInfo,
@ -295,6 +298,13 @@ func (r *Request) SetReaderBody(reader io.ReadSeeker) {
r.ResetBody()
}
// SetStreamingBody set the reader to be used for the request that will stream
// bytes to the server. Request's Body must not be set to any reader.
func (r *Request) SetStreamingBody(reader io.ReadCloser) {
r.streamingBody = reader
r.SetReaderBody(aws.ReadSeekCloser(reader))
}
// Presign returns the request's signed URL. Error will be returned
// if the signing fails. The expire parameter is only used for presigned Amazon
// S3 API requests. All other AWS services will use a fixed expiration
@ -414,11 +424,17 @@ func (r *Request) Sign() error {
return r.Error
}
SanitizeHostForHeader(r.HTTPRequest)
r.Handlers.Sign.Run(r)
return r.Error
}
func (r *Request) getNextRequestBody() (body io.ReadCloser, err error) {
if r.streamingBody != nil {
return r.streamingBody, nil
}
if r.safeBody != nil {
r.safeBody.Close()
}
@ -623,6 +639,10 @@ func getHost(r *http.Request) string {
return r.Host
}
if r.URL == nil {
return ""
}
return r.URL.Host
}

View File

@ -75,7 +75,7 @@ func (d noOpRetryer) RetryRules(_ *Request) time.Duration {
// retryableCodes is a collection of service response codes which are retry-able
// without any further action.
var retryableCodes = map[string]struct{}{
"RequestError": {},
ErrCodeRequestError: {},
"RequestTimeout": {},
ErrCodeResponseTimeout: {},
"RequestTimeoutException": {}, // Glacier's flavor of RequestTimeout
@ -83,6 +83,7 @@ var retryableCodes = map[string]struct{}{
var throttleCodes = map[string]struct{}{
"ProvisionedThroughputExceededException": {},
"ThrottledException": {}, // SNS, XRay, ResourceGroupsTagging API
"Throttling": {},
"ThrottlingException": {},
"RequestLimitExceeded": {},
@ -91,6 +92,7 @@ var throttleCodes = map[string]struct{}{
"TooManyRequestsException": {}, // Lambda functions
"PriorRequestNotComplete": {}, // Route53
"TransactionInProgressException": {},
"EC2ThrottledException": {}, // EC2
}
// credsExpiredCodes is a collection of error codes which signify the credentials
@ -176,8 +178,8 @@ func shouldRetryError(origErr error) bool {
origErr := err.OrigErr()
var shouldRetry bool
if origErr != nil {
shouldRetry := shouldRetryError(origErr)
if err.Code() == "RequestError" && !shouldRetry {
shouldRetry = shouldRetryError(origErr)
if err.Code() == ErrCodeRequestError && !shouldRetry {
return false
}
}

View File

@ -3,6 +3,7 @@ package session
import (
"fmt"
"os"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@ -206,7 +207,14 @@ func credsFromAssumeRole(cfg aws.Config,
sharedCfg.RoleARN,
func(opt *stscreds.AssumeRoleProvider) {
opt.RoleSessionName = sharedCfg.RoleSessionName
opt.Duration = sessOpts.AssumeRoleDuration
if sessOpts.AssumeRoleDuration == 0 &&
sharedCfg.AssumeRoleDuration != nil &&
*sharedCfg.AssumeRoleDuration/time.Minute > 15 {
opt.Duration = *sharedCfg.AssumeRoleDuration
} else if sessOpts.AssumeRoleDuration != 0 {
opt.Duration = sessOpts.AssumeRoleDuration
}
// Assume role with external ID
if len(sharedCfg.ExternalID) > 0 {

View File

@ -241,5 +241,22 @@ over the AWS_CA_BUNDLE environment variable, and will be used if both are set.
Setting a custom HTTPClient in the aws.Config options will override this setting.
To use this option and custom HTTP client, the HTTP client needs to be provided
when creating the session. Not the service client.
The endpoint of the EC2 IMDS client can be configured via the environment
variable, AWS_EC2_METADATA_SERVICE_ENDPOINT when creating the client with a
Session. See Options.EC2IMDSEndpoint for more details.
AWS_EC2_METADATA_SERVICE_ENDPOINT=http://169.254.169.254
If using an URL with an IPv6 address literal, the IPv6 address
component must be enclosed in square brackets.
AWS_EC2_METADATA_SERVICE_ENDPOINT=http://[::1]
The custom EC2 IMDS endpoint can also be specified via the Session options.
sess, err := session.NewSessionWithOptions(session.Options{
EC2IMDSEndpoint: "http://[::1]",
})
*/
package session

View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"strconv"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
@ -141,6 +142,17 @@ type envConfig struct {
// AWS_S3_US_EAST_1_REGIONAL_ENDPOINT=regional
// This can take value as `regional` or `legacy`
S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
// Specifies if the S3 service should allow ARNs to direct the region
// the client's requests are sent to.
//
// AWS_S3_USE_ARN_REGION=true
S3UseARNRegion bool
// Specifies the alternative endpoint to use for EC2 IMDS.
//
// AWS_EC2_METADATA_SERVICE_ENDPOINT=http://[::1]
EC2IMDSEndpoint string
}
var (
@ -201,6 +213,12 @@ var (
s3UsEast1RegionalEndpoint = []string{
"AWS_S3_US_EAST_1_REGIONAL_ENDPOINT",
}
s3UseARNRegionEnvKey = []string{
"AWS_S3_USE_ARN_REGION",
}
ec2IMDSEndpointEnvKey = []string{
"AWS_EC2_METADATA_SERVICE_ENDPOINT",
}
)
// loadEnvConfig retrieves the SDK's environment configuration.
@ -307,6 +325,23 @@ func envConfigLoad(enableSharedConfig bool) (envConfig, error) {
}
}
var s3UseARNRegion string
setFromEnvVal(&s3UseARNRegion, s3UseARNRegionEnvKey)
if len(s3UseARNRegion) != 0 {
switch {
case strings.EqualFold(s3UseARNRegion, "false"):
cfg.S3UseARNRegion = false
case strings.EqualFold(s3UseARNRegion, "true"):
cfg.S3UseARNRegion = true
default:
return envConfig{}, fmt.Errorf(
"invalid value for environment variable, %s=%s, need true or false",
s3UseARNRegionEnvKey[0], s3UseARNRegion)
}
}
setFromEnvVal(&cfg.EC2IMDSEndpoint, ec2IMDSEndpointEnvKey)
return cfg, nil
}

View File

@ -48,6 +48,8 @@ var ErrSharedConfigInvalidCredSource = awserr.New(ErrCodeSharedConfig, "credenti
type Session struct {
Config *aws.Config
Handlers request.Handlers
options Options
}
// New creates a new instance of the handlers merging in the provided configs
@ -99,7 +101,7 @@ func New(cfgs ...*aws.Config) *Session {
return s
}
s := deprecatedNewSession(cfgs...)
s := deprecatedNewSession(envCfg, cfgs...)
if envErr != nil {
msg := "failed to load env config"
s.logDeprecatedNewSessionError(msg, envErr, cfgs)
@ -243,6 +245,23 @@ type Options struct {
// function to initialize this value before changing the handlers to be
// used by the SDK.
Handlers request.Handlers
// Allows specifying a custom endpoint to be used by the EC2 IMDS client
// when making requests to the EC2 IMDS API. The must endpoint value must
// include protocol prefix.
//
// If unset, will the EC2 IMDS client will use its default endpoint.
//
// Can also be specified via the environment variable,
// AWS_EC2_METADATA_SERVICE_ENDPOINT.
//
// AWS_EC2_METADATA_SERVICE_ENDPOINT=http://169.254.169.254
//
// If using an URL with an IPv6 address literal, the IPv6 address
// component must be enclosed in square brackets.
//
// AWS_EC2_METADATA_SERVICE_ENDPOINT=http://[::1]
EC2IMDSEndpoint string
}
// NewSessionWithOptions returns a new Session created from SDK defaults, config files,
@ -329,7 +348,25 @@ func Must(sess *Session, err error) *Session {
return sess
}
func deprecatedNewSession(cfgs ...*aws.Config) *Session {
// Wraps the endpoint resolver with a resolver that will return a custom
// endpoint for EC2 IMDS.
func wrapEC2IMDSEndpoint(resolver endpoints.Resolver, endpoint string) endpoints.Resolver {
return endpoints.ResolverFunc(
func(service, region string, opts ...func(*endpoints.Options)) (
endpoints.ResolvedEndpoint, error,
) {
if service == ec2MetadataServiceID {
return endpoints.ResolvedEndpoint{
URL: endpoint,
SigningName: ec2MetadataServiceID,
SigningRegion: region,
}, nil
}
return resolver.EndpointFor(service, region)
})
}
func deprecatedNewSession(envCfg envConfig, cfgs ...*aws.Config) *Session {
cfg := defaults.Config()
handlers := defaults.Handlers()
@ -341,6 +378,11 @@ func deprecatedNewSession(cfgs ...*aws.Config) *Session {
// endpoints for service client configurations.
cfg.EndpointResolver = endpoints.DefaultResolver()
}
if len(envCfg.EC2IMDSEndpoint) != 0 {
cfg.EndpointResolver = wrapEC2IMDSEndpoint(cfg.EndpointResolver, envCfg.EC2IMDSEndpoint)
}
cfg.Credentials = defaults.CredChain(cfg, handlers)
// Reapply any passed in configs to override credentials if set
@ -349,6 +391,9 @@ func deprecatedNewSession(cfgs ...*aws.Config) *Session {
s := &Session{
Config: cfg,
Handlers: handlers,
options: Options{
EC2IMDSEndpoint: envCfg.EC2IMDSEndpoint,
},
}
initHandlers(s)
@ -418,6 +463,7 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session,
s := &Session{
Config: cfg,
Handlers: handlers,
options: opts,
}
initHandlers(s)
@ -570,6 +616,14 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
endpoints.LegacyS3UsEast1Endpoint,
})
ec2IMDSEndpoint := sessOpts.EC2IMDSEndpoint
if len(ec2IMDSEndpoint) == 0 {
ec2IMDSEndpoint = envCfg.EC2IMDSEndpoint
}
if len(ec2IMDSEndpoint) != 0 {
cfg.EndpointResolver = wrapEC2IMDSEndpoint(cfg.EndpointResolver, ec2IMDSEndpoint)
}
// Configure credentials if not already set by the user when creating the
// Session.
if cfg.Credentials == credentials.AnonymousCredentials && userCfg.Credentials == nil {
@ -580,6 +634,14 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
cfg.Credentials = creds
}
cfg.S3UseARNRegion = userCfg.S3UseARNRegion
if cfg.S3UseARNRegion == nil {
cfg.S3UseARNRegion = &envCfg.S3UseARNRegion
}
if cfg.S3UseARNRegion == nil {
cfg.S3UseARNRegion = &sharedCfg.S3UseARNRegion
}
return nil
}
@ -619,6 +681,7 @@ func (s *Session) Copy(cfgs ...*aws.Config) *Session {
newSession := &Session{
Config: s.Config.Copy(cfgs...),
Handlers: s.Handlers.Copy(),
options: s.options,
}
initHandlers(newSession)
@ -634,15 +697,22 @@ func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Confi
region := aws.StringValue(s.Config.Region)
resolved, err := s.resolveEndpoint(service, region, s.Config)
if err != nil && s.Config.Logger != nil {
s.Config.Logger.Log(fmt.Sprintf(
"ERROR: unable to resolve endpoint for service %q, region %q, err: %v",
service, region, err))
if err != nil {
s.Handlers.Validate.PushBack(func(r *request.Request) {
if len(r.ClientInfo.Endpoint) != 0 {
// Error occurred while resolving endpoint, but the request
// being invoked has had an endpoint specified after the client
// was created.
return
}
r.Error = err
})
}
return client.Config{
Config: s.Config,
Handlers: s.Handlers,
PartitionID: resolved.PartitionID,
Endpoint: resolved.URL,
SigningRegion: resolved.SigningRegion,
SigningNameDerived: resolved.SigningNameDerived,
@ -650,6 +720,8 @@ func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Confi
}
}
const ec2MetadataServiceID = "ec2metadata"
func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endpoints.ResolvedEndpoint, error) {
if ep := aws.StringValue(cfg.Endpoint); len(ep) != 0 {

View File

@ -2,6 +2,7 @@ package session
import (
"fmt"
"time"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
@ -16,12 +17,13 @@ const (
sessionTokenKey = `aws_session_token` // optional
// Assume Role Credentials group
roleArnKey = `role_arn` // group required
sourceProfileKey = `source_profile` // group required (or credential_source)
credentialSourceKey = `credential_source` // group required (or source_profile)
externalIDKey = `external_id` // optional
mfaSerialKey = `mfa_serial` // optional
roleSessionNameKey = `role_session_name` // optional
roleArnKey = `role_arn` // group required
sourceProfileKey = `source_profile` // group required (or credential_source)
credentialSourceKey = `credential_source` // group required (or source_profile)
externalIDKey = `external_id` // optional
mfaSerialKey = `mfa_serial` // optional
roleSessionNameKey = `role_session_name` // optional
roleDurationSecondsKey = "duration_seconds" // optional
// CSM options
csmEnabledKey = `csm_enabled`
@ -51,6 +53,9 @@ const (
// loading configuration from the config files if another profile name
// is not provided.
DefaultSharedConfigProfile = `default`
// S3 ARN Region Usage
s3UseARNRegionKey = "s3_use_arn_region"
)
// sharedConfig represents the configuration fields of the SDK config files.
@ -70,10 +75,11 @@ type sharedConfig struct {
CredentialProcess string
WebIdentityTokenFile string
RoleARN string
RoleSessionName string
ExternalID string
MFASerial string
RoleARN string
RoleSessionName string
ExternalID string
MFASerial string
AssumeRoleDuration *time.Duration
SourceProfileName string
SourceProfile *sharedConfig
@ -89,6 +95,7 @@ type sharedConfig struct {
//
// endpoint_discovery_enabled = true
EnableEndpointDiscovery *bool
// CSM Options
CSMEnabled *bool
CSMHost string
@ -106,6 +113,12 @@ type sharedConfig struct {
// s3_us_east_1_regional_endpoint = regional
// This can take value as `LegacyS3UsEast1Endpoint` or `RegionalS3UsEast1Endpoint`
S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
// Specifies if the S3 service should allow ARNs to direct the region
// the client's requests are sent to.
//
// s3_use_arn_region=true
S3UseARNRegion bool
}
type sharedConfigFile struct {
@ -264,6 +277,11 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
updateString(&cfg.CredentialSource, section, credentialSourceKey)
updateString(&cfg.Region, section, regionKey)
if section.Has(roleDurationSecondsKey) {
d := time.Duration(section.Int(roleDurationSecondsKey)) * time.Second
cfg.AssumeRoleDuration = &d
}
if v := section.String(stsRegionalEndpointSharedKey); len(v) != 0 {
sre, err := endpoints.GetSTSRegionalEndpoint(v)
if err != nil {
@ -306,6 +324,8 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
updateString(&cfg.CSMPort, section, csmPortKey)
updateString(&cfg.CSMClientID, section, csmClientIDKey)
updateBool(&cfg.S3UseARNRegion, section, s3UseARNRegionKey)
return nil
}
@ -398,6 +418,15 @@ func updateString(dst *string, section ini.Section, key string) {
*dst = section.String(key)
}
// updateBool will only update the dst with the value in the section key, key
// is present in the section.
func updateBool(dst *bool, section ini.Section, key string) {
if !section.Has(key) {
return
}
*dst = section.Bool(key)
}
// updateBoolPtr will only update the dst with the value in the section key,
// key is present in the section.
func updateBoolPtr(dst **bool, section ini.Section, key string) {

View File

@ -1,8 +1,7 @@
package v4
import (
"net/http"
"strings"
"github.com/aws/aws-sdk-go/internal/strings"
)
// validator houses a set of rule needed for validation of a
@ -61,7 +60,7 @@ type patterns []string
// been found
func (p patterns) IsValid(value string) bool {
for _, pattern := range p {
if strings.HasPrefix(http.CanonicalHeaderKey(value), pattern) {
if strings.HasPrefixFold(value, pattern) {
return true
}
}

View File

@ -0,0 +1,13 @@
// +build !go1.7
package v4
import (
"net/http"
"github.com/aws/aws-sdk-go/aws"
)
func requestContext(r *http.Request) aws.Context {
return aws.BackgroundContext()
}

View File

@ -0,0 +1,13 @@
// +build go1.7
package v4
import (
"net/http"
"github.com/aws/aws-sdk-go/aws"
)
func requestContext(r *http.Request) aws.Context {
return r.Context()
}

View File

@ -0,0 +1,63 @@
package v4
import (
"encoding/hex"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws/credentials"
)
type credentialValueProvider interface {
Get() (credentials.Value, error)
}
// StreamSigner implements signing of event stream encoded payloads
type StreamSigner struct {
region string
service string
credentials credentialValueProvider
prevSig []byte
}
// NewStreamSigner creates a SigV4 signer used to sign Event Stream encoded messages
func NewStreamSigner(region, service string, seedSignature []byte, credentials *credentials.Credentials) *StreamSigner {
return &StreamSigner{
region: region,
service: service,
credentials: credentials,
prevSig: seedSignature,
}
}
// GetSignature takes an event stream encoded headers and payload and returns a signature
func (s *StreamSigner) GetSignature(headers, payload []byte, date time.Time) ([]byte, error) {
credValue, err := s.credentials.Get()
if err != nil {
return nil, err
}
sigKey := deriveSigningKey(s.region, s.service, credValue.SecretAccessKey, date)
keyPath := buildSigningScope(s.region, s.service, date)
stringToSign := buildEventStreamStringToSign(headers, payload, s.prevSig, keyPath, date)
signature := hmacSHA256(sigKey, []byte(stringToSign))
s.prevSig = signature
return signature, nil
}
func buildEventStreamStringToSign(headers, payload, prevSig []byte, scope string, date time.Time) string {
return strings.Join([]string{
"AWS4-HMAC-SHA256-PAYLOAD",
formatTime(date),
scope,
hex.EncodeToString(prevSig),
hex.EncodeToString(hashSHA256(headers)),
hex.EncodeToString(hashSHA256(payload)),
}, "\n")
}

View File

@ -76,9 +76,14 @@ import (
)
const (
authorizationHeader = "Authorization"
authHeaderSignatureElem = "Signature="
signatureQueryKey = "X-Amz-Signature"
authHeaderPrefix = "AWS4-HMAC-SHA256"
timeFormat = "20060102T150405Z"
shortTimeFormat = "20060102"
awsV4Request = "aws4_request"
// emptyStringSHA256 is a SHA256 of an empty string
emptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
@ -87,9 +92,9 @@ const (
var ignoredHeaders = rules{
blacklist{
mapRule{
"Authorization": struct{}{},
"User-Agent": struct{}{},
"X-Amzn-Trace-Id": struct{}{},
authorizationHeader: struct{}{},
"User-Agent": struct{}{},
"X-Amzn-Trace-Id": struct{}{},
},
},
}
@ -229,11 +234,9 @@ type signingCtx struct {
DisableURIPathEscaping bool
credValues credentials.Value
isPresign bool
formattedTime string
formattedShortTime string
unsignedPayload bool
credValues credentials.Value
isPresign bool
unsignedPayload bool
bodyDigest string
signedHeaders string
@ -337,7 +340,7 @@ func (v4 Signer) signWithBody(r *http.Request, body io.ReadSeeker, service, regi
}
var err error
ctx.credValues, err = v4.Credentials.Get()
ctx.credValues, err = v4.Credentials.GetWithContext(requestContext(r))
if err != nil {
return http.Header{}, err
}
@ -532,39 +535,56 @@ func (ctx *signingCtx) build(disableHeaderHoisting bool) error {
ctx.buildSignature() // depends on string to sign
if ctx.isPresign {
ctx.Request.URL.RawQuery += "&X-Amz-Signature=" + ctx.signature
ctx.Request.URL.RawQuery += "&" + signatureQueryKey + "=" + ctx.signature
} else {
parts := []string{
authHeaderPrefix + " Credential=" + ctx.credValues.AccessKeyID + "/" + ctx.credentialString,
"SignedHeaders=" + ctx.signedHeaders,
"Signature=" + ctx.signature,
authHeaderSignatureElem + ctx.signature,
}
ctx.Request.Header.Set("Authorization", strings.Join(parts, ", "))
ctx.Request.Header.Set(authorizationHeader, strings.Join(parts, ", "))
}
return nil
}
func (ctx *signingCtx) buildTime() {
ctx.formattedTime = ctx.Time.UTC().Format(timeFormat)
ctx.formattedShortTime = ctx.Time.UTC().Format(shortTimeFormat)
// GetSignedRequestSignature attempts to extract the signature of the request.
// Returning an error if the request is unsigned, or unable to extract the
// signature.
func GetSignedRequestSignature(r *http.Request) ([]byte, error) {
if auth := r.Header.Get(authorizationHeader); len(auth) != 0 {
ps := strings.Split(auth, ", ")
for _, p := range ps {
if idx := strings.Index(p, authHeaderSignatureElem); idx >= 0 {
sig := p[len(authHeaderSignatureElem):]
if len(sig) == 0 {
return nil, fmt.Errorf("invalid request signature authorization header")
}
return hex.DecodeString(sig)
}
}
}
if sig := r.URL.Query().Get("X-Amz-Signature"); len(sig) != 0 {
return hex.DecodeString(sig)
}
return nil, fmt.Errorf("request not signed")
}
func (ctx *signingCtx) buildTime() {
if ctx.isPresign {
duration := int64(ctx.ExpireTime / time.Second)
ctx.Query.Set("X-Amz-Date", ctx.formattedTime)
ctx.Query.Set("X-Amz-Date", formatTime(ctx.Time))
ctx.Query.Set("X-Amz-Expires", strconv.FormatInt(duration, 10))
} else {
ctx.Request.Header.Set("X-Amz-Date", ctx.formattedTime)
ctx.Request.Header.Set("X-Amz-Date", formatTime(ctx.Time))
}
}
func (ctx *signingCtx) buildCredentialString() {
ctx.credentialString = strings.Join([]string{
ctx.formattedShortTime,
ctx.Region,
ctx.ServiceName,
"aws4_request",
}, "/")
ctx.credentialString = buildSigningScope(ctx.Region, ctx.ServiceName, ctx.Time)
if ctx.isPresign {
ctx.Query.Set("X-Amz-Credential", ctx.credValues.AccessKeyID+"/"+ctx.credentialString)
@ -588,8 +608,7 @@ func (ctx *signingCtx) buildCanonicalHeaders(r rule, header http.Header) {
var headers []string
headers = append(headers, "host")
for k, v := range header {
canonicalKey := http.CanonicalHeaderKey(k)
if !r.IsValid(canonicalKey) {
if !r.IsValid(k) {
continue // ignored header
}
if ctx.SignedHeaderVals == nil {
@ -653,19 +672,15 @@ func (ctx *signingCtx) buildCanonicalString() {
func (ctx *signingCtx) buildStringToSign() {
ctx.stringToSign = strings.Join([]string{
authHeaderPrefix,
ctx.formattedTime,
formatTime(ctx.Time),
ctx.credentialString,
hex.EncodeToString(makeSha256([]byte(ctx.canonicalString))),
hex.EncodeToString(hashSHA256([]byte(ctx.canonicalString))),
}, "\n")
}
func (ctx *signingCtx) buildSignature() {
secret := ctx.credValues.SecretAccessKey
date := makeHmac([]byte("AWS4"+secret), []byte(ctx.formattedShortTime))
region := makeHmac(date, []byte(ctx.Region))
service := makeHmac(region, []byte(ctx.ServiceName))
credentials := makeHmac(service, []byte("aws4_request"))
signature := makeHmac(credentials, []byte(ctx.stringToSign))
creds := deriveSigningKey(ctx.Region, ctx.ServiceName, ctx.credValues.SecretAccessKey, ctx.Time)
signature := hmacSHA256(creds, []byte(ctx.stringToSign))
ctx.signature = hex.EncodeToString(signature)
}
@ -726,13 +741,13 @@ func (ctx *signingCtx) removePresign() {
ctx.Query.Del("X-Amz-SignedHeaders")
}
func makeHmac(key []byte, data []byte) []byte {
func hmacSHA256(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
func makeSha256(data []byte) []byte {
func hashSHA256(data []byte) []byte {
hash := sha256.New()
hash.Write(data)
return hash.Sum(nil)
@ -804,3 +819,28 @@ func stripExcessSpaces(vals []string) {
vals[i] = string(buf[:m])
}
}
func buildSigningScope(region, service string, dt time.Time) string {
return strings.Join([]string{
formatShortTime(dt),
region,
service,
awsV4Request,
}, "/")
}
func deriveSigningKey(region, service, secretKey string, dt time.Time) []byte {
kDate := hmacSHA256([]byte("AWS4"+secretKey), []byte(formatShortTime(dt)))
kRegion := hmacSHA256(kDate, []byte(region))
kService := hmacSHA256(kRegion, []byte(service))
signingKey := hmacSHA256(kService, []byte(awsV4Request))
return signingKey
}
func formatShortTime(dt time.Time) string {
return dt.UTC().Format(shortTimeFormat)
}
func formatTime(dt time.Time) string {
return dt.UTC().Format(timeFormat)
}

View File

@ -2,6 +2,7 @@ package aws
import (
"io"
"strings"
"sync"
"github.com/aws/aws-sdk-go/internal/sdkio"
@ -205,3 +206,59 @@ func (b *WriteAtBuffer) Bytes() []byte {
defer b.m.Unlock()
return b.buf
}
// MultiCloser is a utility to close multiple io.Closers within a single
// statement.
type MultiCloser []io.Closer
// Close closes all of the io.Closers making up the MultiClosers. Any
// errors that occur while closing will be returned in the order they
// occur.
func (m MultiCloser) Close() error {
var errs errors
for _, c := range m {
err := c.Close()
if err != nil {
errs = append(errs, err)
}
}
if len(errs) != 0 {
return errs
}
return nil
}
type errors []error
func (es errors) Error() string {
var parts []string
for _, e := range es {
parts = append(parts, e.Error())
}
return strings.Join(parts, "\n")
}
// CopySeekableBody copies the seekable body to an io.Writer
func CopySeekableBody(dst io.Writer, src io.ReadSeeker) (int64, error) {
curPos, err := src.Seek(0, sdkio.SeekCurrent)
if err != nil {
return 0, err
}
// copy errors may be assumed to be from the body.
n, err := io.Copy(dst, src)
if err != nil {
return n, err
}
// seek back to the first position after reading to reset
// the body for transmission.
_, err = src.Seek(curPos, sdkio.SeekStart)
if err != nil {
return n, err
}
return n, nil
}

View File

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.25.41"
const SDKVersion = "1.35.3"

View File

@ -0,0 +1,40 @@
// +build !go1.7
package context
import "time"
// An emptyCtx is a copy of the Go 1.7 context.emptyCtx type. This is copied to
// provide a 1.6 and 1.5 safe version of context that is compatible with Go
// 1.7's Context.
//
// An emptyCtx is never canceled, has no values, and has no deadline. It is not
// struct{}, since vars of this type must have distinct addresses.
type emptyCtx int
func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
return
}
func (*emptyCtx) Done() <-chan struct{} {
return nil
}
func (*emptyCtx) Err() error {
return nil
}
func (*emptyCtx) Value(key interface{}) interface{} {
return nil
}
func (e *emptyCtx) String() string {
switch e {
case BackgroundCtx:
return "aws.BackgroundContext"
}
return "unknown empty Context"
}
// BackgroundCtx is the common base context.
var BackgroundCtx = new(emptyCtx)

View File

@ -0,0 +1,50 @@
package arn
import (
"strings"
"github.com/aws/aws-sdk-go/aws/arn"
)
// AccessPointARN provides representation
type AccessPointARN struct {
arn.ARN
AccessPointName string
}
// GetARN returns the base ARN for the Access Point resource
func (a AccessPointARN) GetARN() arn.ARN {
return a.ARN
}
// ParseAccessPointResource attempts to parse the ARN's resource as an
// AccessPoint resource.
//
// Supported Access point resource format:
// - Access point format: arn:{partition}:s3:{region}:{accountId}:accesspoint/{accesspointName}
// - example: arn.aws.s3.us-west-2.012345678901:accesspoint/myaccesspoint
//
func ParseAccessPointResource(a arn.ARN, resParts []string) (AccessPointARN, error) {
if len(a.Region) == 0 {
return AccessPointARN{}, InvalidARNError{ARN: a, Reason: "region not set"}
}
if len(a.AccountID) == 0 {
return AccessPointARN{}, InvalidARNError{ARN: a, Reason: "account-id not set"}
}
if len(resParts) == 0 {
return AccessPointARN{}, InvalidARNError{ARN: a, Reason: "resource-id not set"}
}
if len(resParts) > 1 {
return AccessPointARN{}, InvalidARNError{ARN: a, Reason: "sub resource not supported"}
}
resID := resParts[0]
if len(strings.TrimSpace(resID)) == 0 {
return AccessPointARN{}, InvalidARNError{ARN: a, Reason: "resource-id not set"}
}
return AccessPointARN{
ARN: a,
AccessPointName: resID,
}, nil
}

View File

@ -0,0 +1,74 @@
package arn
import (
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws/arn"
)
// Resource provides the interfaces abstracting ARNs of specific resource
// types.
type Resource interface {
GetARN() arn.ARN
String() string
}
// ResourceParser provides the function for parsing an ARN's resource
// component into a typed resource.
type ResourceParser func(arn.ARN) (Resource, error)
// ParseResource parses an AWS ARN into a typed resource for the S3 API.
func ParseResource(s string, resParser ResourceParser) (resARN Resource, err error) {
a, err := arn.Parse(s)
if err != nil {
return nil, err
}
if len(a.Partition) == 0 {
return nil, InvalidARNError{ARN: a, Reason: "partition not set"}
}
if a.Service != "s3" && a.Service != "s3-outposts" {
return nil, InvalidARNError{ARN: a, Reason: "service is not supported"}
}
if len(a.Resource) == 0 {
return nil, InvalidARNError{ARN: a, Reason: "resource not set"}
}
return resParser(a)
}
// SplitResource splits the resource components by the ARN resource delimiters.
func SplitResource(v string) []string {
var parts []string
var offset int
for offset <= len(v) {
idx := strings.IndexAny(v[offset:], "/:")
if idx < 0 {
parts = append(parts, v[offset:])
break
}
parts = append(parts, v[offset:idx+offset])
offset += idx + 1
}
return parts
}
// IsARN returns whether the given string is an ARN
func IsARN(s string) bool {
return arn.IsARN(s)
}
// InvalidARNError provides the error for an invalid ARN error.
type InvalidARNError struct {
ARN arn.ARN
Reason string
}
// Error returns a string denoting the occurred InvalidARNError
func (e InvalidARNError) Error() string {
return fmt.Sprintf("invalid Amazon %s ARN, %s, %s", e.ARN.Service, e.Reason, e.ARN.String())
}

View File

@ -0,0 +1,126 @@
package arn
import (
"strings"
"github.com/aws/aws-sdk-go/aws/arn"
)
// OutpostARN interface that should be satisfied by outpost ARNs
type OutpostARN interface {
Resource
GetOutpostID() string
}
// ParseOutpostARNResource will parse a provided ARNs resource using the appropriate ARN format
// and return a specific OutpostARN type
//
// Currently supported outpost ARN formats:
// * Outpost AccessPoint ARN format:
// - ARN format: arn:{partition}:s3-outposts:{region}:{accountId}:outpost/{outpostId}/accesspoint/{accesspointName}
// - example: arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint
//
// * Outpost Bucket ARN format:
// - ARN format: arn:{partition}:s3-outposts:{region}:{accountId}:outpost/{outpostId}/bucket/{bucketName}
// - example: arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/bucket/mybucket
//
// Other outpost ARN formats may be supported and added in the future.
//
func ParseOutpostARNResource(a arn.ARN, resParts []string) (OutpostARN, error) {
if len(a.Region) == 0 {
return nil, InvalidARNError{ARN: a, Reason: "region not set"}
}
if len(a.AccountID) == 0 {
return nil, InvalidARNError{ARN: a, Reason: "account-id not set"}
}
// verify if outpost id is present and valid
if len(resParts) == 0 || len(strings.TrimSpace(resParts[0])) == 0 {
return nil, InvalidARNError{ARN: a, Reason: "outpost resource-id not set"}
}
// verify possible resource type exists
if len(resParts) < 3 {
return nil, InvalidARNError{
ARN: a, Reason: "incomplete outpost resource type. Expected bucket or access-point resource to be present",
}
}
// Since we know this is a OutpostARN fetch outpostID
outpostID := strings.TrimSpace(resParts[0])
switch resParts[1] {
case "accesspoint":
accesspointARN, err := ParseAccessPointResource(a, resParts[2:])
if err != nil {
return OutpostAccessPointARN{}, err
}
return OutpostAccessPointARN{
AccessPointARN: accesspointARN,
OutpostID: outpostID,
}, nil
case "bucket":
bucketName, err := parseBucketResource(a, resParts[2:])
if err != nil {
return nil, err
}
return OutpostBucketARN{
ARN: a,
BucketName: bucketName,
OutpostID: outpostID,
}, nil
default:
return nil, InvalidARNError{ARN: a, Reason: "unknown resource set for outpost ARN"}
}
}
// OutpostAccessPointARN represents outpost access point ARN.
type OutpostAccessPointARN struct {
AccessPointARN
OutpostID string
}
// GetOutpostID returns the outpost id of outpost access point arn
func (o OutpostAccessPointARN) GetOutpostID() string {
return o.OutpostID
}
// OutpostBucketARN represents the outpost bucket ARN.
type OutpostBucketARN struct {
arn.ARN
BucketName string
OutpostID string
}
// GetOutpostID returns the outpost id of outpost bucket arn
func (o OutpostBucketARN) GetOutpostID() string {
return o.OutpostID
}
// GetARN retrives the base ARN from outpost bucket ARN resource
func (o OutpostBucketARN) GetARN() arn.ARN {
return o.ARN
}
// parseBucketResource attempts to parse the ARN's bucket resource and retrieve the
// bucket resource id.
//
// parseBucketResource only parses the bucket resource id.
//
func parseBucketResource(a arn.ARN, resParts []string) (bucketName string, err error) {
if len(resParts) == 0 {
return bucketName, InvalidARNError{ARN: a, Reason: "bucket resource-id not set"}
}
if len(resParts) > 1 {
return bucketName, InvalidARNError{ARN: a, Reason: "sub resource not supported"}
}
bucketName = strings.TrimSpace(resParts[0])
if len(bucketName) == 0 {
return bucketName, InvalidARNError{ARN: a, Reason: "bucket resource-id not set"}
}
return bucketName, err
}

View File

@ -0,0 +1,189 @@
package s3shared
import (
"fmt"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
)
const (
invalidARNErrorErrCode = "InvalidARNError"
configurationErrorErrCode = "ConfigurationError"
)
// InvalidARNError denotes the error for Invalid ARN
type InvalidARNError struct {
message string
resource arn.Resource
origErr error
}
// Error returns the InvalidARNError
func (e InvalidARNError) Error() string {
var extra string
if e.resource != nil {
extra = "ARN: " + e.resource.String()
}
return awserr.SprintError(e.Code(), e.Message(), extra, e.origErr)
}
// Code returns the invalid ARN error code
func (e InvalidARNError) Code() string {
return invalidARNErrorErrCode
}
// Message returns the message for Invalid ARN error
func (e InvalidARNError) Message() string {
return e.message
}
// OrigErr is the original error wrapped by Invalid ARN Error
func (e InvalidARNError) OrigErr() error {
return e.origErr
}
// NewInvalidARNError denotes invalid arn error
func NewInvalidARNError(resource arn.Resource, err error) InvalidARNError {
return InvalidARNError{
message: "invalid ARN",
origErr: err,
resource: resource,
}
}
// NewInvalidARNWithCustomEndpointError ARN not supported for custom clients endpoints
func NewInvalidARNWithCustomEndpointError(resource arn.Resource, err error) InvalidARNError {
return InvalidARNError{
message: "resource ARN not supported with custom client endpoints",
origErr: err,
resource: resource,
}
}
// NewInvalidARNWithUnsupportedPartitionError ARN not supported for the target partition
func NewInvalidARNWithUnsupportedPartitionError(resource arn.Resource, err error) InvalidARNError {
return InvalidARNError{
message: "resource ARN not supported for the target ARN partition",
origErr: err,
resource: resource,
}
}
// NewInvalidARNWithFIPSError ARN not supported for FIPS region
func NewInvalidARNWithFIPSError(resource arn.Resource, err error) InvalidARNError {
return InvalidARNError{
message: "resource ARN not supported for FIPS region",
resource: resource,
origErr: err,
}
}
// ConfigurationError is used to denote a client configuration error
type ConfigurationError struct {
message string
resource arn.Resource
clientPartitionID string
clientRegion string
origErr error
}
// Error returns the Configuration error string
func (e ConfigurationError) Error() string {
extra := fmt.Sprintf("ARN: %s, client partition: %s, client region: %s",
e.resource, e.clientPartitionID, e.clientRegion)
return awserr.SprintError(e.Code(), e.Message(), extra, e.origErr)
}
// Code returns configuration error's error-code
func (e ConfigurationError) Code() string {
return configurationErrorErrCode
}
// Message returns the configuration error message
func (e ConfigurationError) Message() string {
return e.message
}
// OrigErr is the original error wrapped by Configuration Error
func (e ConfigurationError) OrigErr() error {
return e.origErr
}
// NewClientPartitionMismatchError stub
func NewClientPartitionMismatchError(resource arn.Resource, clientPartitionID, clientRegion string, err error) ConfigurationError {
return ConfigurationError{
message: "client partition does not match provided ARN partition",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
// NewClientRegionMismatchError denotes cross region access error
func NewClientRegionMismatchError(resource arn.Resource, clientPartitionID, clientRegion string, err error) ConfigurationError {
return ConfigurationError{
message: "client region does not match provided ARN region",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
// NewFailedToResolveEndpointError denotes endpoint resolving error
func NewFailedToResolveEndpointError(resource arn.Resource, clientPartitionID, clientRegion string, err error) ConfigurationError {
return ConfigurationError{
message: "endpoint resolver failed to find an endpoint for the provided ARN region",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
// NewClientConfiguredForFIPSError denotes client config error for unsupported cross region FIPS access
func NewClientConfiguredForFIPSError(resource arn.Resource, clientPartitionID, clientRegion string, err error) ConfigurationError {
return ConfigurationError{
message: "client configured for fips but cross-region resource ARN provided",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
// NewClientConfiguredForAccelerateError denotes client config error for unsupported S3 accelerate
func NewClientConfiguredForAccelerateError(resource arn.Resource, clientPartitionID, clientRegion string, err error) ConfigurationError {
return ConfigurationError{
message: "client configured for S3 Accelerate but is not supported with resource ARN",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
// NewClientConfiguredForCrossRegionFIPSError denotes client config error for unsupported cross region FIPS request
func NewClientConfiguredForCrossRegionFIPSError(resource arn.Resource, clientPartitionID, clientRegion string, err error) ConfigurationError {
return ConfigurationError{
message: "client configured for FIPS with cross-region enabled but is supported with cross-region resource ARN",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
// NewClientConfiguredForDualStackError denotes client config error for unsupported S3 Dual-stack
func NewClientConfiguredForDualStackError(resource arn.Resource, clientPartitionID, clientRegion string, err error) ConfigurationError {
return ConfigurationError{
message: "client configured for S3 Dual-stack but is not supported with resource ARN",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}

View File

@ -0,0 +1,62 @@
package s3shared
import (
"strings"
"github.com/aws/aws-sdk-go/aws"
awsarn "github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
)
// ResourceRequest represents the request and arn resource
type ResourceRequest struct {
Resource arn.Resource
Request *request.Request
}
// ARN returns the resource ARN
func (r ResourceRequest) ARN() awsarn.ARN {
return r.Resource.GetARN()
}
// AllowCrossRegion returns a bool value to denote if S3UseARNRegion flag is set
func (r ResourceRequest) AllowCrossRegion() bool {
return aws.BoolValue(r.Request.Config.S3UseARNRegion)
}
// UseFIPS returns true if request config region is FIPS
func (r ResourceRequest) UseFIPS() bool {
return IsFIPS(aws.StringValue(r.Request.Config.Region))
}
// ResourceConfiguredForFIPS returns true if resource ARNs region is FIPS
func (r ResourceRequest) ResourceConfiguredForFIPS() bool {
return IsFIPS(r.ARN().Region)
}
// IsCrossPartition returns true if client is configured for another partition, than
// the partition that resource ARN region resolves to.
func (r ResourceRequest) IsCrossPartition() bool {
return r.Request.ClientInfo.PartitionID != r.Resource.GetARN().Partition
}
// IsCrossRegion returns true if ARN region is different than client configured region
func (r ResourceRequest) IsCrossRegion() bool {
return IsCrossRegion(r.Request, r.Resource.GetARN().Region)
}
// HasCustomEndpoint returns true if custom client endpoint is provided
func (r ResourceRequest) HasCustomEndpoint() bool {
return len(aws.StringValue(r.Request.Config.Endpoint)) > 0
}
// IsFIPS returns true if region is a fips region
func IsFIPS(clientRegion string) bool {
return strings.HasPrefix(clientRegion, "fips-") || strings.HasSuffix(clientRegion, "-fips")
}
// IsCrossRegion returns true if request signing region is not same as configured region
func IsCrossRegion(req *request.Request, otherRegion string) bool {
return req.ClientInfo.SigningRegion != otherRegion
}

View File

@ -0,0 +1,11 @@
package strings
import (
"strings"
)
// HasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
// under Unicode case-folding.
func HasPrefixFold(s, prefix string) bool {
return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
}

View File

@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -0,0 +1,120 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package singleflight provides a duplicate function call suppression
// mechanism.
package singleflight
import "sync"
// call is an in-flight or completed singleflight.Do call
type call struct {
wg sync.WaitGroup
// These fields are written once before the WaitGroup is done
// and are only read after the WaitGroup is done.
val interface{}
err error
// forgotten indicates whether Forget was called with this call's key
// while the call was still in flight.
forgotten bool
// These fields are read and written with the singleflight
// mutex held before the WaitGroup is done, and are read but
// not written after the WaitGroup is done.
dups int
chans []chan<- Result
}
// Group represents a class of work and forms a namespace in
// which units of work can be executed with duplicate suppression.
type Group struct {
mu sync.Mutex // protects m
m map[string]*call // lazily initialized
}
// Result holds the results of Do, so they can be passed
// on a channel.
type Result struct {
Val interface{}
Err error
Shared bool
}
// Do executes and returns the results of the given function, making
// sure that only one execution is in-flight for a given key at a
// time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
// The return value shared indicates whether v was given to multiple callers.
func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) {
g.mu.Lock()
if g.m == nil {
g.m = make(map[string]*call)
}
if c, ok := g.m[key]; ok {
c.dups++
g.mu.Unlock()
c.wg.Wait()
return c.val, c.err, true
}
c := new(call)
c.wg.Add(1)
g.m[key] = c
g.mu.Unlock()
g.doCall(c, key, fn)
return c.val, c.err, c.dups > 0
}
// DoChan is like Do but returns a channel that will receive the
// results when they are ready.
func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result {
ch := make(chan Result, 1)
g.mu.Lock()
if g.m == nil {
g.m = make(map[string]*call)
}
if c, ok := g.m[key]; ok {
c.dups++
c.chans = append(c.chans, ch)
g.mu.Unlock()
return ch
}
c := &call{chans: []chan<- Result{ch}}
c.wg.Add(1)
g.m[key] = c
g.mu.Unlock()
go g.doCall(c, key, fn)
return ch
}
// doCall handles the single call for a key.
func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) {
c.val, c.err = fn()
c.wg.Done()
g.mu.Lock()
if !c.forgotten {
delete(g.m, key)
}
for _, ch := range c.chans {
ch <- Result{c.val, c.err, c.dups > 0}
}
g.mu.Unlock()
}
// Forget tells the singleflight to forget about a key. Future calls
// to Do for this key will call the function rather than waiting for
// an earlier call to complete.
func (g *Group) Forget(key string) {
g.mu.Lock()
if c, ok := g.m[key]; ok {
c.forgotten = true
}
delete(g.m, key)
g.mu.Unlock()
}

View File

@ -0,0 +1,53 @@
package checksum
import (
"crypto/md5"
"encoding/base64"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
)
const contentMD5Header = "Content-Md5"
// AddBodyContentMD5Handler computes and sets the HTTP Content-MD5 header for requests that
// require it.
func AddBodyContentMD5Handler(r *request.Request) {
// if Content-MD5 header is already present, return
if v := r.HTTPRequest.Header.Get(contentMD5Header); len(v) != 0 {
return
}
// if S3DisableContentMD5Validation flag is set, return
if aws.BoolValue(r.Config.S3DisableContentMD5Validation) {
return
}
// if request is presigned, return
if r.IsPresigned() {
return
}
// if body is not seekable, return
if !aws.IsReaderSeekable(r.Body) {
if r.Config.Logger != nil {
r.Config.Logger.Log(fmt.Sprintf(
"Unable to compute Content-MD5 for unseekable body, S3.%s",
r.Operation.Name))
}
return
}
h := md5.New()
if _, err := aws.CopySeekableBody(h, r.Body); err != nil {
r.Error = awserr.New("ContentMD5", "failed to compute body MD5", err)
return
}
// encode the md5 checksum in base64 and set the request header.
v := base64.StdEncoding.EncodeToString(h.Sum(nil))
r.HTTPRequest.Header.Set(contentMD5Header, v)
}

View File

@ -1,7 +1,7 @@
// Package ec2query provides serialization of AWS EC2 requests and responses.
package ec2query
//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/input/ec2.json build_test.go
//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/input/ec2.json build_test.go
import (
"net/url"

View File

@ -1,6 +1,6 @@
package ec2query
//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/output/ec2.json unmarshal_test.go
//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/output/ec2.json unmarshal_test.go
import (
"encoding/xml"

View File

@ -101,7 +101,7 @@ func (hs *decodedHeaders) UnmarshalJSON(b []byte) error {
}
headers.Set(h.Name, value)
}
(*hs) = decodedHeaders(headers)
*hs = decodedHeaders(headers)
return nil
}

View File

@ -21,10 +21,24 @@ type Decoder struct {
// NewDecoder initializes and returns a Decoder for decoding event
// stream messages from the reader provided.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{
func NewDecoder(r io.Reader, opts ...func(*Decoder)) *Decoder {
d := &Decoder{
r: r,
}
for _, opt := range opts {
opt(d)
}
return d
}
// DecodeWithLogger adds a logger to be used by the decoder when decoding
// stream events.
func DecodeWithLogger(logger aws.Logger) func(*Decoder) {
return func(d *Decoder) {
d.logger = logger
}
}
// Decode attempts to decode a single message from the event stream reader.
@ -40,6 +54,15 @@ func (d *Decoder) Decode(payloadBuf []byte) (m Message, err error) {
}()
}
m, err = Decode(reader, payloadBuf)
return m, err
}
// Decode attempts to decode a single message from the event stream reader.
// Will return the event stream message, or error if Decode fails to read
// the message from the reader.
func Decode(reader io.Reader, payloadBuf []byte) (m Message, err error) {
crc := crc32.New(crc32IEEETable)
hashReader := io.TeeReader(reader, crc)
@ -72,12 +95,6 @@ func (d *Decoder) Decode(payloadBuf []byte) (m Message, err error) {
return m, nil
}
// UseLogger specifies the Logger that that the decoder should use to log the
// message decode to.
func (d *Decoder) UseLogger(logger aws.Logger) {
d.logger = logger
}
func logMessageDecode(logger aws.Logger, msgBuf *bytes.Buffer, msg Message, decodeErr error) {
w := bytes.NewBuffer(nil)
defer func() { logger.Log(w.String()) }()

View File

@ -3,61 +3,107 @@ package eventstream
import (
"bytes"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"hash/crc32"
"io"
"github.com/aws/aws-sdk-go/aws"
)
// Encoder provides EventStream message encoding.
type Encoder struct {
w io.Writer
w io.Writer
logger aws.Logger
headersBuf *bytes.Buffer
}
// NewEncoder initializes and returns an Encoder to encode Event Stream
// messages to an io.Writer.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
func NewEncoder(w io.Writer, opts ...func(*Encoder)) *Encoder {
e := &Encoder{
w: w,
headersBuf: bytes.NewBuffer(nil),
}
for _, opt := range opts {
opt(e)
}
return e
}
// EncodeWithLogger adds a logger to be used by the encode when decoding
// stream events.
func EncodeWithLogger(logger aws.Logger) func(*Encoder) {
return func(d *Encoder) {
d.logger = logger
}
}
// Encode encodes a single EventStream message to the io.Writer the Encoder
// was created with. An error is returned if writing the message fails.
func (e *Encoder) Encode(msg Message) error {
func (e *Encoder) Encode(msg Message) (err error) {
e.headersBuf.Reset()
err := encodeHeaders(e.headersBuf, msg.Headers)
if err != nil {
writer := e.w
if e.logger != nil {
encodeMsgBuf := bytes.NewBuffer(nil)
writer = io.MultiWriter(writer, encodeMsgBuf)
defer func() {
logMessageEncode(e.logger, encodeMsgBuf, msg, err)
}()
}
if err = EncodeHeaders(e.headersBuf, msg.Headers); err != nil {
return err
}
crc := crc32.New(crc32IEEETable)
hashWriter := io.MultiWriter(e.w, crc)
hashWriter := io.MultiWriter(writer, crc)
headersLen := uint32(e.headersBuf.Len())
payloadLen := uint32(len(msg.Payload))
if err := encodePrelude(hashWriter, crc, headersLen, payloadLen); err != nil {
if err = encodePrelude(hashWriter, crc, headersLen, payloadLen); err != nil {
return err
}
if headersLen > 0 {
if _, err := io.Copy(hashWriter, e.headersBuf); err != nil {
if _, err = io.Copy(hashWriter, e.headersBuf); err != nil {
return err
}
}
if payloadLen > 0 {
if _, err := hashWriter.Write(msg.Payload); err != nil {
if _, err = hashWriter.Write(msg.Payload); err != nil {
return err
}
}
msgCRC := crc.Sum32()
return binary.Write(e.w, binary.BigEndian, msgCRC)
return binary.Write(writer, binary.BigEndian, msgCRC)
}
func logMessageEncode(logger aws.Logger, msgBuf *bytes.Buffer, msg Message, encodeErr error) {
w := bytes.NewBuffer(nil)
defer func() { logger.Log(w.String()) }()
fmt.Fprintf(w, "Message to encode:\n")
encoder := json.NewEncoder(w)
if err := encoder.Encode(msg); err != nil {
fmt.Fprintf(w, "Failed to get encoded message, %v\n", err)
}
if encodeErr != nil {
fmt.Fprintf(w, "Encode error: %v\n", encodeErr)
return
}
fmt.Fprintf(w, "Raw message:\n%s\n", hex.Dump(msgBuf.Bytes()))
}
func encodePrelude(w io.Writer, crc hash.Hash32, headersLen, payloadLen uint32) error {
@ -86,7 +132,9 @@ func encodePrelude(w io.Writer, crc hash.Hash32, headersLen, payloadLen uint32)
return nil
}
func encodeHeaders(w io.Writer, headers Headers) error {
// EncodeHeaders writes the header values to the writer encoded in the event
// stream format. Returns an error if a header fails to encode.
func EncodeHeaders(w io.Writer, headers Headers) error {
for _, h := range headers {
hn := headerName{
Len: uint8(len(h.Name)),

View File

@ -1,6 +1,9 @@
package eventstreamapi
import "fmt"
import (
"fmt"
"sync"
)
type messageError struct {
code string
@ -22,3 +25,53 @@ func (e messageError) Error() string {
func (e messageError) OrigErr() error {
return nil
}
// OnceError wraps the behavior of recording an error
// once and signal on a channel when this has occurred.
// Signaling is done by closing of the channel.
//
// Type is safe for concurrent usage.
type OnceError struct {
mu sync.RWMutex
err error
ch chan struct{}
}
// NewOnceError return a new OnceError
func NewOnceError() *OnceError {
return &OnceError{
ch: make(chan struct{}, 1),
}
}
// Err acquires a read-lock and returns an
// error if one has been set.
func (e *OnceError) Err() error {
e.mu.RLock()
err := e.err
e.mu.RUnlock()
return err
}
// SetError acquires a write-lock and will set
// the underlying error value if one has not been set.
func (e *OnceError) SetError(err error) {
if err == nil {
return
}
e.mu.Lock()
if e.err == nil {
e.err = err
close(e.ch)
}
e.mu.Unlock()
}
// ErrorSet returns a channel that will be used to signal
// that an error has been set. This channel will be closed
// when the error value has been set for OnceError.
func (e *OnceError) ErrorSet() <-chan struct{} {
return e.ch
}

View File

@ -2,9 +2,7 @@ package eventstreamapi
import (
"fmt"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
)
@ -15,27 +13,8 @@ type Unmarshaler interface {
UnmarshalEvent(protocol.PayloadUnmarshaler, eventstream.Message) error
}
// EventStream headers with specific meaning to async API functionality.
const (
MessageTypeHeader = `:message-type` // Identifies type of message.
EventMessageType = `event`
ErrorMessageType = `error`
ExceptionMessageType = `exception`
// Message Events
EventTypeHeader = `:event-type` // Identifies message event type e.g. "Stats".
// Message Error
ErrorCodeHeader = `:error-code`
ErrorMessageHeader = `:error-message`
// Message Exception
ExceptionTypeHeader = `:exception-type`
)
// EventReader provides reading from the EventStream of an reader.
type EventReader struct {
reader io.ReadCloser
decoder *eventstream.Decoder
unmarshalerForEventType func(string) (Unmarshaler, error)
@ -47,27 +26,18 @@ type EventReader struct {
// NewEventReader returns a EventReader built from the reader and unmarshaler
// provided. Use ReadStream method to start reading from the EventStream.
func NewEventReader(
reader io.ReadCloser,
decoder *eventstream.Decoder,
payloadUnmarshaler protocol.PayloadUnmarshaler,
unmarshalerForEventType func(string) (Unmarshaler, error),
) *EventReader {
return &EventReader{
reader: reader,
decoder: eventstream.NewDecoder(reader),
decoder: decoder,
payloadUnmarshaler: payloadUnmarshaler,
unmarshalerForEventType: unmarshalerForEventType,
payloadBuf: make([]byte, 10*1024),
}
}
// UseLogger instructs the EventReader to use the logger and log level
// specified.
func (r *EventReader) UseLogger(logger aws.Logger, logLevel aws.LogLevelType) {
if logger != nil && logLevel.Matches(aws.LogDebugWithEventStreamBody) {
r.decoder.UseLogger(logger)
}
}
// ReadEvent attempts to read a message from the EventStream and return the
// unmarshaled event value that the message is for.
//
@ -95,15 +65,27 @@ func (r *EventReader) ReadEvent() (event interface{}, err error) {
case EventMessageType:
return r.unmarshalEventMessage(msg)
case ExceptionMessageType:
err = r.unmarshalEventException(msg)
return nil, err
return nil, r.unmarshalEventException(msg)
case ErrorMessageType:
return nil, r.unmarshalErrorMessage(msg)
default:
return nil, fmt.Errorf("unknown eventstream message type, %v", typ)
return nil, &UnknownMessageTypeError{
Type: typ, Message: msg.Clone(),
}
}
}
// UnknownMessageTypeError provides an error when a message is received from
// the stream, but the reader is unable to determine what kind of message it is.
type UnknownMessageTypeError struct {
Type string
Message eventstream.Message
}
func (e *UnknownMessageTypeError) Error() string {
return "unknown eventstream message type, " + e.Type
}
func (r *EventReader) unmarshalEventMessage(
msg eventstream.Message,
) (event interface{}, err error) {
@ -174,11 +156,6 @@ func (r *EventReader) unmarshalErrorMessage(msg eventstream.Message) (err error)
return msgErr
}
// Close closes the EventReader's EventStream reader.
func (r *EventReader) Close() error {
return r.reader.Close()
}
// GetHeaderString returns the value of the header as a string. If the header
// is not set or the value is not a string an error will be returned.
func GetHeaderString(msg eventstream.Message, headerName string) (string, error) {

View File

@ -0,0 +1,23 @@
package eventstreamapi
// EventStream headers with specific meaning to async API functionality.
const (
ChunkSignatureHeader = `:chunk-signature` // chunk signature for message
DateHeader = `:date` // Date header for signature
// Message header and values
MessageTypeHeader = `:message-type` // Identifies type of message.
EventMessageType = `event`
ErrorMessageType = `error`
ExceptionMessageType = `exception`
// Message Events
EventTypeHeader = `:event-type` // Identifies message event type e.g. "Stats".
// Message Error
ErrorCodeHeader = `:error-code`
ErrorMessageHeader = `:error-message`
// Message Exception
ExceptionTypeHeader = `:exception-type`
)

View File

@ -0,0 +1,123 @@
package eventstreamapi
import (
"bytes"
"strings"
"time"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
)
var timeNow = time.Now
// StreamSigner defines an interface for the implementation of signing of event stream payloads
type StreamSigner interface {
GetSignature(headers, payload []byte, date time.Time) ([]byte, error)
}
// SignEncoder envelopes event stream messages
// into an event stream message payload with included
// signature headers using the provided signer and encoder.
type SignEncoder struct {
signer StreamSigner
encoder Encoder
bufEncoder *BufferEncoder
closeErr error
closed bool
}
// NewSignEncoder returns a new SignEncoder using the provided stream signer and
// event stream encoder.
func NewSignEncoder(signer StreamSigner, encoder Encoder) *SignEncoder {
// TODO: Need to pass down logging
return &SignEncoder{
signer: signer,
encoder: encoder,
bufEncoder: NewBufferEncoder(),
}
}
// Close encodes a final event stream signing envelope with an empty event stream
// payload. This final end-frame is used to mark the conclusion of the stream.
func (s *SignEncoder) Close() error {
if s.closed {
return s.closeErr
}
if err := s.encode([]byte{}); err != nil {
if strings.Contains(err.Error(), "on closed pipe") {
return nil
}
s.closeErr = err
s.closed = true
return s.closeErr
}
return nil
}
// Encode takes the provided message and add envelopes the message
// with the required signature.
func (s *SignEncoder) Encode(msg eventstream.Message) error {
payload, err := s.bufEncoder.Encode(msg)
if err != nil {
return err
}
return s.encode(payload)
}
func (s SignEncoder) encode(payload []byte) error {
date := timeNow()
var msg eventstream.Message
msg.Headers.Set(DateHeader, eventstream.TimestampValue(date))
msg.Payload = payload
var headers bytes.Buffer
if err := eventstream.EncodeHeaders(&headers, msg.Headers); err != nil {
return err
}
sig, err := s.signer.GetSignature(headers.Bytes(), msg.Payload, date)
if err != nil {
return err
}
msg.Headers.Set(ChunkSignatureHeader, eventstream.BytesValue(sig))
return s.encoder.Encode(msg)
}
// BufferEncoder is a utility that provides a buffered
// event stream encoder
type BufferEncoder struct {
encoder Encoder
buffer *bytes.Buffer
}
// NewBufferEncoder returns a new BufferEncoder initialized
// with a 1024 byte buffer.
func NewBufferEncoder() *BufferEncoder {
buf := bytes.NewBuffer(make([]byte, 1024))
return &BufferEncoder{
encoder: eventstream.NewEncoder(buf),
buffer: buf,
}
}
// Encode returns the encoded message as a byte slice.
// The returned byte slice will be modified on the next encode call
// and should not be held onto.
func (e *BufferEncoder) Encode(msg eventstream.Message) ([]byte, error) {
e.buffer.Reset()
if err := e.encoder.Encode(msg); err != nil {
return nil, err
}
return e.buffer.Bytes(), nil
}

View File

@ -0,0 +1,129 @@
package eventstreamapi
import (
"fmt"
"io"
"sync"
"github.com/aws/aws-sdk-go/aws"
)
// StreamWriter provides concurrent safe writing to an event stream.
type StreamWriter struct {
eventWriter *EventWriter
stream chan eventWriteAsyncReport
done chan struct{}
closeOnce sync.Once
err *OnceError
streamCloser io.Closer
}
// NewStreamWriter returns a StreamWriter for the event writer, and stream
// closer provided.
func NewStreamWriter(eventWriter *EventWriter, streamCloser io.Closer) *StreamWriter {
w := &StreamWriter{
eventWriter: eventWriter,
streamCloser: streamCloser,
stream: make(chan eventWriteAsyncReport),
done: make(chan struct{}),
err: NewOnceError(),
}
go w.writeStream()
return w
}
// Close terminates the writers ability to write new events to the stream. Any
// future call to Send will fail with an error.
func (w *StreamWriter) Close() error {
w.closeOnce.Do(w.safeClose)
return w.Err()
}
func (w *StreamWriter) safeClose() {
close(w.done)
}
// ErrorSet returns a channel which will be closed
// if an error occurs.
func (w *StreamWriter) ErrorSet() <-chan struct{} {
return w.err.ErrorSet()
}
// Err returns any error that occurred while attempting to write an event to the
// stream.
func (w *StreamWriter) Err() error {
return w.err.Err()
}
// Send writes a single event to the stream returning an error if the write
// failed.
//
// Send may be called concurrently. Events will be written to the stream
// safely.
func (w *StreamWriter) Send(ctx aws.Context, event Marshaler) error {
if err := w.Err(); err != nil {
return err
}
resultCh := make(chan error)
wrapped := eventWriteAsyncReport{
Event: event,
Result: resultCh,
}
select {
case w.stream <- wrapped:
case <-ctx.Done():
return ctx.Err()
case <-w.done:
return fmt.Errorf("stream closed, unable to send event")
}
select {
case err := <-resultCh:
return err
case <-ctx.Done():
return ctx.Err()
case <-w.done:
return fmt.Errorf("stream closed, unable to send event")
}
}
func (w *StreamWriter) writeStream() {
defer w.Close()
for {
select {
case wrapper := <-w.stream:
err := w.eventWriter.WriteEvent(wrapper.Event)
wrapper.ReportResult(w.done, err)
if err != nil {
w.err.SetError(err)
return
}
case <-w.done:
if err := w.streamCloser.Close(); err != nil {
w.err.SetError(err)
}
return
}
}
}
type eventWriteAsyncReport struct {
Event Marshaler
Result chan<- error
}
func (e eventWriteAsyncReport) ReportResult(cancel <-chan struct{}, err error) bool {
select {
case e.Result <- err:
return true
case <-cancel:
return false
}
}

View File

@ -0,0 +1,109 @@
package eventstreamapi
import (
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
)
// Marshaler provides a marshaling interface for event types to event stream
// messages.
type Marshaler interface {
MarshalEvent(protocol.PayloadMarshaler) (eventstream.Message, error)
}
// Encoder is an stream encoder that will encode an event stream message for
// the transport.
type Encoder interface {
Encode(eventstream.Message) error
}
// EventWriter provides a wrapper around the underlying event stream encoder
// for an io.WriteCloser.
type EventWriter struct {
encoder Encoder
payloadMarshaler protocol.PayloadMarshaler
eventTypeFor func(Marshaler) (string, error)
}
// NewEventWriter returns a new event stream writer, that will write to the
// writer provided. Use the WriteEvent method to write an event to the stream.
func NewEventWriter(encoder Encoder, pm protocol.PayloadMarshaler, eventTypeFor func(Marshaler) (string, error),
) *EventWriter {
return &EventWriter{
encoder: encoder,
payloadMarshaler: pm,
eventTypeFor: eventTypeFor,
}
}
// WriteEvent writes an event to the stream. Returns an error if the event
// fails to marshal into a message, or writing to the underlying writer fails.
func (w *EventWriter) WriteEvent(event Marshaler) error {
msg, err := w.marshal(event)
if err != nil {
return err
}
return w.encoder.Encode(msg)
}
func (w *EventWriter) marshal(event Marshaler) (eventstream.Message, error) {
eventType, err := w.eventTypeFor(event)
if err != nil {
return eventstream.Message{}, err
}
msg, err := event.MarshalEvent(w.payloadMarshaler)
if err != nil {
return eventstream.Message{}, err
}
msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType))
return msg, nil
}
//type EventEncoder struct {
// encoder Encoder
// ppayloadMarshaler protocol.PayloadMarshaler
// eventTypeFor func(Marshaler) (string, error)
//}
//
//func (e EventEncoder) Encode(event Marshaler) error {
// msg, err := e.marshal(event)
// if err != nil {
// return err
// }
//
// return w.encoder.Encode(msg)
//}
//
//func (e EventEncoder) marshal(event Marshaler) (eventstream.Message, error) {
// eventType, err := w.eventTypeFor(event)
// if err != nil {
// return eventstream.Message{}, err
// }
//
// msg, err := event.MarshalEvent(w.payloadMarshaler)
// if err != nil {
// return eventstream.Message{}, err
// }
//
// msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType))
// return msg, nil
//}
//
//func (w *EventWriter) marshal(event Marshaler) (eventstream.Message, error) {
// eventType, err := w.eventTypeFor(event)
// if err != nil {
// return eventstream.Message{}, err
// }
//
// msg, err := event.MarshalEvent(w.payloadMarshaler)
// if err != nil {
// return eventstream.Message{}, err
// }
//
// msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType))
// return msg, nil
//}
//

View File

@ -52,6 +52,15 @@ func (hs *Headers) Del(name string) {
}
}
// Clone returns a deep copy of the headers
func (hs Headers) Clone() Headers {
o := make(Headers, 0, len(hs))
for _, h := range hs {
o.Set(h.Name, h.Value)
}
return o
}
func decodeHeaders(r io.Reader) (Headers, error) {
hs := Headers{}

View File

@ -461,6 +461,11 @@ func (v *TimestampValue) decode(r io.Reader) error {
return nil
}
// MarshalJSON implements the json.Marshaler interface
func (v TimestampValue) MarshalJSON() ([]byte, error) {
return []byte(v.String()), nil
}
func timeFromEpochMilli(t int64) time.Time {
secs := t / 1e3
msec := t % 1e3

View File

@ -27,7 +27,7 @@ func (m *Message) rawMessage() (rawMessage, error) {
if len(m.Headers) > 0 {
var headers bytes.Buffer
if err := encodeHeaders(&headers, m.Headers); err != nil {
if err := EncodeHeaders(&headers, m.Headers); err != nil {
return rawMessage{}, err
}
raw.Headers = headers.Bytes()
@ -57,6 +57,20 @@ func (m *Message) rawMessage() (rawMessage, error) {
return raw, nil
}
// Clone returns a deep copy of the message.
func (m Message) Clone() Message {
var payload []byte
if m.Payload != nil {
payload = make([]byte, len(m.Payload))
copy(payload, m.Payload)
}
return Message{
Headers: m.Headers.Clone(),
Payload: payload,
}
}
type messagePrelude struct {
Length uint32
HeadersLen uint32

View File

@ -6,7 +6,9 @@ import (
"encoding/json"
"fmt"
"io"
"math/big"
"reflect"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
@ -14,6 +16,8 @@ import (
"github.com/aws/aws-sdk-go/private/protocol"
)
var millisecondsFloat = new(big.Float).SetInt64(1e3)
// UnmarshalJSONError unmarshal's the reader's JSON document into the passed in
// type. The value to unmarshal the json document into must be a pointer to the
// type.
@ -38,17 +42,42 @@ func UnmarshalJSONError(v interface{}, stream io.Reader) error {
func UnmarshalJSON(v interface{}, stream io.Reader) error {
var out interface{}
err := json.NewDecoder(stream).Decode(&out)
decoder := json.NewDecoder(stream)
decoder.UseNumber()
err := decoder.Decode(&out)
if err == io.EOF {
return nil
} else if err != nil {
return err
}
return unmarshalAny(reflect.ValueOf(v), out, "")
return unmarshaler{}.unmarshalAny(reflect.ValueOf(v), out, "")
}
func unmarshalAny(value reflect.Value, data interface{}, tag reflect.StructTag) error {
// UnmarshalJSONCaseInsensitive reads a stream and unmarshals the result into the
// object v. Ignores casing for structure members.
func UnmarshalJSONCaseInsensitive(v interface{}, stream io.Reader) error {
var out interface{}
decoder := json.NewDecoder(stream)
decoder.UseNumber()
err := decoder.Decode(&out)
if err == io.EOF {
return nil
} else if err != nil {
return err
}
return unmarshaler{
caseInsensitive: true,
}.unmarshalAny(reflect.ValueOf(v), out, "")
}
type unmarshaler struct {
caseInsensitive bool
}
func (u unmarshaler) unmarshalAny(value reflect.Value, data interface{}, tag reflect.StructTag) error {
vtype := value.Type()
if vtype.Kind() == reflect.Ptr {
vtype = vtype.Elem() // check kind of actual element type
@ -80,17 +109,17 @@ func unmarshalAny(value reflect.Value, data interface{}, tag reflect.StructTag)
if field, ok := vtype.FieldByName("_"); ok {
tag = field.Tag
}
return unmarshalStruct(value, data, tag)
return u.unmarshalStruct(value, data, tag)
case "list":
return unmarshalList(value, data, tag)
return u.unmarshalList(value, data, tag)
case "map":
return unmarshalMap(value, data, tag)
return u.unmarshalMap(value, data, tag)
default:
return unmarshalScalar(value, data, tag)
return u.unmarshalScalar(value, data, tag)
}
}
func unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTag) error {
func (u unmarshaler) unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTag) error {
if data == nil {
return nil
}
@ -114,7 +143,7 @@ func unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTa
// unwrap any payloads
if payload := tag.Get("payload"); payload != "" {
field, _ := t.FieldByName(payload)
return unmarshalAny(value.FieldByName(payload), data, field.Tag)
return u.unmarshalAny(value.FieldByName(payload), data, field.Tag)
}
for i := 0; i < t.NumField(); i++ {
@ -128,9 +157,19 @@ func unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTa
if locName := field.Tag.Get("locationName"); locName != "" {
name = locName
}
if u.caseInsensitive {
if _, ok := mapData[name]; !ok {
// Fallback to uncased name search if the exact name didn't match.
for kn, v := range mapData {
if strings.EqualFold(kn, name) {
mapData[name] = v
}
}
}
}
member := value.FieldByIndex(field.Index)
err := unmarshalAny(member, mapData[name], field.Tag)
err := u.unmarshalAny(member, mapData[name], field.Tag)
if err != nil {
return err
}
@ -138,7 +177,7 @@ func unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTa
return nil
}
func unmarshalList(value reflect.Value, data interface{}, tag reflect.StructTag) error {
func (u unmarshaler) unmarshalList(value reflect.Value, data interface{}, tag reflect.StructTag) error {
if data == nil {
return nil
}
@ -153,7 +192,7 @@ func unmarshalList(value reflect.Value, data interface{}, tag reflect.StructTag)
}
for i, c := range listData {
err := unmarshalAny(value.Index(i), c, "")
err := u.unmarshalAny(value.Index(i), c, "")
if err != nil {
return err
}
@ -162,7 +201,7 @@ func unmarshalList(value reflect.Value, data interface{}, tag reflect.StructTag)
return nil
}
func unmarshalMap(value reflect.Value, data interface{}, tag reflect.StructTag) error {
func (u unmarshaler) unmarshalMap(value reflect.Value, data interface{}, tag reflect.StructTag) error {
if data == nil {
return nil
}
@ -179,14 +218,14 @@ func unmarshalMap(value reflect.Value, data interface{}, tag reflect.StructTag)
kvalue := reflect.ValueOf(k)
vvalue := reflect.New(value.Type().Elem()).Elem()
unmarshalAny(vvalue, v, "")
u.unmarshalAny(vvalue, v, "")
value.SetMapIndex(kvalue, vvalue)
}
return nil
}
func unmarshalScalar(value reflect.Value, data interface{}, tag reflect.StructTag) error {
func (u unmarshaler) unmarshalScalar(value reflect.Value, data interface{}, tag reflect.StructTag) error {
switch d := data.(type) {
case nil:
@ -222,16 +261,31 @@ func unmarshalScalar(value reflect.Value, data interface{}, tag reflect.StructTa
default:
return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type())
}
case float64:
case json.Number:
switch value.Interface().(type) {
case *int64:
di := int64(d)
// Retain the old behavior where we would just truncate the float64
// calling d.Int64() here could cause an invalid syntax error due to the usage of strconv.ParseInt
f, err := d.Float64()
if err != nil {
return err
}
di := int64(f)
value.Set(reflect.ValueOf(&di))
case *float64:
value.Set(reflect.ValueOf(&d))
f, err := d.Float64()
if err != nil {
return err
}
value.Set(reflect.ValueOf(&f))
case *time.Time:
// Time unmarshaled from a float64 can only be epoch seconds
t := time.Unix(int64(d), 0).UTC()
float, ok := new(big.Float).SetString(d.String())
if !ok {
return fmt.Errorf("unsupported float time representation: %v", d.String())
}
float = float.Mul(float, millisecondsFloat)
ms, _ := float.Int64()
t := time.Unix(0, ms*1e6).UTC()
value.Set(reflect.ValueOf(&t))
default:
return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type())

View File

@ -64,7 +64,7 @@ func (h HandlerPayloadMarshal) MarshalPayload(w io.Writer, v interface{}) error
metadata.ClientInfo{},
request.Handlers{},
nil,
&request.Operation{HTTPMethod: "GET"},
&request.Operation{HTTPMethod: "PUT"},
v,
nil,
)

View File

@ -0,0 +1,49 @@
package protocol
import (
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
)
// RequireHTTPMinProtocol request handler is used to enforce that
// the target endpoint supports the given major and minor HTTP protocol version.
type RequireHTTPMinProtocol struct {
Major, Minor int
}
// Handler will mark the request.Request with an error if the
// target endpoint did not connect with the required HTTP protocol
// major and minor version.
func (p RequireHTTPMinProtocol) Handler(r *request.Request) {
if r.Error != nil || r.HTTPResponse == nil {
return
}
if !strings.HasPrefix(r.HTTPResponse.Proto, "HTTP") {
r.Error = newMinHTTPProtoError(p.Major, p.Minor, r)
}
if r.HTTPResponse.ProtoMajor < p.Major || r.HTTPResponse.ProtoMinor < p.Minor {
r.Error = newMinHTTPProtoError(p.Major, p.Minor, r)
}
}
// ErrCodeMinimumHTTPProtocolError error code is returned when the target endpoint
// did not match the required HTTP major and minor protocol version.
const ErrCodeMinimumHTTPProtocolError = "MinimumHTTPProtocolError"
func newMinHTTPProtoError(major, minor int, r *request.Request) error {
return awserr.NewRequestFailure(
awserr.New("MinimumHTTPProtocolError",
fmt.Sprintf(
"operation requires minimum HTTP protocol of HTTP/%d.%d, but was %s",
major, minor, r.HTTPResponse.Proto,
),
nil,
),
r.HTTPResponse.StatusCode, r.RequestID,
)
}

View File

@ -1,7 +1,7 @@
// Package query provides serialization of AWS query requests, and responses.
package query
//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/input/query.json build_test.go
//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/input/query.json build_test.go
import (
"net/url"

View File

@ -1,6 +1,6 @@
package query
//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/output/query.json unmarshal_test.go
//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/output/query.json unmarshal_test.go
import (
"encoding/xml"

View File

@ -15,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
awsStrings "github.com/aws/aws-sdk-go/internal/strings"
"github.com/aws/aws-sdk-go/private/protocol"
)
@ -28,7 +29,9 @@ var UnmarshalMetaHandler = request.NamedHandler{Name: "awssdk.rest.UnmarshalMeta
func Unmarshal(r *request.Request) {
if r.DataFilled() {
v := reflect.Indirect(reflect.ValueOf(r.Data))
unmarshalBody(r, v)
if err := unmarshalBody(r, v); err != nil {
r.Error = err
}
}
}
@ -40,12 +43,21 @@ func UnmarshalMeta(r *request.Request) {
r.RequestID = r.HTTPResponse.Header.Get("X-Amz-Request-Id")
}
if r.DataFilled() {
v := reflect.Indirect(reflect.ValueOf(r.Data))
unmarshalLocationElements(r, v)
if err := UnmarshalResponse(r.HTTPResponse, r.Data, aws.BoolValue(r.Config.LowerCaseHeaderMaps)); err != nil {
r.Error = err
}
}
}
func unmarshalBody(r *request.Request, v reflect.Value) {
// UnmarshalResponse attempts to unmarshal the REST response headers to
// the data type passed in. The type must be a pointer. An error is returned
// with any error unmarshaling the response into the target datatype.
func UnmarshalResponse(resp *http.Response, data interface{}, lowerCaseHeaderMaps bool) error {
v := reflect.Indirect(reflect.ValueOf(data))
return unmarshalLocationElements(resp, v, lowerCaseHeaderMaps)
}
func unmarshalBody(r *request.Request, v reflect.Value) error {
if field, ok := v.Type().FieldByName("_"); ok {
if payloadName := field.Tag.Get("payload"); payloadName != "" {
pfield, _ := v.Type().FieldByName(payloadName)
@ -57,35 +69,38 @@ func unmarshalBody(r *request.Request, v reflect.Value) {
defer r.HTTPResponse.Body.Close()
b, err := ioutil.ReadAll(r.HTTPResponse.Body)
if err != nil {
r.Error = awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
} else {
payload.Set(reflect.ValueOf(b))
return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
}
payload.Set(reflect.ValueOf(b))
case *string:
defer r.HTTPResponse.Body.Close()
b, err := ioutil.ReadAll(r.HTTPResponse.Body)
if err != nil {
r.Error = awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
} else {
str := string(b)
payload.Set(reflect.ValueOf(&str))
return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
}
str := string(b)
payload.Set(reflect.ValueOf(&str))
default:
switch payload.Type().String() {
case "io.ReadCloser":
payload.Set(reflect.ValueOf(r.HTTPResponse.Body))
case "io.ReadSeeker":
b, err := ioutil.ReadAll(r.HTTPResponse.Body)
if err != nil {
r.Error = awserr.New(request.ErrCodeSerialization,
return awserr.New(request.ErrCodeSerialization,
"failed to read response body", err)
return
}
payload.Set(reflect.ValueOf(ioutil.NopCloser(bytes.NewReader(b))))
default:
io.Copy(ioutil.Discard, r.HTTPResponse.Body)
defer r.HTTPResponse.Body.Close()
r.Error = awserr.New(request.ErrCodeSerialization,
r.HTTPResponse.Body.Close()
return awserr.New(request.ErrCodeSerialization,
"failed to decode REST response",
fmt.Errorf("unknown payload type %s", payload.Type()))
}
@ -94,9 +109,11 @@ func unmarshalBody(r *request.Request, v reflect.Value) {
}
}
}
return nil
}
func unmarshalLocationElements(r *request.Request, v reflect.Value) {
func unmarshalLocationElements(resp *http.Response, v reflect.Value, lowerCaseHeaderMaps bool) error {
for i := 0; i < v.NumField(); i++ {
m, field := v.Field(i), v.Type().Field(i)
if n := field.Name; n[0:1] == strings.ToLower(n[0:1]) {
@ -111,26 +128,25 @@ func unmarshalLocationElements(r *request.Request, v reflect.Value) {
switch field.Tag.Get("location") {
case "statusCode":
unmarshalStatusCode(m, r.HTTPResponse.StatusCode)
unmarshalStatusCode(m, resp.StatusCode)
case "header":
err := unmarshalHeader(m, r.HTTPResponse.Header.Get(name), field.Tag)
err := unmarshalHeader(m, resp.Header.Get(name), field.Tag)
if err != nil {
r.Error = awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
break
return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
}
case "headers":
prefix := field.Tag.Get("locationName")
err := unmarshalHeaderMap(m, r.HTTPResponse.Header, prefix)
err := unmarshalHeaderMap(m, resp.Header, prefix, lowerCaseHeaderMaps)
if err != nil {
r.Error = awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
break
awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
}
}
}
if r.Error != nil {
return
}
}
return nil
}
func unmarshalStatusCode(v reflect.Value, statusCode int) {
@ -145,7 +161,7 @@ func unmarshalStatusCode(v reflect.Value, statusCode int) {
}
}
func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string) error {
func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string, normalize bool) error {
if len(headers) == 0 {
return nil
}
@ -153,8 +169,12 @@ func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string) err
case map[string]*string: // we only support string map value types
out := map[string]*string{}
for k, v := range headers {
k = http.CanonicalHeaderKey(k)
if strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) {
if awsStrings.HasPrefixFold(k, prefix) {
if normalize == true {
k = strings.ToLower(k)
} else {
k = http.CanonicalHeaderKey(k)
}
out[k[len(prefix):]] = &v[0]
}
}

View File

@ -2,8 +2,8 @@
// requests and responses.
package restxml
//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/input/rest-xml.json build_test.go
//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/output/rest-xml.json unmarshal_test.go
//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/input/rest-xml.json build_test.go
//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/output/rest-xml.json unmarshal_test.go
import (
"bytes"

View File

@ -27,8 +27,8 @@ const (
// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
ISO8601TimeFormat = "2006-01-02T15:04:05.999999999Z"
// This format is used for output time without seconds precision
ISO8601OutputTimeFormat = "2006-01-02T15:04:05Z"
// This format is used for output time with fractional second precision up to milliseconds
ISO8601OutputTimeFormat = "2006-01-02T15:04:05.999999999Z"
)
// IsKnownTimestampFormat returns if the timestamp format name
@ -48,7 +48,7 @@ func IsKnownTimestampFormat(name string) bool {
// FormatTime returns a string value of the time.
func FormatTime(name string, t time.Time) string {
t = t.UTC()
t = t.UTC().Truncate(time.Millisecond)
switch name {
case RFC822TimeFormatName:
@ -56,7 +56,8 @@ func FormatTime(name string, t time.Time) string {
case ISO8601TimeFormatName:
return t.Format(ISO8601OutputTimeFormat)
case UnixTimeFormatName:
return strconv.FormatInt(t.Unix(), 10)
ms := t.UnixNano() / int64(time.Millisecond)
return strconv.FormatFloat(float64(ms)/1e3, 'f', -1, 64)
default:
panic("unknown timestamp format name, " + name)
}

View File

@ -19,3 +19,9 @@ func UnmarshalDiscardBody(r *request.Request) {
io.Copy(ioutil.Discard, r.HTTPResponse.Body)
r.HTTPResponse.Body.Close()
}
// ResponseMetadata provides the SDK response metadata attributes.
type ResponseMetadata struct {
StatusCode int
RequestID string
}

View File

@ -0,0 +1,65 @@
package protocol
import (
"net/http"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
)
// UnmarshalErrorHandler provides unmarshaling errors API response errors for
// both typed and untyped errors.
type UnmarshalErrorHandler struct {
unmarshaler ErrorUnmarshaler
}
// ErrorUnmarshaler is an abstract interface for concrete implementations to
// unmarshal protocol specific response errors.
type ErrorUnmarshaler interface {
UnmarshalError(*http.Response, ResponseMetadata) (error, error)
}
// NewUnmarshalErrorHandler returns an UnmarshalErrorHandler
// initialized for the set of exception names to the error unmarshalers
func NewUnmarshalErrorHandler(unmarshaler ErrorUnmarshaler) *UnmarshalErrorHandler {
return &UnmarshalErrorHandler{
unmarshaler: unmarshaler,
}
}
// UnmarshalErrorHandlerName is the name of the named handler.
const UnmarshalErrorHandlerName = "awssdk.protocol.UnmarshalError"
// NamedHandler returns a NamedHandler for the unmarshaler using the set of
// errors the unmarshaler was initialized for.
func (u *UnmarshalErrorHandler) NamedHandler() request.NamedHandler {
return request.NamedHandler{
Name: UnmarshalErrorHandlerName,
Fn: u.UnmarshalError,
}
}
// UnmarshalError will attempt to unmarshal the API response's error message
// into either a generic SDK error type, or a typed error corresponding to the
// errors exception name.
func (u *UnmarshalErrorHandler) UnmarshalError(r *request.Request) {
defer r.HTTPResponse.Body.Close()
respMeta := ResponseMetadata{
StatusCode: r.HTTPResponse.StatusCode,
RequestID: r.RequestID,
}
v, err := u.unmarshaler.UnmarshalError(r.HTTPResponse, respMeta)
if err != nil {
r.Error = awserr.NewRequestFailure(
awserr.New(request.ErrCodeSerialization,
"failed to unmarshal response error", err),
respMeta.StatusCode,
respMeta.RequestID,
)
return
}
r.Error = v
}

View File

@ -8,6 +8,7 @@ import (
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go/private/protocol"
@ -60,6 +61,14 @@ func (b *xmlBuilder) buildValue(value reflect.Value, current *XMLNode, tag refle
return nil
}
xml := tag.Get("xml")
if len(xml) != 0 {
name := strings.SplitAfterN(xml, ",", 2)[0]
if name == "-" {
return nil
}
}
t := tag.Get("type")
if t == "" {
switch value.Kind() {

View File

@ -64,6 +64,14 @@ func UnmarshalXML(v interface{}, d *xml.Decoder, wrapper string) error {
// parse deserializes any value from the XMLNode. The type tag is used to infer the type, or reflect
// will be used to determine the type from r.
func parse(r reflect.Value, node *XMLNode, tag reflect.StructTag) error {
xml := tag.Get("xml")
if len(xml) != 0 {
name := strings.SplitAfterN(xml, ",", 2)[0]
if name == "-" {
return nil
}
}
rtype := r.Type()
if rtype.Kind() == reflect.Ptr {
rtype = rtype.Elem() // check kind of actual element type

File diff suppressed because it is too large Load Diff

View File

@ -26,8 +26,12 @@ func init() {
// only set the retryer on request if config doesn't have a retryer
if r.Config.Retryer == nil && (r.Operation.Name == opModifyNetworkInterfaceAttribute || r.Operation.Name == opAssignPrivateIpAddresses) {
maxRetries := client.DefaultRetryerMaxNumRetries
if m := r.Config.MaxRetries; m != nil && *m != aws.UseServiceDefaultRetries {
maxRetries = *m
}
r.Retryer = client.DefaultRetryer{
NumMaxRetries: client.DefaultRetryerMaxNumRetries,
NumMaxRetries: maxRetries,
MinRetryDelay: customRetryerMinRetryDelay,
MinThrottleDelay: customRetryerMinRetryDelay,
MaxRetryDelay: customRetryerMaxRetryDelay,

View File

@ -31,7 +31,7 @@ var initRequest func(*request.Request)
const (
ServiceName = "ec2" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "EC2" // ServiceID is a unique identifer of a specific service.
ServiceID = "EC2" // ServiceID is a unique identifier of a specific service.
)
// New creates a new instance of the EC2 client with a session.

File diff suppressed because it is too large Load Diff

View File

@ -13,7 +13,6 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/sdkio"
)
const (
@ -25,30 +24,6 @@ const (
appendMD5TxEncoding = "append-md5"
)
// contentMD5 computes and sets the HTTP Content-MD5 header for requests that
// require it.
func contentMD5(r *request.Request) {
h := md5.New()
if !aws.IsReaderSeekable(r.Body) {
if r.Config.Logger != nil {
r.Config.Logger.Log(fmt.Sprintf(
"Unable to compute Content-MD5 for unseekable body, S3.%s",
r.Operation.Name))
}
return
}
if _, err := copySeekableBody(h, r.Body); err != nil {
r.Error = awserr.New("ContentMD5", "failed to compute body MD5", err)
return
}
// encode the md5 checksum in base64 and set the request header.
v := base64.StdEncoding.EncodeToString(h.Sum(nil))
r.HTTPRequest.Header.Set(contentMD5Header, v)
}
// computeBodyHashes will add Content MD5 and Content Sha256 hashes to the
// request. If the body is not seekable or S3DisableContentMD5Validation set
// this handler will be ignored.
@ -90,7 +65,7 @@ func computeBodyHashes(r *request.Request) {
dst = io.MultiWriter(hashers...)
}
if _, err := copySeekableBody(dst, r.Body); err != nil {
if _, err := aws.CopySeekableBody(dst, r.Body); err != nil {
r.Error = awserr.New("BodyHashError", "failed to compute body hashes", err)
return
}
@ -119,28 +94,6 @@ const (
sha256HexEncLen = sha256.Size * 2 // hex.EncodedLen
)
func copySeekableBody(dst io.Writer, src io.ReadSeeker) (int64, error) {
curPos, err := src.Seek(0, sdkio.SeekCurrent)
if err != nil {
return 0, err
}
// hash the body. seek back to the first position after reading to reset
// the body for transmission. copy errors may be assumed to be from the
// body.
n, err := io.Copy(dst, src)
if err != nil {
return n, err
}
_, err = src.Seek(curPos, sdkio.SeekStart)
if err != nil {
return n, err
}
return n, nil
}
// Adds the x-amz-te: append_md5 header to the request. This requests the service
// responds with a trailing MD5 checksum.
//

View File

@ -3,7 +3,8 @@ package s3
import (
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3err"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
"github.com/aws/aws-sdk-go/internal/s3shared/s3err"
)
func init() {
@ -13,7 +14,7 @@ func init() {
func defaultInitClientFn(c *client.Client) {
// Support building custom endpoints based on config
c.Handlers.Build.PushFront(updateEndpointForS3Config)
c.Handlers.Build.PushFront(endpointHandler)
// Require SSL when using SSE keys
c.Handlers.Validate.PushBack(validateSSERequiresSSL)
@ -27,17 +28,11 @@ func defaultInitClientFn(c *client.Client) {
}
func defaultInitRequestFn(r *request.Request) {
// Add reuest handlers for specific platforms.
// Add request handlers for specific platforms.
// e.g. 100-continue support for PUT requests using Go 1.6
platformRequestHandlers(r)
switch r.Operation.Name {
case opPutBucketCors, opPutBucketLifecycle, opPutBucketPolicy,
opPutBucketTagging, opDeleteObjects, opPutBucketLifecycleConfiguration,
opPutObjectLegalHold, opPutObjectRetention, opPutObjectLockConfiguration,
opPutBucketReplication:
// These S3 operations require Content-MD5 to be set
r.Handlers.Build.PushBack(contentMD5)
case opGetBucketLocation:
// GetBucketLocation has custom parsing logic
r.Handlers.Unmarshal.PushFront(buildGetBucketLocation)
@ -73,3 +68,10 @@ type sseCustomerKeyGetter interface {
type copySourceSSECustomerKeyGetter interface {
getCopySourceSSECustomerKey() string
}
// endpointARNGetter is an accessor interface to grab the
// the field corresponding to an endpoint ARN input.
type endpointARNGetter interface {
getEndpointARN() (arn.Resource, error)
hasEndpointARN() bool
}

View File

@ -104,19 +104,6 @@
// content from S3. The Encryption and Decryption clients can be used concurrently
// once the client is created.
//
// sess := session.Must(session.NewSession())
//
// // Create the decryption client.
// svc := s3crypto.NewDecryptionClient(sess)
//
// // The object will be downloaded from S3 and decrypted locally. By metadata
// // about the object's encryption will instruct the decryption client how
// // decrypt the content of the object. By default KMS is used for keys.
// result, err := svc.GetObject(&s3.GetObjectInput {
// Bucket: aws.String(myBucket),
// Key: aws.String(myKey),
// })
//
// See the s3crypto package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3crypto/
//

201
vendor/github.com/aws/aws-sdk-go/service/s3/endpoint.go generated vendored Normal file
View File

@ -0,0 +1,201 @@
package s3
import (
"net/url"
"strings"
"github.com/aws/aws-sdk-go/aws"
awsarn "github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3shared"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
)
// Used by shapes with members decorated as endpoint ARN.
func parseEndpointARN(v string) (arn.Resource, error) {
return arn.ParseResource(v, accessPointResourceParser)
}
func accessPointResourceParser(a awsarn.ARN) (arn.Resource, error) {
resParts := arn.SplitResource(a.Resource)
switch resParts[0] {
case "accesspoint":
if a.Service != "s3" {
return arn.AccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3"}
}
return arn.ParseAccessPointResource(a, resParts[1:])
case "outpost":
if a.Service != "s3-outposts" {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3-outposts"}
}
return parseOutpostAccessPointResource(a, resParts[1:])
default:
return nil, arn.InvalidARNError{ARN: a, Reason: "unknown resource type"}
}
}
// parseOutpostAccessPointResource attempts to parse the ARNs resource as an
// outpost access-point resource.
//
// Supported Outpost AccessPoint ARN format:
// - ARN format: arn:{partition}:s3-outposts:{region}:{accountId}:outpost/{outpostId}/accesspoint/{accesspointName}
// - example: arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint
//
func parseOutpostAccessPointResource(a awsarn.ARN, resParts []string) (arn.OutpostAccessPointARN, error) {
// outpost accesspoint arn is only valid if service is s3-outposts
if a.Service != "s3-outposts" {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3-outposts"}
}
if len(resParts) == 0 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"}
}
if len(resParts) < 3 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{
ARN: a, Reason: "access-point resource not set in Outpost ARN",
}
}
resID := strings.TrimSpace(resParts[0])
if len(resID) == 0 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"}
}
var outpostAccessPointARN = arn.OutpostAccessPointARN{}
switch resParts[1] {
case "accesspoint":
accessPointARN, err := arn.ParseAccessPointResource(a, resParts[2:])
if err != nil {
return arn.OutpostAccessPointARN{}, err
}
// set access-point arn
outpostAccessPointARN.AccessPointARN = accessPointARN
default:
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "access-point resource not set in Outpost ARN"}
}
// set outpost id
outpostAccessPointARN.OutpostID = resID
return outpostAccessPointARN, nil
}
func endpointHandler(req *request.Request) {
endpoint, ok := req.Params.(endpointARNGetter)
if !ok || !endpoint.hasEndpointARN() {
updateBucketEndpointFromParams(req)
return
}
resource, err := endpoint.getEndpointARN()
if err != nil {
req.Error = s3shared.NewInvalidARNError(nil, err)
return
}
resReq := s3shared.ResourceRequest{
Resource: resource,
Request: req,
}
if resReq.IsCrossPartition() {
req.Error = s3shared.NewClientPartitionMismatchError(resource,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
return
}
if !resReq.AllowCrossRegion() && resReq.IsCrossRegion() {
req.Error = s3shared.NewClientRegionMismatchError(resource,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
return
}
if resReq.HasCustomEndpoint() {
req.Error = s3shared.NewInvalidARNWithCustomEndpointError(resource, nil)
return
}
switch tv := resource.(type) {
case arn.AccessPointARN:
err = updateRequestAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
case arn.OutpostAccessPointARN:
// outposts does not support FIPS regions
if resReq.ResourceConfiguredForFIPS() {
req.Error = s3shared.NewInvalidARNWithFIPSError(resource, nil)
return
}
err = updateRequestOutpostAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
default:
req.Error = s3shared.NewInvalidARNError(resource, nil)
}
}
func updateBucketEndpointFromParams(r *request.Request) {
bucket, ok := bucketNameFromReqParams(r.Params)
if !ok {
// Ignore operation requests if the bucket name was not provided
// if this is an input validation error the validation handler
// will report it.
return
}
updateEndpointForS3Config(r, bucket)
}
func updateRequestAccessPointEndpoint(req *request.Request, accessPoint arn.AccessPointARN) error {
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points since custom endpoints
// are not supported.
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := accessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
removeBucketFromPath(req.HTTPRequest.URL)
return nil
}
func updateRequestOutpostAccessPointEndpoint(req *request.Request, accessPoint arn.OutpostAccessPointARN) error {
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Dualstack not supported
if aws.BoolValue(req.Config.UseDualStack) {
return s3shared.NewClientConfiguredForDualStackError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points since custom endpoints
// are not supported.
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := outpostAccessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
removeBucketFromPath(req.HTTPRequest.URL)
return nil
}
func removeBucketFromPath(u *url.URL) {
u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1)
if u.Path == "" {
u.Path = "/"
}
}

View File

@ -0,0 +1,177 @@
package s3
import (
"net/url"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3shared"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
"github.com/aws/aws-sdk-go/private/protocol"
)
const (
accessPointPrefixLabel = "accesspoint"
accountIDPrefixLabel = "accountID"
accessPointPrefixTemplate = "{" + accessPointPrefixLabel + "}-{" + accountIDPrefixLabel + "}."
outpostPrefixLabel = "outpost"
outpostAccessPointPrefixTemplate = accessPointPrefixTemplate + "{" + outpostPrefixLabel + "}."
)
// accessPointEndpointBuilder represents the endpoint builder for access point arn
type accessPointEndpointBuilder arn.AccessPointARN
// build builds the endpoint for corresponding access point arn
//
// For building an endpoint from access point arn, format used is:
// - Access point endpoint format : {accesspointName}-{accountId}.s3-accesspoint.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.s3-accesspoint.us-west-2.amazonaws.com
//
// Access Point Endpoint requests are signed using "s3" as signing name.
//
func (a accessPointEndpointBuilder) build(req *request.Request) error {
resolveService := arn.AccessPointARN(a).Service
resolveRegion := arn.AccessPointARN(a).Region
cfgRegion := aws.StringValue(req.Config.Region)
if s3shared.IsFIPS(cfgRegion) {
if aws.BoolValue(req.Config.S3UseARNRegion) && s3shared.IsCrossRegion(req, resolveRegion) {
// FIPS with cross region is not supported, the SDK must fail
// because there is no well defined method for SDK to construct a
// correct FIPS endpoint.
return s3shared.NewClientConfiguredForCrossRegionFIPSError(arn.AccessPointARN(a),
req.ClientInfo.PartitionID, cfgRegion, nil)
}
resolveRegion = cfgRegion
}
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, resolveService)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(arn.AccessPointARN(a),
req.ClientInfo.PartitionID, cfgRegion, err)
}
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
const serviceEndpointLabel = "s3-accesspoint"
// dual stack provided by endpoint resolver
cfgHost := req.HTTPRequest.URL.Host
if strings.HasPrefix(cfgHost, "s3") {
req.HTTPRequest.URL.Host = serviceEndpointLabel + cfgHost[2:]
}
protocol.HostPrefixBuilder{
Prefix: accessPointPrefixTemplate,
LabelsFn: a.hostPrefixLabelValues,
}.Build(req)
// signer redirection
redirectSigner(req, endpoint.SigningName, endpoint.SigningRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(arn.AccessPointARN(a), err)
}
return nil
}
func (a accessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: arn.AccessPointARN(a).AccessPointName,
accountIDPrefixLabel: arn.AccessPointARN(a).AccountID,
}
}
// outpostAccessPointEndpointBuilder represents the Endpoint builder for outpost access point arn.
type outpostAccessPointEndpointBuilder arn.OutpostAccessPointARN
// build builds an endpoint corresponding to the outpost access point arn.
//
// For building an endpoint from outpost access point arn, format used is:
// - Outpost access point endpoint format : {accesspointName}-{accountId}.{outpostId}.s3-outposts.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com
//
// Outpost AccessPoint Endpoint request are signed using "s3-outposts" as signing name.
//
func (o outpostAccessPointEndpointBuilder) build(req *request.Request) error {
resolveRegion := o.Region
resolveService := o.Service
endpointsID := resolveService
if resolveService == "s3-outposts" {
endpointsID = "s3"
}
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, endpointsID)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(o,
req.ClientInfo.PartitionID, resolveRegion, err)
}
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
// add url host as s3-outposts
cfgHost := req.HTTPRequest.URL.Host
if strings.HasPrefix(cfgHost, endpointsID) {
req.HTTPRequest.URL.Host = resolveService + cfgHost[len(endpointsID):]
}
protocol.HostPrefixBuilder{
Prefix: outpostAccessPointPrefixTemplate,
LabelsFn: o.hostPrefixLabelValues,
}.Build(req)
// set the signing region, name to resolved names from ARN
redirectSigner(req, resolveService, resolveRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(o, err)
}
return nil
}
func (o outpostAccessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: o.AccessPointName,
accountIDPrefixLabel: o.AccountID,
outpostPrefixLabel: o.OutpostID,
}
}
func resolveRegionalEndpoint(r *request.Request, region string, endpointsID string) (endpoints.ResolvedEndpoint, error) {
return r.Config.EndpointResolver.EndpointFor(endpointsID, region, func(opts *endpoints.Options) {
opts.DisableSSL = aws.BoolValue(r.Config.DisableSSL)
opts.UseDualStack = aws.BoolValue(r.Config.UseDualStack)
opts.S3UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
})
}
func updateRequestEndpoint(r *request.Request, endpoint string) (err error) {
endpoint = endpoints.AddScheme(endpoint, aws.BoolValue(r.Config.DisableSSL))
r.HTTPRequest.URL, err = url.Parse(endpoint + r.Operation.HTTPPath)
if err != nil {
return awserr.New(request.ErrCodeSerialization,
"failed to parse endpoint URL", err)
}
return nil
}
// redirectSigner sets signing name, signing region for a request
func redirectSigner(req *request.Request, signingName string, signingRegion string) {
req.ClientInfo.SigningName = signingName
req.ClientInfo.SigningRegion = signingRegion
}

View File

@ -8,16 +8,16 @@ const (
// "BucketAlreadyExists".
//
// The requested bucket name is not available. The bucket namespace is shared
// by all users of the system. Please select a different name and try again.
// by all users of the system. Select a different name and try again.
ErrCodeBucketAlreadyExists = "BucketAlreadyExists"
// ErrCodeBucketAlreadyOwnedByYou for service response error code
// "BucketAlreadyOwnedByYou".
//
// The bucket you tried to create already exists, and you own it. Amazon S3
// returns this error in all AWS Regions except in the North Virginia region.
// returns this error in all AWS Regions except in the North Virginia Region.
// For legacy compatibility, if you re-create an existing bucket that you already
// own in the North Virginia region, Amazon S3 returns 200 OK and resets the
// own in the North Virginia Region, Amazon S3 returns 200 OK and resets the
// bucket access control lists (ACLs).
ErrCodeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou"
@ -42,13 +42,13 @@ const (
// ErrCodeObjectAlreadyInActiveTierError for service response error code
// "ObjectAlreadyInActiveTierError".
//
// This operation is not allowed against this storage tier
// This operation is not allowed against this storage tier.
ErrCodeObjectAlreadyInActiveTierError = "ObjectAlreadyInActiveTierError"
// ErrCodeObjectNotInActiveTierError for service response error code
// "ObjectNotInActiveTierError".
//
// The source object of the COPY operation is not in the active tier and is
// only stored in Amazon Glacier.
// only stored in Amazon S3 Glacier.
ErrCodeObjectNotInActiveTierError = "ObjectNotInActiveTierError"
)

View File

@ -30,10 +30,10 @@ var accelerateOpBlacklist = operationBlacklist{
opListBuckets, opCreateBucket, opDeleteBucket,
}
// Request handler to automatically add the bucket name to the endpoint domain
// Automatically add the bucket name to the endpoint domain
// if possible. This style of bucket is valid for all bucket names which are
// DNS compatible and do not contain "."
func updateEndpointForS3Config(r *request.Request) {
func updateEndpointForS3Config(r *request.Request, bucketName string) {
forceHostStyle := aws.BoolValue(r.Config.S3ForcePathStyle)
accelerate := aws.BoolValue(r.Config.S3UseAccelerate)
@ -43,45 +43,29 @@ func updateEndpointForS3Config(r *request.Request) {
r.Config.Logger.Log("ERROR: aws.Config.S3UseAccelerate is not compatible with aws.Config.S3ForcePathStyle, ignoring S3ForcePathStyle.")
}
}
updateEndpointForAccelerate(r)
updateEndpointForAccelerate(r, bucketName)
} else if !forceHostStyle && r.Operation.Name != opGetBucketLocation {
updateEndpointForHostStyle(r)
updateEndpointForHostStyle(r, bucketName)
}
}
func updateEndpointForHostStyle(r *request.Request) {
bucket, ok := bucketNameFromReqParams(r.Params)
if !ok {
// Ignore operation requests if the bucketname was not provided
// if this is an input validation error the validation handler
// will report it.
return
}
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) {
func updateEndpointForHostStyle(r *request.Request, bucketName string) {
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucketName) {
// bucket name must be valid to put into the host
return
}
moveBucketToHost(r.HTTPRequest.URL, bucket)
moveBucketToHost(r.HTTPRequest.URL, bucketName)
}
var (
accelElem = []byte("s3-accelerate.dualstack.")
)
func updateEndpointForAccelerate(r *request.Request) {
bucket, ok := bucketNameFromReqParams(r.Params)
if !ok {
// Ignore operation requests if the bucketname was not provided
// if this is an input validation error the validation handler
// will report it.
return
}
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) {
func updateEndpointForAccelerate(r *request.Request, bucketName string) {
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucketName) {
r.Error = awserr.New("InvalidParameterException",
fmt.Sprintf("bucket name %s is not compatible with S3 Accelerate", bucket),
fmt.Sprintf("bucket name %s is not compatible with S3 Accelerate", bucketName),
nil)
return
}
@ -106,7 +90,7 @@ func updateEndpointForAccelerate(r *request.Request) {
r.HTTPRequest.URL.Host = strings.Join(parts, ".")
moveBucketToHost(r.HTTPRequest.URL, bucket)
moveBucketToHost(r.HTTPRequest.URL, bucketName)
}
// Attempts to retrieve the bucket name from the request input parameters.
@ -148,8 +132,5 @@ func dnsCompatibleBucketName(bucket string) bool {
// moveBucketToHost moves the bucket name from the URI path to URL host.
func moveBucketToHost(u *url.URL, bucket string) {
u.Host = bucket + "." + u.Host
u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1)
if u.Path == "" {
u.Path = "/"
}
removeBucketFromPath(u)
}

View File

@ -31,7 +31,7 @@ var initRequest func(*request.Request)
const (
ServiceName = "s3" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "S3" // ServiceID is a unique identifer of a specific service.
ServiceID = "S3" // ServiceID is a unique identifier of a specific service.
)
// New creates a new instance of the S3 client with a session.
@ -78,6 +78,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint,
svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler)
svc.Handlers.UnmarshalError.PushBackNamed(restxml.UnmarshalErrorHandler)
svc.Handlers.BuildStream.PushBackNamed(restxml.BuildHandler)
svc.Handlers.UnmarshalStream.PushBackNamed(restxml.UnmarshalHandler)
// Run custom client initialization if present

View File

@ -69,7 +69,7 @@ func computeKeyMD5(keyHeader, keyMD5Header, key string, r *http.Request) {
return
}
// In backwards compatiable, the header's value is not base64 encoded,
// In backwards compatible, the header's value is not base64 encoded,
// and needs to be encoded and updated by the SDK's customizations.
b64Key := base64.StdEncoding.EncodeToString([]byte(key))
r.Header.Set(keyHeader, b64Key)

View File

@ -2,6 +2,7 @@ package s3
import (
"bytes"
"io"
"io/ioutil"
"net/http"
@ -24,17 +25,18 @@ func copyMultipartStatusOKUnmarhsalError(r *request.Request) {
r.HTTPResponse.Body = ioutil.NopCloser(body)
defer body.Seek(0, sdkio.SeekStart)
if body.Len() == 0 {
// If there is no body don't attempt to parse the body.
return
}
unmarshalError(r)
if err, ok := r.Error.(awserr.Error); ok && err != nil {
if err.Code() == request.ErrCodeSerialization {
if err.Code() == request.ErrCodeSerialization &&
err.OrigErr() != io.EOF {
r.Error = nil
return
}
r.HTTPResponse.StatusCode = http.StatusServiceUnavailable
// if empty payload
if err.OrigErr() == io.EOF {
r.HTTPResponse.StatusCode = http.StatusInternalServerError
} else {
r.HTTPResponse.StatusCode = http.StatusServiceUnavailable
}
}
}

View File

@ -1,6 +1,7 @@
package s3
import (
"bytes"
"encoding/xml"
"fmt"
"io"
@ -45,17 +46,24 @@ func unmarshalError(r *request.Request) {
// Attempt to parse error from body if it is known
var errResp xmlErrorResponse
err := xmlutil.UnmarshalXMLError(&errResp, r.HTTPResponse.Body)
if err == io.EOF {
// Only capture the error if an unmarshal error occurs that is not EOF,
// because S3 might send an error without a error message which causes
// the XML unmarshal to fail with EOF.
err = nil
var err error
if r.HTTPResponse.StatusCode >= 200 && r.HTTPResponse.StatusCode < 300 {
err = s3unmarshalXMLError(&errResp, r.HTTPResponse.Body)
} else {
err = xmlutil.UnmarshalXMLError(&errResp, r.HTTPResponse.Body)
}
if err != nil {
var errorMsg string
if err == io.EOF {
errorMsg = "empty response payload"
} else {
errorMsg = "failed to unmarshal error message"
}
r.Error = awserr.NewRequestFailure(
awserr.New(request.ErrCodeSerialization,
"failed to unmarshal error message", err),
errorMsg, err),
r.HTTPResponse.StatusCode,
r.RequestID,
)
@ -86,3 +94,21 @@ type RequestFailure interface {
// Host ID is the S3 Host ID needed for debug, and contacting support
HostID() string
}
// s3unmarshalXMLError is s3 specific xml error unmarshaler
// for 200 OK errors and response payloads.
// This function differs from the xmlUtil.UnmarshalXMLError
// func. It does not ignore the EOF error and passes it up.
// Related to bug fix for `s3 200 OK response with empty payload`
func s3unmarshalXMLError(v interface{}, stream io.Reader) error {
var errBuf bytes.Buffer
body := io.TeeReader(stream, &errBuf)
err := xml.NewDecoder(body).Decode(v)
if err != nil && err != io.EOF {
return awserr.NewUnmarshalError(err,
"failed to unmarshal error message", errBuf.Bytes())
}
return err
}

View File

@ -207,6 +207,10 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
// and Deactivating AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
// in the IAM User Guide.
//
// * ErrCodeExpiredTokenException "ExpiredTokenException"
// The web identity token that was passed is expired or is not valid. Get a
// new identity token from the identity provider and then retry the request.
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumeRole
func (c *STS) AssumeRole(input *AssumeRoleInput) (*AssumeRoleOutput, error) {
req, out := c.AssumeRoleRequest(input)
@ -626,7 +630,7 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI
// * Using Web Identity Federation API Operations for Mobile Apps (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc_manual.html)
// and Federation Through a Web-based Identity Provider (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithwebidentity).
//
// * Web Identity Federation Playground (https://web-identity-federation-playground.s3.amazonaws.com/index.html).
// * Web Identity Federation Playground (https://aws.amazon.com/blogs/aws/the-aws-web-identity-federation-playground/).
// Walk through the process of authenticating through Login with Amazon,
// Facebook, or Google, getting temporary security credentials, and then
// using those credentials to make a request to AWS.

View File

@ -3,87 +3,11 @@
// Package sts provides the client and types for making API
// requests to AWS Security Token Service.
//
// The AWS Security Token Service (STS) is a web service that enables you to
// request temporary, limited-privilege credentials for AWS Identity and Access
// Management (IAM) users or for users that you authenticate (federated users).
// This guide provides descriptions of the STS API. For more detailed information
// about using this service, go to Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html).
//
// For information about setting up signatures and authorization through the
// API, go to Signing AWS API Requests (https://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html)
// in the AWS General Reference. For general information about the Query API,
// go to Making Query Requests (https://docs.aws.amazon.com/IAM/latest/UserGuide/IAM_UsingQueryAPI.html)
// in Using IAM. For information about using security tokens with other AWS
// products, go to AWS Services That Work with IAM (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html)
// in the IAM User Guide.
//
// If you're new to AWS and need additional technical information about a specific
// AWS product, you can find the product's technical documentation at http://aws.amazon.com/documentation/
// (http://aws.amazon.com/documentation/).
//
// Endpoints
//
// By default, AWS Security Token Service (STS) is available as a global service,
// and all AWS STS requests go to a single endpoint at https://sts.amazonaws.com.
// Global requests map to the US East (N. Virginia) region. AWS recommends using
// Regional AWS STS endpoints instead of the global endpoint to reduce latency,
// build in redundancy, and increase session token validity. For more information,
// see Managing AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
// in the IAM User Guide.
//
// Most AWS Regions are enabled for operations in all AWS services by default.
// Those Regions are automatically activated for use with AWS STS. Some Regions,
// such as Asia Pacific (Hong Kong), must be manually enabled. To learn more
// about enabling and disabling AWS Regions, see Managing AWS Regions (https://docs.aws.amazon.com/general/latest/gr/rande-manage.html)
// in the AWS General Reference. When you enable these AWS Regions, they are
// automatically activated for use with AWS STS. You cannot activate the STS
// endpoint for a Region that is disabled. Tokens that are valid in all AWS
// Regions are longer than tokens that are valid in Regions that are enabled
// by default. Changing this setting might affect existing systems where you
// temporarily store tokens. For more information, see Managing Global Endpoint
// Session Tokens (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html#sts-regions-manage-tokens)
// in the IAM User Guide.
//
// After you activate a Region for use with AWS STS, you can direct AWS STS
// API calls to that Region. AWS STS recommends that you provide both the Region
// and endpoint when you make calls to a Regional endpoint. You can provide
// the Region alone for manually enabled Regions, such as Asia Pacific (Hong
// Kong). In this case, the calls are directed to the STS Regional endpoint.
// However, if you provide the Region alone for Regions enabled by default,
// the calls are directed to the global endpoint of https://sts.amazonaws.com.
//
// To view the list of AWS STS endpoints and whether they are active by default,
// see Writing Code to Use AWS STS Regions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html#id_credentials_temp_enable-regions_writing_code)
// in the IAM User Guide.
//
// Recording API requests
//
// STS supports AWS CloudTrail, which is a service that records AWS calls for
// your AWS account and delivers log files to an Amazon S3 bucket. By using
// information collected by CloudTrail, you can determine what requests were
// successfully made to STS, who made the request, when it was made, and so
// on.
//
// If you activate AWS STS endpoints in Regions other than the default global
// endpoint, then you must also turn on CloudTrail logging in those Regions.
// This is necessary to record any AWS STS API calls that are made in those
// Regions. For more information, see Turning On CloudTrail in Additional Regions
// (https://docs.aws.amazon.com/awscloudtrail/latest/userguide/aggregating_logs_regions_turn_on_ct.html)
// in the AWS CloudTrail User Guide.
//
// AWS Security Token Service (STS) is a global service with a single endpoint
// at https://sts.amazonaws.com. Calls to this endpoint are logged as calls
// to a global service. However, because this endpoint is physically located
// in the US East (N. Virginia) Region, your logs list us-east-1 as the event
// Region. CloudTrail does not write these logs to the US East (Ohio) Region
// unless you choose to include global service logs in that Region. CloudTrail
// writes calls to all Regional endpoints to their respective Regions. For example,
// calls to sts.us-east-2.amazonaws.com are published to the US East (Ohio)
// Region and calls to sts.eu-central-1.amazonaws.com are published to the EU
// (Frankfurt) Region.
//
// To learn more about CloudTrail, including how to turn it on and find your
// log files, see the AWS CloudTrail User Guide (https://docs.aws.amazon.com/awscloudtrail/latest/userguide/what_is_cloud_trail_top_level.html).
// AWS Security Token Service (STS) enables you to request temporary, limited-privilege
// credentials for AWS Identity and Access Management (IAM) users or for users
// that you authenticate (federated users). This guide provides descriptions
// of the STS API. For more information about using this service, see Temporary
// Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html).
//
// See https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15 for more information on this service.
//

View File

@ -31,7 +31,7 @@ var initRequest func(*request.Request)
const (
ServiceName = "sts" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "STS" // ServiceID is a unique identifer of a specific service.
ServiceID = "STS" // ServiceID is a unique identifier of a specific service.
)
// New creates a new instance of the STS client with a session.

View File

@ -3,7 +3,26 @@ language: go
sudo: false
go:
- 1.4
- 1.5.x
- 1.6.x
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- 1.11.x
- 1.12.x
- 1.13.x
- 1.14.x
- 1.15.x
- tip
install: go get -v -t ./...
script: make test
allow_failures:
- go: tip
script: make build
matrix:
include:
- language: go
go: 1.15.x
script: make test

View File

@ -1,6 +1,8 @@
CMD = jpgo
SRC_PKGS=./ ./cmd/... ./fuzz/...
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " test to run all the tests"
@ -9,21 +11,22 @@ help:
generate:
go generate ./...
go generate ${SRC_PKGS}
build:
rm -f $(CMD)
go build ./...
go build ${SRC_PKGS}
rm -f cmd/$(CMD)/$(CMD) && cd cmd/$(CMD)/ && go build ./...
mv cmd/$(CMD)/$(CMD) .
test:
go test -v ./...
test: test-internal-testify
echo "making tests ${SRC_PKGS}"
go test -v ${SRC_PKGS}
check:
go vet ./...
@echo "golint ./..."
@lint=`golint ./...`; \
go vet ${SRC_PKGS}
@echo "golint ${SRC_PKGS}"
@lint=`golint ${SRC_PKGS}`; \
lint=`echo "$$lint" | grep -v "astnodetype_string.go" | grep -v "toktype_string.go"`; \
echo "$$lint"; \
if [ "$$lint" != "" ]; then exit 1; fi
@ -42,3 +45,7 @@ bench:
pprof-cpu:
go tool pprof ./go-jmespath.test ./cpu.out
test-internal-testify:
cd internal/testify && go test ./...

View File

@ -4,4 +4,84 @@
See http://jmespath.org for more info.
go-jmespath is a GO implementation of JMESPath,
which is a query language for JSON. It will take a JSON
document and transform it into another JSON document
through a JMESPath expression.
Using go-jmespath is really easy. There's a single function
you use, `jmespath.search`:
```go
> import "github.com/jmespath/go-jmespath"
>
> var jsondata = []byte(`{"foo": {"bar": {"baz": [0, 1, 2, 3, 4]}}}`) // your data
> var data interface{}
> err := json.Unmarshal(jsondata, &data)
> result, err := jmespath.Search("foo.bar.baz[2]", data)
result = 2
```
In the example we gave the ``search`` function input data of
`{"foo": {"bar": {"baz": [0, 1, 2, 3, 4]}}}` as well as the JMESPath
expression `foo.bar.baz[2]`, and the `search` function evaluated
the expression against the input data to produce the result ``2``.
The JMESPath language can do a lot more than select an element
from a list. Here are a few more examples:
```go
> var jsondata = []byte(`{"foo": {"bar": {"baz": [0, 1, 2, 3, 4]}}}`) // your data
> var data interface{}
> err := json.Unmarshal(jsondata, &data)
> result, err := jmespath.search("foo.bar", data)
result = { "baz": [ 0, 1, 2, 3, 4 ] }
> var jsondata = []byte(`{"foo": [{"first": "a", "last": "b"},
{"first": "c", "last": "d"}]}`) // your data
> var data interface{}
> err := json.Unmarshal(jsondata, &data)
> result, err := jmespath.search({"foo[*].first", data)
result [ 'a', 'c' ]
> var jsondata = []byte(`{"foo": [{"age": 20}, {"age": 25},
{"age": 30}, {"age": 35},
{"age": 40}]}`) // your data
> var data interface{}
> err := json.Unmarshal(jsondata, &data)
> result, err := jmespath.search("foo[?age > `30`]")
result = [ { age: 35 }, { age: 40 } ]
```
You can also pre-compile your query. This is usefull if
you are going to run multiple searches with it:
```go
> var jsondata = []byte(`{"foo": "bar"}`)
> var data interface{}
> err := json.Unmarshal(jsondata, &data)
> precompiled, err := Compile("foo")
> if err != nil{
> // ... handle the error
> }
> result, err := precompiled.Search(data)
result = "bar"
```
## More Resources
The example above only show a small amount of what
a JMESPath expression can do. If you want to take a
tour of the language, the *best* place to go is the
[JMESPath Tutorial](http://jmespath.org/tutorial.html).
One of the best things about JMESPath is that it is
implemented in many different programming languages including
python, ruby, php, lua, etc. To see a complete list of libraries,
check out the [JMESPath libraries page](http://jmespath.org/libraries.html).
And finally, the full JMESPath specification can be found
on the [JMESPath site](http://jmespath.org/specification.html).

Some files were not shown because too many files have changed in this diff Show More