luxolus/open-vault
2
0
Fork 0
Code Issues 1 Pull requests Releases Activity

Physical S3 backend implementation

Browse source
This commit is contained in:
James Stremick 2015-05-20 10:54:26 -04:00
parent 05e59edb02
commit 53979d6f30
6 changed files with 347 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
Godeps/Godeps.json generated
View file

@ -1,9 +1,6 @@
{
"ImportPath": "github.com/hashicorp/vault",
"GoVersion": "go1.4.2",
"Packages": [
"./..."
],
"Deps": [
{
"ImportPath": "github.com/armon/go-metrics",
@ -13,6 +10,38 @@
"ImportPath": "github.com/armon/go-radix",
"Rev": "0bab926c3433cfd6490c6d3c504a7b471362390c"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/aws",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/internal/endpoints",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/internal/protocol/query",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/internal/protocol/rest",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/internal/protocol/restxml",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/internal/protocol/xml/xmlutil",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/internal/signer/v4",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/awslabs/aws-sdk-go/service/s3",
"Rev": "5e038f730cbb99b144eeb1dbf92cd06c2d00b503"
},
{
"ImportPath": "github.com/go-sql-driver/mysql",
"Comment": "v1.2-88-ga197e5d",
@ -31,11 +60,6 @@
"Comment": "tf0.4.0-3-ge6ea019",
"Rev": "e6ea0192eee4640f32ec73c0cbb71f63e4f2b65a"
},
{
"ImportPath": "github.com/hashicorp/aws-sdk-go/gen/ec2",
"Comment": "tf0.4.0-3-ge6ea019",
"Rev": "e6ea0192eee4640f32ec73c0cbb71f63e4f2b65a"
},
{
"ImportPath": "github.com/hashicorp/aws-sdk-go/gen/endpoints",
"Comment": "tf0.4.0-3-ge6ea019",

1
physical/physical.go
View file

@ -82,4 +82,5 @@ var BuiltinBackends = map[string]Factory{
"consul": newConsulBackend,
"zookeeper": newZookeeperBackend,
"file": newFileBackend,
"s3": newS3Backend,
}

15
physical/physical_test.go
View file

@ -189,6 +189,21 @@ func testBackend_ListPrefix(t *testing.T, b Backend) {
if keys[0] != "baz" {
t.Fatalf("bad: %v", keys)
}
// Delete should recursively remove paths
err = b.Delete("foo")
if err != nil {
t.Fatalf("err: %v", err)
}
// Should now be empty
keys, err = b.List("")
if err != nil {
t.Fatalf("err: %v", err)
}
if len(keys) != 0 {
t.Fatalf("bad: %v", keys)
}
}
func testHABackend(t *testing.T, b HABackend, b2 HABackend) {

207
physical/s3.go Normal file
View file

@ -0,0 +1,207 @@
package physical
import (
"bytes"
"fmt"
"io"
"os"
"sort"
"strings"
"sync"
"time"
"github.com/armon/go-metrics"
"github.com/awslabs/aws-sdk-go/aws"
"github.com/awslabs/aws-sdk-go/aws/credentials"
"github.com/awslabs/aws-sdk-go/service/s3"
)
// S3Backend is a physical backend that stores data
// within an S3 bucket.
type S3Backend struct {
bucket string
client *s3.S3
l sync.Mutex
}
// newS3Backend constructs a S3 backend using a pre-existing
// bucket. Credentials can be provided to the backend, sourced
// from the environment, AWS credential files or by IAM role.
func newS3Backend(conf map[string]string) (Backend, error) {
bucket, ok := conf["bucket"]
if !ok {
return nil, fmt.Errorf("'bucket' must be set")
}
access_key, ok := conf["access_key"]
if !ok {
access_key = ""
}
secret_key, ok := conf["secret_key"]
if !ok {
secret_key = ""
}
region, ok := conf["region"]
if !ok {
region = os.Getenv("AWS_DEFAULT_REGION")
if region == "" {
region = "us-east-1"
}
}
creds := credentials.NewChainCredentials([]credentials.Provider{
&credentials.StaticProvider{Value: credentials.Value{
AccessKeyID: access_key,
SecretAccessKey: secret_key,
}},
&credentials.EnvProvider{},
&credentials.SharedCredentialsProvider{Filename: "", Profile: ""},
&credentials.EC2RoleProvider{},
})
s3conn := s3.New(&aws.Config{
Credentials: creds,
Region: region,
})
_, err := s3conn.HeadBucket(&s3.HeadBucketInput{Bucket: &bucket})
if err != nil {
return nil, fmt.Errorf("unable to access bucket '%s': %v", bucket, err)
}
s := &S3Backend{
client: s3conn,
bucket: bucket,
}
return s, nil
}
// Put is used to insert or update an entry
func (s *S3Backend) Put(entry *Entry) error {
s.l.Lock()
defer s.l.Unlock()
defer metrics.MeasureSince([]string{"s3", "put"}, time.Now())
_, err := s.client.PutObject(&s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(entry.Key),
Body: bytes.NewReader(entry.Value),
})
if err != nil {
return err
}
return nil
}
// Get is used to fetch an entry
func (s *S3Backend) Get(key string) (*Entry, error) {
s.l.Lock()
defer s.l.Unlock()
defer metrics.MeasureSince([]string{"s3", "get"}, time.Now())
resp, err := s.client.GetObject(&s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
})
if awserr := aws.Error(err); awserr != nil {
// Return nil on 404s, error on anything else
if awserr.StatusCode == 404 {
return nil, nil
} else {
return nil, err
}
}
data := make([]byte, *resp.ContentLength)
_, err = io.ReadFull(resp.Body, data)
if err != nil {
return nil, err
}
ent := &Entry{
Key: key,
Value: data,
}
return ent, nil
}
// Delete is used to permanently delete an entry
func (s *S3Backend) Delete(key string) error {
s.l.Lock()
defer s.l.Unlock()
defer metrics.MeasureSince([]string{"s3", "delete"}, time.Now())
listResp, err := s.client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(s.bucket),
Prefix: aws.String(key),
})
if err != nil {
return err
}
objects := &s3.Delete{}
for _, key := range listResp.Contents {
oi := &s3.ObjectIdentifier{Key: key.Key}
objects.Objects = append(objects.Objects, oi)
}
if len(objects.Objects) > 0 {
_, err := s.client.DeleteObjects(&s3.DeleteObjectsInput{
Bucket: aws.String(s.bucket),
Delete: objects,
})
if err != nil {
return err
}
}
return nil
}
// List is used to list all the keys under a given
// prefix, up to the next prefix.
func (s *S3Backend) List(prefix string) ([]string, error) {
s.l.Lock()
defer s.l.Unlock()
defer metrics.MeasureSince([]string{"s3", "list"}, time.Now())
resp, err := s.client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(s.bucket),
Prefix: aws.String(prefix),
})
if err != nil {
return nil, err
}
keys := []string{}
for _, key := range resp.Contents {
key := strings.TrimPrefix(*key.Key, prefix)
if i := strings.Index(key, "/"); i == -1 {
// Add objects only from the current 'folder'
keys = append(keys, key)
} else if i != -1 {
// Add truncated 'folder' paths
keys = appendIfMissing(keys, key[:i+1])
}
}
sort.Strings(keys)
return keys, nil
}
func appendIfMissing(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}

78
physical/s3_test.go Normal file
View file

@ -0,0 +1,78 @@
package physical
import (
"fmt"
"math/rand"
"os"
"testing"
"time"
"github.com/awslabs/aws-sdk-go/aws"
"github.com/awslabs/aws-sdk-go/service/s3"
)
func TestS3Backend(t *testing.T) {
credentialChain := aws.DefaultChainCredentials
creds, err := credentialChain.Get()
if err != nil {
t.Fatalf("err: %s", err)
}
region := os.Getenv("AWS_DEFAULT_REGION")
if region == "" {
region = "us-east-1"
}
s3conn := s3.New(&aws.Config{
Credentials: aws.DefaultChainCredentials,
Region: region,
})
var randInt = rand.New(rand.NewSource(time.Now().UnixNano())).Int()
bucket := fmt.Sprintf("vault-s3-testacc-%d", randInt)
_, err = s3conn.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
t.Fatalf("unable to create test bucket: %s", err)
}
defer func() {
// Gotta list all the objects and delete them
// before being able to delete the bucket
listResp, _ := s3conn.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(bucket),
})
objects := &s3.Delete{}
for _, key := range listResp.Contents {
oi := &s3.ObjectIdentifier{Key: key.Key}
objects.Objects = append(objects.Objects, oi)
}
s3conn.DeleteObjects(&s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: objects,
})
_, err := s3conn.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
if err != nil {
t.Fatalf("err: %s", err)
}
}()
b, err := NewBackend("s3", map[string]string{
"access_key": creds.AccessKeyID,
"secret_key": creds.SecretAccessKey,
"bucket": bucket,
})
if err != nil {
t.Fatalf("err: %s", err)
}
testBackend(t, b)
testBackend_ListPrefix(t, b)
}

14
website/source/docs/config/index.html.md
View file

@ -60,6 +60,9 @@ durability, etc.
* `zookeeper` - Store data within [Zookeeper](https://zookeeper.apache.org/).
This backend does not support HA.
* `s3` - Store data within an S3 bucket [S3](http://aws.amazon.com/s3/).
This backend does not support HA.
* `inmem` - Store data in-memory. This is only really useful for
development and experimentation. Data is lost whenever Vault is
restarted.
@ -104,6 +107,17 @@ For Zookeeper, the following options are supported:
Can be comma separated list (host:port) of many Zookeeper instances.
Defaults to "localhost:2181" if not specified.
#### Backend Reference: S3
For S3, the following options are supported:
* `bucket` (required) - The name of the S3 bucket to use.
* `access_key` - (Required) This is the AWS access key. It must be provided, but it can also be sourced from the AWS_ACCESS_KEY_ID environment variable.
* `secret_key` - (Required) This is the AWS secret key. It must be provided, but it can also be sourced from the AWS_SECRET_ACCESS_KEY environment variable.
* `region` (optional) - This is the AWS region. It can be sourced from the AWS_DEFAULT_REGION environment variable and will default to "us-east-1" if not specified.
#### Backend Reference: Inmem