diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..8dbdd3728 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,173 @@ +version: 2 + +references: + images: + go: &GOLANG_IMAGE golang:1.12.4-stretch # Pin Go to patch version (ex: 1.2.3) + node: &NODE_IMAGE node:10-stretch # Pin Node.js to major version (ex: 10) + + environment: &ENVIRONMENT + CIRCLECI_CLI_VERSION: 0.1.5546 # Pin CircleCI CLI to patch version (ex: 1.2.3) + GO_VERSION: 1.12.4 # Pin Go to patch version (ex: 1.2.3) + GOTESTSUM_VERSION: 0.3.3 # Pin gotestsum to patch version (ex: 1.2.3) + +jobs: + install-ui-dependencies: + docker: + - image: *NODE_IMAGE + working_directory: /src/vault/ui + steps: + - checkout: + path: /src/vault + - restore_cache: + key: yarn-lock-{{ checksum "yarn.lock" }} + - run: + name: Install UI dependencies + command: | + set -eux -o pipefail + + yarn install --ignore-optional + npm rebuild node-sass + - save_cache: + key: yarn-lock-{{ checksum "yarn.lock" }} + paths: + - node_modules + - persist_to_workspace: + root: .. + paths: + - ui/node_modules + + build-go-dev: + docker: + - image: *GOLANG_IMAGE + working_directory: /go/src/github.com/hashicorp/vault + steps: + - checkout + - attach_workspace: + at: . + - run: + name: Build dev binary + command: | + set -eux -o pipefail + + # Move dev UI assets to expected location + rm -rf ./pkg + mkdir ./pkg + + # Build dev binary + make bootstrap dev + - persist_to_workspace: + root: . + paths: + - bin + + test-ui: + docker: + - image: *NODE_IMAGE + working_directory: /src/vault/ui + resource_class: medium+ + steps: + - checkout: + path: /src/vault + - attach_workspace: + at: .. + - run: + name: Test UI + command: | + set -eux -o pipefail + + # Install Chrome + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub \ + | apt-key add - + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" \ + | tee /etc/apt/sources.list.d/google-chrome.list + apt-get update + apt-get -y install google-chrome-stable + rm /etc/apt/sources.list.d/google-chrome.list + rm -rf /var/lib/apt/lists/* /var/cache/apt/* + + # Add ./bin to the PATH so vault binary can be run by Ember tests + export PATH="${PWD}"/../bin:${PATH} + + # Run Ember tests + mkdir -p test-results/qunit + yarn run test-oss + - store_artifacts: + path: test-results + - store_test_results: + path: test-results + + test-go: + machine: true + environment: + <<: *ENVIRONMENT + GO_TAGS: + parallelism: 2 + working_directory: ~/go/src/github.com/hashicorp/vault + steps: + - checkout + - attach_workspace: + at: . + - run: + name: Run Go tests + command: | + set -eux -o pipefail + + # Install Go + curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" + sudo rm -rf /usr/local/go + sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" + rm -f "go${GO_VERSION}.linux-amd64.tar.gz" + export GOPATH="${HOME}/go" + export PATH="${PATH}:${GOPATH}/bin:/usr/local/go/bin" + + # Install CircleCI CLI + curl -sSL \ + "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \ + | sudo tar --overwrite -xz \ + -C /usr/local/bin \ + "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci" + + # Split Go tests by prior test times + package_names=$(go list \ + -tags "${GO_TAGS}" \ + ./... \ + | grep -v /vendor/ \ + | sort \ + | circleci tests split --split-by=timings --timings-type=classname) + + # Install gotestsum + curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" \ + | sudo tar --overwrite -xz -C /usr/local/bin gotestsum + + # Run tests + make prep + mkdir -p test-results/go-test + CGO_ENABLED= \ + VAULT_ADDR= \ + VAULT_TOKEN= \ + VAULT_DEV_ROOT_TOKEN_ID= \ + VAULT_ACC= \ + gotestsum --format=short-verbose --junitfile test-results/go-test/results.xml -- \ + -tags "${GO_TAGS}" \ + -timeout=40m \ + -parallel=20 \ + ${package_names} + - store_artifacts: + path: test-results + - store_test_results: + path: test-results + +workflows: + version: 2 + + ci: + jobs: + - install-ui-dependencies + - build-go-dev + - test-ui: + requires: + - install-ui-dependencies + - build-go-dev + - test-go: + requires: + - build-go-dev diff --git a/.travis.yml b/.travis.yml index 4594ee0e4..1a6a61165 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,7 @@ branches: env: - TEST_COMMAND='make dev test-ember' + - TEST_COMMAND='make dev ember-ci-test' - TEST_COMMAND='travis_wait 75 make testtravis' - TEST_COMMAND='travis_wait 75 make testracetravis' - GO111MODULE=on diff --git a/CHANGELOG.md b/CHANGELOG.md index 65697f4d2..39df4b275 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ BUG FIXES: * auth/okta: Fix handling of group names containing slashes [GH-6665] * core: Correctly honor non-HMAC request keys when auditing requests [GH-6653] * core: Fix the `x-vault-unauthenticated` value in OpenAPI for a number of endpoints [GH-6654] + * core: Fix issue where some OpenAPI parameters were incorrectly listed as being sent + as a header [GH-6679] * pki: fix a panic when a client submits a null value [GH-5679] * replication: Fix an issue causing startup problems if a namespace policy wasn't replicated properly diff --git a/Makefile b/Makefile index 0f344aaba..d288f549f 100644 --- a/Makefile +++ b/Makefile @@ -128,6 +128,12 @@ test-ember: @echo "--> Running ember tests" @cd ui && yarn run test-oss +ember-ci-test: + @echo "--> Installing JavaScript assets" + @cd ui && yarn --ignore-optional + @echo "--> Running ember tests in Browserstack" + @cd ui && yarn run test:browserstack + ember-dist: @echo "--> Installing JavaScript assets" @cd ui && yarn --ignore-optional diff --git a/command/agent.go b/command/agent.go index 70df21252..288400444 100644 --- a/command/agent.go +++ b/command/agent.go @@ -21,6 +21,7 @@ import ( "github.com/hashicorp/vault/command/agent/auth/approle" "github.com/hashicorp/vault/command/agent/auth/aws" "github.com/hashicorp/vault/command/agent/auth/azure" + "github.com/hashicorp/vault/command/agent/auth/cert" "github.com/hashicorp/vault/command/agent/auth/gcp" "github.com/hashicorp/vault/command/agent/auth/jwt" "github.com/hashicorp/vault/command/agent/auth/kubernetes" @@ -331,6 +332,8 @@ func (c *AgentCommand) Run(args []string) int { method, err = aws.NewAWSAuthMethod(authConfig) case "azure": method, err = azure.NewAzureAuthMethod(authConfig) + case "cert": + method, err = cert.NewCertAuthMethod(authConfig) case "gcp": method, err = gcp.NewGCPAuthMethod(authConfig) case "jwt": diff --git a/command/agent/auth/cert/cert.go b/command/agent/auth/cert/cert.go new file mode 100644 index 000000000..fc1f42606 --- /dev/null +++ b/command/agent/auth/cert/cert.go @@ -0,0 +1,65 @@ +package cert + +import ( + "context" + "errors" + "fmt" + + hclog "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/command/agent/auth" +) + +type certMethod struct { + logger hclog.Logger + mountPath string + name string +} + +func NewCertAuthMethod(conf *auth.AuthConfig) (auth.AuthMethod, error) { + if conf == nil { + return nil, errors.New("empty config") + } + + // Not concerned if the conf.Config is empty as the 'name' + // parameter is optional when using TLS Auth + + c := &certMethod{ + logger: conf.Logger, + mountPath: conf.MountPath, + name: "", + } + + if conf.Config != nil { + nameRaw, ok := conf.Config["name"] + if !ok { + nameRaw = "" + } + c.name, ok = nameRaw.(string) + if !ok { + return nil, errors.New("could not convert 'name' config value to string") + } + } + + return c, nil +} + +func (c *certMethod) Authenticate(_ context.Context, client *api.Client) (string, map[string]interface{}, error) { + c.logger.Trace("beginning authentication") + + authMap := map[string]interface{}{} + + if c.name != "" { + authMap["name"] = c.name + } + + return fmt.Sprintf("%s/login", c.mountPath), authMap, nil +} + +func (c *certMethod) NewCreds() chan struct{} { + return nil +} + +func (c *certMethod) CredSuccess() {} + +func (c *certMethod) Shutdown() {} diff --git a/command/agent/cert_with_name_end_to_end_test.go b/command/agent/cert_with_name_end_to_end_test.go new file mode 100644 index 000000000..54135b49f --- /dev/null +++ b/command/agent/cert_with_name_end_to_end_test.go @@ -0,0 +1,244 @@ +package agent + +import ( + "context" + "encoding/pem" + "io/ioutil" + "os" + "testing" + "time" + + hclog "github.com/hashicorp/go-hclog" + + "github.com/hashicorp/vault/api" + vaultcert "github.com/hashicorp/vault/builtin/credential/cert" + "github.com/hashicorp/vault/command/agent/auth" + agentcert "github.com/hashicorp/vault/command/agent/auth/cert" + "github.com/hashicorp/vault/command/agent/sink" + "github.com/hashicorp/vault/command/agent/sink/file" + "github.com/hashicorp/vault/helper/dhutil" + vaulthttp "github.com/hashicorp/vault/http" + "github.com/hashicorp/vault/sdk/helper/jsonutil" + "github.com/hashicorp/vault/sdk/helper/logging" + "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/vault" +) + +func TestCertWithNameEndToEnd(t *testing.T) { + testCertWithNameEndToEnd(t, false) + testCertWithNameEndToEnd(t, true) +} + +func testCertWithNameEndToEnd(t *testing.T, ahWrapping bool) { + logger := logging.NewVaultLogger(hclog.Trace) + coreConfig := &vault.CoreConfig{ + Logger: logger, + CredentialBackends: map[string]logical.Factory{ + "cert": vaultcert.Factory, + }, + } + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + vault.TestWaitActive(t, cluster.Cores[0].Core) + client := cluster.Cores[0].Client + + // Setup Vault + err := client.Sys().EnableAuthWithOptions("cert", &api.EnableAuthOptions{ + Type: "cert", + }) + if err != nil { + t.Fatal(err) + } + + certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.CACert.Raw}) + + _, err = client.Logical().Write("auth/cert/certs/test", map[string]interface{}{ + "name": "test", + "certificate": string(certificatePEM), + "policies": "default", + }) + if err != nil { + t.Fatal(err) + } + + // Generate encryption params + pub, pri, err := dhutil.GeneratePublicPrivateKey() + if err != nil { + t.Fatal(err) + } + + ouf, err := ioutil.TempFile("", "auth.tokensink.test.") + if err != nil { + t.Fatal(err) + } + out := ouf.Name() + ouf.Close() + os.Remove(out) + t.Logf("output: %s", out) + + dhpathf, err := ioutil.TempFile("", "auth.dhpath.test.") + if err != nil { + t.Fatal(err) + } + dhpath := dhpathf.Name() + dhpathf.Close() + os.Remove(dhpath) + + // Write DH public key to file + mPubKey, err := jsonutil.EncodeJSON(&dhutil.PublicKeyInfo{ + Curve25519PublicKey: pub, + }) + if err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(dhpath, mPubKey, 0600); err != nil { + t.Fatal(err) + } else { + logger.Trace("wrote dh param file", "path", dhpath) + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + timer := time.AfterFunc(30*time.Second, func() { + cancelFunc() + }) + defer timer.Stop() + + am, err := agentcert.NewCertAuthMethod(&auth.AuthConfig{ + Logger: logger.Named("auth.cert"), + MountPath: "auth/cert", + Config: map[string]interface{}{ + "name": "test", + }, + }) + if err != nil { + t.Fatal(err) + } + + ahConfig := &auth.AuthHandlerConfig{ + Logger: logger.Named("auth.handler"), + Client: client, + EnableReauthOnNewCredentials: true, + } + if ahWrapping { + ahConfig.WrapTTL = 10 * time.Second + } + ah := auth.NewAuthHandler(ahConfig) + go ah.Run(ctx, am) + defer func() { + <-ah.DoneCh + }() + + config := &sink.SinkConfig{ + Logger: logger.Named("sink.file"), + AAD: "foobar", + DHType: "curve25519", + DHPath: dhpath, + Config: map[string]interface{}{ + "path": out, + }, + } + if !ahWrapping { + config.WrapTTL = 10 * time.Second + } + fs, err := file.NewFileSink(config) + if err != nil { + t.Fatal(err) + } + config.Sink = fs + + ss := sink.NewSinkServer(&sink.SinkServerConfig{ + Logger: logger.Named("sink.server"), + Client: client, + }) + go ss.Run(ctx, ah.OutputCh, []*sink.SinkConfig{config}) + defer func() { + <-ss.DoneCh + }() + + // This has to be after the other defers so it happens first + defer cancelFunc() + + cloned, err := client.Clone() + if err != nil { + t.Fatal(err) + } + + checkToken := func() string { + timeout := time.Now().Add(5 * time.Second) + for { + if time.Now().After(timeout) { + t.Fatal("did not find a written token after timeout") + } + val, err := ioutil.ReadFile(out) + if err == nil { + os.Remove(out) + if len(val) == 0 { + t.Fatal("written token was empty") + } + + // First decrypt it + resp := new(dhutil.Envelope) + if err := jsonutil.DecodeJSON(val, resp); err != nil { + continue + } + + aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey) + if err != nil { + t.Fatal(err) + } + if len(aesKey) == 0 { + t.Fatal("got empty aes key") + } + + val, err = dhutil.DecryptAES(aesKey, resp.EncryptedPayload, resp.Nonce, []byte("foobar")) + if err != nil { + t.Fatalf("error: %v\nresp: %v", err, string(val)) + } + + // Now unwrap it + wrapInfo := new(api.SecretWrapInfo) + if err := jsonutil.DecodeJSON(val, wrapInfo); err != nil { + t.Fatal(err) + } + switch { + case wrapInfo.TTL != 10: + t.Fatalf("bad wrap info: %v", wrapInfo.TTL) + case !ahWrapping && wrapInfo.CreationPath != "sys/wrapping/wrap": + t.Fatalf("bad wrap path: %v", wrapInfo.CreationPath) + case ahWrapping && wrapInfo.CreationPath != "auth/cert/login": + t.Fatalf("bad wrap path: %v", wrapInfo.CreationPath) + case wrapInfo.Token == "": + t.Fatal("wrap token is empty") + } + cloned.SetToken(wrapInfo.Token) + secret, err := cloned.Logical().Unwrap("") + if err != nil { + t.Fatal(err) + } + if ahWrapping { + switch { + case secret.Auth == nil: + t.Fatal("unwrap secret auth is nil") + case secret.Auth.ClientToken == "": + t.Fatal("unwrap token is nil") + } + return secret.Auth.ClientToken + } else { + switch { + case secret.Data == nil: + t.Fatal("unwrap secret data is nil") + case secret.Data["token"] == nil: + t.Fatal("unwrap token is nil") + } + return secret.Data["token"].(string) + } + } + time.Sleep(250 * time.Millisecond) + } + } + checkToken() +} diff --git a/command/agent/cert_with_no_name_end_to_end_test.go b/command/agent/cert_with_no_name_end_to_end_test.go new file mode 100644 index 000000000..e6bb683ca --- /dev/null +++ b/command/agent/cert_with_no_name_end_to_end_test.go @@ -0,0 +1,241 @@ +package agent + +import ( + "context" + "encoding/pem" + "io/ioutil" + "os" + "testing" + "time" + + hclog "github.com/hashicorp/go-hclog" + + "github.com/hashicorp/vault/api" + vaultcert "github.com/hashicorp/vault/builtin/credential/cert" + "github.com/hashicorp/vault/command/agent/auth" + agentcert "github.com/hashicorp/vault/command/agent/auth/cert" + "github.com/hashicorp/vault/command/agent/sink" + "github.com/hashicorp/vault/command/agent/sink/file" + "github.com/hashicorp/vault/helper/dhutil" + vaulthttp "github.com/hashicorp/vault/http" + "github.com/hashicorp/vault/sdk/helper/jsonutil" + "github.com/hashicorp/vault/sdk/helper/logging" + "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/vault" +) + +func TestCertWithNoNAmeEndToEnd(t *testing.T) { + testCertWithNoNAmeEndToEnd(t, false) + testCertWithNoNAmeEndToEnd(t, true) +} + +func testCertWithNoNAmeEndToEnd(t *testing.T, ahWrapping bool) { + logger := logging.NewVaultLogger(hclog.Trace) + coreConfig := &vault.CoreConfig{ + Logger: logger, + CredentialBackends: map[string]logical.Factory{ + "cert": vaultcert.Factory, + }, + } + cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + vault.TestWaitActive(t, cluster.Cores[0].Core) + client := cluster.Cores[0].Client + + // Setup Vault + err := client.Sys().EnableAuthWithOptions("cert", &api.EnableAuthOptions{ + Type: "cert", + }) + if err != nil { + t.Fatal(err) + } + + certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.CACert.Raw}) + + _, err = client.Logical().Write("auth/cert/certs/test", map[string]interface{}{ + "name": "test", + "certificate": string(certificatePEM), + "policies": "default", + }) + if err != nil { + t.Fatal(err) + } + + // Generate encryption params + pub, pri, err := dhutil.GeneratePublicPrivateKey() + if err != nil { + t.Fatal(err) + } + + ouf, err := ioutil.TempFile("", "auth.tokensink.test.") + if err != nil { + t.Fatal(err) + } + out := ouf.Name() + ouf.Close() + os.Remove(out) + t.Logf("output: %s", out) + + dhpathf, err := ioutil.TempFile("", "auth.dhpath.test.") + if err != nil { + t.Fatal(err) + } + dhpath := dhpathf.Name() + dhpathf.Close() + os.Remove(dhpath) + + // Write DH public key to file + mPubKey, err := jsonutil.EncodeJSON(&dhutil.PublicKeyInfo{ + Curve25519PublicKey: pub, + }) + if err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(dhpath, mPubKey, 0600); err != nil { + t.Fatal(err) + } else { + logger.Trace("wrote dh param file", "path", dhpath) + } + + ctx, cancelFunc := context.WithCancel(context.Background()) + timer := time.AfterFunc(30*time.Second, func() { + cancelFunc() + }) + defer timer.Stop() + + am, err := agentcert.NewCertAuthMethod(&auth.AuthConfig{ + Logger: logger.Named("auth.cert"), + MountPath: "auth/cert", + }) + if err != nil { + t.Fatal(err) + } + + ahConfig := &auth.AuthHandlerConfig{ + Logger: logger.Named("auth.handler"), + Client: client, + EnableReauthOnNewCredentials: true, + } + if ahWrapping { + ahConfig.WrapTTL = 10 * time.Second + } + ah := auth.NewAuthHandler(ahConfig) + go ah.Run(ctx, am) + defer func() { + <-ah.DoneCh + }() + + config := &sink.SinkConfig{ + Logger: logger.Named("sink.file"), + AAD: "foobar", + DHType: "curve25519", + DHPath: dhpath, + Config: map[string]interface{}{ + "path": out, + }, + } + if !ahWrapping { + config.WrapTTL = 10 * time.Second + } + fs, err := file.NewFileSink(config) + if err != nil { + t.Fatal(err) + } + config.Sink = fs + + ss := sink.NewSinkServer(&sink.SinkServerConfig{ + Logger: logger.Named("sink.server"), + Client: client, + }) + go ss.Run(ctx, ah.OutputCh, []*sink.SinkConfig{config}) + defer func() { + <-ss.DoneCh + }() + + // This has to be after the other defers so it happens first + defer cancelFunc() + + cloned, err := client.Clone() + if err != nil { + t.Fatal(err) + } + + checkToken := func() string { + timeout := time.Now().Add(5 * time.Second) + for { + if time.Now().After(timeout) { + t.Fatal("did not find a written token after timeout") + } + val, err := ioutil.ReadFile(out) + if err == nil { + os.Remove(out) + if len(val) == 0 { + t.Fatal("written token was empty") + } + + // First decrypt it + resp := new(dhutil.Envelope) + if err := jsonutil.DecodeJSON(val, resp); err != nil { + continue + } + + aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey) + if err != nil { + t.Fatal(err) + } + if len(aesKey) == 0 { + t.Fatal("got empty aes key") + } + + val, err = dhutil.DecryptAES(aesKey, resp.EncryptedPayload, resp.Nonce, []byte("foobar")) + if err != nil { + t.Fatalf("error: %v\nresp: %v", err, string(val)) + } + + // Now unwrap it + wrapInfo := new(api.SecretWrapInfo) + if err := jsonutil.DecodeJSON(val, wrapInfo); err != nil { + t.Fatal(err) + } + switch { + case wrapInfo.TTL != 10: + t.Fatalf("bad wrap info: %v", wrapInfo.TTL) + case !ahWrapping && wrapInfo.CreationPath != "sys/wrapping/wrap": + t.Fatalf("bad wrap path: %v", wrapInfo.CreationPath) + case ahWrapping && wrapInfo.CreationPath != "auth/cert/login": + t.Fatalf("bad wrap path: %v", wrapInfo.CreationPath) + case wrapInfo.Token == "": + t.Fatal("wrap token is empty") + } + cloned.SetToken(wrapInfo.Token) + secret, err := cloned.Logical().Unwrap("") + if err != nil { + t.Fatal(err) + } + if ahWrapping { + switch { + case secret.Auth == nil: + t.Fatal("unwrap secret auth is nil") + case secret.Auth.ClientToken == "": + t.Fatal("unwrap token is nil") + } + return secret.Auth.ClientToken + } else { + switch { + case secret.Data == nil: + t.Fatal("unwrap secret data is nil") + case secret.Data["token"] == nil: + t.Fatal("unwrap token is nil") + } + return secret.Data["token"].(string) + } + } + time.Sleep(250 * time.Millisecond) + } + } + checkToken() +} diff --git a/command/kv_helpers.go b/command/kv_helpers.go index 654584089..d9246f7a0 100644 --- a/command/kv_helpers.go +++ b/command/kv_helpers.go @@ -1,6 +1,7 @@ package command import ( + "errors" "fmt" "io" "path" @@ -69,6 +70,9 @@ func kvPreflightVersionRequest(client *api.Client, path string) (string, int, er if err != nil { return "", 0, err } + if secret == nil { + return "", 0, errors.New("nil response from pre-flight request") + } var mountPath string if mountPathRaw, ok := secret.Data["path"]; ok { mountPath = mountPathRaw.(string) diff --git a/command/operator_init.go b/command/operator_init.go index e47c19499..6c7d41925 100644 --- a/command/operator_init.go +++ b/command/operator_init.go @@ -30,6 +30,7 @@ type OperatorInitCommand struct { flagRecoveryShares int flagRecoveryThreshold int flagRecoveryPGPKeys []string + flagStoredShares int // Consul flagConsulAuto bool @@ -139,6 +140,13 @@ func (c *OperatorInitCommand) Flags() *FlagSets { "key.", }) + f.IntVar(&IntVar{ + Name: "stored-shares", + Target: &c.flagStoredShares, + Default: -1, + Usage: "DEPRECATED: This flag does nothing. It will be removed in Vault 1.3.", + }) + // Consul Options f = set.NewFlagSet("Consul Options") @@ -220,6 +228,10 @@ func (c *OperatorInitCommand) Run(args []string) int { return 1 } + if c.flagStoredShares != -1 { + c.UI.Warn("-stored-shares has no effect and will be removed in Vault 1.3.\n") + } + // Build the initial init request initReq := &api.InitRequest{ SecretShares: c.flagKeyShares, diff --git a/physical/gcs/gcs.go b/physical/gcs/gcs.go index 134a3773e..da9c3b1fb 100644 --- a/physical/gcs/gcs.go +++ b/physical/gcs/gcs.go @@ -158,9 +158,9 @@ func NewBackend(c map[string]string, logger log.Logger) (physical.Backend, error } return &Backend{ - bucket: bucket, - haEnabled: haEnabled, - + bucket: bucket, + haEnabled: haEnabled, + chunkSize: chunkSize, client: client, permitPool: physical.NewPermitPool(maxParallel), logger: logger, diff --git a/physical/gcs/gcs_test.go b/physical/gcs/gcs_test.go index 64aeaaeb4..4caab730f 100644 --- a/physical/gcs/gcs_test.go +++ b/physical/gcs/gcs_test.go @@ -5,6 +5,7 @@ import ( "fmt" "math/rand" "os" + "strconv" "testing" "time" @@ -57,6 +58,17 @@ func TestBackend(t *testing.T) { t.Fatal(err) } + // Verify chunkSize is set correctly on the Backend + be := backend.(*Backend) + expectedChunkSize, err := strconv.Atoi(defaultChunkSize) + if err != nil { + t.Fatalf("failed to convert defaultChunkSize to int: %s", err) + } + expectedChunkSize = expectedChunkSize * 1024 + if be.chunkSize != expectedChunkSize { + t.Fatalf("expected chunkSize to be %d. got=%d", expectedChunkSize, be.chunkSize) + } + physical.ExerciseBackend(t, backend) physical.ExerciseBackend_ListPrefix(t, backend) } diff --git a/scripts/gen_openapi.sh b/scripts/gen_openapi.sh index de956e21b..0472b6df6 100755 --- a/scripts/gen_openapi.sh +++ b/scripts/gen_openapi.sh @@ -61,7 +61,11 @@ vault secrets enable ssh vault secrets enable totp vault secrets enable transit -curl -H "X-Vault-Token: root" "http://127.0.0.1:8200/v1/sys/internal/specs/openapi" > openapi.json +if [ "$1" == "-p" ]; then + curl -H "X-Vault-Token: root" "http://127.0.0.1:8200/v1/sys/internal/specs/openapi" | jq > openapi.json +else + curl -H "X-Vault-Token: root" "http://127.0.0.1:8200/v1/sys/internal/specs/openapi" > openapi.json +fi kill $VAULT_PID sleep 1 diff --git a/sdk/framework/openapi.go b/sdk/framework/openapi.go index 9d97337f1..37de53fdb 100644 --- a/sdk/framework/openapi.go +++ b/sdk/framework/openapi.go @@ -257,13 +257,6 @@ func documentPath(p *Path, specialPaths *logical.Paths, backendType logical.Back required = false } - // Header parameters are part of the Parameters group but with - // a dedicated "header" location, a header parameter is not required. - if field.Type == TypeHeader { - location = "header" - required = false - } - t := convertType(field.Type) p := OASParameter{ Name: name, @@ -608,8 +601,7 @@ func splitFields(allFields map[string]*FieldSchema, pattern string) (pathFields, for name, field := range allFields { if _, ok := pathFields[name]; !ok { - // Header fields are in "parameters" with other path fields - if field.Type == TypeHeader || field.Query { + if field.Query { pathFields[name] = field } else { bodyFields[name] = field diff --git a/sdk/framework/openapi_test.go b/sdk/framework/openapi_test.go index a9966ca31..b9990902f 100644 --- a/sdk/framework/openapi_test.go +++ b/sdk/framework/openapi_test.go @@ -151,8 +151,8 @@ func TestOpenAPI_Regex(t *testing.T) { func TestOpenAPI_ExpandPattern(t *testing.T) { tests := []struct { - in_pattern string - out_pathlets []string + inPattern string + outPathlets []string }{ {"rekey/backup", []string{"rekey/backup"}}, {"rekey/backup$", []string{"rekey/backup"}}, @@ -203,10 +203,10 @@ func TestOpenAPI_ExpandPattern(t *testing.T) { } for i, test := range tests { - out := expandPattern(test.in_pattern) + out := expandPattern(test.inPattern) sort.Strings(out) - if !reflect.DeepEqual(out, test.out_pathlets) { - t.Fatalf("Test %d: Expected %v got %v", i, test.out_pathlets, out) + if !reflect.DeepEqual(out, test.outPathlets) { + t.Fatalf("Test %d: Expected %v got %v", i, test.outPathlets, out) } } } @@ -266,7 +266,10 @@ func TestOpenAPI_SpecialPaths(t *testing.T) { Root: test.rootPaths, Unauthenticated: test.unauthPaths, } - documentPath(&path, sp, logical.TypeLogical, doc) + err := documentPath(&path, sp, logical.TypeLogical, doc) + if err != nil { + t.Fatal(err) + } result := test.root if doc.Paths["/"+test.pattern].Sudo != result { t.Fatalf("Test (root) %d: Expected %v got %v", i, test.root, result) @@ -288,11 +291,11 @@ func TestOpenAPI_Paths(t *testing.T) { Pattern: "lookup/" + GenericNameRegex("id"), Fields: map[string]*FieldSchema{ - "id": &FieldSchema{ + "id": { Type: TypeString, Description: "My id parameter", }, - "token": &FieldSchema{ + "token": { Type: TypeString, Description: "My token", }, @@ -442,8 +445,14 @@ func TestOpenAPI_OperationID(t *testing.T) { for _, context := range []string{"", "bar"} { doc := NewOASDocument() - documentPath(path1, nil, logical.TypeLogical, doc) - documentPath(path2, nil, logical.TypeLogical, doc) + err := documentPath(path1, nil, logical.TypeLogical, doc) + if err != nil { + t.Fatal(err) + } + err = documentPath(path2, nil, logical.TypeLogical, doc) + if err != nil { + t.Fatal(err) + } doc.CreateOperationIDs(context) tests := []struct { @@ -500,7 +509,10 @@ func TestOpenAPI_CustomDecoder(t *testing.T) { } docOrig := NewOASDocument() - documentPath(p, nil, logical.TypeLogical, docOrig) + err := documentPath(p, nil, logical.TypeLogical, docOrig) + if err != nil { + t.Fatal(err) + } docJSON := mustJSONMarshal(t, docOrig) diff --git a/sdk/framework/testdata/operations.json b/sdk/framework/testdata/operations.json index ad36dfaa5..f889f1182 100644 --- a/sdk/framework/testdata/operations.json +++ b/sdk/framework/testdata/operations.json @@ -31,15 +31,6 @@ "type": "string" }, "required": true - }, - { - "name": "x-abc-token", - "description": "a header value", - "in": "header", - "schema": { - "type": "string", - "enum": ["a", "b", "c"] - } } ], "get": { @@ -95,6 +86,11 @@ "description": "the name", "default": "Larry", "pattern": "\\w([\\w-.]*\\w)?" + }, + "x-abc-token": { + "type": "string", + "description": "a header value", + "enum": ["a", "b", "c"] } } } diff --git a/ui/README.md b/ui/README.md index 0134868d4..10cab89ad 100644 --- a/ui/README.md +++ b/ui/README.md @@ -7,6 +7,8 @@ - [Running / Development](#running--development) - [Code Generators](#code-generators) - [Running Tests](#running-tests) + - [Automated Cross-Browser Testing](#automated-cross-browser-testing) + - [Running Browserstack Locally](#running-browserstack-locally) - [Linting](#linting) - [Building Vault UI into a Vault Binary](#building-vault-ui-into-a-vault-binary) - [Vault Storybook](#vault-storybook) @@ -71,6 +73,17 @@ acceptance tests then run, proxing requests back to that server. - `yarn run test-oss -s` to keep the test server running after the initial run. - `yarn run test -f="policies"` to filter the tests that are run. `-f` gets passed into [QUnit's `filter` config](https://api.qunitjs.com/config/QUnit.config#qunitconfigfilter-string--default-undefined) +- `yarn run test:browserstack` to run the kv acceptance tests in Browserstack + +#### Automated Cross-Browser Testing + +Vault uses [Browserstack Automate](https://automate.browserstack.com/) to run all the kv acceptance tests on various browsers. You can view the list of browsers we test by viewing `testem.browserstack.js`. + +##### Running Browserstack Locally + +To run the Browserstack tests locally you will need to add your `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY` to your environment. Then run `yarn run test:browserstack`. You can view the currently running tests at `localhost:7357` or log in to [Browserstack Automate](https://automate.browserstack.com/) to view a previous build. + +To run the tests locally in a browser other than IE11, swap out `launch_in_ci: ['BS_IE_11']` inside `testem.browserstack.js`. ### Linting @@ -157,3 +170,4 @@ It is important to add all new components into Storybook and to keep the story a - [Storybook for Ember Live Example](https://storybooks-ember.netlify.com/?path=/story/addon-centered--button) - [Storybook Addons](https://github.com/storybooks/storybook/tree/master/addons/) - [Storybook Docs](https://storybook.js.org/docs/basics/introduction/) +- [Browserstack Automate](https://automate.browserstack.com/) diff --git a/ui/app/components/auth-form.js b/ui/app/components/auth-form.js index aadc85ab4..fc8596404 100644 --- a/ui/app/components/auth-form.js +++ b/ui/app/components/auth-form.js @@ -198,7 +198,7 @@ export default Component.extend(DEFAULTS, { let transition = this.router.transitionTo(targetRoute, { queryParams: { namespace } }); // returning this w/then because if we keep it // in the task, it will get cancelled when the component in un-rendered - return transition.followRedirects().then(() => { + yield transition.followRedirects().then(() => { if (isRoot) { this.flashMessages.warning( 'You have logged in with a root token. As a security precaution, this root token will not be stored by your browser and you will need to re-authenticate after the window is closed or refreshed.' diff --git a/ui/app/components/auth-jwt.js b/ui/app/components/auth-jwt.js index 97564ef15..acaeb98ae 100644 --- a/ui/app/components/auth-jwt.js +++ b/ui/app/components/auth-jwt.js @@ -1,7 +1,7 @@ import Ember from 'ember'; import { inject as service } from '@ember/service'; import Component from './outer-html'; -import { next, later } from '@ember/runloop'; +import { later } from '@ember/runloop'; import { task, timeout, waitForEvent } from 'ember-concurrency'; import { computed } from '@ember/object'; @@ -27,18 +27,16 @@ export default Component.extend({ onNamespace() {}, didReceiveAttrs() { - next(() => { - let { oldSelectedAuthPath, selectedAuthPath } = this; - let shouldDebounce = !oldSelectedAuthPath && !selectedAuthPath; - if (oldSelectedAuthPath !== selectedAuthPath) { - this.set('role', null); - this.onRoleName(this.roleName); - this.fetchRole.perform(null, { debounce: false }); - } else if (shouldDebounce) { - this.fetchRole.perform(this.roleName); - } - this.set('oldSelectedAuthPath', selectedAuthPath); - }); + let { oldSelectedAuthPath, selectedAuthPath } = this; + let shouldDebounce = !oldSelectedAuthPath && !selectedAuthPath; + if (oldSelectedAuthPath !== selectedAuthPath) { + this.set('role', null); + this.onRoleName(this.roleName); + this.fetchRole.perform(null, { debounce: false }); + } else if (shouldDebounce) { + this.fetchRole.perform(this.roleName); + } + this.set('oldSelectedAuthPath', selectedAuthPath); }, // OIDC roles in the JWT/OIDC backend are those with an authUrl, @@ -68,7 +66,9 @@ export default Component.extend({ } } this.set('role', role); - }).restartable(), + }) + .restartable() + .withTestWaiter(), handleOIDCError(err) { this.onLoading(false); diff --git a/ui/app/components/config-pki-ca.js b/ui/app/components/config-pki-ca.js index 7c4c63e86..a5669a5e6 100644 --- a/ui/app/components/config-pki-ca.js +++ b/ui/app/components/config-pki-ca.js @@ -107,7 +107,8 @@ export default Component.extend({ if (!pem) { return []; } - const pemFile = new File([pem], { type: 'text/plain' }); + + const pemFile = new Blob([pem], { type: 'text/plain' }); const links = [ { display: 'Download CA Certificate in PEM format', @@ -121,7 +122,7 @@ export default Component.extend({ }, ]; if (caChain) { - const caChainFile = new File([caChain], { type: 'text/plain' }); + const caChainFile = new Blob([caChain], { type: 'text/plain' }); links.push({ display: 'Download CA Certificate Chain', name: `${backend}_ca_chain.pem`, diff --git a/ui/app/components/pgp-file.js b/ui/app/components/pgp-file.js index a8e1d1c55..caf5d835c 100644 --- a/ui/app/components/pgp-file.js +++ b/ui/app/components/pgp-file.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import { set } from '@ember/object'; +import { task } from 'ember-concurrency'; const BASE_64_REGEX = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/gi; export default Component.extend({ @@ -34,12 +35,12 @@ export default Component.extend({ readFile(file) { const reader = new FileReader(); - reader.onload = () => this.setPGPKey(reader.result, file.name); + reader.onload = () => this.setPGPKey.perform(reader.result, file.name); // this gives us a base64-encoded string which is important in the onload reader.readAsDataURL(file); }, - setPGPKey(dataURL, filename) { + setPGPKey: task(function*(dataURL, filename) { const b64File = dataURL.split(',')[1].trim(); const decoded = atob(b64File).trim(); @@ -48,8 +49,8 @@ export default Component.extend({ // If after decoding it's not b64, we want // the original as it was only encoded when we used `readAsDataURL`. const fileData = decoded.match(BASE_64_REGEX) ? decoded : b64File; - this.get('onChange')(this.get('index'), { value: fileData, fileName: filename }); - }, + yield this.get('onChange')(this.get('index'), { value: fileData, fileName: filename }); + }).withTestWaiter(), actions: { pickedFile(e) { diff --git a/ui/app/components/secret-link.js b/ui/app/components/secret-link.js index 0b2b12d0a..c755e9501 100644 --- a/ui/app/components/secret-link.js +++ b/ui/app/components/secret-link.js @@ -21,6 +21,8 @@ export function linkParams({ mode, secret, queryParams }) { export default Component.extend({ tagName: '', + // so that ember-test-selectors doesn't log a warning + supportsDataTestProperties: true, mode: 'list', secret: null, diff --git a/ui/app/components/wizard-content.js b/ui/app/components/wizard-content.js index bf2e03621..6a929b5db 100644 --- a/ui/app/components/wizard-content.js +++ b/ui/app/components/wizard-content.js @@ -2,6 +2,7 @@ import { inject as service } from '@ember/service'; import Component from '@ember/component'; import { computed } from '@ember/object'; import { FEATURE_MACHINE_STEPS, INIT_STEPS } from 'vault/helpers/wizard-constants'; +import { htmlSafe } from '@ember/template'; export default Component.extend({ wizard: service(), @@ -66,25 +67,25 @@ export default Component.extend({ let bar = []; if (this.currentTutorialProgress) { bar.push({ - style: `width:${this.currentTutorialProgress.percentage}%;`, + style: htmlSafe(`width:${this.currentTutorialProgress.percentage}%;`), completed: false, showIcon: true, }); } else { if (this.currentFeatureProgress) { this.completedFeatures.forEach(feature => { - bar.push({ style: 'width:100%;', completed: true, feature: feature, showIcon: true }); + bar.push({ style: htmlSafe('width:100%;'), completed: true, feature: feature, showIcon: true }); }); this.wizard.featureList.forEach(feature => { if (feature === this.currentMachine) { bar.push({ - style: `width:${this.currentFeatureProgress.percentage}%;`, + style: htmlSafe(`width:${this.currentFeatureProgress.percentage}%;`), completed: this.currentFeatureProgress.percentage == 100 ? true : false, feature: feature, showIcon: true, }); } else { - bar.push({ style: 'width:0%;', completed: false, feature: feature, showIcon: true }); + bar.push({ style: htmlSafe('width:0%;'), completed: false, feature: feature, showIcon: true }); } }); } diff --git a/ui/app/components/wizard/features-selection.js b/ui/app/components/wizard/features-selection.js index bb5919d28..f57d7f751 100644 --- a/ui/app/components/wizard/features-selection.js +++ b/ui/app/components/wizard/features-selection.js @@ -2,6 +2,7 @@ import { inject as service } from '@ember/service'; import Component from '@ember/component'; import { computed } from '@ember/object'; import { FEATURE_MACHINE_TIME } from 'vault/helpers/wizard-constants'; +import { htmlSafe } from '@ember/template'; export default Component.extend({ wizard: service(), @@ -48,10 +49,10 @@ export default Component.extend({ }), selectProgress: computed('selectedFeatures', function() { let bar = this.selectedFeatures.map(feature => { - return { style: 'width:0%;', completed: false, showIcon: true, feature: feature }; + return { style: htmlSafe('width:0%;'), completed: false, showIcon: true, feature: feature }; }); if (bar.length === 0) { - bar = [{ style: 'width:0%;', showIcon: false }]; + bar = [{ style: htmlSafe('width:0%;'), showIcon: false }]; } return bar; }), diff --git a/ui/app/routes/loading.js b/ui/app/routes/loading.js index a327169dd..06fbc703b 100644 --- a/ui/app/routes/loading.js +++ b/ui/app/routes/loading.js @@ -2,7 +2,7 @@ import Route from '@ember/routing/route'; export default Route.extend({ renderTemplate() { - let { targetName } = this.router.currentState.routerJs.activeTransition; + let { targetName } = this._router.currentState.routerJs.activeTransition; let isCallback = targetName === 'vault.cluster.oidc-callback' || targetName === 'vault.cluster.oidc-callback-namespace'; if (isCallback) { diff --git a/ui/app/serializers/identity/entity.js b/ui/app/serializers/identity/entity.js index 4ff22baac..6060bb03e 100644 --- a/ui/app/serializers/identity/entity.js +++ b/ui/app/serializers/identity/entity.js @@ -2,6 +2,8 @@ import DS from 'ember-data'; import IdentitySerializer from './_base'; export default IdentitySerializer.extend(DS.EmbeddedRecordsMixin, { + // we don't need to serialize relationships here + serializeHasMany() {}, attrs: { aliases: { embedded: 'always' }, }, diff --git a/ui/app/services/auth.js b/ui/app/services/auth.js index 87a4286e5..f9704de33 100644 --- a/ui/app/services/auth.js +++ b/ui/app/services/auth.js @@ -12,7 +12,7 @@ import { supportedAuthBackends } from 'vault/helpers/supported-auth-backends'; import { task, timeout } from 'ember-concurrency'; const TOKEN_SEPARATOR = '☃'; const TOKEN_PREFIX = 'vault-'; -const ROOT_PREFIX = '🗝'; +const ROOT_PREFIX = '_root_'; const BACKENDS = supportedAuthBackends(); export { TOKEN_SEPARATOR, TOKEN_PREFIX, ROOT_PREFIX }; @@ -296,29 +296,23 @@ export default Service.extend({ if (this.environment() === 'development') { return; } + this.getTokensFromStorage().forEach(key => { const data = this.getTokenData(key); - if (data.policies.includes('root')) { + if (data && data.policies.includes('root')) { this.removeTokenData(key); } }); }, - authenticate(/*{clusterId, backend, data}*/) { + async authenticate(/*{clusterId, backend, data}*/) { const [options] = arguments; const adapter = this.clusterAdapter(); - return adapter.authenticate(options).then(resp => { - return this.persistAuthData(options, resp.auth || resp.data, this.get('namespace.path')).then( - authData => { - return this.get('permissions') - .getPaths.perform() - .then(() => { - return authData; - }); - } - ); - }); + let resp = await adapter.authenticate(options); + let authData = await this.persistAuthData(options, resp.auth || resp.data, this.get('namespace.path')); + await this.get('permissions').getPaths.perform(); + return authData; }, deleteCurrentToken() { diff --git a/ui/app/templates/components/pgp-file.hbs b/ui/app/templates/components/pgp-file.hbs index 8526d9e37..51aafd84a 100644 --- a/ui/app/templates/components/pgp-file.hbs +++ b/ui/app/templates/components/pgp-file.hbs @@ -1,6 +1,6 @@
-