open-vault/command/commands.go
Brian Kassouf ed14061578
Raft Storage Backend (#6888)
* Work on raft backend

* Add logstore locally

* Add encryptor and unsealable interfaces

* Add clustering support to raft

* Remove client and handler

* Bootstrap raft on init

* Cleanup raft logic a bit

* More raft work

* Work on TLS config

* More work on bootstrapping

* Fix build

* More work on bootstrapping

* More bootstrapping work

* fix build

* Remove consul dep

* Fix build

* merged oss/master into raft-storage

* Work on bootstrapping

* Get bootstrapping to work

* Clean up FMS and node-id

* Update local node ID logic

* Cleanup node-id change

* Work on snapshotting

* Raft: Add remove peer API (#906)

* Add remove peer API

* Add some comments

* Fix existing snapshotting (#909)

* Raft get peers API (#912)

* Read raft configuration

* address review feedback

* Use the Leadership Transfer API to step-down the active node (#918)

* Raft join and unseal using Shamir keys (#917)

* Raft join using shamir

* Store AEAD instead of master key

* Split the raft join process to answer the challenge after a successful unseal

* get the follower to standby state

* Make unseal work

* minor changes

* Some input checks

* reuse the shamir seal access instead of new default seal access

* refactor joinRaftSendAnswer function

* Synchronously send answer in auto-unseal case

* Address review feedback

* Raft snapshots (#910)

* Fix existing snapshotting

* implement the noop snapshotting

* Add comments and switch log libraries

* add some snapshot tests

* add snapshot test file

* add TODO

* More work on raft snapshotting

* progress on the ConfigStore strategy

* Don't use two buckets

* Update the snapshot store logic to hide the file logic

* Add more backend tests

* Cleanup code a bit

* [WIP] Raft recovery (#938)

* Add recovery functionality

* remove fmt.Printfs

* Fix a few fsm bugs

* Add max size value for raft backend (#942)

* Add max size value for raft backend

* Include physical.ErrValueTooLarge in the message

* Raft snapshot Take/Restore API  (#926)

* Inital work on raft snapshot APIs

* Always redirect snapshot install/download requests

* More work on the snapshot APIs

* Cleanup code a bit

* On restore handle special cases

* Use the seal to encrypt the sha sum file

* Add sealer mechanism and fix some bugs

* Call restore while state lock is held

* Send restore cb trigger through raft log

* Make error messages nicer

* Add test helpers

* Add snapshot test

* Add shamir unseal test

* Add more raft snapshot API tests

* Fix locking

* Change working to initalize

* Add underlying raw object to test cluster core

* Move leaderUUID to core

* Add raft TLS rotation logic (#950)

* Add TLS rotation logic

* Cleanup logic a bit

* Add/Remove from follower state on add/remove peer

* add comments

* Update more comments

* Update request_forwarding_service.proto

* Make sure we populate all nodes in the followerstate obj

* Update times

* Apply review feedback

* Add more raft config setting (#947)

* Add performance config setting

* Add more config options and fix tests

* Test Raft Recovery (#944)

* Test raft recovery

* Leave out a node during recovery

* remove unused struct

* Update physical/raft/snapshot_test.go

* Update physical/raft/snapshot_test.go

* fix vendoring

* Switch to new raft interface

* Remove unused files

* Switch a gogo -> proto instance

* Remove unneeded vault dep in go.sum

* Update helper/testhelpers/testhelpers.go

Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com>

* Update vault/cluster/cluster.go

* track active key within the keyring itself (#6915)

* track active key within the keyring itself

* lookup and store using the active key ID

* update docstring

* minor refactor

* Small text fixes (#6912)

* Update physical/raft/raft.go

Co-Authored-By: Calvin Leung Huang <cleung2010@gmail.com>

* review feedback

* Move raft logical system into separate file

* Update help text a bit

* Enforce cluster addr is set and use it for raft bootstrapping

* Fix tests

* fix http test panic

* Pull in latest raft-snapshot library

* Add comment
2019-06-20 12:14:58 -07:00

636 lines
19 KiB
Go

package command
import (
"os"
"os/signal"
"syscall"
"github.com/hashicorp/vault/audit"
"github.com/hashicorp/vault/builtin/plugin"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/sdk/physical"
"github.com/hashicorp/vault/sdk/version"
"github.com/mitchellh/cli"
/*
The builtinplugins package is initialized here because it, in turn,
initializes the database plugins.
They register multiple database drivers for the "database/sql" package.
*/
_ "github.com/hashicorp/vault/helper/builtinplugins"
auditFile "github.com/hashicorp/vault/builtin/audit/file"
auditSocket "github.com/hashicorp/vault/builtin/audit/socket"
auditSyslog "github.com/hashicorp/vault/builtin/audit/syslog"
credAliCloud "github.com/hashicorp/vault-plugin-auth-alicloud"
credCentrify "github.com/hashicorp/vault-plugin-auth-centrify"
credGcp "github.com/hashicorp/vault-plugin-auth-gcp/plugin"
credOIDC "github.com/hashicorp/vault-plugin-auth-jwt"
credPCF "github.com/hashicorp/vault-plugin-auth-pcf"
credAws "github.com/hashicorp/vault/builtin/credential/aws"
credCert "github.com/hashicorp/vault/builtin/credential/cert"
credGitHub "github.com/hashicorp/vault/builtin/credential/github"
credLdap "github.com/hashicorp/vault/builtin/credential/ldap"
credOkta "github.com/hashicorp/vault/builtin/credential/okta"
credToken "github.com/hashicorp/vault/builtin/credential/token"
credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
logicalKv "github.com/hashicorp/vault-plugin-secrets-kv"
logicalDb "github.com/hashicorp/vault/builtin/logical/database"
physAliCloudOSS "github.com/hashicorp/vault/physical/alicloudoss"
physAzure "github.com/hashicorp/vault/physical/azure"
physCassandra "github.com/hashicorp/vault/physical/cassandra"
physCockroachDB "github.com/hashicorp/vault/physical/cockroachdb"
physConsul "github.com/hashicorp/vault/physical/consul"
physCouchDB "github.com/hashicorp/vault/physical/couchdb"
physDynamoDB "github.com/hashicorp/vault/physical/dynamodb"
physEtcd "github.com/hashicorp/vault/physical/etcd"
physFoundationDB "github.com/hashicorp/vault/physical/foundationdb"
physGCS "github.com/hashicorp/vault/physical/gcs"
physManta "github.com/hashicorp/vault/physical/manta"
physMSSQL "github.com/hashicorp/vault/physical/mssql"
physMySQL "github.com/hashicorp/vault/physical/mysql"
physPostgreSQL "github.com/hashicorp/vault/physical/postgresql"
physRaft "github.com/hashicorp/vault/physical/raft"
physS3 "github.com/hashicorp/vault/physical/s3"
physSpanner "github.com/hashicorp/vault/physical/spanner"
physSwift "github.com/hashicorp/vault/physical/swift"
physZooKeeper "github.com/hashicorp/vault/physical/zookeeper"
physFile "github.com/hashicorp/vault/sdk/physical/file"
physInmem "github.com/hashicorp/vault/sdk/physical/inmem"
)
const (
// EnvVaultCLINoColor is an env var that toggles colored UI output.
EnvVaultCLINoColor = `VAULT_CLI_NO_COLOR`
// EnvVaultFormat is the output format
EnvVaultFormat = `VAULT_FORMAT`
// flagNameAddress is the flag used in the base command to read in the
// address of the Vault server.
flagNameAddress = "address"
// flagnameCACert is the flag used in the base command to read in the CA
// cert.
flagNameCACert = "ca-cert"
// flagnameCAPath is the flag used in the base command to read in the CA
// cert path.
flagNameCAPath = "ca-path"
//flagNameClientCert is the flag used in the base command to read in the
//client key
flagNameClientKey = "client-key"
//flagNameClientCert is the flag used in the base command to read in the
//client cert
flagNameClientCert = "client-cert"
// flagNameTLSSkipVerify is the flag used in the base command to read in
// the option to ignore TLS certificate verification.
flagNameTLSSkipVerify = "tls-skip-verify"
// flagNameAuditNonHMACRequestKeys is the flag name used for auth/secrets enable
flagNameAuditNonHMACRequestKeys = "audit-non-hmac-request-keys"
// flagNameAuditNonHMACResponseKeys is the flag name used for auth/secrets enable
flagNameAuditNonHMACResponseKeys = "audit-non-hmac-response-keys"
// flagNameDescription is the flag name used for tuning the secret and auth mount description parameter
flagNameDescription = "description"
// flagListingVisibility is the flag to toggle whether to show the mount in the UI-specific listing endpoint
flagNameListingVisibility = "listing-visibility"
// flagNamePassthroughRequestHeaders is the flag name used to set passthrough request headers to the backend
flagNamePassthroughRequestHeaders = "passthrough-request-headers"
// flagNameAllowedResponseHeaders is used to set allowed response headers from a plugin
flagNameAllowedResponseHeaders = "allowed-response-headers"
// flagNameTokenType is the flag name used to force a specific token type
flagNameTokenType = "token-type"
)
var (
auditBackends = map[string]audit.Factory{
"file": auditFile.Factory,
"socket": auditSocket.Factory,
"syslog": auditSyslog.Factory,
}
credentialBackends = map[string]logical.Factory{
"plugin": plugin.Factory,
}
logicalBackends = map[string]logical.Factory{
"plugin": plugin.Factory,
"database": logicalDb.Factory,
// This is also available in the plugin catalog, but is here due to the need to
// automatically mount it.
"kv": logicalKv.Factory,
}
physicalBackends = map[string]physical.Factory{
"alicloudoss": physAliCloudOSS.NewAliCloudOSSBackend,
"azure": physAzure.NewAzureBackend,
"cassandra": physCassandra.NewCassandraBackend,
"cockroachdb": physCockroachDB.NewCockroachDBBackend,
"consul": physConsul.NewConsulBackend,
"couchdb_transactional": physCouchDB.NewTransactionalCouchDBBackend,
"couchdb": physCouchDB.NewCouchDBBackend,
"dynamodb": physDynamoDB.NewDynamoDBBackend,
"etcd": physEtcd.NewEtcdBackend,
"file_transactional": physFile.NewTransactionalFileBackend,
"file": physFile.NewFileBackend,
"foundationdb": physFoundationDB.NewFDBBackend,
"gcs": physGCS.NewBackend,
"inmem_ha": physInmem.NewInmemHA,
"inmem_transactional_ha": physInmem.NewTransactionalInmemHA,
"inmem_transactional": physInmem.NewTransactionalInmem,
"inmem": physInmem.NewInmem,
"manta": physManta.NewMantaBackend,
"mssql": physMSSQL.NewMSSQLBackend,
"mysql": physMySQL.NewMySQLBackend,
"postgresql": physPostgreSQL.NewPostgreSQLBackend,
"s3": physS3.NewS3Backend,
"spanner": physSpanner.NewBackend,
"swift": physSwift.NewSwiftBackend,
"raft": physRaft.NewRaftBackend,
"zookeeper": physZooKeeper.NewZooKeeperBackend,
}
)
// Commands is the mapping of all the available commands.
var Commands map[string]cli.CommandFactory
func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
loginHandlers := map[string]LoginHandler{
"alicloud": &credAliCloud.CLIHandler{},
"aws": &credAws.CLIHandler{},
"centrify": &credCentrify.CLIHandler{},
"cert": &credCert.CLIHandler{},
"gcp": &credGcp.CLIHandler{},
"github": &credGitHub.CLIHandler{},
"ldap": &credLdap.CLIHandler{},
"oidc": &credOIDC.CLIHandler{},
"okta": &credOkta.CLIHandler{},
"pcf": &credPCF.CLIHandler{},
"radius": &credUserpass.CLIHandler{
DefaultMount: "radius",
},
"token": &credToken.CLIHandler{},
"userpass": &credUserpass.CLIHandler{
DefaultMount: "userpass",
},
}
getBaseCommand := func() *BaseCommand {
return &BaseCommand{
UI: ui,
tokenHelper: runOpts.TokenHelper,
flagAddress: runOpts.Address,
client: runOpts.Client,
}
}
Commands = map[string]cli.CommandFactory{
"agent": func() (cli.Command, error) {
return &AgentCommand{
BaseCommand: &BaseCommand{
UI: serverCmdUi,
},
ShutdownCh: MakeShutdownCh(),
}, nil
},
"audit": func() (cli.Command, error) {
return &AuditCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"audit disable": func() (cli.Command, error) {
return &AuditDisableCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"audit enable": func() (cli.Command, error) {
return &AuditEnableCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"audit list": func() (cli.Command, error) {
return &AuditListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"auth tune": func() (cli.Command, error) {
return &AuthTuneCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"auth": func() (cli.Command, error) {
return &AuthCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"auth disable": func() (cli.Command, error) {
return &AuthDisableCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"auth enable": func() (cli.Command, error) {
return &AuthEnableCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"auth help": func() (cli.Command, error) {
return &AuthHelpCommand{
BaseCommand: getBaseCommand(),
Handlers: loginHandlers,
}, nil
},
"auth list": func() (cli.Command, error) {
return &AuthListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"delete": func() (cli.Command, error) {
return &DeleteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"lease": func() (cli.Command, error) {
return &LeaseCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"lease renew": func() (cli.Command, error) {
return &LeaseRenewCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"lease revoke": func() (cli.Command, error) {
return &LeaseRevokeCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"list": func() (cli.Command, error) {
return &ListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"login": func() (cli.Command, error) {
return &LoginCommand{
BaseCommand: getBaseCommand(),
Handlers: loginHandlers,
}, nil
},
"namespace": func() (cli.Command, error) {
return &NamespaceCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"namespace list": func() (cli.Command, error) {
return &NamespaceListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"namespace lookup": func() (cli.Command, error) {
return &NamespaceLookupCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"namespace create": func() (cli.Command, error) {
return &NamespaceCreateCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"namespace delete": func() (cli.Command, error) {
return &NamespaceDeleteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator": func() (cli.Command, error) {
return &OperatorCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator generate-root": func() (cli.Command, error) {
return &OperatorGenerateRootCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator init": func() (cli.Command, error) {
return &OperatorInitCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator key-status": func() (cli.Command, error) {
return &OperatorKeyStatusCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator migrate": func() (cli.Command, error) {
return &OperatorMigrateCommand{
BaseCommand: getBaseCommand(),
PhysicalBackends: physicalBackends,
ShutdownCh: MakeShutdownCh(),
}, nil
},
"operator rekey": func() (cli.Command, error) {
return &OperatorRekeyCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator rotate": func() (cli.Command, error) {
return &OperatorRotateCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator seal": func() (cli.Command, error) {
return &OperatorSealCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator step-down": func() (cli.Command, error) {
return &OperatorStepDownCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"operator unseal": func() (cli.Command, error) {
return &OperatorUnsealCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"path-help": func() (cli.Command, error) {
return &PathHelpCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"plugin": func() (cli.Command, error) {
return &PluginCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"plugin deregister": func() (cli.Command, error) {
return &PluginDeregisterCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"plugin info": func() (cli.Command, error) {
return &PluginInfoCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"plugin list": func() (cli.Command, error) {
return &PluginListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"plugin register": func() (cli.Command, error) {
return &PluginRegisterCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"policy": func() (cli.Command, error) {
return &PolicyCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"policy delete": func() (cli.Command, error) {
return &PolicyDeleteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"policy fmt": func() (cli.Command, error) {
return &PolicyFmtCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"policy list": func() (cli.Command, error) {
return &PolicyListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"policy read": func() (cli.Command, error) {
return &PolicyReadCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"policy write": func() (cli.Command, error) {
return &PolicyWriteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"print": func() (cli.Command, error) {
return &PrintCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"print token": func() (cli.Command, error) {
return &PrintTokenCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"read": func() (cli.Command, error) {
return &ReadCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"secrets": func() (cli.Command, error) {
return &SecretsCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"secrets disable": func() (cli.Command, error) {
return &SecretsDisableCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"secrets enable": func() (cli.Command, error) {
return &SecretsEnableCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"secrets list": func() (cli.Command, error) {
return &SecretsListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"secrets move": func() (cli.Command, error) {
return &SecretsMoveCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"secrets tune": func() (cli.Command, error) {
return &SecretsTuneCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"server": func() (cli.Command, error) {
return &ServerCommand{
BaseCommand: &BaseCommand{
UI: serverCmdUi,
tokenHelper: runOpts.TokenHelper,
flagAddress: runOpts.Address,
},
AuditBackends: auditBackends,
CredentialBackends: credentialBackends,
LogicalBackends: logicalBackends,
PhysicalBackends: physicalBackends,
ShutdownCh: MakeShutdownCh(),
SighupCh: MakeSighupCh(),
SigUSR2Ch: MakeSigUSR2Ch(),
}, nil
},
"ssh": func() (cli.Command, error) {
return &SSHCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"status": func() (cli.Command, error) {
return &StatusCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"token": func() (cli.Command, error) {
return &TokenCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"token create": func() (cli.Command, error) {
return &TokenCreateCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"token capabilities": func() (cli.Command, error) {
return &TokenCapabilitiesCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"token lookup": func() (cli.Command, error) {
return &TokenLookupCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"token renew": func() (cli.Command, error) {
return &TokenRenewCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"token revoke": func() (cli.Command, error) {
return &TokenRevokeCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"unwrap": func() (cli.Command, error) {
return &UnwrapCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"version": func() (cli.Command, error) {
return &VersionCommand{
VersionInfo: version.GetVersion(),
BaseCommand: getBaseCommand(),
}, nil
},
"write": func() (cli.Command, error) {
return &WriteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv": func() (cli.Command, error) {
return &KVCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv put": func() (cli.Command, error) {
return &KVPutCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv patch": func() (cli.Command, error) {
return &KVPatchCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv rollback": func() (cli.Command, error) {
return &KVRollbackCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv get": func() (cli.Command, error) {
return &KVGetCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv delete": func() (cli.Command, error) {
return &KVDeleteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv list": func() (cli.Command, error) {
return &KVListCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv destroy": func() (cli.Command, error) {
return &KVDestroyCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv undelete": func() (cli.Command, error) {
return &KVUndeleteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv enable-versioning": func() (cli.Command, error) {
return &KVEnableVersioningCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv metadata": func() (cli.Command, error) {
return &KVMetadataCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv metadata put": func() (cli.Command, error) {
return &KVMetadataPutCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv metadata get": func() (cli.Command, error) {
return &KVMetadataGetCommand{
BaseCommand: getBaseCommand(),
}, nil
},
"kv metadata delete": func() (cli.Command, error) {
return &KVMetadataDeleteCommand{
BaseCommand: getBaseCommand(),
}, nil
},
}
}
// MakeShutdownCh returns a channel that can be used for shutdown
// notifications for commands. This channel will send a message for every
// SIGINT or SIGTERM received.
func MakeShutdownCh() chan struct{} {
resultCh := make(chan struct{})
shutdownCh := make(chan os.Signal, 4)
signal.Notify(shutdownCh, os.Interrupt, syscall.SIGTERM)
go func() {
<-shutdownCh
close(resultCh)
}()
return resultCh
}
// MakeSighupCh returns a channel that can be used for SIGHUP
// reloading. This channel will send a message for every
// SIGHUP received.
func MakeSighupCh() chan struct{} {
resultCh := make(chan struct{})
signalCh := make(chan os.Signal, 4)
signal.Notify(signalCh, syscall.SIGHUP)
go func() {
for {
<-signalCh
resultCh <- struct{}{}
}
}()
return resultCh
}