cli: Add KV `export` and `import`
This commit adds two new commands to the Consul KV CLI, which export and import a JSON formatted representation of the Consul KV tree. It is useful to migrate parts of the KV tree between unrelated Consul clusters, and could also be used for initial data population of the KV store.
This commit is contained in:
James Nugent
parent
a08672c71a
commit
3b1f523b85
|
|
@ -0,0 +1,123 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/consul/api"
|
||||
"github.com/mitchellh/cli"
|
||||
)
|
||||
|
||||
// KVExportCommand is a Command implementation that is used to export
|
||||
// a KV tree as JSON
|
||||
type KVExportCommand struct {
|
||||
Ui cli.Ui
|
||||
}
|
||||
|
||||
func (c *KVExportCommand) Synopsis() string {
|
||||
return "Exports a tree from the KV store as JSON"
|
||||
}
|
||||
|
||||
func (c *KVExportCommand) Help() string {
|
||||
helpText := `
|
||||
Usage: consul kv export [KEY_OR_PREFIX]
|
||||
|
||||
Retrieves key-value pairs for the given prefix from Consul's key-value store,
|
||||
and writes a JSON representation to stdout. This can be used with the command
|
||||
"consul kv import" to move entire trees between Consul clusters.
|
||||
|
||||
$ consul kv export vault
|
||||
|
||||
For a full list of options and examples, please see the Consul documentation.
|
||||
|
||||
` + apiOptsText + `
|
||||
|
||||
KV Export Options:
|
||||
|
||||
None.
|
||||
`
|
||||
return strings.TrimSpace(helpText)
|
||||
}
|
||||
|
||||
func (c *KVExportCommand) Run(args []string) int {
|
||||
cmdFlags := flag.NewFlagSet("export", flag.ContinueOnError)
|
||||
|
||||
datacenter := cmdFlags.String("datacenter", "", "")
|
||||
token := cmdFlags.String("token", "", "")
|
||||
stale := cmdFlags.Bool("stale", false, "")
|
||||
httpAddr := HTTPAddrFlag(cmdFlags)
|
||||
if err := cmdFlags.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
key := ""
|
||||
// Check for arg validation
|
||||
args = cmdFlags.Args()
|
||||
switch len(args) {
|
||||
case 0:
|
||||
key = ""
|
||||
case 1:
|
||||
key = args[0]
|
||||
default:
|
||||
c.Ui.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
|
||||
return 1
|
||||
}
|
||||
|
||||
// This is just a "nice" thing to do. Since pairs cannot start with a /, but
|
||||
// users will likely put "/" or "/foo", lets go ahead and strip that for them
|
||||
// here.
|
||||
if len(key) > 0 && key[0] == '/' {
|
||||
key = key[1:]
|
||||
}
|
||||
|
||||
// Create and test the HTTP client
|
||||
conf := api.DefaultConfig()
|
||||
conf.Address = *httpAddr
|
||||
conf.Token = *token
|
||||
client, err := api.NewClient(conf)
|
||||
if err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
pairs, _, err := client.KV().List(key, &api.QueryOptions{
|
||||
Datacenter: *datacenter,
|
||||
AllowStale: *stale,
|
||||
})
|
||||
if err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error querying Consul agent: %s", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
exported := make([]*kvExportEntry, len(pairs))
|
||||
for i, pair := range pairs {
|
||||
exported[i] = toExportEntry(pair)
|
||||
}
|
||||
|
||||
marshaled, err := json.MarshalIndent(exported, "", "\t")
|
||||
if err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error exporting KV data: %s", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
c.Ui.Info(string(marshaled))
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
type kvExportEntry struct {
|
||||
Key string `json:"key"`
|
||||
Flags uint64 `json:"flags"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
func toExportEntry(pair *api.KVPair) *kvExportEntry {
|
||||
return &kvExportEntry{
|
||||
Key: pair.Key,
|
||||
Flags: pair.Flags,
|
||||
Value: base64.StdEncoding.EncodeToString(pair.Value),
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/consul/api"
|
||||
"github.com/mitchellh/cli"
|
||||
)
|
||||
|
||||
func TestKVExportCommand_Run(t *testing.T) {
|
||||
srv, client := testAgentWithAPIClient(t)
|
||||
defer srv.Shutdown()
|
||||
waitForLeader(t, srv.httpAddr)
|
||||
|
||||
ui := new(cli.MockUi)
|
||||
c := &KVExportCommand{Ui: ui}
|
||||
|
||||
keys := map[string]string{
|
||||
"foo/a": "a",
|
||||
"foo/b": "b",
|
||||
"foo/c": "c",
|
||||
"bar": "d",
|
||||
}
|
||||
for k, v := range keys {
|
||||
pair := &api.KVPair{Key: k, Value: []byte(v)}
|
||||
if _, err := client.KV().Put(pair, nil); err != nil {
|
||||
t.Fatalf("err: %#v", err)
|
||||
}
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-http-addr=" + srv.httpAddr,
|
||||
"foo",
|
||||
}
|
||||
|
||||
code := c.Run(args)
|
||||
if code != 0 {
|
||||
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
|
||||
}
|
||||
|
||||
output := ui.OutputWriter.String()
|
||||
|
||||
var exported []*kvExportEntry
|
||||
err := json.Unmarshal([]byte(output), &exported)
|
||||
if err != nil {
|
||||
t.Fatalf("bad: %s", code)
|
||||
}
|
||||
|
||||
if len(exported) != 3 {
|
||||
t.Fatalf("bad: expected 3, got %d", len(exported))
|
||||
}
|
||||
|
||||
for _, entry := range exported {
|
||||
if base64.StdEncoding.EncodeToString([]byte(keys[entry.Key])) != entry.Value {
|
||||
t.Fatalf("bad: expected %s, got %s", keys[entry.Key], entry.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/consul/api"
|
||||
"github.com/mitchellh/cli"
|
||||
)
|
||||
|
||||
// KVImportCommand is a Command implementation that is used to import
|
||||
// a KV tree stored as JSON
|
||||
type KVImportCommand struct {
|
||||
Ui cli.Ui
|
||||
|
||||
// testStdin is the input for testing.
|
||||
testStdin io.Reader
|
||||
}
|
||||
|
||||
func (c *KVImportCommand) Synopsis() string {
|
||||
return "Imports a tree stored as JSON to the KV store"
|
||||
}
|
||||
|
||||
func (c *KVImportCommand) Help() string {
|
||||
helpText := `
|
||||
Usage: consul kv import [DATA]
|
||||
|
||||
Imports key-value pairs to the key-value store from the JSON representation
|
||||
generated by the "consul kv export" command.
|
||||
|
||||
The data can be read from a file by prefixing the filename with the "@"
|
||||
symbol. For example:
|
||||
|
||||
$ consul kv import @filename.json
|
||||
|
||||
Or it can be read from stdin using the "-" symbol:
|
||||
|
||||
$ cat filename.json | consul kv import config/program/license -
|
||||
|
||||
Alternatively the data may be provided as the final parameter to the command,
|
||||
though care must be taken with regards to shell escaping.
|
||||
|
||||
For a full list of options and examples, please see the Consul documentation.
|
||||
|
||||
` + apiOptsText + `
|
||||
|
||||
KV Import Options:
|
||||
|
||||
None.
|
||||
`
|
||||
return strings.TrimSpace(helpText)
|
||||
}
|
||||
|
||||
func (c *KVImportCommand) Run(args []string) int {
|
||||
cmdFlags := flag.NewFlagSet("import", flag.ContinueOnError)
|
||||
|
||||
datacenter := cmdFlags.String("datacenter", "", "")
|
||||
token := cmdFlags.String("token", "", "")
|
||||
httpAddr := HTTPAddrFlag(cmdFlags)
|
||||
if err := cmdFlags.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
// Check for arg validation
|
||||
args = cmdFlags.Args()
|
||||
data, err := c.dataFromArgs(args)
|
||||
if err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error! %s", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
// Create and test the HTTP client
|
||||
conf := api.DefaultConfig()
|
||||
conf.Address = *httpAddr
|
||||
conf.Token = *token
|
||||
client, err := api.NewClient(conf)
|
||||
if err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
var entries []*kvExportEntry
|
||||
if err := json.Unmarshal([]byte(data), &entries); err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Cannot unmarshal data: %s", err))
|
||||
return 1
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
value, err := base64.StdEncoding.DecodeString(entry.Value)
|
||||
if err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error base 64 decoding value for key %s: %s", entry.Key, err))
|
||||
return 1
|
||||
}
|
||||
|
||||
pair := &api.KVPair{
|
||||
Key: entry.Key,
|
||||
Flags: entry.Flags,
|
||||
Value: value,
|
||||
}
|
||||
|
||||
wo := &api.WriteOptions{
|
||||
Datacenter: *datacenter,
|
||||
Token: *token,
|
||||
}
|
||||
|
||||
if _, err := client.KV().Put(pair, wo); err != nil {
|
||||
c.Ui.Error(fmt.Sprintf("Error! Failed writing data for key %s: %s", pair.Key, err))
|
||||
return 1
|
||||
}
|
||||
|
||||
c.Ui.Info(fmt.Sprintf("Imported: %s", pair.Key))
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *KVImportCommand) dataFromArgs(args []string) (string, error) {
|
||||
var stdin io.Reader = os.Stdin
|
||||
if c.testStdin != nil {
|
||||
stdin = c.testStdin
|
||||
}
|
||||
|
||||
switch len(args) {
|
||||
case 0:
|
||||
return "", errors.New("Missing DATA argument")
|
||||
case 1:
|
||||
default:
|
||||
return "", fmt.Errorf("Too many arguments (expected 1 or 2, got %d)", len(args))
|
||||
}
|
||||
|
||||
data := args[0]
|
||||
|
||||
if len(data) == 0 {
|
||||
return "", errors.New("Empty DATA argument")
|
||||
}
|
||||
|
||||
switch data[0] {
|
||||
case '@':
|
||||
data, err := ioutil.ReadFile(data[1:])
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Failed to read file: %s", err)
|
||||
}
|
||||
return string(data), nil
|
||||
case '-':
|
||||
if len(data) > 1 {
|
||||
return data, nil
|
||||
} else {
|
||||
var b bytes.Buffer
|
||||
if _, err := io.Copy(&b, stdin); err != nil {
|
||||
return "", fmt.Errorf("Failed to read stdin: %s", err)
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
default:
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mitchellh/cli"
|
||||
)
|
||||
|
||||
func TestKVImportCommand_Run(t *testing.T) {
|
||||
srv, client := testAgentWithAPIClient(t)
|
||||
defer srv.Shutdown()
|
||||
waitForLeader(t, srv.httpAddr)
|
||||
|
||||
const json = `[
|
||||
{
|
||||
"key": "foo",
|
||||
"flags": 0,
|
||||
"value": "YmFyCg=="
|
||||
},
|
||||
{
|
||||
"key": "foo/a",
|
||||
"flags": 0,
|
||||
"value": "YmF6Cg=="
|
||||
}
|
||||
]`
|
||||
|
||||
ui := new(cli.MockUi)
|
||||
c := &KVImportCommand{
|
||||
Ui: ui,
|
||||
testStdin: strings.NewReader(json),
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-http-addr=" + srv.httpAddr,
|
||||
"-",
|
||||
}
|
||||
|
||||
code := c.Run(args)
|
||||
if code != 0 {
|
||||
t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
|
||||
}
|
||||
|
||||
pair, _, err := client.KV().Get("foo", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(string(pair.Value)) != "bar" {
|
||||
t.Fatalf("bad: expected: bar, got %s", pair.Value)
|
||||
}
|
||||
|
||||
pair, _, err = client.KV().Get("foo/a", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(string(pair.Value)) != "baz" {
|
||||
t.Fatalf("bad: expected: baz, got %s", pair.Value)
|
||||
}
|
||||
}
|
||||
12
commands.go
12
commands.go
|
|
@ -78,6 +78,18 @@ func init() {
|
|||
}, nil
|
||||
},
|
||||
|
||||
"kv export": func() (cli.Command, error) {
|
||||
return &command.KVExportCommand{
|
||||
Ui: ui,
|
||||
}, nil
|
||||
},
|
||||
|
||||
"kv import": func() (cli.Command, error) {
|
||||
return &command.KVImportCommand{
|
||||
Ui: ui,
|
||||
}, nil
|
||||
},
|
||||
|
||||
"join": func() (cli.Command, error) {
|
||||
return &command.JoinCommand{
|
||||
Ui: ui,
|
||||
|
|
|
|||
|
|
@ -31,7 +31,9 @@ Usage: consul kv <subcommand> [options] [args]
|
|||
Subcommands:
|
||||
|
||||
delete Removes data from the KV store
|
||||
export Exports part of the KV tree in JSON format
|
||||
get Retrieves or lists data from the KV store
|
||||
import Imports part of the KV tree in JSON format
|
||||
put Sets or updates data in the KV store
|
||||
```
|
||||
|
||||
|
|
@ -39,7 +41,9 @@ For more information, examples, and usage about a subcommand, click on the name
|
|||
of the subcommand in the sidebar or one of the links below:
|
||||
|
||||
- [delete](/docs/commands/kv/delete.html)
|
||||
- [export](/docs/commands/kv/export.html)
|
||||
- [get](/docs/commands/kv/get.html)
|
||||
- [import](/docs/commands/kv/import.html)
|
||||
- [put](/docs/commands/kv/put.html)
|
||||
|
||||
## Basic Examples
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
---
|
||||
layout: "docs"
|
||||
page_title: "Commands: KV Export"
|
||||
sidebar_current: "docs-commands-kv-export"
|
||||
---
|
||||
|
||||
# Consul KV Export
|
||||
|
||||
Command: `consul kv export`
|
||||
|
||||
The `kv export` command is used to retrieve key-value pairs for the given
|
||||
prefix from Consul's key-value store, and write a JSON representation to
|
||||
stdout. This can be used with the command "consul kv import" to move entire
|
||||
trees between Consul clusters.
|
||||
|
||||
## Usage
|
||||
|
||||
Usage: `consul kv export [PREFIX]`
|
||||
|
||||
#### API Options
|
||||
|
||||
<%= partial "docs/commands/http_api_options" %>
|
||||
|
||||
## Examples
|
||||
|
||||
To export the tree at "vault/" in the key value store:
|
||||
|
||||
```
|
||||
$ consul kv export vault/
|
||||
# JSON output
|
||||
```
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
---
|
||||
layout: "docs"
|
||||
page_title: "Commands: KV Import"
|
||||
sidebar_current: "docs-commands-kv-import"
|
||||
---
|
||||
|
||||
# Consul KV Import
|
||||
|
||||
Command: `consul kv import`
|
||||
|
||||
The `kv import` command is used to import KV pairs from the JSON representation
|
||||
generated by the `kv export` command.
|
||||
|
||||
## Usage
|
||||
|
||||
Usage: `consul kv import [DATA]`
|
||||
|
||||
#### API Options
|
||||
|
||||
<%= partial "docs/commands/http_api_options" %>
|
||||
|
||||
## Examples
|
||||
|
||||
To import from a file, prepend the filename with `@`:
|
||||
|
||||
```
|
||||
$ consul kv import @values.json
|
||||
# Output
|
||||
```
|
||||
|
||||
To import from stdin, use `-` as the data parameter:
|
||||
|
||||
```
|
||||
$ cat values.json | consul kv import -
|
||||
# Output
|
||||
```
|
||||
|
||||
You can also pass the JSON directly, however care must be taken with shell
|
||||
escaping:
|
||||
|
||||
```
|
||||
$ consul kv import "$(cat values.json)"
|
||||
# Output
|
||||
```
|
||||
|
||||
|
||||
Loading…
Reference in New Issue