From 22cf348c246506fa79d5765b9e71c8d648991bb7 Mon Sep 17 00:00:00 2001 From: Nathan Coleman Date: Wed, 31 May 2023 14:27:35 -0400 Subject: [PATCH] Export peering cli (#15654) * Sujata's peering-cli branch * Added error message for connecting to cluster * We can export service to peer * export handling multiple peers * export handles multiple peers * export now can handle multiple services * Export after 1st cleanup * Successful export * Added the namespace option * Add .changelog entry * go mod tidy * Stub unit tests for peering export command * added export in peering.go * Adding export_test * Moved the code to services from peers and cleaned the serviceNamespace * Added support for exporting to partitions * Fixed partition bug * Added unit tests for export command * Add multi-tenancy flags * gofmt * Add some helpful comments * Exclude namespace + partition flags when running OSS * cleaned up partition stuff * Validate required flags differently for OSS vs. ENT * Update success output to include only the requested consumers * cleaned up * fixed broken test * gofmt * Include all flags in OSS build * Remove example previously added to peering command * Move stray import into correct block * Update changelog entry to include support for exporting to a partition * Add required-ness label to consumer-peers flag description * Update command/services/export/export.go Co-authored-by: Dan Stough * Add docs placeholder for new services export command * Moved piece of code to OSS * Break config entry init + update into separate functions * fixed * Vary existing service export comparison for OSS vs. ENT * Move OSS-specific test to export_oss_test.go * Set config entry name based on partition being exported from * Set namespace on added services * Adding namespace * Remove export documentation We will include documentation in a followup PR * Consolidate code from export_oss into export.go * Consolidated export_oss_test.go and export_test.go * Add example of partition export to command synopsis * Allow empty peers flag if partitions flag provided * Add test coverage for -consumer-partitions flag * Update command/services/export/export.go Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com> * Update command/services/export/export.go Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com> * Update changelog entry * Use "cluster peers" to clear up any possible confusion * Update test assertions --------- Co-authored-by: 20sr20 Co-authored-by: Dan Stough Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com> --- .changelog/15654.txt | 3 + command/flags/http.go | 4 + command/registry.go | 4 +- command/services/export/export.go | 260 +++++++++++++++++++++++++ command/services/export/export_test.go | 152 +++++++++++++++ 5 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 .changelog/15654.txt create mode 100644 command/services/export/export.go create mode 100644 command/services/export/export_test.go diff --git a/.changelog/15654.txt b/.changelog/15654.txt new file mode 100644 index 000000000..169191377 --- /dev/null +++ b/.changelog/15654.txt @@ -0,0 +1,3 @@ +```release-note:feature +cli: Adds new command - `consul services export` - for exporting a service to a peer or partition +``` diff --git a/command/flags/http.go b/command/flags/http.go index 4c3a34392..82e59431a 100644 --- a/command/flags/http.go +++ b/command/flags/http.go @@ -101,6 +101,10 @@ func (f *HTTPFlags) Datacenter() string { return f.datacenter.String() } +func (f *HTTPFlags) Namespace() string { + return f.namespace.String() +} + func (f *HTTPFlags) Partition() string { return f.partition.String() } diff --git a/command/registry.go b/command/registry.go index 55b9e9f8a..b370f14bc 100644 --- a/command/registry.go +++ b/command/registry.go @@ -111,6 +111,7 @@ import ( "github.com/hashicorp/consul/command/rtt" "github.com/hashicorp/consul/command/services" svcsderegister "github.com/hashicorp/consul/command/services/deregister" + svcsexport "github.com/hashicorp/consul/command/services/export" svcsregister "github.com/hashicorp/consul/command/services/register" "github.com/hashicorp/consul/command/snapshot" snapinspect "github.com/hashicorp/consul/command/snapshot/inspect" @@ -121,7 +122,7 @@ import ( tlscacreate "github.com/hashicorp/consul/command/tls/ca/create" tlscert "github.com/hashicorp/consul/command/tls/cert" tlscertcreate "github.com/hashicorp/consul/command/tls/cert/create" - troubleshoot "github.com/hashicorp/consul/command/troubleshoot" + "github.com/hashicorp/consul/command/troubleshoot" troubleshootproxy "github.com/hashicorp/consul/command/troubleshoot/proxy" troubleshootupstreams "github.com/hashicorp/consul/command/troubleshoot/upstreams" "github.com/hashicorp/consul/command/validate" @@ -241,6 +242,7 @@ func RegisteredCommands(ui cli.Ui) map[string]mcli.CommandFactory { entry{"services", func(cli.Ui) (cli.Command, error) { return services.New(), nil }}, entry{"services register", func(ui cli.Ui) (cli.Command, error) { return svcsregister.New(ui), nil }}, entry{"services deregister", func(ui cli.Ui) (cli.Command, error) { return svcsderegister.New(ui), nil }}, + entry{"services export", func(ui cli.Ui) (cli.Command, error) { return svcsexport.New(ui), nil }}, entry{"snapshot", func(cli.Ui) (cli.Command, error) { return snapshot.New(), nil }}, entry{"snapshot inspect", func(ui cli.Ui) (cli.Command, error) { return snapinspect.New(ui), nil }}, entry{"snapshot restore", func(ui cli.Ui) (cli.Command, error) { return snaprestore.New(ui), nil }}, diff --git a/command/services/export/export.go b/command/services/export/export.go new file mode 100644 index 000000000..637553f93 --- /dev/null +++ b/command/services/export/export.go @@ -0,0 +1,260 @@ +package export + +import ( + "errors" + "flag" + "fmt" + "strings" + + "github.com/mitchellh/cli" + + "github.com/hashicorp/consul/agent" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/command/flags" +) + +func New(ui cli.Ui) *cmd { + c := &cmd{UI: ui} + c.init() + return c +} + +type cmd struct { + UI cli.Ui + flags *flag.FlagSet + http *flags.HTTPFlags + help string + + serviceName string + peerNames string + partitionNames string +} + +func (c *cmd) init() { + c.flags = flag.NewFlagSet("", flag.ContinueOnError) + + c.flags.StringVar(&c.serviceName, "name", "", "(Required) Specify the name of the service you want to export.") + c.flags.StringVar(&c.peerNames, "consumer-peers", "", "(Required) A comma-separated list of cluster peers to export the service to. In Consul Enterprise, this flag is optional if -consumer-partitions is specified.") + c.flags.StringVar(&c.partitionNames, "consumer-partitions", "", "(Enterprise only) A comma-separated list of admin partitions within the same datacenter to export the service to. This flag is optional if -consumer-peers is specified.") + + c.http = &flags.HTTPFlags{} + flags.Merge(c.flags, c.http.ClientFlags()) + flags.Merge(c.flags, c.http.MultiTenancyFlags()) + c.help = flags.Usage(help, c.flags) +} + +func (c *cmd) Run(args []string) int { + if err := c.flags.Parse(args); err != nil { + return 1 + } + + if err := c.validateFlags(); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + peerNames, err := c.getPeerNames() + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + partitionNames, err := c.getPartitionNames() + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + client, err := c.http.APIClient() + if err != nil { + c.UI.Error(fmt.Sprintf("Error connect to Consul agent: %s", err)) + return 1 + } + + // Name matches partition, so "default" if none specified + cfgName := "default" + if c.http.Partition() != "" { + cfgName = c.http.Partition() + } + + entry, _, err := client.ConfigEntries().Get(api.ExportedServices, cfgName, &api.QueryOptions{Namespace: ""}) + if err != nil && !strings.Contains(err.Error(), agent.ConfigEntryNotFoundErr) { + c.UI.Error(fmt.Sprintf("Error reading config entry %s/%s: %v", "exported-services", "default", err)) + return 1 + } + + var cfg *api.ExportedServicesConfigEntry + if entry == nil { + cfg = c.initializeConfigEntry(cfgName, peerNames, partitionNames) + } else { + existingCfg, ok := entry.(*api.ExportedServicesConfigEntry) + if !ok { + c.UI.Error(fmt.Sprintf("Existing config entry has incorrect type: %t", entry)) + return 1 + } + + cfg = c.updateConfigEntry(existingCfg, peerNames, partitionNames) + } + + ok, _, err := client.ConfigEntries().CAS(cfg, cfg.GetModifyIndex(), nil) + if err != nil { + c.UI.Error(fmt.Sprintf("Error writing config entry: %s", err)) + return 1 + } else if !ok { + c.UI.Error(fmt.Sprintf("Config entry was changed during update. Please try again")) + return 1 + } + + switch { + case len(c.peerNames) > 0 && len(c.partitionNames) > 0: + c.UI.Info(fmt.Sprintf("Successfully exported service %q to cluster peers %q and to partitions %q", c.serviceName, c.peerNames, c.partitionNames)) + case len(c.peerNames) > 0: + c.UI.Info(fmt.Sprintf("Successfully exported service %q to cluster peers %q", c.serviceName, c.peerNames)) + case len(c.partitionNames) > 0: + c.UI.Info(fmt.Sprintf("Successfully exported service %q to partitions %q", c.serviceName, c.partitionNames)) + } + + return 0 +} + +func (c *cmd) validateFlags() error { + if c.serviceName == "" { + return errors.New("Missing the required -name flag") + } + + if c.peerNames == "" && c.partitionNames == "" { + return errors.New("Missing the required -consumer-peers or -consumer-partitions flag") + } + + return nil +} + +func (c *cmd) getPeerNames() ([]string, error) { + var peerNames []string + if c.peerNames != "" { + peerNames = strings.Split(c.peerNames, ",") + for _, peerName := range peerNames { + if peerName == "" { + return nil, fmt.Errorf("Invalid peer %q", peerName) + } + } + } + return peerNames, nil +} + +func (c *cmd) getPartitionNames() ([]string, error) { + var partitionNames []string + if c.partitionNames != "" { + partitionNames = strings.Split(c.partitionNames, ",") + for _, partitionName := range partitionNames { + if partitionName == "" { + return nil, fmt.Errorf("Invalid partition %q", partitionName) + } + } + } + return partitionNames, nil +} + +func (c *cmd) initializeConfigEntry(cfgName string, peerNames, partitionNames []string) *api.ExportedServicesConfigEntry { + return &api.ExportedServicesConfigEntry{ + Name: cfgName, + Services: []api.ExportedService{ + { + Name: c.serviceName, + Namespace: c.http.Namespace(), + Consumers: buildConsumers(peerNames, partitionNames), + }, + }, + } +} + +func (c *cmd) updateConfigEntry(cfg *api.ExportedServicesConfigEntry, peerNames, partitionNames []string) *api.ExportedServicesConfigEntry { + serviceExists := false + + for i, service := range cfg.Services { + if service.Name == c.serviceName && service.Namespace == c.http.Namespace() { + serviceExists = true + + // Add a consumer for each peer where one doesn't already exist + for _, peerName := range peerNames { + peerExists := false + for _, consumer := range service.Consumers { + if consumer.Peer == peerName { + peerExists = true + break + } + } + if !peerExists { + cfg.Services[i].Consumers = append(cfg.Services[i].Consumers, api.ServiceConsumer{Peer: peerName}) + } + } + + // Add a consumer for each partition where one doesn't already exist + for _, partitionName := range partitionNames { + partitionExists := false + + for _, consumer := range service.Consumers { + if consumer.Partition == partitionName { + partitionExists = true + break + } + } + if !partitionExists { + cfg.Services[i].Consumers = append(cfg.Services[i].Consumers, api.ServiceConsumer{Partition: partitionName}) + } + } + } + } + + if !serviceExists { + cfg.Services = append(cfg.Services, api.ExportedService{ + Name: c.serviceName, + Namespace: c.http.Namespace(), + Consumers: buildConsumers(peerNames, partitionNames), + }) + } + + return cfg +} + +func buildConsumers(peerNames []string, partitionNames []string) []api.ServiceConsumer { + var consumers []api.ServiceConsumer + for _, peer := range peerNames { + consumers = append(consumers, api.ServiceConsumer{ + Peer: peer, + }) + } + for _, partition := range partitionNames { + consumers = append(consumers, api.ServiceConsumer{ + Partition: partition, + }) + } + return consumers +} + +//======== + +func (c *cmd) Synopsis() string { + return synopsis +} + +func (c *cmd) Help() string { + return flags.Usage(c.help, nil) +} + +const ( + synopsis = "Export a service from one peer or admin partition to another" + help = ` +Usage: consul services export [options] -name -consumer-peers + + Export a service to a peered cluster. + + $ consul services export -name=web -consumer-peers=other-cluster + + Use the -consumer-partitions flag instead of -consumer-peers to export to a different partition in the same cluster. + + $ consul services export -name=web -consumer-partitions=other-partition + + Additional flags and more advanced use cases are detailed below. +` +) diff --git a/command/services/export/export_test.go b/command/services/export/export_test.go new file mode 100644 index 000000000..6a2dfa9d6 --- /dev/null +++ b/command/services/export/export_test.go @@ -0,0 +1,152 @@ +package export + +import ( + "strings" + "testing" + + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" + + "github.com/hashicorp/consul/agent" + "github.com/hashicorp/consul/testrpc" +) + +func TestExportCommand(t *testing.T) { + + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + + t.Run("help output should have no tabs", func(t *testing.T) { + if strings.ContainsRune(New(cli.NewMockUi()).Help(), '\t') { + t.Fatal("help has tabs") + } + }) + + a := agent.NewTestAgent(t, ``) + t.Cleanup(func() { _ = a.Shutdown() }) + testrpc.WaitForTestAgent(t, a.RPC, "dc1") + t.Run("peer or partition is required", func(t *testing.T) { + + ui := cli.NewMockUi() + cmd := New(ui) + + args := []string{ + "-name=testservice", + } + + code := cmd.Run(args) + require.Equal(t, 1, code, "err: %s", ui.ErrorWriter.String()) + require.Contains(t, ui.ErrorWriter.String(), "Missing the required -consumer-peers or -consumer-partitions flag") + }) + t.Run("service name is required", func(t *testing.T) { + + ui := cli.NewMockUi() + cmd := New(ui) + + args := []string{} + + code := cmd.Run(args) + require.Equal(t, 1, code, "err: %s", ui.ErrorWriter.String()) + require.Contains(t, ui.ErrorWriter.String(), "Missing the required -name flag") + }) + + t.Run("valid peer name is required", func(t *testing.T) { + + ui := cli.NewMockUi() + cmd := New(ui) + + args := []string{ + "-name=testservice", + "-consumer-peers=a,", + } + + code := cmd.Run(args) + require.Equal(t, 1, code, "err: %s", ui.ErrorWriter.String()) + require.Contains(t, ui.ErrorWriter.String(), "Invalid peer") + }) + + t.Run("valid partition name is required", func(t *testing.T) { + + ui := cli.NewMockUi() + cmd := New(ui) + + args := []string{ + "-name=testservice", + "-consumer-partitions=a,", + } + + code := cmd.Run(args) + require.Equal(t, 1, code, "err: %s", ui.ErrorWriter.String()) + require.Contains(t, ui.ErrorWriter.String(), "Invalid partition") + }) + + t.Run("initial config entry should be created w/ partitions", func(t *testing.T) { + + ui := cli.NewMockUi() + cmd := New(ui) + + args := []string{ + "-http-addr=" + a.HTTPAddr(), + "-name=testservice", + "-consumer-partitions=a,b", + } + + code := cmd.Run(args) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), "Successfully exported service") + }) + + t.Run("initial config entry should be created w/ peers", func(t *testing.T) { + + ui := cli.NewMockUi() + cmd := New(ui) + + args := []string{ + "-http-addr=" + a.HTTPAddr(), + "-name=testservice", + "-consumer-peers=a,b", + } + + code := cmd.Run(args) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), "Successfully exported service") + }) + + t.Run("existing config entry should be updated w/ new peers and partitions", func(t *testing.T) { + + ui := cli.NewMockUi() + + args := []string{ + "-http-addr=" + a.HTTPAddr(), + "-name=testservice", + "-consumer-peers=a,b", + } + + code := New(ui).Run(args) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), `Successfully exported service "testservice" to cluster peers "a,b"`) + + args = []string{ + "-http-addr=" + a.HTTPAddr(), + "-name=testservice", + "-consumer-peers=c", + } + + code = New(ui).Run(args) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), `Successfully exported service "testservice" to cluster peers "c"`) + + args = []string{ + "-http-addr=" + a.HTTPAddr(), + "-name=testservice", + "-consumer-partitions=d", + } + + code = New(ui).Run(args) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), `Successfully exported service "testservice" to partitions "d"`) + }) +}