luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Require Connect and TLS to generate peering tokens 

By requiring Connect and a gRPC TLS listener we can automatically
configure TLS for all peering control-plane traffic.
This commit is contained in:
freddygv 2022-09-29 15:49:58 -06:00
parent a21e5799f7
commit 6ef8d329d2
11 changed files with 185 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
agent/consul/leader_peering_test.go
View File

@ -43,18 +43,6 @@ const (
)
func TestLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T) {
t.Run("without-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeNone)
})
t.Run("manual-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeManual)
})
t.Run("auto-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeAuto)
})
}
func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, mode tlsMode) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
@ -64,22 +52,14 @@ func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, mode tlsMode)
c.NodeName = "acceptor"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
if mode == tlsModeManual {
c.ConnectEnabled = false
c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
}
if mode == tlsModeAuto {
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptor.RPC, "dc1")
@ -364,18 +344,6 @@ func TestLeader_PeeringSync_Lifecycle_UnexportWhileDown(t *testing.T) {
}
func TestLeader_PeeringSync_Lifecycle_ServerDeletion(t *testing.T) {
t.Run("without-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeNone)
})
t.Run("manual-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeManual)
})
t.Run("auto-tls", func(t *testing.T) {
testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeAuto)
})
}
func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, mode tlsMode) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
@ -385,22 +353,14 @@ func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, mode tlsMod
c.NodeName = "acceptor"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
if mode == tlsModeManual {
c.ConnectEnabled = false
c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
}
if mode == tlsModeAuto {
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptor.RPC, "dc1")
@ -507,7 +467,7 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) {
t.Run("server-name-validation", func(t *testing.T) {
testLeader_PeeringSync_failsForTLSError(t, func(token *structs.PeeringToken) {
token.ServerName = "wrong.name"
}, `transport: authentication handshake failed: x509: certificate is valid for server.dc1.consul, bob.server.dc1.consul, not wrong.name`)
}, `transport: authentication handshake failed: x509: certificate is valid for server.dc1.peering.11111111-2222-3333-4444-555555555555.consul, not wrong.name`)
})
t.Run("bad-ca-roots", func(t *testing.T) {
wrongRoot, err := ioutil.ReadFile("../../test/client_certs/rootca.crt")
@ -522,15 +482,20 @@ func TestLeader_PeeringSync_FailsForTLSError(t *testing.T) {
func testLeader_PeeringSync_failsForTLSError(t *testing.T, tokenMutateFn func(token *structs.PeeringToken), expectErr string) {
require.NotNil(t, tokenMutateFn)
ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "bob"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.ConnectEnabled = false
c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")
@ -573,10 +538,6 @@ func testLeader_PeeringSync_failsForTLSError(t *testing.T, tokenMutateFn func(to
c.NodeName = "betty"
c.Datacenter = "dc2"
c.PrimaryDatacenter = "dc2"
c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
c.TLSConfig.GRPC.CertFile = "../../test/hostname/Betty.crt"
c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Betty.key"
})
testrpc.WaitForLeader(t, s2.RPC, "dc2")
@ -615,11 +576,11 @@ func TestLeader_Peering_DeferredDeletion(t *testing.T) {
t.Skip("too slow for testing.Short")
}
// TODO(peering): Configure with TLS
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "s1.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCTLSPort = freeport.GetOne(t)
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")
@ -694,15 +655,21 @@ func TestLeader_Peering_DialerReestablishesConnectionOnError(t *testing.T) {
}
// Reserve a gRPC port so we can restart the accepting server with the same port.
ports := freeport.GetN(t, 1)
acceptingServerPort := ports[0]
acceptingServerPort := freeport.GetOne(t)
ca := connect.TestCA(t, nil)
_, acceptingServer := testServerWithConfig(t, func(c *Config) {
c.NodeName = "acceptingServer.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCPort = acceptingServerPort
c.PeeringEnabled = true
c.GRPCTLSPort = acceptingServerPort
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptingServer.RPC, "dc1")
@ -805,9 +772,17 @@ func TestLeader_Peering_DialerReestablishesConnectionOnError(t *testing.T) {
c.NodeName = "acceptingServer.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCPort = acceptingServerPort
c.DataDir = acceptingServer.config.DataDir
c.NodeID = acceptingServer.config.NodeID
c.GRPCTLSPort = acceptingServerPort
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptingServerRestart.RPC, "dc1")
@ -902,11 +877,19 @@ func TestLeader_Peering_ImportedExportedServicesCount(t *testing.T) {
t.Skip("too slow for testing.Short")
}
ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "s1.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.PeeringEnabled = true
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")
@ -1204,11 +1187,19 @@ func TestLeader_PeeringMetrics_emitPeeringMetrics(t *testing.T) {
lastIdx = uint64(0)
)
// TODO(peering): Configure with TLS
ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.NodeName = "s1.dc1"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")
@ -1615,10 +1606,20 @@ func Test_Leader_PeeringSync_ServerAddressUpdates(t *testing.T) {
maxRetryBackoff = 1
t.Cleanup(func() { maxRetryBackoff = orig })
ca := connect.TestCA(t, nil)
_, acceptor := testServerWithConfig(t, func(c *Config) {
c.NodeName = "acceptor"
c.Datacenter = "dc1"
c.TLSConfig.Domain = "consul"
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, acceptor.RPC, "dc1")

23
agent/consul/peering_backend.go
View File

@ -57,26 +57,21 @@ func (b *PeeringBackend) GetLeaderAddress() string {
// GetTLSMaterials returns the TLS materials for the dialer to dial the acceptor using TLS.
// It returns the server name to validate, and the CA certificate to validate with.
func (b *PeeringBackend) GetTLSMaterials() (string, []string, error) {
// Do not send TLS materials to the dialer if we to not have TLS configured for gRPC.
if b.srv.config.GRPCTLSPort <= 0 && !b.srv.tlsConfigurator.GRPCServerUseTLS() {
return "", nil, nil
}
// If the Connect CA is not in use we rely on the manually configured certs.
// Otherwise we rely on the internally managed server certificate.
if !b.srv.config.ConnectEnabled {
serverName := b.srv.tlsConfigurator.ServerSNI(b.srv.config.Datacenter, "")
caPems := b.srv.tlsConfigurator.GRPCManualCAPems()
return serverName, caPems, nil
func (b *PeeringBackend) GetTLSMaterials(generatingToken bool) (string, []string, error) {
if generatingToken {
if !b.srv.config.ConnectEnabled {
return "", nil, fmt.Errorf("connect.enabled must be set to true in the server's configuration when generating peering tokens")
}
if b.srv.config.GRPCTLSPort <= 0 && !b.srv.tlsConfigurator.GRPCServerUseTLS() {
return "", nil, fmt.Errorf("TLS for gRPC must be enabled when generating peering tokens")
}
}
roots, err := b.srv.getCARoots(nil, b.srv.fsm.State())
if err != nil {
return "", nil, fmt.Errorf("failed to fetch roots: %w", err)
}
if len(roots.Roots) == 0 {
if len(roots.Roots) == 0 || roots.TrustDomain == "" {
return "", nil, fmt.Errorf("CA has not finished initializing")
}

28
agent/consul/peering_backend_oss_test.go
View File

@ -11,7 +11,10 @@ import (
"github.com/stretchr/testify/require"
gogrpc "google.golang.org/grpc"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/proto/pbpeering"
"github.com/hashicorp/consul/sdk/freeport"
"github.com/hashicorp/consul/testrpc"
)
@ -21,9 +24,18 @@ func TestPeeringBackend_RejectsPartition(t *testing.T) {
}
t.Parallel()
ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc1"
c.Bootstrap = true
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")
@ -55,9 +67,17 @@ func TestPeeringBackend_IgnoresDefaultPartition(t *testing.T) {
}
t.Parallel()
ca := connect.TestCA(t, nil)
_, s1 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc1"
c.Bootstrap = true
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
testrpc.WaitForLeader(t, s1.RPC, "dc1")

59
agent/consul/peering_backend_test.go
View File

@ -7,17 +7,18 @@ import (
"testing"
"time"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/sdk/freeport"
"github.com/hashicorp/consul/types"
"github.com/stretchr/testify/require"
gogrpc "google.golang.org/grpc"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/pool"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/proto/pbpeering"
"github.com/hashicorp/consul/proto/pbpeerstream"
"github.com/hashicorp/consul/sdk/freeport"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/hashicorp/consul/testrpc"
"github.com/hashicorp/consul/types"
"github.com/stretchr/testify/require"
)
func TestPeeringBackend_ForwardToLeader(t *testing.T) {
@ -25,17 +26,26 @@ func TestPeeringBackend_ForwardToLeader(t *testing.T) {
t.Skip("too slow for testing.Short")
}
_, conf1 := testServerConfig(t)
server1, err := newServer(t, conf1)
require.NoError(t, err)
_, conf2 := testServerConfig(t)
conf2.Bootstrap = false
server2, err := newServer(t, conf2)
require.NoError(t, err)
ca := connect.TestCA(t, nil)
_, server1 := testServerWithConfig(t, func(c *Config) {
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
_, server2 := testServerWithConfig(t, func(c *Config) {
c.Bootstrap = false
})
// Join a 2nd server (not the leader)
testrpc.WaitForLeader(t, server1.RPC, "dc1")
testrpc.WaitForActiveCARoot(t, server1.RPC, "dc1", nil)
joinLAN(t, server2, server1)
testrpc.WaitForLeader(t, server2.RPC, "dc1")
@ -166,17 +176,26 @@ func TestPeerStreamService_ForwardToLeader(t *testing.T) {
t.Skip("too slow for testing.Short")
}
_, conf1 := testServerConfig(t)
server1, err := newServer(t, conf1)
require.NoError(t, err)
_, conf2 := testServerConfig(t)
conf2.Bootstrap = false
server2, err := newServer(t, conf2)
require.NoError(t, err)
ca := connect.TestCA(t, nil)
_, server1 := testServerWithConfig(t, func(c *Config) {
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
_, server2 := testServerWithConfig(t, func(c *Config) {
c.Bootstrap = false
})
// server1 is leader, server2 follower
testrpc.WaitForLeader(t, server1.RPC, "dc1")
testrpc.WaitForActiveCARoot(t, server1.RPC, "dc1", nil)
joinLAN(t, server2, server1)
testrpc.WaitForLeader(t, server2.RPC, "dc1")

12
agent/consul/prepared_query_endpoint_test.go
View File

@ -21,12 +21,14 @@ import (
"github.com/hashicorp/consul-net-rpc/net/rpc"
"github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/connect"
grpcexternal "github.com/hashicorp/consul/agent/grpc-external"
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/agent/structs/aclfilter"
tokenStore "github.com/hashicorp/consul/agent/token"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/proto/pbpeering"
"github.com/hashicorp/consul/sdk/freeport"
"github.com/hashicorp/consul/sdk/testutil/retry"
"github.com/hashicorp/consul/testrpc"
"github.com/hashicorp/consul/types"
@ -1463,10 +1465,20 @@ func TestPreparedQuery_Execute(t *testing.T) {
s2.tokens.UpdateReplicationToken("root", tokenStore.TokenSourceConfig)
ca := connect.TestCA(t, nil)
dir3, s3 := testServerWithConfig(t, func(c *Config) {
c.Datacenter = "dc3"
c.PrimaryDatacenter = "dc3"
c.NodeName = "acceptingServer.dc3"
c.GRPCTLSPort = freeport.GetOne(t)
c.CAConfig = &structs.CAConfiguration{
ClusterID: connect.TestClusterID,
Provider: structs.ConsulCAProvider,
Config: map[string]interface{}{
"PrivateKey": ca.SigningKey,
"RootCert": ca.RootCert,
},
}
})
defer os.RemoveAll(dir3)
defer s3.Shutdown()

1
agent/grpc-external/services/peerstream/stream_test.go
View File

@ -43,7 +43,6 @@ import (
const (
testPeerID = "caf067a6-f112-4907-9101-d45857d2b149"
testActiveStreamSecretID = "e778c518-f0db-473a-9224-24b357da971d"
testPendingStreamSecretID = "522c0daf-2ef2-4dab-bc78-5e04e3daf552"
testEstablishmentSecretID = "f6569d37-1c5b-4415-aae5-26f4594f7f60"
)

16
agent/rpc/peering/service.go
View File

@ -116,7 +116,7 @@ type Backend interface {
// GetTLSMaterials returns the TLS materials for the dialer to dial the acceptor using TLS.
// It returns the server name to validate, and the CA certificate to validate with.
GetTLSMaterials() (string, []string, error)
GetTLSMaterials(generatingToken bool) (string, []string, error)
// GetServerAddresses returns the addresses used for establishing a peering connection.
// These may be server addresses or mesh gateway addresses if peering through mesh gateways.
@ -221,6 +221,11 @@ func (s *Server) GenerateToken(
return nil, err
}
serverName, caPEMs, err := s.Backend.GetTLSMaterials(true)
if err != nil {
return nil, err
}
var (
peering *pbpeering.Peering
secretID string
@ -288,11 +293,6 @@ func (s *Server) GenerateToken(
break
}
serverName, caPEMs, err := s.Backend.GetTLSMaterials()
if err != nil {
return nil, err
}
// ServerExternalAddresses must be formatted as addr:port.
var serverAddrs []string
if len(req.ServerExternalAddresses) > 0 {
@ -484,12 +484,12 @@ func (s *Server) validatePeeringLocality(token *structs.PeeringToken, partition
// If the token has the same server name as this cluster, but we can't find the peering
// in our store, it indicates a naming conflict.
serverName, _, err := s.Backend.GetTLSMaterials()
serverName, _, err := s.Backend.GetTLSMaterials(false)
if err != nil {
return fmt.Errorf("failed to fetch TLS materials: %w", err)
}
if serverName != "" && token.ServerName != "" && serverName == token.ServerName && peering == nil {
if serverName == token.ServerName && peering == nil {
return fmt.Errorf("conflict - peering token's server name matches the current cluster's server name, %q, but there is no record in the database", serverName)
}

2
agent/rpc/peering/service_test.go
View File

@ -398,7 +398,7 @@ func TestPeeringService_Establish_serverNameConflict(t *testing.T) {
id, err := uuid.GenerateUUID()
require.NoError(t, err, "could not generate uuid")
serverName, _, err := s.Server.GetPeeringBackend().GetTLSMaterials()
serverName, _, err := s.Server.GetPeeringBackend().GetTLSMaterials(true)
require.NoError(t, err)
peeringToken := structs.PeeringToken{

6
test/integration/connect/envoy/consul-base-cfg/peering_server.hcl Normal file
View File

@ -0,0 +1,6 @@
ports {
grpc_tls = 8503
}
connect {
enabled = true
}

13
test/integration/connect/envoy/run-tests.sh
View File

@ -104,6 +104,13 @@ function init_workdir {
mv workdir/${CLUSTER}/consul/server.hcl workdir/${CLUSTER}/consul-server/server.hcl
fi
if test -f "workdir/${CLUSTER}/consul/peering_server.hcl" -a $REQUIRE_PEERS = "1"
then
mv workdir/${CLUSTER}/consul/peering_server.hcl workdir/${CLUSTER}/consul-server/peering_server.hcl
else
rm workdir/${CLUSTER}/consul/peering_server.hcl
fi
# copy the ca-certs for SDS so we can verify the right ones are served
mkdir -p workdir/test-sds-server/certs
cp test-sds-server/certs/ca-root.crt workdir/test-sds-server/certs/ca-root.crt
@ -216,11 +223,6 @@ function start_consul {
docker_kill_rm consul-${DC}-server
docker_kill_rm consul-${DC}
server_grpc_port="-1"
if is_set $REQUIRE_PEERS; then
server_grpc_port="8502"
fi
docker run -d --name envoy_consul-${DC}-server_1 \
--net=envoy-tests \
$WORKDIR_SNIPPET \
@ -231,7 +233,6 @@ function start_consul {
agent -dev -datacenter "${DC}" \
-config-dir "/workdir/${DC}/consul" \
-config-dir "/workdir/${DC}/consul-server" \
-grpc-port $server_grpc_port \
-client "0.0.0.0" \
-bind "0.0.0.0" >/dev/null