package consul import ( "fmt" "os" "strings" "testing" "time" "github.com/hashicorp/go-uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" msgpackrpc "github.com/hashicorp/consul-net-rpc/net-rpc-msgpackrpc" "github.com/hashicorp/consul-net-rpc/net/rpc" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/acl/resolver" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/lib/stringslice" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/testrpc" "github.com/hashicorp/consul/types" ) func TestCatalog_Register(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() arg := structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "db", Tags: []string{"primary"}, Port: 8000, }, Check: &structs.HealthCheck{ CheckID: types.CheckID("db-check"), ServiceID: "db", }, } var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out) if err != nil { t.Fatalf("err: %v", err) } } func TestCatalog_RegisterService_InvalidAddress(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() for _, addr := range []string{"0.0.0.0", "::", "[::]"} { t.Run("addr "+addr, func(t *testing.T) { arg := structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "db", Address: addr, Port: 8000, }, } var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out) if err == nil || err.Error() != "Invalid service address" { t.Fatalf("got error %v want 'Invalid service address'", err) } }) } } func TestCatalog_RegisterService_SkipNodeUpdate(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() // Register a node arg := structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", } var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out) if err != nil { t.Fatal(err) } // Update it with a blank address, should fail. arg.Address = "" arg.Service = &structs.NodeService{ Service: "db", Port: 8000, } err = msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out) if err == nil || err.Error() != "Must provide address if SkipNodeUpdate is not set" { t.Fatalf("got error %v want 'Must provide address...'", err) } // Set SkipNodeUpdate, should succeed arg.SkipNodeUpdate = true err = msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out) if err != nil { t.Fatal(err) } } func TestCatalog_Register_NodeID(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() arg := structs.RegisterRequest{ Datacenter: "dc1", ID: "nope", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "db", Tags: []string{"primary"}, Port: 8000, }, Check: &structs.HealthCheck{ CheckID: types.CheckID("db-check"), ServiceID: "db", }, } var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out) if err == nil || !strings.Contains(err.Error(), "Bad node ID") { t.Fatalf("err: %v", err) } arg.ID = types.NodeID("adf4238a-882b-9ddc-4a9d-5b6758e4159e") if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out); err != nil { t.Fatalf("err: %v", err) } } func TestCatalog_Register_ACLDeny(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) defer os.RemoveAll(dir1) defer s1.Shutdown() testrpc.WaitForTestAgent(t, s1.RPC, "dc1", testrpc.WithToken("root")) codec := rpcClient(t, s1) defer codec.Close() rules := ` service "foo" { policy = "write" } node "foo" { policy = "write" } ` id := createToken(t, codec, rules) argR := structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "db", Tags: []string{"primary"}, Port: 8000, }, WriteRequest: structs.WriteRequest{Token: id}, } var outR struct{} // This should fail since we are writing to the "db" service, which isn't // allowed. err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &argR, &outR) if !acl.IsErrPermissionDenied(err) { t.Fatalf("err: %v", err) } // The "foo" service should work, though. argR.Service.Service = "foo" err = msgpackrpc.CallWithCodec(codec, "Catalog.Register", &argR, &outR) if err != nil { t.Fatalf("err: %v", err) } // Try the former special case for the "consul" service. argR.Service.Service = "consul" err = msgpackrpc.CallWithCodec(codec, "Catalog.Register", &argR, &outR) if !acl.IsErrPermissionDenied(err) { t.Fatalf("err: %v", err) } // Register a db service using the root token. argR.Service.Service = "db" argR.Service.ID = "my-id" argR.Token = "root" err = msgpackrpc.CallWithCodec(codec, "Catalog.Register", &argR, &outR) if err != nil { t.Fatalf("err: %v", err) } // Prove that we are properly looking up the node services and passing // that to the ACL helper. We can vet the helper independently in its // own unit test after this. This is trying to register over the db // service we created above, which is a check that depends on looking // at the existing registration data with that service ID. This is a new // check for version 8. argR.Service.Service = "foo" argR.Service.ID = "my-id" argR.Token = id err = msgpackrpc.CallWithCodec(codec, "Catalog.Register", &argR, &outR) if !acl.IsErrPermissionDenied(err) { t.Fatalf("err: %v", err) } } func createTokenFull(t *testing.T, cc rpc.ClientCodec, policyRules string) *structs.ACLToken { t.Helper() return createTokenWithPolicyNameFull(t, cc, "the-policy", policyRules, "root") } func createToken(t *testing.T, cc rpc.ClientCodec, policyRules string) string { return createTokenFull(t, cc, policyRules).SecretID } func createTokenWithPolicyNameFull(t *testing.T, cc rpc.ClientCodec, policyName string, policyRules string, token string) *structs.ACLToken { t.Helper() reqPolicy := structs.ACLPolicySetRequest{ Datacenter: "dc1", Policy: structs.ACLPolicy{ Name: policyName, Rules: policyRules, }, WriteRequest: structs.WriteRequest{Token: token}, } err := msgpackrpc.CallWithCodec(cc, "ACL.PolicySet", &reqPolicy, &structs.ACLPolicy{}) require.NoError(t, err) secretId, err := uuid.GenerateUUID() require.NoError(t, err) reqToken := structs.ACLTokenSetRequest{ Datacenter: "dc1", ACLToken: structs.ACLToken{ SecretID: secretId, Policies: []structs.ACLTokenPolicyLink{{Name: policyName}}, }, WriteRequest: structs.WriteRequest{Token: token}, } resp := &structs.ACLToken{} err = msgpackrpc.CallWithCodec(cc, "ACL.TokenSet", &reqToken, &resp) require.NoError(t, err) return resp } func createTokenWithPolicyName(t *testing.T, cc rpc.ClientCodec, policyName string, policyRules string, token string) string { return createTokenWithPolicyNameFull(t, cc, policyName, policyRules, token).SecretID } func TestCatalog_Register_ForwardLeader(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec1 := rpcClient(t, s1) defer codec1.Close() dir2, s2 := testServer(t) defer os.RemoveAll(dir2) defer s2.Shutdown() codec2 := rpcClient(t, s2) defer codec2.Close() // Try to join joinLAN(t, s2, s1) testrpc.WaitForLeader(t, s1.RPC, "dc1") testrpc.WaitForLeader(t, s2.RPC, "dc1") // Use the follower as the client var codec rpc.ClientCodec if !s1.IsLeader() { codec = codec1 } else { codec = codec2 } arg := structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "db", Tags: []string{"primary"}, Port: 8000, }, } var out struct{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out); err != nil { t.Fatalf("err: %v", err) } } func TestCatalog_Register_ForwardDC(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() dir2, s2 := testServerDC(t, "dc2") defer os.RemoveAll(dir2) defer s2.Shutdown() // Try to join joinWAN(t, s2, s1) testrpc.WaitForLeader(t, s1.RPC, "dc2") arg := structs.RegisterRequest{ Datacenter: "dc2", // Should forward through s1 Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "db", Tags: []string{"primary"}, Port: 8000, }, } var out struct{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out); err != nil { t.Fatalf("err: %v", err) } } func TestCatalog_Register_ConnectProxy(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.TestRegisterRequestProxy(t) // Register var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // List req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: args.Service.Service, } var resp structs.IndexedServiceNodes assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) assert.Len(t, resp.ServiceNodes, 1) v := resp.ServiceNodes[0] assert.Equal(t, structs.ServiceKindConnectProxy, v.ServiceKind) assert.Equal(t, args.Service.Proxy.DestinationServiceName, v.ServiceProxy.DestinationServiceName) } // Test an invalid ConnectProxy. We don't need to exhaustively test because // this is all tested in structs on the Validate method. func TestCatalog_Register_ConnectProxy_invalid(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.TestRegisterRequestProxy(t) args.Service.Proxy.DestinationServiceName = "" // Register var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out) assert.NotNil(t, err) assert.Contains(t, err.Error(), "DestinationServiceName") } // Test that write is required for the proxy destination to register a proxy. func TestCatalog_Register_ConnectProxy_ACLDestinationServiceName(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") rules := ` service "foo" { policy = "write" } node "foo" { policy = "write" } ` token := createToken(t, codec, rules) // Register should fail because we don't have permission on the destination args := structs.TestRegisterRequestProxy(t) args.Service.Service = "foo" args.Service.Proxy.DestinationServiceName = "bar" args.WriteRequest.Token = token var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out) assert.True(t, acl.IsErrPermissionDenied(err)) // Register should fail with the right destination but wrong name args = structs.TestRegisterRequestProxy(t) args.Service.Service = "bar" args.Service.Proxy.DestinationServiceName = "foo" args.WriteRequest.Token = token err = msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out) assert.True(t, acl.IsErrPermissionDenied(err)) // Register should work with the right destination args = structs.TestRegisterRequestProxy(t) args.Service.Service = "foo" args.Service.Proxy.DestinationServiceName = "foo" args.WriteRequest.Token = token assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) } func TestCatalog_Register_ConnectNative(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.TestRegisterRequest(t) args.Service.Connect.Native = true // Register var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // List req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: args.Service.Service, } var resp structs.IndexedServiceNodes assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) assert.Len(t, resp.ServiceNodes, 1) v := resp.ServiceNodes[0] assert.Equal(t, structs.ServiceKindTypical, v.ServiceKind) assert.True(t, v.ServiceConnect.Native) } func TestCatalog_Deregister(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() arg := structs.DeregisterRequest{ Datacenter: "dc1", Node: "foo", } var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &arg, &out) if err != nil { t.Fatalf("err: %v", err) } testrpc.WaitForLeader(t, s1.RPC, "dc1") if err := msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &arg, &out); err != nil { t.Fatalf("err: %v", err) } } func TestCatalog_Deregister_ACLDeny(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") rules := ` node "node" { policy = "write" } service "service" { policy = "write" } ` id := createToken(t, codec, rules) // Register a node, node check, service, and service check. argR := structs.RegisterRequest{ Datacenter: "dc1", Node: "node", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "service", Port: 8000, }, Checks: structs.HealthChecks{ &structs.HealthCheck{ Node: "node", CheckID: "node-check", }, &structs.HealthCheck{ Node: "node", CheckID: "service-check", ServiceID: "service", }, }, WriteRequest: structs.WriteRequest{Token: id}, } var outR struct{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &argR, &outR); err != nil { t.Fatalf("err: %v", err) } // We should be not be able to deregister everything without a token. var err error var out struct{} err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node", CheckID: "service-check"}, &out) if !acl.IsErrPermissionDenied(err) { t.Fatalf("err: %v", err) } err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node", CheckID: "node-check"}, &out) if !acl.IsErrPermissionDenied(err) { t.Fatalf("err: %v", err) } err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node", ServiceID: "service"}, &out) if !acl.IsErrPermissionDenied(err) { t.Fatalf("err: %v", err) } err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node"}, &out) if !acl.IsErrPermissionDenied(err) { t.Fatalf("err: %v", err) } // Second pass these should all go through with the token set. err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node", CheckID: "service-check", WriteRequest: structs.WriteRequest{ Token: id, }}, &out) if err != nil { t.Fatalf("err: %v", err) } err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node", CheckID: "node-check", WriteRequest: structs.WriteRequest{ Token: id, }}, &out) if err != nil { t.Fatalf("err: %v", err) } err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node", ServiceID: "service", WriteRequest: structs.WriteRequest{ Token: id, }}, &out) if err != nil { t.Fatalf("err: %v", err) } err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "node", WriteRequest: structs.WriteRequest{ Token: id, }}, &out) if err != nil { t.Fatalf("err: %v", err) } // Try a few error cases. err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "nope", ServiceID: "nope", WriteRequest: structs.WriteRequest{ Token: id, }}, &out) if err == nil || !strings.Contains(err.Error(), "Unknown service") { t.Fatalf("err: %v", err) } err = msgpackrpc.CallWithCodec(codec, "Catalog.Deregister", &structs.DeregisterRequest{ Datacenter: "dc1", Node: "nope", CheckID: "nope", WriteRequest: structs.WriteRequest{ Token: id, }}, &out) if err == nil || !strings.Contains(err.Error(), "Unknown check") { t.Fatalf("err: %v", err) } } func TestCatalog_ListDatacenters(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() dir2, s2 := testServerDC(t, "dc2") defer os.RemoveAll(dir2) defer s2.Shutdown() // Try to join joinWAN(t, s2, s1) testrpc.WaitForLeader(t, s1.RPC, "dc1") var out []string if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListDatacenters", struct{}{}, &out); err != nil { t.Fatalf("err: %v", err) } // The DCs should come out sorted by default. if len(out) != 2 { t.Fatalf("bad: %v", out) } if out[0] != "dc1" { t.Fatalf("bad: %v", out) } if out[1] != "dc2" { t.Fatalf("bad: %v", out) } } func TestCatalog_ListDatacenters_DistanceSort(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() dir2, s2 := testServerDC(t, "dc2") defer os.RemoveAll(dir2) defer s2.Shutdown() dir3, s3 := testServerDC(t, "acdc") defer os.RemoveAll(dir3) defer s3.Shutdown() // Try to join joinWAN(t, s2, s1) joinWAN(t, s3, s1) testrpc.WaitForLeader(t, s1.RPC, "dc1") var out []string if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListDatacenters", struct{}{}, &out); err != nil { t.Fatalf("err: %v", err) } // It's super hard to force the Serfs into a known configuration of // coordinates, so the best we can do is make sure that the sorting // function is getting called (it's tested extensively in rtt_test.go). // Since this is relative to dc1, it will be listed first (proving we // went into the sort fn). if len(out) != 3 { t.Fatalf("bad: %v", out) } if out[0] != "dc1" { t.Fatalf("bad: %v", out) } } func TestCatalog_ListNodes(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.DCSpecificRequest{ Datacenter: "dc1", } var out structs.IndexedNodes err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if err != nil { t.Fatalf("err: %v", err) } testrpc.WaitForLeader(t, s1.RPC, "dc1") // Just add a node if err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } retry.Run(t, func(r *retry.R) { msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if got, want := len(out.Nodes), 2; got != want { r.Fatalf("got %d nodes want %d", got, want) } }) // Server node is auto added from Serf if out.Nodes[1].Node != s1.config.NodeName { t.Fatalf("bad: %v", out) } if out.Nodes[0].Node != "foo" { t.Fatalf("bad: %v", out) } if out.Nodes[0].Address != "127.0.0.1" { t.Fatalf("bad: %v", out) } require.False(t, out.QueryMeta.NotModified) t.Run("with option AllowNotModifiedResponse", func(t *testing.T) { args.QueryOptions = structs.QueryOptions{ MinQueryIndex: out.QueryMeta.Index, MaxQueryTime: 20 * time.Millisecond, AllowNotModifiedResponse: true, } err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) require.NoError(t, err) require.Equal(t, out.Index, out.QueryMeta.Index) require.Len(t, out.Nodes, 0) require.True(t, out.QueryMeta.NotModified, "NotModified should be true") }) } func TestCatalog_ListNodes_NodeMetaFilter(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") // Add a new node with the right meta k/v pair node := &structs.Node{Node: "foo", Address: "127.0.0.1", Meta: map[string]string{"somekey": "somevalue"}} if err := s1.fsm.State().EnsureNode(1, node); err != nil { t.Fatalf("err: %v", err) } // Filter by a specific meta k/v pair args := structs.DCSpecificRequest{ Datacenter: "dc1", NodeMetaFilters: map[string]string{ "somekey": "somevalue", }, } var out structs.IndexedNodes retry.Run(t, func(r *retry.R) { msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if got, want := len(out.Nodes), 1; got != want { r.Fatalf("got %d nodes want %d", got, want) } }) // Verify that only the correct node was returned if out.Nodes[0].Node != "foo" { t.Fatalf("bad: %v", out) } if out.Nodes[0].Address != "127.0.0.1" { t.Fatalf("bad: %v", out) } if v, ok := out.Nodes[0].Meta["somekey"]; !ok || v != "somevalue" { t.Fatalf("bad: %v", out) } // Now filter on a nonexistent meta k/v pair args = structs.DCSpecificRequest{ Datacenter: "dc1", NodeMetaFilters: map[string]string{ "somekey": "invalid", }, } out = structs.IndexedNodes{} err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if err != nil { t.Fatalf("err: %v", err) } // Should get an empty list of nodes back retry.Run(t, func(r *retry.R) { msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if len(out.Nodes) != 0 { r.Fatal(nil) } }) } func TestCatalog_RPC_Filter(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") // prep the cluster with some data we can use in our filters registerTestCatalogEntries(t, codec) // Run the tests against the test server t.Run("ListNodes", func(t *testing.T) { args := structs.DCSpecificRequest{ Datacenter: "dc1", QueryOptions: structs.QueryOptions{Filter: "Meta.os == linux"}, } out := new(structs.IndexedNodes) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, out)) require.Len(t, out.Nodes, 2) require.Condition(t, func() bool { return (out.Nodes[0].Node == "foo" && out.Nodes[1].Node == "baz") || (out.Nodes[0].Node == "baz" && out.Nodes[1].Node == "foo") }) args.Filter = "Meta.os == linux and Meta.env == qa" out = new(structs.IndexedNodes) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, out)) require.Len(t, out.Nodes, 1) require.Equal(t, "baz", out.Nodes[0].Node) }) t.Run("ServiceNodes", func(t *testing.T) { args := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "redis", QueryOptions: structs.QueryOptions{Filter: "ServiceMeta.version == 1"}, } out := new(structs.IndexedServiceNodes) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out)) require.Len(t, out.ServiceNodes, 2) require.Condition(t, func() bool { return (out.ServiceNodes[0].Node == "foo" && out.ServiceNodes[1].Node == "bar") || (out.ServiceNodes[0].Node == "bar" && out.ServiceNodes[1].Node == "foo") }) args.Filter = "ServiceMeta.version == 2" out = new(structs.IndexedServiceNodes) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out)) require.Len(t, out.ServiceNodes, 1) require.Equal(t, "foo", out.ServiceNodes[0].Node) }) t.Run("NodeServices", func(t *testing.T) { args := structs.NodeSpecificRequest{ Datacenter: "dc1", Node: "baz", QueryOptions: structs.QueryOptions{Filter: "Service == web"}, } out := new(structs.IndexedNodeServices) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &args, &out)) require.Len(t, out.NodeServices.Services, 2) args.Filter = "Service == web and Meta.version == 2" out = new(structs.IndexedNodeServices) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &args, &out)) require.Len(t, out.NodeServices.Services, 1) }) } func TestCatalog_ListNodes_StaleRead(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec1 := rpcClient(t, s1) defer codec1.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") dir2, s2 := testServerDCBootstrap(t, "dc1", false) defer os.RemoveAll(dir2) defer s2.Shutdown() codec2 := rpcClient(t, s2) defer codec2.Close() // Try to join joinLAN(t, s2, s1) testrpc.WaitForTestAgent(t, s1.RPC, "dc1") testrpc.WaitForTestAgent(t, s2.RPC, "dc1") // Use the follower as the client var codec rpc.ClientCodec if !s1.IsLeader() { codec = codec1 // Inject fake data on the follower! if err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } } else { codec = codec2 // Inject fake data on the follower! if err := s2.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } } args := structs.DCSpecificRequest{ Datacenter: "dc1", QueryOptions: structs.QueryOptions{AllowStale: true}, } var out structs.IndexedNodes retry.Run(t, func(r *retry.R) { if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out); err != nil { r.Fatalf("err: %v", err) } found := false for _, n := range out.Nodes { if n.Node == "foo" { found = true } } if !found { r.Fatalf("failed to find foo in %#v", out.Nodes) } if out.QueryMeta.LastContact == 0 { r.Fatalf("should have a last contact time") } if !out.QueryMeta.KnownLeader { r.Fatalf("should have known leader") } }) } func TestCatalog_ListNodes_ConsistentRead_Fail(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() dir2, s2 := testServerDCBootstrap(t, "dc1", false) defer os.RemoveAll(dir2) defer s2.Shutdown() dir3, s3 := testServerDCBootstrap(t, "dc1", false) defer os.RemoveAll(dir3) defer s3.Shutdown() // Try to join and wait for all servers to get promoted to voters. joinLAN(t, s2, s1) joinLAN(t, s3, s2) servers := []*Server{s1, s2, s3} retry.Run(t, func(r *retry.R) { r.Check(wantRaft(servers)) for _, s := range servers { r.Check(wantPeers(s, 3)) } }) // Use the leader as the client, kill the followers. var codec rpc.ClientCodec for _, s := range servers { if s.IsLeader() { codec = rpcClient(t, s) defer codec.Close() } else { s.Shutdown() } } if codec == nil { t.Fatalf("no leader") } args := structs.DCSpecificRequest{ Datacenter: "dc1", QueryOptions: structs.QueryOptions{RequireConsistent: true}, } var out structs.IndexedNodes err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if err == nil || !strings.HasPrefix(err.Error(), "leadership lost") { t.Fatalf("err: %v", err) } if out.QueryMeta.LastContact != 0 { t.Fatalf("should not have a last contact time") } if out.QueryMeta.KnownLeader { t.Fatalf("should have no known leader") } } func TestCatalog_ListNodes_ConsistentRead(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec1 := rpcClient(t, s1) defer codec1.Close() dir2, s2 := testServerDCBootstrap(t, "dc1", false) defer os.RemoveAll(dir2) defer s2.Shutdown() codec2 := rpcClient(t, s2) defer codec2.Close() // Try to join joinLAN(t, s2, s1) testrpc.WaitForLeader(t, s1.RPC, "dc1") testrpc.WaitForLeader(t, s2.RPC, "dc1") // Use the leader as the client, kill the follower var codec rpc.ClientCodec if s1.IsLeader() { codec = codec1 } else { codec = codec2 } args := structs.DCSpecificRequest{ Datacenter: "dc1", QueryOptions: structs.QueryOptions{RequireConsistent: true}, } var out structs.IndexedNodes if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out); err != nil { t.Fatalf("err: %v", err) } if out.QueryMeta.LastContact != 0 { t.Fatalf("should not have a last contact time") } if !out.QueryMeta.KnownLeader { t.Fatalf("should have known leader") } } func TestCatalog_ListNodes_DistanceSort(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") if err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "aaa", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureNode(2, &structs.Node{Node: "foo", Address: "127.0.0.2"}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureNode(3, &structs.Node{Node: "bar", Address: "127.0.0.3"}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureNode(4, &structs.Node{Node: "baz", Address: "127.0.0.4"}); err != nil { t.Fatalf("err: %v", err) } // Set all but one of the nodes to known coordinates. updates := structs.Coordinates{ {Node: "foo", Coord: lib.GenerateCoordinate(2 * time.Millisecond)}, {Node: "bar", Coord: lib.GenerateCoordinate(5 * time.Millisecond)}, {Node: "baz", Coord: lib.GenerateCoordinate(1 * time.Millisecond)}, } if err := s1.fsm.State().CoordinateBatchUpdate(5, updates); err != nil { t.Fatalf("err: %v", err) } // Query with no given source node, should get the natural order from // the index. args := structs.DCSpecificRequest{ Datacenter: "dc1", } var out structs.IndexedNodes retry.Run(t, func(r *retry.R) { msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if got, want := len(out.Nodes), 5; got != want { r.Fatalf("got %d nodes want %d", got, want) } }) if out.Nodes[0].Node != "aaa" { t.Fatalf("bad: %v", out) } if out.Nodes[1].Node != "bar" { t.Fatalf("bad: %v", out) } if out.Nodes[2].Node != "baz" { t.Fatalf("bad: %v", out) } if out.Nodes[3].Node != "foo" { t.Fatalf("bad: %v", out) } if out.Nodes[4].Node != s1.config.NodeName { t.Fatalf("bad: %v", out) } // Query relative to foo, note that there's no known coordinate for the // default-added Serf node nor "aaa" so they will go at the end. args = structs.DCSpecificRequest{ Datacenter: "dc1", Source: structs.QuerySource{Datacenter: "dc1", Node: "foo"}, } retry.Run(t, func(r *retry.R) { msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out) if got, want := len(out.Nodes), 5; got != want { r.Fatalf("got %d nodes want %d", got, want) } }) if out.Nodes[0].Node != "foo" { t.Fatalf("bad: %v", out) } if out.Nodes[1].Node != "baz" { t.Fatalf("bad: %v", out) } if out.Nodes[2].Node != "bar" { t.Fatalf("bad: %v", out) } if out.Nodes[3].Node != "aaa" { t.Fatalf("bad: %v", out) } if out.Nodes[4].Node != s1.config.NodeName { t.Fatalf("bad: %v", out) } } func TestCatalog_ListNodes_ACLFilter(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") token := func(policy string) string { rules := fmt.Sprintf( `node "%s" { policy = "%s" }`, s1.config.NodeName, policy, ) return createTokenWithPolicyName(t, codec, policy, rules, "root") } args := structs.DCSpecificRequest{ Datacenter: "dc1", } t.Run("deny", func(t *testing.T) { args.Token = token("deny") var reply structs.IndexedNodes if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &reply); err != nil { t.Fatalf("err: %v", err) } if len(reply.Nodes) != 0 { t.Fatalf("bad: %v", reply.Nodes) } if !reply.QueryMeta.ResultsFilteredByACLs { t.Fatal("ResultsFilteredByACLs should be true") } }) t.Run("allow", func(t *testing.T) { args.Token = token("read") var reply structs.IndexedNodes if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &reply); err != nil { t.Fatalf("err: %v", err) } if len(reply.Nodes) != 1 { t.Fatalf("bad: %v", reply.Nodes) } if reply.QueryMeta.ResultsFilteredByACLs { t.Fatal("ResultsFilteredByACLs should not true") } }) } func Benchmark_Catalog_ListNodes(t *testing.B) { dir1, s1 := testServer(nil) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(nil, s1) defer codec.Close() // Just add a node if err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } args := structs.DCSpecificRequest{ Datacenter: "dc1", } for i := 0; i < t.N; i++ { var out structs.IndexedNodes if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListNodes", &args, &out); err != nil { t.Fatalf("err: %v", err) } } } func TestCatalog_ListServices(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.DCSpecificRequest{ Datacenter: "dc1", } var out structs.IndexedServices err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out) if err != nil { t.Fatalf("err: %v", err) } testrpc.WaitForLeader(t, s1.RPC, "dc1") // Just add a node if err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}); err != nil { t.Fatalf("err: %v", err) } if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { t.Fatalf("err: %v", err) } if len(out.Services) != 2 { t.Fatalf("bad: %v", out) } for _, s := range out.Services { if s == nil { t.Fatalf("bad: %v", s) } } // Consul service should auto-register if _, ok := out.Services["consul"]; !ok { t.Fatalf("bad: %v", out) } if len(out.Services["db"]) != 1 { t.Fatalf("bad: %v", out) } if out.Services["db"][0] != "primary" { t.Fatalf("bad: %v", out) } require.False(t, out.QueryMeta.NotModified) require.False(t, out.QueryMeta.ResultsFilteredByACLs) t.Run("with option AllowNotModifiedResponse", func(t *testing.T) { args.QueryOptions = structs.QueryOptions{ MinQueryIndex: out.QueryMeta.Index, MaxQueryTime: 20 * time.Millisecond, AllowNotModifiedResponse: true, } err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out) require.NoError(t, err) require.Equal(t, out.Index, out.QueryMeta.Index) require.Len(t, out.Services, 0) require.True(t, out.QueryMeta.NotModified, "NotModified should be true") }) } func TestCatalog_ListServices_NodeMetaFilter(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") // Add a new node with the right meta k/v pair node := &structs.Node{Node: "foo", Address: "127.0.0.1", Meta: map[string]string{"somekey": "somevalue"}} if err := s1.fsm.State().EnsureNode(1, node); err != nil { t.Fatalf("err: %v", err) } // Add a service to the new node if err := s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}); err != nil { t.Fatalf("err: %v", err) } // Filter by a specific meta k/v pair args := structs.DCSpecificRequest{ Datacenter: "dc1", NodeMetaFilters: map[string]string{ "somekey": "somevalue", }, } var out structs.IndexedServices if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { t.Fatalf("err: %v", err) } if len(out.Services) != 1 { t.Fatalf("bad: %v", out) } if out.Services["db"] == nil { t.Fatalf("bad: %v", out.Services["db"]) } if len(out.Services["db"]) != 1 { t.Fatalf("bad: %v", out) } if out.Services["db"][0] != "primary" { t.Fatalf("bad: %v", out) } // Now filter on a nonexistent meta k/v pair args = structs.DCSpecificRequest{ Datacenter: "dc1", NodeMetaFilters: map[string]string{ "somekey": "invalid", }, } out = structs.IndexedServices{} err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out) if err != nil { t.Fatalf("err: %v", err) } // Should get an empty list of nodes back if len(out.Services) != 0 { t.Fatalf("bad: %v", out.Services) } } func TestCatalog_ListServices_Filter(t *testing.T) { t.Parallel() _, s1 := testServer(t) codec := rpcClient(t, s1) testrpc.WaitForTestAgent(t, s1.RPC, "dc1") // prep the cluster with some data we can use in our filters registerTestCatalogEntries(t, codec) // Run the tests against the test server t.Run("ListServices", func(t *testing.T) { args := structs.DCSpecificRequest{ Datacenter: "dc1", } args.Filter = "ServiceName == redis" out := new(structs.IndexedServices) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) require.Contains(t, out.Services, "redis") require.ElementsMatch(t, []string{"v1", "v2"}, out.Services["redis"]) args.Filter = "NodeMeta.os == NoSuchOS" out = new(structs.IndexedServices) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) require.Len(t, out.Services, 0) args.Filter = "NodeMeta.NoSuchMetadata == linux" out = new(structs.IndexedServices) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) require.Len(t, out.Services, 0) args.Filter = "InvalidField == linux" out = new(structs.IndexedServices) require.Error(t, msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, out)) }) } func TestCatalog_ListServices_Blocking(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.DCSpecificRequest{ Datacenter: "dc1", } var out structs.IndexedServices testrpc.WaitForLeader(t, s1.RPC, "dc1") // Run the query if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { t.Fatalf("err: %v", err) } // Setup a blocking query args.MinQueryIndex = out.Index args.MaxQueryTime = time.Second // Async cause a change idx := out.Index start := time.Now() go func() { time.Sleep(100 * time.Millisecond) if err := s1.fsm.State().EnsureNode(idx+1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Errorf("err: %v", err) return } if err := s1.fsm.State().EnsureService(idx+2, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}); err != nil { t.Errorf("err: %v", err) } }() // Re-run the query out = structs.IndexedServices{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { t.Fatalf("err: %v", err) } // Should block at least 100ms if time.Since(start) < 100*time.Millisecond { t.Fatalf("too fast") } // Check the indexes if out.Index != idx+2 { t.Fatalf("bad: %v", out) } // Should find the service if len(out.Services) != 2 { t.Fatalf("bad: %v", out) } } func TestCatalog_ListServices_Timeout(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.DCSpecificRequest{ Datacenter: "dc1", } var out structs.IndexedServices testrpc.WaitForLeader(t, s1.RPC, "dc1") // Run the query if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { t.Fatalf("err: %v", err) } // Setup a blocking query args.MinQueryIndex = out.Index args.MaxQueryTime = 100 * time.Millisecond // Re-run the query start := time.Now() out = structs.IndexedServices{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { t.Fatalf("err: %v", err) } // Should block at least 100ms if time.Since(start) < 100*time.Millisecond { t.Fatalf("too fast") } // Check the indexes, should not change if out.Index != args.MinQueryIndex { t.Fatalf("bad: %v", out) } } func TestCatalog_ListServices_Stale(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true }) defer os.RemoveAll(dir1) defer s1.Shutdown() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") dir2, s2 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" // Enable ACLs! c.ACLsEnabled = true c.Bootstrap = false // Disable bootstrap }) defer os.RemoveAll(dir2) defer s2.Shutdown() args := structs.DCSpecificRequest{ Datacenter: "dc1", } args.AllowStale = true var out structs.IndexedServices // Inject a node if err := s1.fsm.State().EnsureNode(3, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } codec := rpcClient(t, s2) defer codec.Close() // Run the query, do not wait for leader, never any contact with leader, should fail if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err.Error() != structs.ErrNoLeader.Error() { t.Fatalf("expected %v but got err: %v and %v", structs.ErrNoLeader, err, out) } // Try to join joinLAN(t, s2, s1) retry.Run(t, func(r *retry.R) { r.Check(wantRaft([]*Server{s1, s2})) }) waitForLeader(s1, s2) testrpc.WaitForLeader(t, s2.RPC, "dc1") retry.Run(t, func(r *retry.R) { out = structs.IndexedServices{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { r.Fatalf("err: %v", err) } // Should find the services if len(out.Services) != 1 { r.Fatalf("bad: %#v", out.Services) } if !out.KnownLeader { r.Fatalf("should have a leader: %v", out) } }) s1.Leave() s1.Shutdown() testrpc.WaitUntilNoLeader(t, s2.RPC, "dc1") args.AllowStale = false // Since the leader is now down, non-stale query should fail now out = structs.IndexedServices{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err.Error() != structs.ErrLeaderNotTracked.Error() { t.Fatalf("expected %v but got err: %v and %v", structs.ErrNoLeader, err, out) } if out.KnownLeader { t.Fatalf("should not have a leader anymore: %#v", out) } // With stale, request should still work args.AllowStale = true retry.Run(t, func(r *retry.R) { out = structs.IndexedServices{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &args, &out); err != nil { r.Fatalf("err: %v", err) } if out.KnownLeader || len(out.Services) != 1 { r.Fatalf("got %t nodes want %d", out.KnownLeader, len(out.Services)) } }) } func TestCatalog_ListServiceNodes(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", ServiceTags: []string{"replica"}, TagFilter: false, } var out structs.IndexedServiceNodes err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out) if err != nil { t.Fatalf("err: %v", err) } testrpc.WaitForLeader(t, s1.RPC, "dc1") // Just add a node if err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}); err != nil { t.Fatalf("err: %v", err) } if err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out); err != nil { t.Fatalf("err: %v", err) } if len(out.ServiceNodes) != 1 { t.Fatalf("bad: %v", out) } // Try with a filter args.TagFilter = true out = structs.IndexedServiceNodes{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out); err != nil { t.Fatalf("err: %v", err) } if len(out.ServiceNodes) != 0 { t.Fatalf("bad: %v", out) } } func TestCatalog_ListServiceNodes_ByAddress(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") args := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", } var out structs.IndexedServiceNodes require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out)) fooAddress := "10.1.2.3" fooPort := 1111 fooTaggedAddresses := map[string]structs.ServiceAddress{ "lan": { Address: "10.1.2.3", Port: fooPort, }, "wan": { Address: "198.18.1.2", Port: fooPort, }, } barAddress := "10.1.2.3" barPort := 2222 barTaggedAddresses := map[string]structs.ServiceAddress{ "lan": { Address: "10.1.2.3", Port: barPort, }, "wan": { Address: "198.18.2.3", Port: barPort, }, } bazAddress := "192.168.1.35" bazPort := 2222 bazTaggedAddresses := map[string]structs.ServiceAddress{ "lan": { Address: "192.168.1.35", Port: barPort, }, "wan": { Address: "198.18.2.4", Port: barPort, }, } // Just add a node require.NoError(t, s1.fsm.State().EnsureNode(1, &structs.Node{Node: "node", Address: "127.0.0.1"})) require.NoError(t, s1.fsm.State().EnsureService(2, "node", &structs.NodeService{ID: "foo", Service: "db", Address: fooAddress, TaggedAddresses: fooTaggedAddresses, Port: fooPort})) require.NoError(t, s1.fsm.State().EnsureService(2, "node", &structs.NodeService{ID: "bar", Service: "db", Address: barAddress, TaggedAddresses: barTaggedAddresses, Port: barPort})) require.NoError(t, s1.fsm.State().EnsureService(2, "node", &structs.NodeService{ID: "baz", Service: "db", Address: bazAddress, TaggedAddresses: bazTaggedAddresses, Port: bazPort})) require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out)) require.Len(t, out.ServiceNodes, 3) // Try with an address that would match foo & bar args.ServiceAddress = "10.1.2.3" out = structs.IndexedServiceNodes{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out)) require.Len(t, out.ServiceNodes, 2) for _, sn := range out.ServiceNodes { require.True(t, sn.ServiceID == "foo" || sn.ServiceID == "bar") } // Try with an address that would match just bar args.ServiceAddress = "198.18.2.3" out = structs.IndexedServiceNodes{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out)) require.Len(t, out.ServiceNodes, 1) require.Equal(t, "bar", out.ServiceNodes[0].ServiceID) } // TestCatalog_ListServiceNodes_ServiceTags_V1_2_3Compat asserts the compatibility between <=v1.2.3 agents and >=v1.3.0 servers // see https://github.com/hashicorp/consul/issues/4922 func TestCatalog_ListServiceNodes_ServiceTags_V1_2_3Compat(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}) require.NoError(t, err) // register two service instances with different tags err = s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{ID: "db1", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}) require.NoError(t, err) err = s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{ID: "db2", Service: "db", Tags: []string{"secondary"}, Address: "127.0.0.1", Port: 5001}) require.NoError(t, err) // DEPRECATED (singular-service-tag) - remove this when backwards RPC compat // with 1.2.x is not required. // make a request with the <=1.2.3 ServiceTag tag field (vs ServiceTags) args := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", ServiceTag: "primary", TagFilter: true, } var out structs.IndexedServiceNodes err = msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out) require.NoError(t, err) // nodes should be filtered, even when using the old ServiceTag field require.Equal(t, 1, len(out.ServiceNodes)) require.Equal(t, "db1", out.ServiceNodes[0].ServiceID) // DEPRECATED (singular-service-tag) - remove this when backwards RPC compat // with 1.2.x is not required. // test with the other tag args = structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", ServiceTag: "secondary", TagFilter: true, } out = structs.IndexedServiceNodes{} err = msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out) require.NoError(t, err) require.Equal(t, 1, len(out.ServiceNodes)) require.Equal(t, "db2", out.ServiceNodes[0].ServiceID) // no tag, both instances args = structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", } out = structs.IndexedServiceNodes{} err = msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out) require.NoError(t, err) require.Equal(t, 2, len(out.ServiceNodes)) // DEPRECATED (singular-service-tag) - remove this when backwards RPC compat // with 1.2.x is not required. // when both ServiceTag and ServiceTags fields are populated, use ServiceTag args = structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", ServiceTag: "primary", ServiceTags: []string{"secondary"}, TagFilter: true, } out = structs.IndexedServiceNodes{} err = msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out) require.NoError(t, err) require.Equal(t, "db1", out.ServiceNodes[0].ServiceID) } func TestCatalog_ListServiceNodes_NodeMetaFilter(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") // Add 2 nodes with specific meta maps node := &structs.Node{Node: "foo", Address: "127.0.0.1", Meta: map[string]string{"somekey": "somevalue", "common": "1"}} if err := s1.fsm.State().EnsureNode(1, node); err != nil { t.Fatalf("err: %v", err) } node2 := &structs.Node{Node: "bar", Address: "127.0.0.2", Meta: map[string]string{"common": "1"}} if err := s1.fsm.State().EnsureNode(2, node2); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureService(3, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary", "v2"}, Address: "127.0.0.1", Port: 5000}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureService(4, "bar", &structs.NodeService{ID: "db2", Service: "db", Tags: []string{"secondary", "v2"}, Address: "127.0.0.2", Port: 5000}); err != nil { t.Fatalf("err: %v", err) } cases := []struct { filters map[string]string tags []string services structs.ServiceNodes }{ // Basic meta filter { filters: map[string]string{"somekey": "somevalue"}, services: structs.ServiceNodes{&structs.ServiceNode{Node: "foo", ServiceID: "db"}}, }, // Basic meta filter, tag { filters: map[string]string{"somekey": "somevalue"}, tags: []string{"primary"}, services: structs.ServiceNodes{&structs.ServiceNode{Node: "foo", ServiceID: "db"}}, }, // Common meta filter { filters: map[string]string{"common": "1"}, services: structs.ServiceNodes{ &structs.ServiceNode{Node: "bar", ServiceID: "db2"}, &structs.ServiceNode{Node: "foo", ServiceID: "db"}, }, }, // Common meta filter, tag { filters: map[string]string{"common": "1"}, tags: []string{"secondary"}, services: structs.ServiceNodes{ &structs.ServiceNode{Node: "bar", ServiceID: "db2"}, }, }, // Invalid meta filter { filters: map[string]string{"invalid": "nope"}, services: structs.ServiceNodes{}, }, // Multiple filter values { filters: map[string]string{"somekey": "somevalue", "common": "1"}, services: structs.ServiceNodes{&structs.ServiceNode{Node: "foo", ServiceID: "db"}}, }, // Multiple filter values, tag { filters: map[string]string{"somekey": "somevalue", "common": "1"}, tags: []string{"primary"}, services: structs.ServiceNodes{&structs.ServiceNode{Node: "foo", ServiceID: "db"}}, }, // Common meta filter, single tag { filters: map[string]string{"common": "1"}, tags: []string{"v2"}, services: structs.ServiceNodes{ &structs.ServiceNode{Node: "bar", ServiceID: "db2"}, &structs.ServiceNode{Node: "foo", ServiceID: "db"}, }, }, // Common meta filter, multiple tags { filters: map[string]string{"common": "1"}, tags: []string{"v2", "primary"}, services: structs.ServiceNodes{&structs.ServiceNode{Node: "foo", ServiceID: "db"}}, }, } for _, tc := range cases { args := structs.ServiceSpecificRequest{ Datacenter: "dc1", NodeMetaFilters: tc.filters, ServiceName: "db", ServiceTags: tc.tags, TagFilter: len(tc.tags) > 0, } var out structs.IndexedServiceNodes require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out)) require.Len(t, out.ServiceNodes, len(tc.services)) for i, serviceNode := range out.ServiceNodes { if serviceNode.Node != tc.services[i].Node || serviceNode.ServiceID != tc.services[i].ServiceID { t.Fatalf("bad: %v, %v filters: %v", serviceNode, tc.services[i], tc.filters) } } } } func TestCatalog_ListServiceNodes_DistanceSort(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() args := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", } var out structs.IndexedServiceNodes err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out) if err != nil { t.Fatalf("err: %v", err) } testrpc.WaitForLeader(t, s1.RPC, "dc1") // Add a few nodes for the associated services. s1.fsm.State().EnsureNode(1, &structs.Node{Node: "aaa", Address: "127.0.0.1"}) s1.fsm.State().EnsureService(2, "aaa", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}) s1.fsm.State().EnsureNode(3, &structs.Node{Node: "foo", Address: "127.0.0.2"}) s1.fsm.State().EnsureService(4, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.2", Port: 5000}) s1.fsm.State().EnsureNode(5, &structs.Node{Node: "bar", Address: "127.0.0.3"}) s1.fsm.State().EnsureService(6, "bar", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.3", Port: 5000}) s1.fsm.State().EnsureNode(7, &structs.Node{Node: "baz", Address: "127.0.0.4"}) s1.fsm.State().EnsureService(8, "baz", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.4", Port: 5000}) // Set all but one of the nodes to known coordinates. updates := structs.Coordinates{ {Node: "foo", Coord: lib.GenerateCoordinate(2 * time.Millisecond)}, {Node: "bar", Coord: lib.GenerateCoordinate(5 * time.Millisecond)}, {Node: "baz", Coord: lib.GenerateCoordinate(1 * time.Millisecond)}, } if err := s1.fsm.State().CoordinateBatchUpdate(9, updates); err != nil { t.Fatalf("err: %v", err) } // Query with no given source node, should get the natural order from // the index. if err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out); err != nil { t.Fatalf("err: %v", err) } if len(out.ServiceNodes) != 4 { t.Fatalf("bad: %v", out) } if out.ServiceNodes[0].Node != "aaa" { t.Fatalf("bad: %v", out) } if out.ServiceNodes[1].Node != "bar" { t.Fatalf("bad: %v", out) } if out.ServiceNodes[2].Node != "baz" { t.Fatalf("bad: %v", out) } if out.ServiceNodes[3].Node != "foo" { t.Fatalf("bad: %v", out) } // Query relative to foo, note that there's no known coordinate for "aaa" // so it will go at the end. args = structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "db", Source: structs.QuerySource{Datacenter: "dc1", Node: "foo"}, } if err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &args, &out); err != nil { t.Fatalf("err: %v", err) } if len(out.ServiceNodes) != 4 { t.Fatalf("bad: %v", out) } if out.ServiceNodes[0].Node != "foo" { t.Fatalf("bad: %v", out) } if out.ServiceNodes[1].Node != "baz" { t.Fatalf("bad: %v", out) } if out.ServiceNodes[2].Node != "bar" { t.Fatalf("bad: %v", out) } if out.ServiceNodes[3].Node != "aaa" { t.Fatalf("bad: %v", out) } } func TestCatalog_ListServiceNodes_ConnectProxy(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") // Register the service args := structs.TestRegisterRequestProxy(t) var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", args, &out)) // List req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: args.Service.Service, TagFilter: false, } var resp structs.IndexedServiceNodes assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) assert.Len(t, resp.ServiceNodes, 1) v := resp.ServiceNodes[0] assert.Equal(t, structs.ServiceKindConnectProxy, v.ServiceKind) assert.Equal(t, args.Service.Proxy.DestinationServiceName, v.ServiceProxy.DestinationServiceName) } func TestCatalog_ServiceNodes_Gateway(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") { var out struct{} // Register a service "api" args := structs.TestRegisterRequest(t) args.Service.Service = "api" args.Check = &structs.HealthCheck{ Name: "api", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a proxy for api args = structs.TestRegisterRequestProxy(t) args.Service.Service = "api-proxy" args.Service.Proxy.DestinationServiceName = "api" args.Check = &structs.HealthCheck{ Name: "api-proxy", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a service "web" args = structs.TestRegisterRequest(t) args.Check = &structs.HealthCheck{ Name: "web", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a proxy for web args = structs.TestRegisterRequestProxy(t) args.Check = &structs.HealthCheck{ Name: "web-proxy", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a gateway for web args = &structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Kind: structs.ServiceKindTerminatingGateway, Service: "gateway", Port: 443, }, Check: &structs.HealthCheck{ Name: "gateway", Status: api.HealthPassing, ServiceID: args.Service.Service, }, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) entryArgs := &structs.ConfigEntryRequest{ Op: structs.ConfigEntryUpsert, Datacenter: "dc1", Entry: &structs.TerminatingGatewayConfigEntry{ Kind: "terminating-gateway", Name: "gateway", Services: []structs.LinkedService{ { Name: "web", }, }, }, } var entryResp bool assert.Nil(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &entryArgs, &entryResp)) } retry.Run(t, func(r *retry.R) { // List should return both the terminating-gateway and the connect-proxy associated with web req := structs.ServiceSpecificRequest{ Connect: true, Datacenter: "dc1", ServiceName: "web", } var resp structs.IndexedServiceNodes assert.Nil(r, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) assert.Len(r, resp.ServiceNodes, 2) // Check sidecar assert.Equal(r, structs.ServiceKindConnectProxy, resp.ServiceNodes[0].ServiceKind) assert.Equal(r, "foo", resp.ServiceNodes[0].Node) assert.Equal(r, "web-proxy", resp.ServiceNodes[0].ServiceName) assert.Equal(r, "web-proxy", resp.ServiceNodes[0].ServiceID) assert.Equal(r, "web", resp.ServiceNodes[0].ServiceProxy.DestinationServiceName) assert.Equal(r, 2222, resp.ServiceNodes[0].ServicePort) // Check gateway assert.Equal(r, structs.ServiceKindTerminatingGateway, resp.ServiceNodes[1].ServiceKind) assert.Equal(r, "foo", resp.ServiceNodes[1].Node) assert.Equal(r, "gateway", resp.ServiceNodes[1].ServiceName) assert.Equal(r, "gateway", resp.ServiceNodes[1].ServiceID) assert.Equal(r, 443, resp.ServiceNodes[1].ServicePort) }) } func TestCatalog_ListServiceNodes_ConnectDestination(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") // Register the proxy service args := structs.TestRegisterRequestProxy(t) var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", args, &out)) // Register the service { dst := args.Service.Proxy.DestinationServiceName args := structs.TestRegisterRequest(t) args.Service.Service = dst var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", args, &out)) } // List req := structs.ServiceSpecificRequest{ Connect: true, Datacenter: "dc1", ServiceName: args.Service.Proxy.DestinationServiceName, } var resp structs.IndexedServiceNodes assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) assert.Len(t, resp.ServiceNodes, 1) v := resp.ServiceNodes[0] assert.Equal(t, structs.ServiceKindConnectProxy, v.ServiceKind) assert.Equal(t, args.Service.Proxy.DestinationServiceName, v.ServiceProxy.DestinationServiceName) // List by non-Connect req = structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: args.Service.Proxy.DestinationServiceName, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) assert.Len(t, resp.ServiceNodes, 1) v = resp.ServiceNodes[0] assert.Equal(t, args.Service.Proxy.DestinationServiceName, v.ServiceName) assert.Equal(t, "", v.ServiceProxy.DestinationServiceName) } // Test that calling ServiceNodes with Connect: true will return // Connect native services. func TestCatalog_ListServiceNodes_ConnectDestinationNative(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") // Register the native service args := structs.TestRegisterRequest(t) args.Service.Connect.Native = true var out struct{} require.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", args, &out)) // List req := structs.ServiceSpecificRequest{ Connect: true, Datacenter: "dc1", ServiceName: args.Service.Service, } var resp structs.IndexedServiceNodes require.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) require.Len(t, resp.ServiceNodes, 1) v := resp.ServiceNodes[0] require.Equal(t, args.Service.Service, v.ServiceName) // List by non-Connect req = structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: args.Service.Service, } require.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) require.Len(t, resp.ServiceNodes, 1) v = resp.ServiceNodes[0] require.Equal(t, args.Service.Service, v.ServiceName) } func TestCatalog_ListServiceNodes_ConnectProxy_ACL(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") rules := ` service "foo-proxy" { policy = "write" } service "foo" { policy = "write" } node "foo" { policy = "read" } ` token := createToken(t, codec, rules) { // Register a proxy args := structs.TestRegisterRequestProxy(t) args.Service.Service = "foo-proxy" args.Service.Proxy.DestinationServiceName = "bar" args.WriteRequest.Token = "root" var out struct{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a proxy args = structs.TestRegisterRequestProxy(t) args.Service.Service = "foo-proxy" args.Service.Proxy.DestinationServiceName = "foo" args.WriteRequest.Token = "root" require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a proxy args = structs.TestRegisterRequestProxy(t) args.Service.Service = "another-proxy" args.Service.Proxy.DestinationServiceName = "foo" args.WriteRequest.Token = "root" require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) } // List w/ token. This should disallow because we don't have permission // to read "bar" req := structs.ServiceSpecificRequest{ Connect: true, Datacenter: "dc1", ServiceName: "bar", QueryOptions: structs.QueryOptions{Token: token}, } var resp structs.IndexedServiceNodes require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) require.Len(t, resp.ServiceNodes, 0) // List w/ token. This should work since we're requesting "foo", but should // also only contain the proxies with names that adhere to our ACL. req = structs.ServiceSpecificRequest{ Connect: true, Datacenter: "dc1", ServiceName: "foo", QueryOptions: structs.QueryOptions{Token: token}, } resp = structs.IndexedServiceNodes{} require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) require.Len(t, resp.ServiceNodes, 1) v := resp.ServiceNodes[0] require.Equal(t, "foo-proxy", v.ServiceName) require.True(t, resp.QueryMeta.ResultsFilteredByACLs, "ResultsFilteredByACLs should be true") } func TestCatalog_ListServiceNodes_ConnectNative(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") // Register the service args := structs.TestRegisterRequest(t) args.Service.Connect.Native = true var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", args, &out)) // List req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: args.Service.Service, TagFilter: false, } var resp structs.IndexedServiceNodes assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &req, &resp)) assert.Len(t, resp.ServiceNodes, 1) v := resp.ServiceNodes[0] assert.Equal(t, args.Service.Connect.Native, v.ServiceConnect.Native) } func TestCatalog_NodeServices(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") args := structs.NodeSpecificRequest{ Datacenter: "dc1", Node: "foo", } var out structs.IndexedNodeServices err := msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &args, &out) if err != nil { t.Fatalf("err: %v", err) } testrpc.WaitForLeader(t, s1.RPC, "dc1") // Just add a node if err := s1.fsm.State().EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: []string{"primary"}, Address: "127.0.0.1", Port: 5000}); err != nil { t.Fatalf("err: %v", err) } if err := s1.fsm.State().EnsureService(3, "foo", &structs.NodeService{ID: "web", Service: "web", Tags: nil, Address: "127.0.0.1", Port: 80}); err != nil { t.Fatalf("err: %v", err) } if err := msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &args, &out); err != nil { t.Fatalf("err: %v", err) } if out.NodeServices.Node.Address != "127.0.0.1" { t.Fatalf("bad: %v", out) } if len(out.NodeServices.Services) != 2 { t.Fatalf("bad: %v", out) } services := out.NodeServices.Services if !stringslice.Contains(services["db"].Tags, "primary") || services["db"].Port != 5000 { t.Fatalf("bad: %v", out) } if len(services["web"].Tags) != 0 || services["web"].Port != 80 { t.Fatalf("bad: %v", out) } } func TestCatalog_NodeServices_ConnectProxy(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") // Register the service args := structs.TestRegisterRequestProxy(t) var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", args, &out)) // List req := structs.NodeSpecificRequest{ Datacenter: "dc1", Node: args.Node, } var resp structs.IndexedNodeServices assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &req, &resp)) assert.Len(t, resp.NodeServices.Services, 1) v := resp.NodeServices.Services[args.Service.Service] assert.Equal(t, structs.ServiceKindConnectProxy, v.Kind) assert.Equal(t, args.Service.Proxy.DestinationServiceName, v.Proxy.DestinationServiceName) } func TestCatalog_NodeServices_ConnectNative(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") // Register the service args := structs.TestRegisterRequest(t) var out struct{} assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", args, &out)) // List req := structs.NodeSpecificRequest{ Datacenter: "dc1", Node: args.Node, } var resp structs.IndexedNodeServices assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &req, &resp)) assert.Len(t, resp.NodeServices.Services, 1) v := resp.NodeServices.Services[args.Service.Service] assert.Equal(t, args.Service.Connect.Native, v.Connect.Native) } // Used to check for a regression against a known bug func TestCatalog_Register_FailedCase1(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() arg := structs.RegisterRequest{ Datacenter: "dc1", Node: "bar", Address: "127.0.0.2", Service: &structs.NodeService{ Service: "web", Tags: nil, Port: 8000, }, } var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out) if err != nil { t.Fatalf("err: %v", err) } testrpc.WaitForLeader(t, s1.RPC, "dc1") if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &arg, &out); err != nil { t.Fatalf("err: %v", err) } // Check we can get this back query := &structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "web", } var out2 structs.IndexedServiceNodes if err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", query, &out2); err != nil { t.Fatalf("err: %v", err) } // Check the output if len(out2.ServiceNodes) != 1 { t.Fatalf("Bad: %v", out2) } } func testACLFilterServer(t *testing.T) (dir, token string, srv *Server, codec rpc.ClientCodec) { dir, srv = testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) codec = rpcClient(t, srv) testrpc.WaitForTestAgent(t, srv.RPC, "dc1", testrpc.WithToken("root")) rules := ` service "foo" { policy = "write" } node_prefix "" { policy = "read" } ` token = createToken(t, codec, rules) // Register a service regArg := structs.RegisterRequest{ Datacenter: "dc1", Node: srv.config.NodeName, Address: "127.0.0.1", Service: &structs.NodeService{ ID: "foo", Service: "foo", }, Check: &structs.HealthCheck{ CheckID: "service:foo", Name: "service:foo", ServiceID: "foo", Status: api.HealthPassing, }, WriteRequest: structs.WriteRequest{Token: "root"}, } if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", ®Arg, nil); err != nil { t.Fatalf("err: %s", err) } // Register a service which should be denied regArg = structs.RegisterRequest{ Datacenter: "dc1", Node: srv.config.NodeName, Address: "127.0.0.1", Service: &structs.NodeService{ ID: "bar", Service: "bar", }, Check: &structs.HealthCheck{ CheckID: "service:bar", Name: "service:bar", ServiceID: "bar", Status: api.HealthPassing, }, WriteRequest: structs.WriteRequest{Token: "root"}, } if err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", ®Arg, nil); err != nil { t.Fatalf("err: %s", err) } return } // TestCatalog_Register_DenyPeeringRegistration makes sure that users cannot send structs.RegisterRequest // with a PeerName in any part of the request. func TestCatalog_Register_DenyPeeringRegistration(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() _, s := testServerWithConfig(t) codec := rpcClient(t, s) // we will add PeerName to copies of arg arg := structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "db", Tags: []string{"primary"}, Port: 8000, }, Check: &structs.HealthCheck{ CheckID: types.CheckID("db-check"), ServiceID: "db", }, Checks: structs.HealthChecks{ &structs.HealthCheck{ CheckID: types.CheckID("db-check"), ServiceID: "db", }, }, } type testcase struct { name string reqCopyFn func(arg *structs.RegisterRequest) structs.RegisterRequest } testCases := []testcase{ { name: "peer name on top level", reqCopyFn: func(arg *structs.RegisterRequest) structs.RegisterRequest { copyR := *arg copyR.PeerName = "foo" return copyR }, }, { name: "peer name in service", reqCopyFn: func(arg *structs.RegisterRequest) structs.RegisterRequest { copyR := *arg copyR.Service.PeerName = "foo" return copyR }, }, { name: "peer name in check", reqCopyFn: func(arg *structs.RegisterRequest) structs.RegisterRequest { copyR := *arg copyR.Check.PeerName = "foo" return copyR }, }, { name: "peer name in checks", reqCopyFn: func(arg *structs.RegisterRequest) structs.RegisterRequest { copyR := *arg copyR.Checks[0].PeerName = "foo" return copyR }, }, { name: "peer name everywhere", reqCopyFn: func(arg *structs.RegisterRequest) structs.RegisterRequest { copyR := *arg copyR.PeerName = "foo1" copyR.Service.PeerName = "foo2" copyR.Check.PeerName = "foo3" copyR.Checks[0].PeerName = "foo4" return copyR }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { req := tc.reqCopyFn(&arg) var out struct{} err := msgpackrpc.CallWithCodec(codec, "Catalog.Register", &req, &out) require.Error(t, err) require.Contains(t, err.Error(), "cannot register requests with PeerName in them") }) } } func TestCatalog_ListServices_FilterACL(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir, token, srv, codec := testACLFilterServer(t) defer os.RemoveAll(dir) defer srv.Shutdown() defer codec.Close() testrpc.WaitForTestAgent(t, srv.RPC, "dc1", testrpc.WithToken("root")) opt := structs.DCSpecificRequest{ Datacenter: "dc1", QueryOptions: structs.QueryOptions{Token: token}, } reply := structs.IndexedServices{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ListServices", &opt, &reply); err != nil { t.Fatalf("err: %s", err) } if _, ok := reply.Services["foo"]; !ok { t.Fatalf("bad: %#v", reply.Services) } if _, ok := reply.Services["bar"]; ok { t.Fatalf("bad: %#v", reply.Services) } if !reply.QueryMeta.ResultsFilteredByACLs { t.Fatal("ResultsFilteredByACLs should be true") } } func TestCatalog_ServiceNodes_FilterACL(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir, token, srv, codec := testACLFilterServer(t) defer os.RemoveAll(dir) defer srv.Shutdown() defer codec.Close() opt := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "foo", QueryOptions: structs.QueryOptions{Token: token}, } reply := structs.IndexedServiceNodes{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &opt, &reply); err != nil { t.Fatalf("err: %s", err) } found := false for _, sn := range reply.ServiceNodes { if sn.ServiceID == "foo" { found = true break } } if !found { t.Fatalf("bad: %#v", reply.ServiceNodes) } // Filters services we can't access opt = structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "bar", QueryOptions: structs.QueryOptions{Token: token}, } reply = structs.IndexedServiceNodes{} if err := msgpackrpc.CallWithCodec(codec, "Catalog.ServiceNodes", &opt, &reply); err != nil { t.Fatalf("err: %s", err) } for _, sn := range reply.ServiceNodes { if sn.ServiceID == "bar" { t.Fatalf("bad: %#v", reply.ServiceNodes) } } require.True(t, reply.QueryMeta.ResultsFilteredByACLs, "ResultsFilteredByACLs should be true") // We've already proven that we call the ACL filtering function so we // test node filtering down in acl.go for node cases. This also proves // that we respect the version 8 ACL flag, since the test server sets // that to false (the regression value of *not* changing this is better // for now until we change the sense of the version 8 ACL flag). } func TestCatalog_NodeServices_ACL(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1", testrpc.WithToken("root")) token := func(policy string) string { rules := fmt.Sprintf(` node "%s" { policy = "%s" } service "consul" { policy = "%s" } `, s1.config.NodeName, policy, policy, ) return createTokenWithPolicyName(t, codec, policy, rules, "root") } args := structs.NodeSpecificRequest{ Datacenter: "dc1", Node: s1.config.NodeName, } t.Run("deny", func(t *testing.T) { args.Token = token("deny") var reply structs.IndexedNodeServices err := msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &args, &reply) require.NoError(t, err) require.Nil(t, reply.NodeServices) require.True(t, reply.QueryMeta.ResultsFilteredByACLs, "ResultsFilteredByACLs should be true") }) t.Run("allow", func(t *testing.T) { args.Token = token("read") var reply structs.IndexedNodeServices err := msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &args, &reply) require.NoError(t, err) require.NotNil(t, reply.NodeServices) require.False(t, reply.QueryMeta.ResultsFilteredByACLs, "ResultsFilteredByACLs should be false") }) } func TestCatalog_NodeServices_FilterACL(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir, token, srv, codec := testACLFilterServer(t) defer os.RemoveAll(dir) defer srv.Shutdown() defer codec.Close() testrpc.WaitForTestAgent(t, srv.RPC, "dc1", testrpc.WithToken("root")) opt := structs.NodeSpecificRequest{ Datacenter: "dc1", Node: srv.config.NodeName, QueryOptions: structs.QueryOptions{Token: token}, } var reply structs.IndexedNodeServices require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.NodeServices", &opt, &reply)) require.NotNil(t, reply.NodeServices) require.Len(t, reply.NodeServices.Services, 1) svc, ok := reply.NodeServices.Services["foo"] require.True(t, ok) require.Equal(t, "foo", svc.ID) } func TestCatalog_GatewayServices_TerminatingGateway(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") { var out struct{} // Register a service "api" args := structs.TestRegisterRequest(t) args.Service.Service = "api" args.Check = &structs.HealthCheck{ Name: "api", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a service "db" args = structs.TestRegisterRequest(t) args.Service.Service = "db" args.Check = &structs.HealthCheck{ Name: "db", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a service "redis" args = structs.TestRegisterRequest(t) args.Service.Service = "redis" args.Check = &structs.HealthCheck{ Name: "redis", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a gateway args = &structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Kind: structs.ServiceKindTerminatingGateway, Service: "gateway", Port: 443, }, Check: &structs.HealthCheck{ Name: "gateway", Status: api.HealthPassing, ServiceID: "gateway", }, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) entryArgs := &structs.ConfigEntryRequest{ Op: structs.ConfigEntryUpsert, Datacenter: "dc1", Entry: &structs.TerminatingGatewayConfigEntry{ Kind: "terminating-gateway", Name: "gateway", Services: []structs.LinkedService{ { Name: "api", CAFile: "api/ca.crt", CertFile: "api/client.crt", KeyFile: "api/client.key", SNI: "my-domain", }, { Name: "db", }, { Name: "*", CAFile: "ca.crt", CertFile: "client.crt", KeyFile: "client.key", SNI: "my-alt-domain", }, }, }, } var entryResp bool assert.Nil(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &entryArgs, &entryResp)) } retry.Run(t, func(r *retry.R) { // List should return all three services req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "gateway", } var resp structs.IndexedGatewayServices assert.Nil(r, msgpackrpc.CallWithCodec(codec, "Catalog.GatewayServices", &req, &resp)) assert.Len(r, resp.Services, 3) expect := structs.GatewayServices{ { Service: structs.NewServiceName("api", nil), Gateway: structs.NewServiceName("gateway", nil), GatewayKind: structs.ServiceKindTerminatingGateway, CAFile: "api/ca.crt", CertFile: "api/client.crt", KeyFile: "api/client.key", SNI: "my-domain", ServiceKind: structs.GatewayServiceKindService, }, { Service: structs.NewServiceName("db", nil), Gateway: structs.NewServiceName("gateway", nil), GatewayKind: structs.ServiceKindTerminatingGateway, CAFile: "", CertFile: "", KeyFile: "", ServiceKind: structs.GatewayServiceKindService, }, { Service: structs.NewServiceName("redis", nil), Gateway: structs.NewServiceName("gateway", nil), GatewayKind: structs.ServiceKindTerminatingGateway, CAFile: "ca.crt", CertFile: "client.crt", KeyFile: "client.key", SNI: "my-alt-domain", FromWildcard: true, }, } // Ignore raft index for equality for _, s := range resp.Services { s.RaftIndex = structs.RaftIndex{} } assert.Equal(r, expect, resp.Services) }) } func TestCatalog_GatewayServices_BothGateways(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServer(t) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1") { var out struct{} // Register a service "api" args := structs.TestRegisterRequest(t) args.Service.Service = "api" args.Check = &structs.HealthCheck{ Name: "api", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a terminating gateway args = &structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Kind: structs.ServiceKindTerminatingGateway, Service: "gateway", Port: 443, }, Check: &structs.HealthCheck{ Name: "gateway", Status: api.HealthPassing, ServiceID: "gateway", }, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) entryArgs := &structs.ConfigEntryRequest{ Op: structs.ConfigEntryUpsert, Datacenter: "dc1", Entry: &structs.TerminatingGatewayConfigEntry{ Kind: "terminating-gateway", Name: "gateway", Services: []structs.LinkedService{ { Name: "api", }, }, }, } var entryResp bool assert.Nil(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &entryArgs, &entryResp)) // Register a service "db" args = structs.TestRegisterRequest(t) args.Service.Service = "db" args.Check = &structs.HealthCheck{ Name: "db", Status: api.HealthPassing, ServiceID: args.Service.Service, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register an ingress gateway args = &structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.2", Service: &structs.NodeService{ Kind: structs.ServiceKindTerminatingGateway, Service: "ingress", Port: 444, }, Check: &structs.HealthCheck{ Name: "ingress", Status: api.HealthPassing, ServiceID: "ingress", }, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) entryArgs = &structs.ConfigEntryRequest{ Op: structs.ConfigEntryUpsert, Datacenter: "dc1", Entry: &structs.IngressGatewayConfigEntry{ Kind: "ingress-gateway", Name: "ingress", Listeners: []structs.IngressListener{ { Port: 8888, Services: []structs.IngressService{ {Name: "db"}, }, }, }, }, } assert.Nil(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &entryArgs, &entryResp)) } retry.Run(t, func(r *retry.R) { req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "gateway", } var resp structs.IndexedGatewayServices assert.Nil(r, msgpackrpc.CallWithCodec(codec, "Catalog.GatewayServices", &req, &resp)) assert.Len(r, resp.Services, 1) expect := structs.GatewayServices{ { Service: structs.NewServiceName("api", nil), Gateway: structs.NewServiceName("gateway", nil), GatewayKind: structs.ServiceKindTerminatingGateway, ServiceKind: structs.GatewayServiceKindService, }, } // Ignore raft index for equality for _, s := range resp.Services { s.RaftIndex = structs.RaftIndex{} } assert.Equal(r, expect, resp.Services) req.ServiceName = "ingress" assert.Nil(r, msgpackrpc.CallWithCodec(codec, "Catalog.GatewayServices", &req, &resp)) assert.Len(r, resp.Services, 1) expect = structs.GatewayServices{ { Service: structs.NewServiceName("db", nil), Gateway: structs.NewServiceName("ingress", nil), GatewayKind: structs.ServiceKindIngressGateway, Protocol: "tcp", Port: 8888, }, } // Ignore raft index for equality for _, s := range resp.Services { s.RaftIndex = structs.RaftIndex{} } assert.Equal(r, expect, resp.Services) }) // Test a non-gateway service being requested req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "api", } var resp structs.IndexedGatewayServices err := msgpackrpc.CallWithCodec(codec, "Catalog.GatewayServices", &req, &resp) assert.NoError(t, err) assert.Empty(t, resp.Services) // Ensure that the index is not zero so that a blocking query still gets the // latest GatewayServices index assert.NotEqual(t, 0, resp.Index) } func TestCatalog_GatewayServices_ACLFiltering(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForTestAgent(t, s1.RPC, "dc1", testrpc.WithToken("root")) { var out struct{} // Register a service "api" args := structs.TestRegisterRequest(t) args.Service.Service = "api" args.Check = &structs.HealthCheck{ Name: "api", Status: api.HealthPassing, ServiceID: args.Service.Service, } args.Token = "root" assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a service "db" args = structs.TestRegisterRequest(t) args.Service.Service = "db" args.Check = &structs.HealthCheck{ Name: "db", Status: api.HealthPassing, ServiceID: args.Service.Service, } args.Token = "root" assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a service "redis" args = structs.TestRegisterRequest(t) args.Service.Service = "redis" args.Check = &structs.HealthCheck{ Name: "redis", Status: api.HealthPassing, ServiceID: args.Service.Service, } args.Token = "root" assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) // Register a gateway args = &structs.RegisterRequest{ Datacenter: "dc1", Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Kind: structs.ServiceKindTerminatingGateway, Service: "gateway", Port: 443, }, Check: &structs.HealthCheck{ Name: "gateway", Status: api.HealthPassing, ServiceID: "gateway", }, } args.Token = "root" assert.Nil(t, msgpackrpc.CallWithCodec(codec, "Catalog.Register", &args, &out)) entryArgs := &structs.ConfigEntryRequest{ Op: structs.ConfigEntryUpsert, Datacenter: "dc1", Entry: &structs.TerminatingGatewayConfigEntry{ Kind: "terminating-gateway", Name: "gateway", Services: []structs.LinkedService{ { Name: "api", CAFile: "api/ca.crt", CertFile: "api/client.crt", KeyFile: "api/client.key", }, { Name: "db", }, { Name: "db_replica", }, { Name: "*", CAFile: "ca.crt", CertFile: "client.crt", KeyFile: "client.key", }, }, }, WriteRequest: structs.WriteRequest{Token: "root"}, } var entryResp bool assert.Nil(t, msgpackrpc.CallWithCodec(codec, "ConfigEntry.Apply", &entryArgs, &entryResp)) } rules := ` service_prefix "db" { policy = "read" } ` svcToken, err := upsertTestTokenWithPolicyRules(codec, "root", "dc1", rules) require.NoError(t, err) retry.Run(t, func(r *retry.R) { // List should return an empty list, since we do not have read on the gateway req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "gateway", QueryOptions: structs.QueryOptions{Token: svcToken.SecretID}, } var resp structs.IndexedGatewayServices err := msgpackrpc.CallWithCodec(codec, "Catalog.GatewayServices", &req, &resp) require.True(r, acl.IsErrPermissionDenied(err)) }) rules = ` service "gateway" { policy = "read" } ` gwToken, err := upsertTestTokenWithPolicyRules(codec, "root", "dc1", rules) require.NoError(t, err) retry.Run(t, func(r *retry.R) { // List should return an empty list, since we do not have read on db req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "gateway", QueryOptions: structs.QueryOptions{Token: gwToken.SecretID}, } var resp structs.IndexedGatewayServices assert.Nil(r, msgpackrpc.CallWithCodec(codec, "Catalog.GatewayServices", &req, &resp)) assert.Len(r, resp.Services, 0) assert.True(r, resp.QueryMeta.ResultsFilteredByACLs, "ResultsFilteredByACLs should be true") }) rules = ` service_prefix "db" { policy = "read" } service "gateway" { policy = "read" } ` validToken, err := upsertTestTokenWithPolicyRules(codec, "root", "dc1", rules) require.NoError(t, err) retry.Run(t, func(r *retry.R) { // List should return db entry since we have read on db and gateway req := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "gateway", QueryOptions: structs.QueryOptions{Token: validToken.SecretID}, } var resp structs.IndexedGatewayServices assert.Nil(r, msgpackrpc.CallWithCodec(codec, "Catalog.GatewayServices", &req, &resp)) assert.Len(r, resp.Services, 2) assert.True(r, resp.QueryMeta.ResultsFilteredByACLs, "ResultsFilteredByACLs should be true") expect := structs.GatewayServices{ { Service: structs.NewServiceName("db", nil), Gateway: structs.NewServiceName("gateway", nil), GatewayKind: structs.ServiceKindTerminatingGateway, ServiceKind: structs.GatewayServiceKindService, }, { Service: structs.NewServiceName("db_replica", nil), Gateway: structs.NewServiceName("gateway", nil), GatewayKind: structs.ServiceKindTerminatingGateway, ServiceKind: structs.GatewayServiceKindUnknown, }, } // Ignore raft index for equality for _, s := range resp.Services { s.RaftIndex = structs.RaftIndex{} } assert.Equal(r, expect, resp.Services) }) } func TestVetRegisterWithACL(t *testing.T) { appendAuthz := func(t *testing.T, defaultAuthz acl.Authorizer, rules string) acl.Authorizer { policy, err := acl.NewPolicyFromSource(rules, nil, nil) require.NoError(t, err) authz, err := acl.NewPolicyAuthorizerWithDefaults(defaultAuthz, []*acl.Policy{policy}, nil) require.NoError(t, err) return authz } t.Run("With an 'allow all' authorizer the update should be allowed", func(t *testing.T) { args := &structs.RegisterRequest{ Node: "nope", Address: "127.0.0.1", } // With an "allow all" authorizer the update should be allowed. require.NoError(t, vetRegisterWithACL(resolver.Result{Authorizer: acl.ManageAll()}, args, nil)) }) var perms acl.Authorizer = acl.DenyAll() var resolvedPerms resolver.Result args := &structs.RegisterRequest{ Node: "nope", Address: "127.0.0.1", } // Create a basic node policy. perms = appendAuthz(t, perms, ` node "node" { policy = "write" } `) resolvedPerms = resolver.Result{Authorizer: perms} // With that policy, the update should now be blocked for node reasons. err := vetRegisterWithACL(resolvedPerms, args, nil) require.True(t, acl.IsErrPermissionDenied(err)) // Now use a permitted node name. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", } require.NoError(t, vetRegisterWithACL(resolvedPerms, args, nil)) // Build some node info that matches what we have now. ns := &structs.NodeServices{ Node: &structs.Node{ Node: "node", Address: "127.0.0.1", }, Services: make(map[string]*structs.NodeService), } // Try to register a service, which should be blocked. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "service", ID: "my-id", }, } err = vetRegisterWithACL(resolver.Result{Authorizer: perms}, args, ns) require.True(t, acl.IsErrPermissionDenied(err)) // Chain on a basic service policy. perms = appendAuthz(t, perms, ` service "service" { policy = "write" } `) resolvedPerms = resolver.Result{Authorizer: perms} // With the service ACL, the update should go through. require.NoError(t, vetRegisterWithACL(resolvedPerms, args, ns)) // Add an existing service that they are clobbering and aren't allowed // to write to. ns = &structs.NodeServices{ Node: &structs.Node{ Node: "node", Address: "127.0.0.1", }, Services: map[string]*structs.NodeService{ "my-id": { Service: "other", ID: "my-id", }, }, } err = vetRegisterWithACL(resolvedPerms, args, ns) require.True(t, acl.IsErrPermissionDenied(err)) // Chain on a policy that allows them to write to the other service. perms = appendAuthz(t, perms, ` service "other" { policy = "write" } `) resolvedPerms = resolver.Result{Authorizer: perms} // Now it should go through. require.NoError(t, vetRegisterWithACL(resolvedPerms, args, ns)) // Try creating the node and the service at once by having no existing // node record. This should be ok since we have node and service // permissions. require.NoError(t, vetRegisterWithACL(resolvedPerms, args, nil)) // Add a node-level check to the member, which should be rejected. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "service", ID: "my-id", }, Check: &structs.HealthCheck{ Node: "node", }, } err = vetRegisterWithACL(resolvedPerms, args, ns) testutil.RequireErrorContains(t, err, "check member must be nil") // Move the check into the slice, but give a bad node name. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "service", ID: "my-id", }, Checks: []*structs.HealthCheck{ { Node: "nope", }, }, } err = vetRegisterWithACL(resolvedPerms, args, ns) testutil.RequireErrorContains(t, err, "doesn't match register request node") // Fix the node name, which should now go through. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "service", ID: "my-id", }, Checks: []*structs.HealthCheck{ { Node: "node", }, }, } require.NoError(t, vetRegisterWithACL(resolvedPerms, args, ns)) // Add a service-level check. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "service", ID: "my-id", }, Checks: []*structs.HealthCheck{ { Node: "node", }, { Node: "node", ServiceID: "my-id", }, }, } require.NoError(t, vetRegisterWithACL(resolvedPerms, args, ns)) // Try creating everything at once. This should be ok since we have all // the permissions we need. It also makes sure that we can register a // new node, service, and associated checks. require.NoError(t, vetRegisterWithACL(resolvedPerms, args, nil)) // Nil out the service registration, which'll skip the special case // and force us to look at the ns data (it will look like we are // writing to the "other" service which also has "my-id"). args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", Checks: []*structs.HealthCheck{ { Node: "node", }, { Node: "node", ServiceID: "my-id", }, }, } require.NoError(t, vetRegisterWithACL(resolvedPerms, args, ns)) // Chain on a policy that forbids them to write to the other service. perms = appendAuthz(t, perms, ` service "other" { policy = "deny" } `) resolvedPerms = resolver.Result{Authorizer: perms} // This should get rejected. err = vetRegisterWithACL(resolvedPerms, args, ns) require.True(t, acl.IsErrPermissionDenied(err)) // Change the existing service data to point to a service name they // can write to. This should go through. ns = &structs.NodeServices{ Node: &structs.Node{ Node: "node", Address: "127.0.0.1", }, Services: map[string]*structs.NodeService{ "my-id": { Service: "service", ID: "my-id", }, }, } require.NoError(t, vetRegisterWithACL(resolvedPerms, args, ns)) // Chain on a policy that forbids them to write to the node. perms = appendAuthz(t, perms, ` node "node" { policy = "deny" } `) resolvedPerms = resolver.Result{Authorizer: perms} // This should get rejected because there's a node-level check in here. err = vetRegisterWithACL(resolvedPerms, args, ns) require.True(t, acl.IsErrPermissionDenied(err)) // Change the node-level check into a service check, and then it should // go through. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.1", Checks: []*structs.HealthCheck{ { Node: "node", ServiceID: "my-id", }, { Node: "node", ServiceID: "my-id", }, }, } require.NoError(t, vetRegisterWithACL(resolvedPerms, args, ns)) // Finally, attempt to update the node part of the data and make sure // that gets rejected since they no longer have permissions. args = &structs.RegisterRequest{ Node: "node", Address: "127.0.0.2", Checks: []*structs.HealthCheck{ { Node: "node", ServiceID: "my-id", }, { Node: "node", ServiceID: "my-id", }, }, } err = vetRegisterWithACL(resolvedPerms, args, ns) require.True(t, acl.IsErrPermissionDenied(err)) } func TestVetDeregisterWithACL(t *testing.T) { t.Parallel() args := &structs.DeregisterRequest{ Node: "nope", } // With an "allow all" authorizer the update should be allowed. if err := vetDeregisterWithACL(resolver.Result{Authorizer: acl.ManageAll()}, args, nil, nil); err != nil { t.Fatalf("err: %v", err) } // Create a basic node policy. policy, err := acl.NewPolicyFromSource(` node_prefix "node" { policy = "write" } `, nil, nil) if err != nil { t.Fatalf("err %v", err) } nodePerms, err := acl.NewPolicyAuthorizerWithDefaults(acl.DenyAll(), []*acl.Policy{policy}, nil) if err != nil { t.Fatalf("err: %v", err) } policy, err = acl.NewPolicyFromSource(` service "my-service" { policy = "write" } `, nil, nil) if err != nil { t.Fatalf("err %v", err) } servicePerms, err := acl.NewPolicyAuthorizerWithDefaults(acl.DenyAll(), []*acl.Policy{policy}, nil) if err != nil { t.Fatalf("err: %v", err) } for _, args := range []struct { DeregisterRequest structs.DeregisterRequest Service *structs.NodeService Check *structs.HealthCheck Perms acl.Authorizer Expected bool Name string }{ { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", }, Perms: nodePerms, Expected: false, Name: "no right on node", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", }, Perms: servicePerms, Expected: false, Name: "right on service but node dergister request", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", ServiceID: "my-service-id", }, Service: &structs.NodeService{ Service: "my-service", }, Perms: nodePerms, Expected: false, Name: "no rights on node nor service", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", ServiceID: "my-service-id", }, Service: &structs.NodeService{ Service: "my-service", }, Perms: servicePerms, Expected: true, Name: "no rights on node but rights on service", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", ServiceID: "my-service-id", CheckID: "my-check", }, Service: &structs.NodeService{ Service: "my-service", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: nodePerms, Expected: false, Name: "no right on node nor service for check", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", ServiceID: "my-service-id", CheckID: "my-check", }, Service: &structs.NodeService{ Service: "my-service", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: servicePerms, Expected: true, Name: "no rights on node but rights on service for check", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", CheckID: "my-check", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: nodePerms, Expected: false, Name: "no right on node for node check", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "nope", CheckID: "my-check", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: servicePerms, Expected: false, Name: "rights on service but no right on node for node check", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", }, Perms: nodePerms, Expected: true, Name: "rights on node for node", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", }, Perms: servicePerms, Expected: false, Name: "rights on service but not on node for node", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", ServiceID: "my-service-id", }, Service: &structs.NodeService{ Service: "my-service", }, Perms: nodePerms, Expected: true, Name: "rights on node for service", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", ServiceID: "my-service-id", }, Service: &structs.NodeService{ Service: "my-service", }, Perms: servicePerms, Expected: true, Name: "rights on service for service", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", ServiceID: "my-service-id", CheckID: "my-check", }, Service: &structs.NodeService{ Service: "my-service", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: nodePerms, Expected: true, Name: "right on node for check", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", ServiceID: "my-service-id", CheckID: "my-check", }, Service: &structs.NodeService{ Service: "my-service", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: servicePerms, Expected: true, Name: "rights on service for check", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", CheckID: "my-check", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: nodePerms, Expected: true, Name: "rights on node for check", }, { DeregisterRequest: structs.DeregisterRequest{ Node: "node", CheckID: "my-check", }, Check: &structs.HealthCheck{ CheckID: "my-check", }, Perms: servicePerms, Expected: false, Name: "rights on service for node check", }, } { t.Run(args.Name, func(t *testing.T) { err = vetDeregisterWithACL(resolver.Result{Authorizer: args.Perms}, &args.DeregisterRequest, args.Service, args.Check) if !args.Expected { if err == nil { t.Errorf("expected error with %+v", args.DeregisterRequest) } if !acl.IsErrPermissionDenied(err) { t.Errorf("expected permission denied error with %+v, instead got %+v", args.DeregisterRequest, err) } } else if err != nil { t.Errorf("expected no error with %+v", args.DeregisterRequest) } }) } } func TestCatalog_VirtualIPForService(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.Build = "1.11.0" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") err := s1.fsm.State().EnsureRegistration(1, &structs.RegisterRequest{ Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "api", Connect: structs.ServiceConnect{ Native: true, }, }, }) require.NoError(t, err) args := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "api", } var out string require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.VirtualIPForService", &args, &out)) require.Equal(t, "240.0.0.1", out) } func TestCatalog_VirtualIPForService_ACLDeny(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } t.Parallel() dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "deny" c.Build = "1.11.0" }) defer os.RemoveAll(dir1) defer s1.Shutdown() codec := rpcClient(t, s1) defer codec.Close() testrpc.WaitForLeader(t, s1.RPC, "dc1") err := s1.fsm.State().EnsureRegistration(1, &structs.RegisterRequest{ Node: "foo", Address: "127.0.0.1", Service: &structs.NodeService{ Service: "api", Connect: structs.ServiceConnect{ Native: true, }, }, }) require.NoError(t, err) // Call the endpoint with no token and expect permission denied. args := structs.ServiceSpecificRequest{ Datacenter: "dc1", ServiceName: "api", } var out string err = msgpackrpc.CallWithCodec(codec, "Catalog.VirtualIPForService", &args, &out) require.Contains(t, err.Error(), acl.ErrPermissionDenied.Error()) require.Equal(t, "", out) id := createToken(t, codec, ` service "api" { policy = "read" }`) // Now try with the token and it will go through. args.Token = id require.NoError(t, msgpackrpc.CallWithCodec(codec, "Catalog.VirtualIPForService", &args, &out)) require.Equal(t, "240.0.0.1", out) // Make sure we still get permission denied for an unknown service. args.ServiceName = "nope" var out2 string err = msgpackrpc.CallWithCodec(codec, "Catalog.VirtualIPForService", &args, &out2) require.Contains(t, err.Error(), acl.ErrPermissionDenied.Error()) require.Equal(t, "", out2) }