Backport of parse config protocol on write to optimize disco-chain compilation into release/1.16.x (#19858)

* parse config protocol on write to optimize disco-chain compilation (#19829)

* parse config protocol on write to optimize disco-chain compilation

* add changelog

* add test fixes from PR

* add missing config field

---------

Co-authored-by: Dhia Ayachi <dhia@hashicorp.com>
This commit is contained in:
hc-github-team-consul-core 2023-12-07 14:37:52 -06:00 committed by GitHub
parent c279233d2b
commit b2a57ae0fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 77 additions and 35 deletions

3
.changelog/19829.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:enhancement
mesh: parse the proxy-defaults protocol when write the config-entry to avoid parsing it when compiling the discovery chain.
```

View File

@ -8,14 +8,12 @@ import (
"strings" "strings"
"time" "time"
"github.com/mitchellh/hashstructure"
"github.com/mitchellh/mapstructure"
"github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/configentry" "github.com/hashicorp/consul/agent/configentry"
"github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/proto/private/pbpeering" "github.com/hashicorp/consul/proto/private/pbpeering"
"github.com/mitchellh/hashstructure"
) )
type CompileRequest struct { type CompileRequest struct {
@ -244,21 +242,13 @@ func (c *compiler) recordServiceProtocol(sid structs.ServiceID) error {
return c.recordProtocol(sid, serviceDefault.Protocol) return c.recordProtocol(sid, serviceDefault.Protocol)
} }
if proxyDefault := c.entries.GetProxyDefaults(sid.PartitionOrDefault()); proxyDefault != nil { if proxyDefault := c.entries.GetProxyDefaults(sid.PartitionOrDefault()); proxyDefault != nil {
var cfg proxyConfig if proxyDefault.Protocol != "" {
// Ignore errors and fallback on defaults if it does happen. return c.recordProtocol(sid, proxyDefault.Protocol)
_ = mapstructure.WeakDecode(proxyDefault.Config, &cfg)
if cfg.Protocol != "" {
return c.recordProtocol(sid, cfg.Protocol)
} }
} }
return c.recordProtocol(sid, "") return c.recordProtocol(sid, "")
} }
// proxyConfig is a snippet from agent/xds/config.go:ProxyConfig
type proxyConfig struct {
Protocol string `mapstructure:"protocol"`
}
func (c *compiler) recordProtocol(fromService structs.ServiceID, protocol string) error { func (c *compiler) recordProtocol(fromService structs.ServiceID, protocol string) error {
if protocol == "" { if protocol == "" {
protocol = "tcp" protocol = "tcp"

View File

@ -1142,8 +1142,9 @@ func testcase_PeerRedirect() compileTestCase {
func testcase_PeerRedirectProxyDefHTTP() compileTestCase { func testcase_PeerRedirectProxyDefHTTP() compileTestCase {
entries := newEntries() entries := newEntries()
entries.AddProxyDefaults(&structs.ProxyConfigEntry{ entries.AddProxyDefaults(&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"Protocol": "http", "Protocol": "http",
}, },
@ -1768,8 +1769,9 @@ func testcase_DefaultResolver_WithProxyDefaults() compileTestCase {
entries := newEntries() entries := newEntries()
entries.AddProxyDefaults(&structs.ProxyConfigEntry{ entries.AddProxyDefaults(&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "grpc",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "grpc", "protocol": "grpc",
}, },
@ -3263,8 +3265,9 @@ func newSimpleRoute(name string, muts ...func(*structs.ServiceRoute)) structs.Se
func setGlobalProxyProtocol(entries *configentry.DiscoveryChainSet, protocol string) { func setGlobalProxyProtocol(entries *configentry.DiscoveryChainSet, protocol string) {
entries.AddProxyDefaults(&structs.ProxyConfigEntry{ entries.AddProxyDefaults(&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: protocol,
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": protocol, "protocol": protocol,
}, },

View File

@ -781,8 +781,9 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) {
}}, }},
}, },
&structs.ProxyConfigEntry{ &structs.ProxyConfigEntry{
Kind: structs.ProxyConfigGlobal, Kind: structs.ProxyConfigGlobal,
Name: "global", Name: "global",
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },

View File

@ -6,6 +6,7 @@ package state
import ( import (
"errors" "errors"
"fmt" "fmt"
"github.com/mitchellh/mapstructure"
"strings" "strings"
memdb "github.com/hashicorp/go-memdb" memdb "github.com/hashicorp/go-memdb"
@ -519,6 +520,11 @@ func insertConfigEntryWithTxn(tx WriteTxn, idx uint64, conf structs.ConfigEntry)
if err != nil { if err != nil {
return err return err
} }
case structs.ProxyDefaults:
err := addProtocol(conf.(*structs.ProxyConfigEntry))
if err != nil {
return err
}
} }
// Assign virtual-ips, if needed // Assign virtual-ips, if needed
@ -544,6 +550,21 @@ func insertConfigEntryWithTxn(tx WriteTxn, idx uint64, conf structs.ConfigEntry)
return nil return nil
} }
// proxyConfig is a snippet from agent/xds/config.go:ProxyConfig
type proxyConfig struct {
Protocol string `mapstructure:"protocol"`
}
func addProtocol(conf *structs.ProxyConfigEntry) error {
var cfg proxyConfig
err := mapstructure.WeakDecode(conf.Config, &cfg)
if err != nil {
return err
}
conf.Protocol = cfg.Protocol
return nil
}
func configEntryHasVirtualIP(c structs.ConfigEntry) bool { func configEntryHasVirtualIP(c structs.ConfigEntry) bool {
if c == nil || c.GetName() == "" { if c == nil || c.GetName() == "" {
return false return false

View File

@ -24,16 +24,27 @@ import (
func TestStore_ConfigEntry(t *testing.T) { func TestStore_ConfigEntry(t *testing.T) {
s := testConfigStateStore(t) s := testConfigStateStore(t)
cfg := &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: "global",
Config: map[string]interface{}{
"DestinationServiceName": "foo",
"protocol": "udp",
},
}
expected := &structs.ProxyConfigEntry{ expected := &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: "global", Name: "global",
Config: map[string]interface{}{ Config: map[string]interface{}{
"DestinationServiceName": "foo", "DestinationServiceName": "foo",
"protocol": "udp",
}, },
Protocol: "udp",
} }
// Create // Create
require.NoError(t, s.EnsureConfigEntry(0, expected)) require.NoError(t, s.EnsureConfigEntry(0, cfg))
idx, config, err := s.ConfigEntry(nil, structs.ProxyDefaults, "global", nil) idx, config, err := s.ConfigEntry(nil, structs.ProxyDefaults, "global", nil)
require.NoError(t, err) require.NoError(t, err)

View File

@ -72,8 +72,9 @@ func TestManager_BasicLifecycle(t *testing.T) {
dbSplitChain := func() *structs.CompiledDiscoveryChain { dbSplitChain := func() *structs.CompiledDiscoveryChain {
set := configentry.NewDiscoveryChainSet() set := configentry.NewDiscoveryChainSet()
set.AddEntries(&structs.ProxyConfigEntry{ set.AddEntries(&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },

View File

@ -586,6 +586,7 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func(
) )
case "default-services-http": case "default-services-http":
proxyDefaults := &structs.ProxyConfigEntry{ proxyDefaults := &structs.ProxyConfigEntry{
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },
@ -687,8 +688,9 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func(
case "chain-and-l7-stuff": case "chain-and-l7-stuff":
entries = []structs.ConfigEntry{ entries = []structs.ConfigEntry{
&structs.ProxyConfigEntry{ &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },

View File

@ -131,8 +131,9 @@ func TestConfigSnapshotTransparentProxy(t testing.T) *ConfigSnapshot {
func TestConfigSnapshotTransparentProxyHTTPUpstream(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot { func TestConfigSnapshotTransparentProxyHTTPUpstream(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
// Set default service protocol to HTTP // Set default service protocol to HTTP
entries := append(additionalEntries, &structs.ProxyConfigEntry{ entries := append(additionalEntries, &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },

View File

@ -478,6 +478,7 @@ func setupTestVariationDiscoveryChain(
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
EnterpriseMeta: em, EnterpriseMeta: em,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },
@ -538,6 +539,7 @@ func setupTestVariationDiscoveryChain(
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
EnterpriseMeta: entMeta, EnterpriseMeta: entMeta,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },
@ -593,6 +595,7 @@ func setupTestVariationDiscoveryChain(
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
EnterpriseMeta: entMeta, EnterpriseMeta: entMeta,
Protocol: "grpc",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "grpc", "protocol": "grpc",
}, },
@ -628,6 +631,7 @@ func setupTestVariationDiscoveryChain(
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
EnterpriseMeta: entMeta, EnterpriseMeta: entMeta,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },
@ -879,6 +883,7 @@ func setupTestVariationDiscoveryChain(
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
EnterpriseMeta: entMeta, EnterpriseMeta: entMeta,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },

View File

@ -397,6 +397,7 @@ type ProxyConfigEntry struct {
Kind string Kind string
Name string Name string
Config map[string]interface{} Config map[string]interface{}
Protocol string `json:"-"`
Mode ProxyMode `json:",omitempty"` Mode ProxyMode `json:",omitempty"`
TransparentProxy TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"` TransparentProxy TransparentProxyConfig `json:",omitempty" alias:"transparent_proxy"`
MutualTLSMode MutualTLSMode `json:",omitempty" alias:"mutual_tls_mode"` MutualTLSMode MutualTLSMode `json:",omitempty" alias:"mutual_tls_mode"`

View File

@ -116,7 +116,7 @@ func (s *DiscoveryGraphNode) IsResolver() bool {
} }
func (s *DiscoveryGraphNode) MapKey() string { func (s *DiscoveryGraphNode) MapKey() string {
return fmt.Sprintf("%s:%s", s.Type, s.Name) return s.Type + ":" + s.Name
} }
// compiled form of ServiceResolverConfigEntry // compiled form of ServiceResolverConfigEntry

View File

@ -76,8 +76,9 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase {
create: func(t testinf.T) *proxycfg.ConfigSnapshot { create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil, return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil,
&structs.ProxyConfigEntry{ &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "http",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http", "protocol": "http",
}, },
@ -90,8 +91,9 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase {
create: func(t testinf.T) *proxycfg.ConfigSnapshot { create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil, return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil,
&structs.ProxyConfigEntry{ &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "http2",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "http2", "protocol": "http2",
}, },
@ -104,8 +106,9 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase {
create: func(t testinf.T) *proxycfg.ConfigSnapshot { create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil, return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil,
&structs.ProxyConfigEntry{ &structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults, Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal, Name: structs.ProxyConfigGlobal,
Protocol: "grpc",
Config: map[string]interface{}{ Config: map[string]interface{}{
"protocol": "grpc", "protocol": "grpc",
}, },