luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Update enterprise configurations to be in OSS 

This will emit warnings about the configs not doing anything but still allow them to be parsed.

This also added the warnings for enterprise fields that we already had in OSS but didn’t change their enforcement behavior. For example, attempting to use a network segment will cause a hard error in OSS.
This commit is contained in:
Matt Keeler 2020-04-28 09:45:33 -04:00
parent fbb5c28c80
commit 64baf36b60
No known key found for this signature in database
GPG Key ID: 04DBAE1857E0081B
9 changed files with 309 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
agent/config/builder.go
View File

@ -278,11 +278,14 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
if s.Name == "" || s.Data == "" {
continue
}
c2, err := Parse(s.Data, s.Format)
c2, keys, err := Parse(s.Data, s.Format)
if err != nil {
return RuntimeConfig{}, fmt.Errorf("Error parsing %s: %s", s.Name, err)
}
// for now this is a soft failure that will cause warnings but not actual problems
b.validateEnterpriseConfigKeys(&c2, keys)
// if we have a single 'check' or 'service' we need to add them to the
// list of checks and services first since we cannot merge them
// generically and later values would clobber earlier ones.

66
agent/config/builder_oss.go
View File

@ -2,6 +2,72 @@
package config
import (
"fmt"
"github.com/hashicorp/go-multierror"
)
var (
enterpriseConfigMap map[string]func(*Config) = map[string]func(c *Config){
"non_voting_server": func(c *Config) {
// to maintain existing compatibility we don't nullify the value
},
"segment": func(c *Config) {
// to maintain existing compatibility we don't nullify the value
},
"segments": func(c *Config) {
// to maintain existing compatibility we don't nullify the value
},
"autopilot.redundancy_zone_tag": func(c *Config) {
// to maintain existing compatibility we don't nullify the value
},
"autopilot.upgrade_version_tag": func(c *Config) {
// to maintain existing compatibility we don't nullify the value
},
"autopilot.disable_upgrade_migration": func(c *Config) {
// to maintain existing compatibility we don't nullify the value
},
"dns_config.prefer_namespace": func(c *Config) {
c.DNS.PreferNamespace = nil
},
"acl.msp_disable_bootstrap": func(c *Config) {
c.ACL.MSPDisableBootstrap = nil
},
"acl.tokens.managed_service_provider": func(c *Config) {
c.ACL.Tokens.ManagedServiceProvider = nil
},
}
)
type enterpriseConfigKeyError struct {
key string
}
func (e enterpriseConfigKeyError) Error() string {
return fmt.Sprintf("%q is a Consul Enterprise configuration and will have no effect", e.key)
}
func (_ *Builder) BuildEnterpriseRuntimeConfig(_ *Config) (EnterpriseRuntimeConfig, error) {
return EnterpriseRuntimeConfig{}, nil
}
// validateEnterpriseConfig is a function to validate the enterprise specific
// configuration items after Parsing but before merging into the overall
// configuration. The original intent is to use it to ensure that we warn
// for enterprise configurations used in OSS.
func (b *Builder) validateEnterpriseConfigKeys(config *Config, keys []string) error {
var err error
for _, k := range keys {
if unset, ok := enterpriseConfigMap[k]; ok {
keyErr := enterpriseConfigKeyError{key: k}
b.warn(keyErr.Error())
err = multierror.Append(err, keyErr)
unset(config)
}
}
return err
}

159
agent/config/builder_oss_test.go Normal file
View File

@ -0,0 +1,159 @@
// +build !consulent
package config
import (
"testing"
"github.com/hashicorp/go-multierror"
"github.com/stretchr/testify/require"
)
func TestBuilder_validateEnterpriseConfigKeys(t *testing.T) {
// ensure that all the enterprise configurations
type testCase struct {
config Config
keys []string
badKeys []string
check func(t *testing.T, c *Config)
}
boolVal := true
stringVal := "string"
cases := map[string]testCase{
"non_voting_server": {
config: Config{
NonVotingServer: &boolVal,
},
keys: []string{"non_voting_server"},
badKeys: []string{"non_voting_server"},
},
"segment": {
config: Config{
SegmentName: &stringVal,
},
keys: []string{"segment"},
badKeys: []string{"segment"},
},
"segments": {
config: Config{
Segments: []Segment{
{Name: &stringVal},
},
},
keys: []string{"segments"},
badKeys: []string{"segments"},
},
"autopilot.redundancy_zone_tag": {
config: Config{
Autopilot: Autopilot{
RedundancyZoneTag: &stringVal,
},
},
keys: []string{"autopilot.redundancy_zone_tag"},
badKeys: []string{"autopilot.redundancy_zone_tag"},
},
"autopilot.upgrade_version_tag": {
config: Config{
Autopilot: Autopilot{
UpgradeVersionTag: &stringVal,
},
},
keys: []string{"autopilot.upgrade_version_tag"},
badKeys: []string{"autopilot.upgrade_version_tag"},
},
"autopilot.disable_upgrade_migration": {
config: Config{
Autopilot: Autopilot{
DisableUpgradeMigration: &boolVal,
},
},
keys: []string{"autopilot.disable_upgrade_migration"},
badKeys: []string{"autopilot.disable_upgrade_migration"},
},
"dns_config.prefer_namespace": {
config: Config{
DNS: DNS{
PreferNamespace: &boolVal,
},
},
keys: []string{"dns_config.prefer_namespace"},
badKeys: []string{"dns_config.prefer_namespace"},
check: func(t *testing.T, c *Config) {
require.Nil(t, c.DNS.PreferNamespace)
},
},
"acl.msp_disable_bootstrap": {
config: Config{
ACL: ACL{
MSPDisableBootstrap: &boolVal,
},
},
keys: []string{"acl.msp_disable_bootstrap"},
badKeys: []string{"acl.msp_disable_bootstrap"},
check: func(t *testing.T, c *Config) {
require.Nil(t, c.ACL.MSPDisableBootstrap)
},
},
"acl.tokens.managed_service_provider": {
config: Config{
ACL: ACL{
Tokens: Tokens{
ManagedServiceProvider: []ServiceProviderToken{
{
AccessorID: &stringVal,
SecretID: &stringVal,
},
},
},
},
},
keys: []string{"acl.tokens.managed_service_provider"},
badKeys: []string{"acl.tokens.managed_service_provider"},
check: func(t *testing.T, c *Config) {
require.Empty(t, c.ACL.Tokens.ManagedServiceProvider)
require.Nil(t, c.ACL.Tokens.ManagedServiceProvider)
},
},
"multi": {
config: Config{
NonVotingServer: &boolVal,
SegmentName: &stringVal,
},
keys: []string{"non_voting_server", "segment", "acl.tokens.agent_master"},
badKeys: []string{"non_voting_server", "segment"},
},
}
for name, tcase := range cases {
t.Run(name, func(t *testing.T) {
b := &Builder{}
err := b.validateEnterpriseConfigKeys(&tcase.config, tcase.keys)
if len(tcase.badKeys) > 0 {
require.Error(t, err)
multiErr, ok := err.(*multierror.Error)
require.True(t, ok)
var badKeys []string
for _, e := range multiErr.Errors {
if keyErr, ok := e.(enterpriseConfigKeyError); ok {
badKeys = append(badKeys, keyErr.key)
require.Contains(t, b.Warnings, keyErr.Error())
}
}
require.ElementsMatch(t, tcase.badKeys, badKeys)
if tcase.check != nil {
tcase.check(t, &tcase.config)
}
} else {
require.NoError(t, err)
}
})
}
}

82
agent/config/config.go
View File

@ -34,7 +34,7 @@ func FormatFrom(name string) string {
}
// Parse parses a config fragment in either JSON or HCL format.
func Parse(data string, format string) (c Config, err error) {
func Parse(data string, format string) (c Config, keys []string, err error) {
var raw map[string]interface{}
switch format {
case "json":
@ -45,7 +45,7 @@ func Parse(data string, format string) (c Config, err error) {
err = fmt.Errorf("invalid format: %s", format)
}
if err != nil {
return Config{}, err
return Config{}, nil, err
}
// We want to be able to report fields which we cannot map as an
@ -136,15 +136,20 @@ func Parse(data string, format string) (c Config, err error) {
Result: &c,
})
if err != nil {
return Config{}, err
return Config{}, nil, err
}
if err := d.Decode(m); err != nil {
return Config{}, err
return Config{}, nil, err
}
for _, k := range md.Unused {
err = multierror.Append(err, fmt.Errorf("invalid config key %s", k))
}
// Don't check these here. The builder can emit warnings for fields it
// doesn't like
keys = md.Keys
return
}
@ -245,7 +250,6 @@ type Config struct {
NodeID *string `json:"node_id,omitempty" hcl:"node_id" mapstructure:"node_id"`
NodeMeta map[string]string `json:"node_meta,omitempty" hcl:"node_meta" mapstructure:"node_meta"`
NodeName *string `json:"node_name,omitempty" hcl:"node_name" mapstructure:"node_name"`
NonVotingServer *bool `json:"non_voting_server,omitempty" hcl:"non_voting_server" mapstructure:"non_voting_server"`
Performance Performance `json:"performance,omitempty" hcl:"performance" mapstructure:"performance"`
PidFile *string `json:"pid_file,omitempty" hcl:"pid_file" mapstructure:"pid_file"`
Ports Ports `json:"ports,omitempty" hcl:"ports" mapstructure:"ports"`
@ -266,8 +270,6 @@ type Config struct {
RetryJoinMaxAttemptsLAN *int `json:"retry_max,omitempty" hcl:"retry_max" mapstructure:"retry_max"`
RetryJoinMaxAttemptsWAN *int `json:"retry_max_wan,omitempty" hcl:"retry_max_wan" mapstructure:"retry_max_wan"`
RetryJoinWAN []string `json:"retry_join_wan,omitempty" hcl:"retry_join_wan" mapstructure:"retry_join_wan"`
SegmentName *string `json:"segment,omitempty" hcl:"segment" mapstructure:"segment"`
Segments []Segment `json:"segments,omitempty" hcl:"segments" mapstructure:"segments"`
SerfBindAddrLAN *string `json:"serf_lan,omitempty" hcl:"serf_lan" mapstructure:"serf_lan"`
SerfBindAddrWAN *string `json:"serf_wan,omitempty" hcl:"serf_wan" mapstructure:"serf_wan"`
ServerMode *bool `json:"server,omitempty" hcl:"server" mapstructure:"server"`
@ -317,6 +319,13 @@ type Config struct {
Version *string `json:"version,omitempty" hcl:"version" mapstructure:"version"`
VersionPrerelease *string `json:"version_prerelease,omitempty" hcl:"version_prerelease" mapstructure:"version_prerelease"`
// Enterprise Only
NonVotingServer *bool `json:"non_voting_server,omitempty" hcl:"non_voting_server" mapstructure:"non_voting_server"`
// Enterprise Only
SegmentName *string `json:"segment,omitempty" hcl:"segment" mapstructure:"segment"`
// Enterprise Only
Segments []Segment `json:"segments,omitempty" hcl:"segments" mapstructure:"segments"`
// enterpriseConfig embeds fields that we only access in consul-enterprise builds
EnterpriseConfig `hcl:",squash" mapstructure:",squash"`
}
@ -372,13 +381,17 @@ type AdvertiseAddrsConfig struct {
type Autopilot struct {
CleanupDeadServers *bool `json:"cleanup_dead_servers,omitempty" hcl:"cleanup_dead_servers" mapstructure:"cleanup_dead_servers"`
DisableUpgradeMigration *bool `json:"disable_upgrade_migration,omitempty" hcl:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"`
LastContactThreshold *string `json:"last_contact_threshold,omitempty" hcl:"last_contact_threshold" mapstructure:"last_contact_threshold"`
MaxTrailingLogs *int `json:"max_trailing_logs,omitempty" hcl:"max_trailing_logs" mapstructure:"max_trailing_logs"`
MinQuorum *uint `json:"min_quorum,omitempty" hcl:"min_quorum" mapstructure:"min_quorum"`
RedundancyZoneTag *string `json:"redundancy_zone_tag,omitempty" hcl:"redundancy_zone_tag" mapstructure:"redundancy_zone_tag"`
ServerStabilizationTime *string `json:"server_stabilization_time,omitempty" hcl:"server_stabilization_time" mapstructure:"server_stabilization_time"`
UpgradeVersionTag *string `json:"upgrade_version_tag,omitempty" hcl:"upgrade_version_tag" mapstructure:"upgrade_version_tag"`
// Enterprise Only
DisableUpgradeMigration *bool `json:"disable_upgrade_migration,omitempty" hcl:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"`
// Enterprise Only
RedundancyZoneTag *string `json:"redundancy_zone_tag,omitempty" hcl:"redundancy_zone_tag" mapstructure:"redundancy_zone_tag"`
// Enterprise Only
UpgradeVersionTag *string `json:"upgrade_version_tag,omitempty" hcl:"upgrade_version_tag" mapstructure:"upgrade_version_tag"`
}
// ServiceWeights defines the registration of weights used in DNS for a Service
@ -606,21 +619,23 @@ type SOA struct {
}
type DNS struct {
AllowStale *bool `json:"allow_stale,omitempty" hcl:"allow_stale" mapstructure:"allow_stale"`
ARecordLimit *int `json:"a_record_limit,omitempty" hcl:"a_record_limit" mapstructure:"a_record_limit"`
DisableCompression *bool `json:"disable_compression,omitempty" hcl:"disable_compression" mapstructure:"disable_compression"`
EnableTruncate *bool `json:"enable_truncate,omitempty" hcl:"enable_truncate" mapstructure:"enable_truncate"`
MaxStale *string `json:"max_stale,omitempty" hcl:"max_stale" mapstructure:"max_stale"`
NodeTTL *string `json:"node_ttl,omitempty" hcl:"node_ttl" mapstructure:"node_ttl"`
OnlyPassing *bool `json:"only_passing,omitempty" hcl:"only_passing" mapstructure:"only_passing"`
RecursorTimeout *string `json:"recursor_timeout,omitempty" hcl:"recursor_timeout" mapstructure:"recursor_timeout"`
ServiceTTL map[string]string `json:"service_ttl,omitempty" hcl:"service_ttl" mapstructure:"service_ttl"`
UDPAnswerLimit *int `json:"udp_answer_limit,omitempty" hcl:"udp_answer_limit" mapstructure:"udp_answer_limit"`
NodeMetaTXT *bool `json:"enable_additional_node_meta_txt,omitempty" hcl:"enable_additional_node_meta_txt" mapstructure:"enable_additional_node_meta_txt"`
SOA *SOA `json:"soa,omitempty" hcl:"soa" mapstructure:"soa"`
UseCache *bool `json:"use_cache,omitempty" hcl:"use_cache" mapstructure:"use_cache"`
CacheMaxAge *string `json:"cache_max_age,omitempty" hcl:"cache_max_age" mapstructure:"cache_max_age"`
EnterpriseDNSConfig `hcl:",squash" mapstructure:",squash"`
AllowStale *bool `json:"allow_stale,omitempty" hcl:"allow_stale" mapstructure:"allow_stale"`
ARecordLimit *int `json:"a_record_limit,omitempty" hcl:"a_record_limit" mapstructure:"a_record_limit"`
DisableCompression *bool `json:"disable_compression,omitempty" hcl:"disable_compression" mapstructure:"disable_compression"`
EnableTruncate *bool `json:"enable_truncate,omitempty" hcl:"enable_truncate" mapstructure:"enable_truncate"`
MaxStale *string `json:"max_stale,omitempty" hcl:"max_stale" mapstructure:"max_stale"`
NodeTTL *string `json:"node_ttl,omitempty" hcl:"node_ttl" mapstructure:"node_ttl"`
OnlyPassing *bool `json:"only_passing,omitempty" hcl:"only_passing" mapstructure:"only_passing"`
RecursorTimeout *string `json:"recursor_timeout,omitempty" hcl:"recursor_timeout" mapstructure:"recursor_timeout"`
ServiceTTL map[string]string `json:"service_ttl,omitempty" hcl:"service_ttl" mapstructure:"service_ttl"`
UDPAnswerLimit *int `json:"udp_answer_limit,omitempty" hcl:"udp_answer_limit" mapstructure:"udp_answer_limit"`
NodeMetaTXT *bool `json:"enable_additional_node_meta_txt,omitempty" hcl:"enable_additional_node_meta_txt" mapstructure:"enable_additional_node_meta_txt"`
SOA *SOA `json:"soa,omitempty" hcl:"soa" mapstructure:"soa"`
UseCache *bool `json:"use_cache,omitempty" hcl:"use_cache" mapstructure:"use_cache"`
CacheMaxAge *string `json:"cache_max_age,omitempty" hcl:"cache_max_age" mapstructure:"cache_max_age"`
// Enterprise Only
PreferNamespace *bool `json:"prefer_namespace,omitempty" hcl:"prefer_namespace" mapstructure:"prefer_namespace"`
}
type HTTPConfig struct {
@ -713,18 +728,23 @@ type ACL struct {
Tokens Tokens `json:"tokens,omitempty" hcl:"tokens" mapstructure:"tokens"`
DisabledTTL *string `json:"disabled_ttl,omitempty" hcl:"disabled_ttl" mapstructure:"disabled_ttl"`
EnableTokenPersistence *bool `json:"enable_token_persistence" hcl:"enable_token_persistence" mapstructure:"enable_token_persistence"`
EnterpriseACLConfig `hcl:",squash" mapstructure:",squash"`
// Enterprise Only
MSPDisableBootstrap *bool `json:"msp_disable_bootstrap" hcl:"msp_disable_bootstrap" mapstructure:"msp_disable_bootstrap"`
}
type Tokens struct {
Master *string `json:"master,omitempty" hcl:"master" mapstructure:"master"`
Replication *string `json:"replication,omitempty" hcl:"replication" mapstructure:"replication"`
AgentMaster *string `json:"agent_master,omitempty" hcl:"agent_master" mapstructure:"agent_master"`
Default *string `json:"default,omitempty" hcl:"default" mapstructure:"default"`
Agent *string `json:"agent,omitempty" hcl:"agent" mapstructure:"agent"`
Master *string `json:"master,omitempty" hcl:"master" mapstructure:"master"`
Replication *string `json:"replication,omitempty" hcl:"replication" mapstructure:"replication"`
AgentMaster *string `json:"agent_master,omitempty" hcl:"agent_master" mapstructure:"agent_master"`
Default *string `json:"default,omitempty" hcl:"default" mapstructure:"default"`
Agent *string `json:"agent,omitempty" hcl:"agent" mapstructure:"agent"`
// Enterprise Only
ManagedServiceProvider []ServiceProviderToken `json:"managed_service_provider,omitempty" hcl:"managed_service_provider" mapstructure:"managed_service_provider"`
}
// ServiceProviderToken groups an accessor and secret for a service provider token. Enterprise Only
type ServiceProviderToken struct {
AccessorID *string `json:"accessor_id,omitempty" hcl:"accessor_id" mapstructure:"accessor_id"`
SecretID *string `json:"secret_id,omitempty" hcl:"secret_id" mapstructure:"secret_id"`

6
agent/config/config_oss.go
View File

@ -13,9 +13,3 @@ type EnterpriseMeta struct{}
func (_ *EnterpriseMeta) ToStructs() structs.EnterpriseMeta {
return *structs.DefaultEnterpriseMeta()
}
// EnterpriseDNSConfig OSS stub
type EnterpriseDNSConfig struct{}
// EnterpriseACLConfig OSS stub
type EnterpriseACLConfig struct{}

2
agent/config/default.go
View File

@ -12,7 +12,7 @@ import (
func DefaultRPCProtocol() (int, error) {
src := DefaultSource()
c, err := Parse(src.Data, src.Format)
c, _, err := Parse(src.Data, src.Format)
if err != nil {
return 0, fmt.Errorf("Error parsing default config: %s", err)
}

10
agent/config/runtime_oss_test.go
View File

@ -11,3 +11,13 @@ var entFullDNSJSONConfig = ``
var entFullDNSHCLConfig = ``
var entFullRuntimeConfig = EnterpriseRuntimeConfig{}
var enterpriseNonVotingServerWarnings []string = []string{enterpriseConfigKeyError{key: "non_voting_server"}.Error()}
var enterpriseConfigKeyWarnings []string
func init() {
for k, _ := range enterpriseConfigMap {
enterpriseConfigKeyWarnings = append(enterpriseConfigKeyWarnings, enterpriseConfigKeyError{key: k}.Error())
}
}

14
agent/config/runtime_test.go
View File

@ -609,6 +609,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
rt.NonVotingServer = true
rt.DataDir = dataDir
},
warns: enterpriseNonVotingServerWarnings,
},
{
desc: "-pid-file",
@ -3915,6 +3916,7 @@ func TestFullConfig(t *testing.T) {
"role_ttl": "9876s",
"token_ttl": "3321s",
"enable_token_replication" : true,
"msp_disable_bootstrap": true,
"tokens" : {
"master" : "8a19ac27",
"agent_master" : "64fd0e08",
@ -4110,7 +4112,8 @@ func TestFullConfig(t *testing.T) {
},
"udp_answer_limit": 29909,
"use_cache": true,
"cache_max_age": "5m"` + entFullDNSJSONConfig + `
"cache_max_age": "5m",
"prefer_namespace": true
},
"enable_acl_replication": true,
"enable_agent_tls_for_checks": true,
@ -4546,6 +4549,7 @@ func TestFullConfig(t *testing.T) {
role_ttl = "9876s"
token_ttl = "3321s"
enable_token_replication = true
msp_disable_bootstrap = true
tokens = {
master = "8a19ac27",
agent_master = "64fd0e08",
@ -4743,7 +4747,7 @@ func TestFullConfig(t *testing.T) {
udp_answer_limit = 29909
use_cache = true
cache_max_age = "5m"
` + entFullDNSHCLConfig + `
prefer_namespace = true
}
enable_acl_replication = true
enable_agent_tls_for_checks = true
@ -5885,6 +5889,8 @@ func TestFullConfig(t *testing.T) {
`bootstrap_expect > 0: expecting 53 servers`,
}
warns = append(warns, enterpriseConfigKeyWarnings...)
// ensure that all fields are set to unique non-zero values
// todo(fs): This currently fails since ServiceDefinition.Check is not used
// todo(fs): not sure on how to work around this. Possible options are:
@ -5947,9 +5953,7 @@ func TestFullConfig(t *testing.T) {
}
// check the warnings
if got, want := b.Warnings, warns; !verify.Values(t, "warnings", got, want) {
t.FailNow()
}
require.ElementsMatch(t, warns, b.Warnings, "Warnings: %v", b.Warnings)
})
}
}

9
agent/config/segment_oss_test.go
View File

@ -22,6 +22,9 @@ func TestSegments(t *testing.T) {
json: []string{`{ "server": true, "segment": "a" }`},
hcl: []string{` server = true segment = "a" `},
err: `Network segments are not supported in this version of Consul`,
warns: []string{
enterpriseConfigKeyError{key: "segment"}.Error(),
},
},
{
desc: "segment port must be set",
@ -31,6 +34,9 @@ func TestSegments(t *testing.T) {
json: []string{`{ "segments":[{ "name":"x" }] }`},
hcl: []string{`segments = [{ name = "x" }]`},
err: `Port for segment "x" cannot be <= 0`,
warns: []string{
enterpriseConfigKeyError{key: "segments"}.Error(),
},
},
{
desc: "segments not in OSS",
@ -40,6 +46,9 @@ func TestSegments(t *testing.T) {
json: []string{`{ "segments":[{ "name":"x", "port": 123 }] }`},
hcl: []string{`segments = [{ name = "x" port = 123 }]`},
err: `Network segments are not supported in this version of Consul`,
warns: []string{
enterpriseConfigKeyError{key: "segments"}.Error(),
},
},
}