luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

backport of commit c0afba3a0c2ae093fee756a9019d49db25367d69 (#17975) 

Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
This commit is contained in:
hc-github-team-consul-core 2023-06-30 09:20:50 -05:00 committed by GitHub
parent 5e3ad77fc0
commit 08547ba585
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 2 additions and 1020 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

199
agent/envoyextensions/builtin/http/localratelimit/ratelimit.go
View File

@ -1,199 +0,0 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package localratelimit
import (
"errors"
"fmt"
"time"
envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_ratelimit "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/local_ratelimit/v3"
envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
envoy_resource_v3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3"
"github.com/hashicorp/go-multierror"
"github.com/mitchellh/mapstructure"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/wrapperspb"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/envoyextensions/extensioncommon"
)
type ratelimit struct {
extensioncommon.BasicExtensionAdapter
ProxyType string
// Token bucket of the rate limit
MaxTokens *int
TokensPerFill *int
FillInterval *int
// Percent of requests to be rate limited
FilterEnabled *uint32
FilterEnforced *uint32
}
var _ extensioncommon.BasicExtension = (*ratelimit)(nil)
// Constructor follows a specific function signature required for the extension registration.
func Constructor(ext api.EnvoyExtension) (extensioncommon.EnvoyExtender, error) {
var r ratelimit
if name := ext.Name; name != api.BuiltinLocalRatelimitExtension {
return nil, fmt.Errorf("expected extension name 'ratelimit' but got %q", name)
}
if err := r.fromArguments(ext.Arguments); err != nil {
return nil, err
}
return &extensioncommon.BasicEnvoyExtender{
Extension: &r,
}, nil
}
func (r *ratelimit) fromArguments(args map[string]interface{}) error {
if err := mapstructure.Decode(args, r); err != nil {
return fmt.Errorf("error decoding extension arguments: %v", err)
}
if r.ProxyType == "" {
r.ProxyType = string(api.ServiceKindConnectProxy)
}
return r.validate()
}
func (r *ratelimit) validate() error {
var resultErr error
// NOTE: Envoy requires FillInterval value must be greater than 0.
// If unset, it is considered as 0.
if r.FillInterval == nil {
resultErr = multierror.Append(resultErr, fmt.Errorf("FillInterval(in second) is missing"))
} else if *r.FillInterval <= 0 {
resultErr = multierror.Append(resultErr, fmt.Errorf("FillInterval(in second) must be greater than 0, got %d", *r.FillInterval))
}
// NOTE: Envoy requires MaxToken value must be greater than 0.
// If unset, it is considered as 0.
if r.MaxTokens == nil {
resultErr = multierror.Append(resultErr, fmt.Errorf("MaxTokens is missing"))
} else if *r.MaxTokens <= 0 {
resultErr = multierror.Append(resultErr, fmt.Errorf("MaxTokens must be greater than 0, got %d", r.MaxTokens))
}
// TokensPerFill is allowed to unset. In this case, envoy
// uses its default value, which is 1.
if r.TokensPerFill != nil && *r.TokensPerFill <= 0 {
resultErr = multierror.Append(resultErr, fmt.Errorf("TokensPerFill must be greater than 0, got %d", *r.TokensPerFill))
}
if err := validateProxyType(r.ProxyType); err != nil {
resultErr = multierror.Append(resultErr, err)
}
return resultErr
}
// CanApply determines if the extension can apply to the given extension configuration.
func (p *ratelimit) CanApply(config *extensioncommon.RuntimeConfig) bool {
return string(config.Kind) == p.ProxyType
}
// PatchFilter inserts a http local rate_limit filter at the head of
// envoy.filters.network.http_connection_manager filters
func (r ratelimit) PatchFilter(p extensioncommon.FilterPayload) (*envoy_listener_v3.Filter, bool, error) {
filter := p.Message
// rate limit is only applied to the inbound listener of the service itself
// since the limit is aggregated from all downstream connections.
if !p.IsInbound() {
return filter, false, nil
}
if filter.Name != "envoy.filters.network.http_connection_manager" {
return filter, false, nil
}
if typedConfig := filter.GetTypedConfig(); typedConfig == nil {
return filter, false, errors.New("error getting typed config for http filter")
}
config := envoy_resource_v3.GetHTTPConnectionManager(filter)
if config == nil {
return filter, false, errors.New("error unmarshalling filter")
}
tokenBucket := envoy_type_v3.TokenBucket{}
if r.TokensPerFill != nil {
tokenBucket.TokensPerFill = &wrapperspb.UInt32Value{
Value: uint32(*r.TokensPerFill),
}
}
if r.MaxTokens != nil {
tokenBucket.MaxTokens = uint32(*r.MaxTokens)
}
if r.FillInterval != nil {
tokenBucket.FillInterval = durationpb.New(time.Duration(*r.FillInterval) * time.Second)
}
var FilterEnabledDefault *envoy_core_v3.RuntimeFractionalPercent
if r.FilterEnabled != nil {
FilterEnabledDefault = &envoy_core_v3.RuntimeFractionalPercent{
DefaultValue: &envoy_type_v3.FractionalPercent{
Numerator: *r.FilterEnabled,
Denominator: envoy_type_v3.FractionalPercent_HUNDRED,
},
}
}
var FilterEnforcedDefault *envoy_core_v3.RuntimeFractionalPercent
if r.FilterEnforced != nil {
FilterEnforcedDefault = &envoy_core_v3.RuntimeFractionalPercent{
DefaultValue: &envoy_type_v3.FractionalPercent{
Numerator: *r.FilterEnforced,
Denominator: envoy_type_v3.FractionalPercent_HUNDRED,
},
}
}
ratelimitHttpFilter, err := extensioncommon.MakeEnvoyHTTPFilter(
"envoy.filters.http.local_ratelimit",
&envoy_ratelimit.LocalRateLimit{
TokenBucket: &tokenBucket,
StatPrefix: "local_ratelimit",
FilterEnabled: FilterEnabledDefault,
FilterEnforced: FilterEnforcedDefault,
},
)
if err != nil {
return filter, false, err
}
changedFilters := make([]*envoy_http_v3.HttpFilter, 0, len(config.HttpFilters)+1)
// The ratelimitHttpFilter is inserted as the first element of the http
// filter chain.
changedFilters = append(changedFilters, ratelimitHttpFilter)
changedFilters = append(changedFilters, config.HttpFilters...)
config.HttpFilters = changedFilters
newFilter, err := extensioncommon.MakeFilter("envoy.filters.network.http_connection_manager", config)
if err != nil {
return filter, false, errors.New("error making new filter")
}
return newFilter, true, nil
}
func validateProxyType(t string) error {
if t != string(api.ServiceKindConnectProxy) {
return fmt.Errorf("unexpected ProxyType %q", t)
}
return nil
}

187
agent/envoyextensions/builtin/http/localratelimit/ratelimit_test.go
View File

@ -1,187 +0,0 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package localratelimit
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/envoyextensions/extensioncommon"
)
func TestConstructor(t *testing.T) {
makeArguments := func(overrides map[string]interface{}) map[string]interface{} {
m := map[string]interface{}{
"ProxyType": "connect-proxy",
}
for k, v := range overrides {
m[k] = v
}
return m
}
cases := map[string]struct {
extensionName string
arguments map[string]interface{}
expected ratelimit
ok bool
expectedErrMsg string
}{
"with no arguments": {
arguments: nil,
ok: false,
},
"with an invalid name": {
arguments: makeArguments(map[string]interface{}{}),
extensionName: "bad",
ok: false,
},
"MaxToken is missing": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"FillInterval": 30,
"TokensPerFill": 5,
}),
expectedErrMsg: "MaxTokens is missing",
ok: false,
},
"MaxTokens <= 0": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"FillInterval": 30,
"TokensPerFill": 5,
"MaxTokens": 0,
}),
expectedErrMsg: "MaxTokens must be greater than 0",
ok: false,
},
"FillInterval is missing": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"TokensPerFill": 5,
"MaxTokens": 10,
}),
expectedErrMsg: "FillInterval(in second) is missing",
ok: false,
},
"FillInterval <= 0": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"FillInterval": 0,
"TokensPerFill": 5,
"MaxTokens": 10,
}),
expectedErrMsg: "FillInterval(in second) must be greater than 0",
ok: false,
},
"TokensPerFill <= 0": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"FillInterval": 30,
"TokensPerFill": 0,
"MaxTokens": 10,
}),
expectedErrMsg: "TokensPerFill must be greater than 0",
ok: false,
},
"FilterEnabled < 0": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"FillInterval": 30,
"TokensPerFill": 5,
"MaxTokens": 10,
"FilterEnabled": -1,
}),
expectedErrMsg: "cannot parse 'FilterEnabled', -1 overflows uint",
ok: false,
},
"FilterEnforced < 0": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"FillInterval": 30,
"TokensPerFill": 5,
"MaxTokens": 10,
"FilterEnforced": -1,
}),
expectedErrMsg: "cannot parse 'FilterEnforced', -1 overflows uint",
ok: false,
},
"invalid proxy type": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "invalid",
"FillInterval": 30,
"MaxTokens": 20,
"TokensPerFill": 5,
}),
expectedErrMsg: `unexpected ProxyType "invalid"`,
ok: false,
},
"default proxy type": {
arguments: makeArguments(map[string]interface{}{
"FillInterval": 30,
"MaxTokens": 20,
"TokensPerFill": 5,
}),
expected: ratelimit{
ProxyType: "connect-proxy",
MaxTokens: intPointer(20),
FillInterval: intPointer(30),
TokensPerFill: intPointer(5),
},
ok: true,
},
"valid everything": {
arguments: makeArguments(map[string]interface{}{
"ProxyType": "connect-proxy",
"FillInterval": 30,
"MaxTokens": 20,
"TokensPerFill": 5,
}),
expected: ratelimit{
ProxyType: "connect-proxy",
MaxTokens: intPointer(20),
FillInterval: intPointer(30),
TokensPerFill: intPointer(5),
},
ok: true,
},
}
for n, tc := range cases {
t.Run(n, func(t *testing.T) {
extensionName := api.BuiltinLocalRatelimitExtension
if tc.extensionName != "" {
extensionName = tc.extensionName
}
svc := api.CompoundServiceName{Name: "svc"}
ext := extensioncommon.RuntimeConfig{
ServiceName: svc,
EnvoyExtension: api.EnvoyExtension{
Name: extensionName,
Arguments: tc.arguments,
},
}
e, err := Constructor(ext.EnvoyExtension)
if tc.ok {
require.NoError(t, err)
require.Equal(t, &extensioncommon.BasicEnvoyExtender{Extension: &tc.expected}, e)
} else {
require.Error(t, err)
require.Contains(t, err.Error(), tc.expectedErrMsg)
}
})
}
}
func intPointer(i int) *int {
return &i
}

2
agent/envoyextensions/registered_extensions.go
View File

@ -11,7 +11,6 @@ import (
awslambda "github.com/hashicorp/consul/agent/envoyextensions/builtin/aws-lambda" awslambda "github.com/hashicorp/consul/agent/envoyextensions/builtin/aws-lambda"
extauthz "github.com/hashicorp/consul/agent/envoyextensions/builtin/ext-authz" extauthz "github.com/hashicorp/consul/agent/envoyextensions/builtin/ext-authz"
"github.com/hashicorp/consul/agent/envoyextensions/builtin/http/localratelimit"
"github.com/hashicorp/consul/agent/envoyextensions/builtin/lua" "github.com/hashicorp/consul/agent/envoyextensions/builtin/lua"
propertyoverride "github.com/hashicorp/consul/agent/envoyextensions/builtin/property-override" propertyoverride "github.com/hashicorp/consul/agent/envoyextensions/builtin/property-override"
"github.com/hashicorp/consul/agent/envoyextensions/builtin/wasm" "github.com/hashicorp/consul/agent/envoyextensions/builtin/wasm"
@ -24,7 +23,6 @@ type extensionConstructor func(api.EnvoyExtension) (extensioncommon.EnvoyExtende
var extensionConstructors = map[string]extensionConstructor{ var extensionConstructors = map[string]extensionConstructor{
api.BuiltinLuaExtension: lua.Constructor, api.BuiltinLuaExtension: lua.Constructor,
api.BuiltinAWSLambdaExtension: awslambda.Constructor, api.BuiltinAWSLambdaExtension: awslambda.Constructor,
api.BuiltinLocalRatelimitExtension: localratelimit.Constructor,
api.BuiltinPropertyOverrideExtension: propertyoverride.Constructor, api.BuiltinPropertyOverrideExtension: propertyoverride.Constructor,
api.BuiltinWasmExtension: wasm.Constructor, api.BuiltinWasmExtension: wasm.Constructor,
api.BuiltinExtAuthzExtension: extauthz.Constructor, api.BuiltinExtAuthzExtension: extauthz.Constructor,

24
agent/xds/delta_envoy_extender_oss_test.go
View File

@ -15,13 +15,14 @@ import (
envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"github.com/hashicorp/consul/agent/xds/testcommon"
"github.com/hashicorp/go-hclog" "github.com/hashicorp/go-hclog"
goversion "github.com/hashicorp/go-version" goversion "github.com/hashicorp/go-version"
testinf "github.com/mitchellh/go-testing-interface" testinf "github.com/mitchellh/go-testing-interface"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
"github.com/hashicorp/consul/agent/xds/testcommon"
propertyoverride "github.com/hashicorp/consul/agent/envoyextensions/builtin/property-override" propertyoverride "github.com/hashicorp/consul/agent/envoyextensions/builtin/property-override"
"github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/structs"
@ -579,27 +580,6 @@ end`,
return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nsFunc, nil, makeLambdaServiceDefaults(true)) return proxycfg.TestConfigSnapshotDiscoveryChain(t, "default", false, nsFunc, nil, makeLambdaServiceDefaults(true))
}, },
}, },
{
name: "http-local-ratelimit-applyto-filter",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) {
ns.Proxy.Config["protocol"] = "http"
ns.Proxy.EnvoyExtensions = []structs.EnvoyExtension{
{
Name: api.BuiltinLocalRatelimitExtension,
Arguments: map[string]interface{}{
"ProxyType": "connect-proxy",
"MaxTokens": 3,
"TokensPerFill": 2,
"FillInterval": 10,
"FilterEnabled": 100,
"FilterEnforced": 100,
},
},
}
}, nil)
},
},
{ {
name: "wasm-http-local-file", name: "wasm-http-local-file",
create: func(t testinf.T) *proxycfg.ConfigSnapshot { create: func(t testinf.T) *proxycfg.ConfigSnapshot {

127
agent/xds/testdata/builtin_extension/clusters/http-local-ratelimit-applyto-filter.latest.golden vendored
View File

@ -1,127 +0,0 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"altStatName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"circuitBreakers": {},
"outlierDetection": {},
"commonLbConfig": {
"healthyPanicThreshold": {}
},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db"
}
]
}
},
"sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"circuitBreakers": {},
"outlierDetection": {},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
},
"matchSubjectAltNames": [
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/geo-cache-target"
},
{
"exact": "spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc2/svc/geo-cache-target"
}
]
}
},
"sni": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "local_app",
"type": "STATIC",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "local_app",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 8080
}
}
}
}
]
}
]
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

75
agent/xds/testdata/builtin_extension/endpoints/http-local-ratelimit-applyto-filter.latest.golden vendored
View File

@ -1,75 +0,0 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.2",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.10.1.1",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "10.20.1.2",
"portValue": 8080
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"nonce": "00000001"
}

256
agent/xds/testdata/builtin_extension/listeners/http-local-ratelimit-applyto-filter.latest.golden vendored
View File

@ -1,256 +0,0 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "db:127.0.0.1:9191",
"address": {
"socketAddress": {
"address": "127.0.0.1",
"portValue": 9191
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.db.default.default.dc1",
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "prepared_query:geo-cache:127.10.10.10:8181",
"address": {
"socketAddress": {
"address": "127.10.10.10",
"portValue": 8181
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.prepared_query_geo-cache",
"cluster": "geo-cache.default.dc1.query.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"trafficDirection": "OUTBOUND"
},
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "public_listener:0.0.0.0:9999",
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 9999
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"statPrefix": "public_listener",
"routeConfig": {
"name": "public_listener",
"virtualHosts": [
{
"name": "public_listener",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "local_app"
}
}
]
}
]
},
"httpFilters": [
{
"name": "envoy.filters.http.local_ratelimit",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit",
"statPrefix": "local_ratelimit",
"tokenBucket": {
"maxTokens": 3,
"tokensPerFill": 2,
"fillInterval": "10s"
},
"filterEnabled": {
"defaultValue": {
"numerator": 100
}
},
"filterEnforced": {
"defaultValue": {
"numerator": 100
}
}
}
},
{
"name": "envoy.filters.http.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC",
"rules": {}
}
},
{
"name": "envoy.filters.http.header_to_metadata",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.header_to_metadata.v3.Config",
"requestRules": [
{
"header": "x-forwarded-client-cert",
"onHeaderPresent": {
"metadataNamespace": "consul",
"key": "trust-domain",
"regexValueRewrite": {
"pattern": {
"googleRe2": {},
"regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*"
},
"substitution": "\\1"
}
}
},
{
"header": "x-forwarded-client-cert",
"onHeaderPresent": {
"metadataNamespace": "consul",
"key": "partition",
"regexValueRewrite": {
"pattern": {
"googleRe2": {},
"regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*"
},
"substitution": "\\2"
}
}
},
{
"header": "x-forwarded-client-cert",
"onHeaderPresent": {
"metadataNamespace": "consul",
"key": "namespace",
"regexValueRewrite": {
"pattern": {
"googleRe2": {},
"regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*"
},
"substitution": "\\3"
}
}
},
{
"header": "x-forwarded-client-cert",
"onHeaderPresent": {
"metadataNamespace": "consul",
"key": "datacenter",
"regexValueRewrite": {
"pattern": {
"googleRe2": {},
"regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*"
},
"substitution": "\\4"
}
}
},
{
"header": "x-forwarded-client-cert",
"onHeaderPresent": {
"metadataNamespace": "consul",
"key": "service",
"regexValueRewrite": {
"pattern": {
"googleRe2": {},
"regex": ".*URI=spiffe://([^/]+.[^/]+)(?:/ap/([^/]+))?/ns/([^/]+)/dc/([^/]+)/svc/([^/;,]+).*"
},
"substitution": "\\5"
}
}
}
]
}
},
{
"name": "envoy.filters.http.router",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"tracing": {
"randomSampling": {}
},
"forwardClientCertDetails": "APPEND_FORWARD",
"setCurrentClientCertDetails": {
"subject": true,
"cert": true,
"chain": true,
"dns": true,
"uri": true
}
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
},
"privateKey": {
"inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
},
"alpnProtocols": [
"http/1.1"
]
},
"requireClientCertificate": true
}
}
}
],
"trafficDirection": "INBOUND"
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}

5
agent/xds/testdata/builtin_extension/routes/http-local-ratelimit-applyto-filter.latest.golden vendored
View File

@ -1,5 +0,0 @@
{
"versionInfo": "00000001",
"typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"nonce": "00000001"
}

1
api/config_entry.go
View File

@ -42,7 +42,6 @@ const (
BuiltinAWSLambdaExtension string = "builtin/aws/lambda" BuiltinAWSLambdaExtension string = "builtin/aws/lambda"
BuiltinExtAuthzExtension string = "builtin/ext-authz" BuiltinExtAuthzExtension string = "builtin/ext-authz"
BuiltinLuaExtension string = "builtin/lua" BuiltinLuaExtension string = "builtin/lua"
BuiltinLocalRatelimitExtension string = "builtin/http/localratelimit"
BuiltinPropertyOverrideExtension string = "builtin/property-override" BuiltinPropertyOverrideExtension string = "builtin/property-override"
BuiltinWasmExtension string = "builtin/wasm" BuiltinWasmExtension string = "builtin/wasm"
// BuiltinValidateExtension should not be exposed directly or accepted as a valid configured // BuiltinValidateExtension should not be exposed directly or accepted as a valid configured

7
test/integration/connect/envoy/case-envoyext-ratelimit/capture.sh
View File

@ -1,7 +0,0 @@
#!/bin/bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
snapshot_envoy_admin localhost:19000 s1 primary || true
snapshot_envoy_admin localhost:19001 s2 primary || true

19
test/integration/connect/envoy/case-envoyext-ratelimit/service_s1.hcl
View File

@ -1,19 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
services {
name = "s1"
port = 8080
connect {
sidecar_service {
proxy {
upstreams = [
{
destination_name = "s2"
local_bind_port = 5000
}
]
}
}
}
}

8
test/integration/connect/envoy/case-envoyext-ratelimit/service_s2.hcl
View File

@ -1,8 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
services {
name = "s2"
port = 8181
connect { sidecar_service {} }
}

49
test/integration/connect/envoy/case-envoyext-ratelimit/setup.sh
View File

@ -1,49 +0,0 @@
#!/bin/bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
set -eEuo pipefail
upsert_config_entry primary '
Kind = "service-defaults"
Name = "s2"
Protocol = "http"
EnvoyExtensions = [
{
Name = "builtin/http/localratelimit",
Arguments = {
ProxyType = "connect-proxy"
MaxTokens = 1,
TokensPerFill = 1,
FillInterval = 120,
FilterEnabled = 100,
FilterEnforced = 100,
}
}
]
'
upsert_config_entry primary '
Kind = "service-defaults"
Name = "s1"
Protocol = "tcp"
EnvoyExtensions = [
{
Name = "builtin/http/localratelimit",
Arguments = {
ProxyType = "connect-proxy"
MaxTokens = 1,
TokensPerFill = 1,
FillInterval = 120,
FilterEnabled = 100,
FilterEnforced = 100,
}
}
]
'
register_services primary
gen_envoy_bootstrap s1 19000 primary
gen_envoy_bootstrap s2 19001 primary

6
test/integration/connect/envoy/case-envoyext-ratelimit/vars.sh
View File

@ -1,6 +0,0 @@
#!/bin/bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0
export REQUIRED_SERVICES="s1 s1-sidecar-proxy s2 s2-sidecar-proxy"

57
test/integration/connect/envoy/case-envoyext-ratelimit/verify.bats
View File

@ -1,57 +0,0 @@
#!/usr/bin/env bats
load helpers
@test "s1 proxy admin is up on :19000" {
retry_default curl -f -s localhost:19000/stats -o /dev/null
}
@test "s2 proxy admin is up on :19001" {
retry_default curl -f -s localhost:19001/stats -o /dev/null
}
@test "s1 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21000 s1
}
@test "s2 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21001 s2
}
@test "s2 proxy should be healthy" {
assert_service_has_healthy_instances s2 1
}
@test "s1 upstream should have healthy endpoints for s2" {
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1
}
@test "s2 proxy should have been configured with http local ratelimit filters" {
HTTP_FILTERS=$(get_envoy_http_filters localhost:19001)
PUB=$(echo "$HTTP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ')
echo "HTTP_FILTERS = $HTTP_FILTERS"
echo "PUB = $PUB"
[ "$PUB" = "envoy.filters.http.local_ratelimit,envoy.filters.http.rbac,envoy.filters.http.header_to_metadata,envoy.filters.http.router" ]
}
@test "s1(tcp) proxy should not be changed by http/localratelimit extension" {
TCP_FILTERS=$(get_envoy_listener_filters localhost:19000)
PUB=$(echo "$TCP_FILTERS" | grep -E "^public_listener:" | cut -f 2 -d ' ')
echo "TCP_FILTERS = $TCP_FILTERS"
echo "PUB = $PUB"
[ "$PUB" = "envoy.filters.network.rbac,envoy.filters.network.tcp_proxy" ]
}
@test "first connection to s2 - success" {
run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ]
[[ "$output" == *"hello"* ]]
}
@test "ratelimit to s2 is in effect - return code 429" {
retry_default must_fail_http_connection localhost:5000 429
}