Enhancement: Consul Compatibility Checking (#15818)

* add functions for returning the max and min Envoy major versions
- added an UnsupportedEnvoyVersions list
- removed an unused error from TestDetermineSupportedProxyFeaturesFromString
- modified minSupportedVersion to use the function for getting the Min Envoy major version. Using just the major version without the patch is equivalent to using `.0`

* added a function for executing the envoy --version command
- added a new exec.go file to not be locked to unix system

* added envoy version check when using consul connect envoy

* added changelog entry

* added docs change

.changelog/15818.txt

@@ -0,0 +1,6 @@
+```release-note:enhancement
+connect: for early awareness of Envoy incompatibilities, when using the `consul connect envoy` command the Envoy version will now be checked for compatibility. If incompatible Consul will error and exit.
+```
+```release-note:breaking-change
+connect: Consul will now error and exit when using the `consul connect envoy` command if the Envoy version is incompatible. To ignore this check use flag `--ignore-envoy-compatibility`
+```

agent/xds/envoy_versioning.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	"github.com/hashicorp/consul/agent/xds/proxysupport"
 
 	"github.com/hashicorp/go-version"
 )
@@ -11,7 +12,7 @@ import (
 var (
 	// minSupportedVersion is the oldest mainline version we support. This should always be
 	// the zero'th point release of the last element of proxysupport.EnvoyVersions.
-	minSupportedVersion = version.Must(version.NewVersion("1.21.0"))
+	minSupportedVersion = version.Must(version.NewVersion(proxysupport.GetMinEnvoyMinorVersion()))
 
 	specificUnsupportedVersions = []unsupportedVersion{}
 )

agent/xds/envoy_versioning_test.go

@@ -68,7 +68,6 @@ func TestDetermineEnvoyVersionFromNode(t *testing.T) {
 
 func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) {
 	const (
-		err1_13   = "is too old of a point release and is not supported by Consul because it does not support RBAC rules using url_path. Please upgrade to version 1.13.1+."
 		errTooOld = "is too old and is not supported by Consul"
 	)
 

agent/xds/proxysupport/proxysupport.go

@@ -1,5 +1,7 @@
 package proxysupport
 
+import "strings"
+
 // EnvoyVersions lists the latest officially supported versions of envoy.
 //
 // This list must be sorted by semver descending. Only one point release for
@@ -12,3 +14,28 @@ var EnvoyVersions = []string{
 	"1.22.5",
 	"1.21.5",
 }
+
+// UnsupportedEnvoyVersions lists any unsupported Envoy versions (mainly minor versions) that fall
+// within the range of EnvoyVersions above.
+// For example, if patch 1.21.3 (patch 3) had a breaking change, and was not supported
+// even though 1.21 is a supported major release, you would then add 1.21.3 to this list.
+// This list will be empty in most cases.
+//
+// see: https://www.consul.io/docs/connect/proxies/envoy#supported-versions
+var UnsupportedEnvoyVersions = []string{}
+
+// GetMaxEnvoyMinorVersion grabs the first value in EnvoyVersions and strips the patch number off in order
+// to return the maximum supported Envoy minor version
+// For example, if the input string is "1.14.1", the function would return "1.14".
+func GetMaxEnvoyMinorVersion() string {
+	s := strings.Split(EnvoyVersions[0], ".")
+	return s[0] + "." + s[1]
+}
+
+// GetMinEnvoyMinorVersion grabs the last value in EnvoyVersions and strips the patch number off in order
+// to return the minimum supported Envoy minor version
+// For example, if the input string is "1.12.1", the function would return "1.12".
+func GetMinEnvoyMinorVersion() string {
+	s := strings.Split(EnvoyVersions[len(EnvoyVersions)-1], ".")
+	return s[0] + "." + s[1]
+}

agent/xds/proxysupport/proxysupport_test.go

@@ -0,0 +1,30 @@
+package proxysupport
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/hashicorp/go-version"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProxySupportOrder(t *testing.T) {
+	versions := make([]*version.Version, len(EnvoyVersions))
+	beforeSort := make([]*version.Version, len(EnvoyVersions))
+	for i, raw := range EnvoyVersions {
+		v, _ := version.NewVersion(raw)
+		versions[i] = v
+		beforeSort[i] = v
+	}
+
+	// After this, the versions are properly sorted
+	// go-version has a collection container, but it only allows for sorting in ascending order
+	sort.Slice(versions, func(i, j int) bool {
+		return versions[j].LessThan(versions[i])
+	})
+
+	// Check that we already have a sorted list
+	for i := range EnvoyVersions {
+		assert.True(t, versions[i].Equal(beforeSort[i]))
+	}
+}

command/connect/envoy/envoy.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 
@@ -21,6 +22,7 @@ import (
 	"github.com/hashicorp/consul/command/flags"
 	"github.com/hashicorp/consul/ipaddr"
 	"github.com/hashicorp/consul/tlsutil"
+	"github.com/hashicorp/go-version"
 )
 
 func New(ui cli.Ui) *cmd {
@@ -39,26 +41,27 @@ type cmd struct {
 	client *api.Client
 
 	// flags
-	meshGateway           bool
+	meshGateway              bool
-	gateway               string
+	gateway                  string
-	proxyID               string
+	proxyID                  string
-	nodeName              string
+	nodeName                 string
-	sidecarFor            string
+	sidecarFor               string
-	adminAccessLogPath    string
+	adminAccessLogPath       string
-	adminBind             string
+	adminBind                string
-	envoyBin              string
+	envoyBin                 string
-	bootstrap             bool
+	bootstrap                bool
-	disableCentralConfig  bool
+	disableCentralConfig     bool
-	grpcAddr              string
+	grpcAddr                 string
-	grpcCAFile            string
+	grpcCAFile               string
-	grpcCAPath            string
+	grpcCAPath               string
-	envoyVersion          string
+	envoyVersion             string
-	prometheusBackendPort string
+	prometheusBackendPort    string
-	prometheusScrapePath  string
+	prometheusScrapePath     string
-	prometheusCAFile      string
+	prometheusCAFile         string
-	prometheusCAPath      string
+	prometheusCAPath         string
-	prometheusCertFile    string
+	prometheusCertFile       string
-	prometheusKeyFile     string
+	prometheusKeyFile        string
+	ignoreEnvoyCompatibility bool
 
 	// mesh gateway registration information
 	register           bool
@@ -204,6 +207,10 @@ func (c *cmd) init() {
 	c.flags.StringVar(&c.prometheusKeyFile, "prometheus-key-file", "",
 		"Path to a private key file for Envoy to use when serving TLS on the Prometheus metrics endpoint. "+
 			"Only applicable when envoy_prometheus_bind_addr is set in proxy config.")
+	c.flags.BoolVar(&c.ignoreEnvoyCompatibility, "ignore-envoy-compatibility", false,
+		"If set to `true`, this flag ignores the Envoy version compatibility check. We recommend setting this "+
+			"flag to `false` to ensure compatibility with Envoy and prevent potential issues. "+
+			"Default is `false`.")
 
 	c.http = &flags.HTTPFlags{}
 	flags.Merge(c.flags, c.http.ClientFlags())
@@ -455,6 +462,27 @@ func (c *cmd) run(args []string) int {
 		return 1
 	}
 
+	// Check if envoy version is supported
+	if !c.ignoreEnvoyCompatibility {
+		v, err := execEnvoyVersion(binary)
+		if err != nil {
+			c.UI.Warn("Couldn't get envoy version for compatibility check: " + err.Error())
+			return 1
+		}
+
+		ok, err := checkEnvoyVersionCompatibility(v, proxysupport.UnsupportedEnvoyVersions)
+
+		if err != nil {
+			c.UI.Warn("There was an error checking the compatibility of the envoy version: " + err.Error())
+		} else if !ok {
+			c.UI.Error(fmt.Sprintf("Envoy version %s is not supported. If there is a reason you need to use "+
+				"this version of envoy use the ignore-envoy-compatibility flag. Using an unsupported version of Envoy "+
+				"is not recommended and your experience may vary. For more information on compatibility "+
+				"see https://developer.hashicorp.com/consul/docs/connect/proxies/envoy#envoy-and-consul-client-agent", v))
+			return 1
+		}
+	}
+
 	err = execEnvoy(binary, nil, args, bootstrapJson)
 	if err == errUnsupportedOS {
 		c.UI.Error("Directly running Envoy is only supported on linux and macOS " +
@@ -834,3 +862,35 @@ Usage: consul connect envoy [options] [-- pass-through options]
     $ consul connect envoy -sidecar-for web -- --log-level debug
 `
 )
+
+func checkEnvoyVersionCompatibility(envoyVersion string, unsupportedList []string) (bool, error) {
+	// Now compare the versions to the list of supported versions
+	v, err := version.NewVersion(envoyVersion)
+	if err != nil {
+		return false, err
+	}
+
+	var cs strings.Builder
+
+	// Add one to the max minor version so that we accept all patches
+	splitS := strings.Split(proxysupport.GetMaxEnvoyMinorVersion(), ".")
+	minor, err := strconv.Atoi(splitS[1])
+	if err != nil {
+		return false, err
+	}
+	minor++
+	maxSupported := fmt.Sprintf("%s.%d", splitS[0], minor)
+
+	// Build the constraint string, make sure that we are less than but not equal to maxSupported since we added 1
+	cs.WriteString(fmt.Sprintf(">= %s, < %s", proxysupport.GetMinEnvoyMinorVersion(), maxSupported))
+	for _, s := range unsupportedList {
+		cs.WriteString(fmt.Sprintf(", != %s", s))
+	}
+
+	constraints, err := version.NewConstraint(cs.String())
+	if err != nil {
+		return false, err
+	}
+
+	return constraints.Check(v), nil
+}

command/connect/envoy/envoy_test.go

@@ -3,15 +3,18 @@ package envoy
 import (
 	"encoding/json"
 	"flag"
+	"fmt"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/hashicorp/consul/agent/xds/proxysupport"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/mitchellh/cli"
@@ -1522,3 +1525,83 @@ func testMockAgentSelf(wantXDSPorts agent.GRPCPorts, agentSelf110 bool) http.Han
 		w.Write(selfJSON)
 	}
 }
+
+func TestCheckEnvoyVersionCompatibility(t *testing.T) {
+	tests := []struct {
+		name            string
+		envoyVersion    string
+		unsupportedList []string
+		expectedSupport bool
+		isErrorExpected bool
+	}{
+		{
+			name:            "supported-using-proxy-support-defined",
+			envoyVersion:    proxysupport.EnvoyVersions[1],
+			unsupportedList: proxysupport.UnsupportedEnvoyVersions,
+			expectedSupport: true,
+		},
+		{
+			name:            "supported-at-max",
+			envoyVersion:    proxysupport.GetMaxEnvoyMinorVersion(),
+			unsupportedList: proxysupport.UnsupportedEnvoyVersions,
+			expectedSupport: true,
+		},
+		{
+			name:            "supported-patch-higher",
+			envoyVersion:    addNPatchVersion(proxysupport.EnvoyVersions[0], 1),
+			unsupportedList: proxysupport.UnsupportedEnvoyVersions,
+			expectedSupport: true,
+		},
+		{
+			name:            "not-supported-minor-higher",
+			envoyVersion:    addNMinorVersion(proxysupport.EnvoyVersions[0], 1),
+			unsupportedList: proxysupport.UnsupportedEnvoyVersions,
+			expectedSupport: false,
+		},
+		{
+			name:            "not-supported-minor-lower",
+			envoyVersion:    addNMinorVersion(proxysupport.EnvoyVersions[len(proxysupport.EnvoyVersions)-1], -1),
+			unsupportedList: proxysupport.UnsupportedEnvoyVersions,
+			expectedSupport: false,
+		},
+		{
+			name:            "not-supported-explicitly-unsupported-version",
+			envoyVersion:    addNPatchVersion(proxysupport.EnvoyVersions[0], 1),
+			unsupportedList: []string{"1.23.1", addNPatchVersion(proxysupport.EnvoyVersions[0], 1)},
+			expectedSupport: false,
+		},
+		{
+			name:            "error-bad-input",
+			envoyVersion:    "1.abc.3",
+			unsupportedList: proxysupport.UnsupportedEnvoyVersions,
+			expectedSupport: false,
+			isErrorExpected: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			actual, err := checkEnvoyVersionCompatibility(tc.envoyVersion, tc.unsupportedList)
+			if tc.isErrorExpected {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, tc.expectedSupport, actual)
+		})
+	}
+}
+
+func addNPatchVersion(s string, n int) string {
+	splitS := strings.Split(s, ".")
+	minor, _ := strconv.Atoi(splitS[2])
+	minor += n
+	return fmt.Sprintf("%s.%s.%d", splitS[0], splitS[1], minor)
+}
+
+func addNMinorVersion(s string, n int) string {
+	splitS := strings.Split(s, ".")
+	major, _ := strconv.Atoi(splitS[1])
+	major += n
+	return fmt.Sprintf("%s.%d.%s", splitS[0], major, splitS[2])
+}

command/connect/envoy/exec.go

@@ -0,0 +1,44 @@
+package envoy
+
+import (
+	"errors"
+	"os/exec"
+	"regexp"
+)
+
+const (
+	envoyVersionFlag = "--version"
+)
+
+// execCommand lets us mock out the exec.Command function
+var execCommand = exec.Command
+
+func execEnvoyVersion(binary string) (string, error) {
+	cmd := execCommand(binary, envoyVersionFlag)
+
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	version, err := parseEnvoyVersionNumber(string(output))
+	if err != nil {
+		return "", err
+	}
+	return version, nil
+}
+
+func parseEnvoyVersionNumber(fullVersion string) (string, error) {
+	// Use a regular expression to match the major.minor.patch version string in the fullVersion
+	// Example input:
+	// `envoy  version: 69958e4fe32da561376d8b1d367b5e6942dfba24/1.24.1/Distribution/RELEASE/BoringSSL`
+	re := regexp.MustCompile(`(\d+\.\d+\.\d+)`)
+	matches := re.FindStringSubmatch(fullVersion)
+
+	// If no matches were found, return an error
+	if len(matches) == 0 {
+		return "", errors.New("unable to parse Envoy version from output")
+	}
+
+	// Return the first match (the major.minor.patch version string)
+	return matches[0], nil
+}

command/connect/envoy/exec_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -133,6 +134,79 @@ func TestExecEnvoy(t *testing.T) {
 	}
 }
 
+func TestExecEnvoyVersion(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
+	tests := []struct {
+		name           string
+		cmdOutput      string
+		expectedOutput string
+	}{
+		{
+			name:           "actual-version-output-1-24-1",
+			cmdOutput:      `envoy  version: 69958e4fe32da561376d8b1d367b5e6942dfba24/1.24.1/Distribution/RELEASE/BoringSSL`,
+			expectedOutput: "1.24.1",
+		},
+		{
+			name:           "format-change",
+			cmdOutput:      `envoy  version: (69958e4fe32da561376d8b1d367b5e6942dfba24)__(1.24.1)/Distribution/RELEASE/BoringSSL`,
+			expectedOutput: "1.24.1",
+		},
+		{
+			name:           "zeroes",
+			cmdOutput:      `envoy  version: 69958e4fe32da561376d8b1d367b5e6942dfba24/0.0.0/Distribution/RELEASE/BoringSSL`,
+			expectedOutput: "0.0.0",
+		},
+		{
+			name:           "test-multi-digit",
+			cmdOutput:      `envoy  version: 69958e4fe32da561376d8b1d367b5e6942dfba24/1246390.9401081.1238495/Distribution/RELEASE/BoringSSL`,
+			expectedOutput: "1246390.9401081.1238495",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			fe := fakeEnvoy{
+				desiredOutput: tc.cmdOutput,
+			}
+			execCommand = fe.ExecCommand
+			// Reset back to base exec.Command
+			defer func() { execCommand = exec.Command }()
+			version, err := execEnvoyVersion("fake-envoy")
+
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.expectedOutput, version)
+		})
+	}
+}
+
+type fakeEnvoy struct {
+	desiredOutput string
+}
+
+func (fe fakeEnvoy) ExecCommand(command string, args ...string) *exec.Cmd {
+	cs := []string{"-test.run=TestEnvoyExecHelperProcess", "--", command}
+	cs = append(cs, args...)
+	// last argument will be the output
+	cs = append(cs, fe.desiredOutput)
+	cmd := exec.Command(os.Args[0], cs...)
+	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
+	return cmd
+}
+
+func TestEnvoyExecHelperProcess(t *testing.T) {
+	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
+		return
+	}
+
+	output := os.Args[len(os.Args)-1]
+	fmt.Fprint(os.Stdout, output)
+	os.Exit(0)
+}
+
 type FakeEnvoyExecData struct {
 	Args       []string `json:"args"`
 	ConfigPath string   `json:"configPath"`

website/content/commands/connect/envoy.mdx

@@ -104,6 +104,10 @@ Usage: `consul connect envoy [options] [-- pass-through options]`
   TLS on the Prometheus metrics endpoint. Only applicable when `envoy_prometheus_bind_addr`
   is set in proxy config.
 
+- `-ignore-envoy-compatibility` - If set to `true`, this flag ignores the Envoy version
+compatibility check. We recommend setting this flag to `false` to ensure
+compatibility with Envoy and prevent potential issues. Default is `false`.
+
 - `-- [pass-through options]` - Any options given after a double dash are passed
   directly through to the `envoy` invocation. See [Envoy's
   documentation](https://www.envoyproxy.io/docs) for more details. The command