Backport of e2e: modernize vaultcompat testing into release/1.6.x (#18182)
This pull request was automerged via backport-assistant
This commit is contained in:
hc-github-team-nomad-core
GitHub
parent
8bf5067f86
commit
c25c04816d
|
|
@ -48,7 +48,11 @@ jobs:
|
|||
run: git config --global url.'https://${{ env.ELEVATED_GITHUB_TOKEN }}@github.com'.insteadOf 'https://github.com'
|
||||
- uses: hashicorp/setup-golang@v1
|
||||
- run: make deps
|
||||
- run: make integration-test
|
||||
- name: Vault Compatability
|
||||
run: |
|
||||
sudo sed -i 's!Defaults!#Defaults!g' /etc/sudoers
|
||||
sudo -E env "PATH=$PATH" make integration-test
|
||||
sudo -E env "PATH=$PATH" make clean
|
||||
- run: make e2e-test
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
10
GNUmakefile
10
GNUmakefile
|
|
@ -311,13 +311,13 @@ e2e-test: dev ## Run the Nomad e2e test suite
|
|||
.PHONY: integration-test
|
||||
integration-test: dev ## Run Nomad integration tests
|
||||
@echo "==> Running Nomad integration test suites:"
|
||||
go test \
|
||||
$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
|
||||
-cover \
|
||||
NOMAD_E2E_VAULTCOMPAT=1 go test \
|
||||
-v \
|
||||
-race \
|
||||
-timeout=900s \
|
||||
-count=1 \
|
||||
-tags "$(GO_TAGS)" \
|
||||
github.com/hashicorp/nomad/e2e/vaultcompat/ \
|
||||
-integration
|
||||
github.com/hashicorp/nomad/e2e/vaultcompat
|
||||
|
||||
.PHONY: clean
|
||||
clean: GOPATH=$(shell go env GOPATH)
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
# Vault Integration Test
|
||||
|
||||
Not run as part of nightly e2e suite at this point.
|
||||
|
||||
Downloads, caches, and tests Nomad against open source Vault binaries. Runs
|
||||
only when `-integration` is set.
|
||||
|
||||
Run with:
|
||||
|
||||
```
|
||||
cd e2e/vault/
|
||||
go test -integration
|
||||
```
|
||||
|
||||
**Warning: Downloads a lot of Vault versions!**
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package vaultcompat
|
||||
|
||||
import (
|
||||
"github.com/hashicorp/nomad/api"
|
||||
"github.com/hashicorp/nomad/helper/pointer"
|
||||
)
|
||||
|
||||
const (
|
||||
// policy is the recommended Nomad Vault policy
|
||||
policy = `path "auth/token/create/nomad-cluster" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
path "auth/token/roles/nomad-cluster" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
path "auth/token/lookup-self" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "auth/token/lookup" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
path "auth/token/revoke-accessor" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
path "sys/capabilities-self" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
path "auth/token/renew-self" {
|
||||
capabilities = ["update"]
|
||||
}`
|
||||
)
|
||||
|
||||
var (
|
||||
// role is the recommended nomad cluster role
|
||||
role = map[string]interface{}{
|
||||
"disallowed_policies": "nomad-server",
|
||||
"explicit_max_ttl": 0, // use old name for vault compatibility
|
||||
"name": "nomad-cluster",
|
||||
"orphan": false,
|
||||
"period": 259200, // use old name for vault compatibility
|
||||
"renewable": true,
|
||||
}
|
||||
|
||||
// job is a test job that is used to request a Vault token and cat the token
|
||||
// out before exiting.
|
||||
job = &api.Job{
|
||||
ID: pointer.Of("test"),
|
||||
Type: pointer.Of("batch"),
|
||||
Datacenters: []string{"dc1"},
|
||||
TaskGroups: []*api.TaskGroup{
|
||||
{
|
||||
Name: pointer.Of("test"),
|
||||
Tasks: []*api.Task{
|
||||
{
|
||||
Name: "test",
|
||||
Driver: "raw_exec",
|
||||
Config: map[string]interface{}{
|
||||
"command": "cat",
|
||||
"args": []string{"${NOMAD_SECRETS_DIR}/vault_token"},
|
||||
},
|
||||
Vault: &api.Vault{
|
||||
Policies: []string{"default"},
|
||||
},
|
||||
},
|
||||
},
|
||||
RestartPolicy: &api.RestartPolicy{
|
||||
Attempts: pointer.Of(0),
|
||||
Mode: pointer.Of("fail"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
// Package vaultcompat contains vault version compatibility matrix tests.
|
||||
package vaultcompat
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# Copyright (c) HashiCorp, Inc.
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
job "cat" {
|
||||
type = "batch"
|
||||
group "testcase" {
|
||||
task "cat" {
|
||||
driver = "raw_exec"
|
||||
|
||||
config {
|
||||
command = "cat"
|
||||
args = ["${NOMAD_SECRETS_DIR}/vault_token"]
|
||||
}
|
||||
|
||||
vault {
|
||||
policies = ["default"]
|
||||
}
|
||||
}
|
||||
|
||||
restart {
|
||||
attempts = 0
|
||||
mode = "fail"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# Copyright (c) HashiCorp, Inc.
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
path "auth/token/create/nomad-cluster" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
|
||||
path "auth/token/roles/nomad-cluster" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "auth/token/lookup-self" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "auth/token/lookup" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
|
||||
path "auth/token/revoke-accessor" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
|
||||
path "sys/capabilities-self" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
|
||||
path "auth/token/renew-self" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package vaultcompat
|
||||
|
||||
// role is the recommended nomad cluster role
|
||||
var role = map[string]interface{}{
|
||||
"disallowed_policies": "nomad-server",
|
||||
"explicit_max_ttl": 0, // use old name for vault compatibility
|
||||
"name": "nomad-cluster",
|
||||
"orphan": false,
|
||||
"period": 259200, // use old name for vault compatibility
|
||||
"renewable": true,
|
||||
}
|
||||
|
|
@ -1,430 +0,0 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package vaultcompat
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-version"
|
||||
"github.com/hashicorp/nomad/command/agent"
|
||||
"github.com/hashicorp/nomad/helper/pointer"
|
||||
"github.com/hashicorp/nomad/nomad/structs/config"
|
||||
"github.com/hashicorp/nomad/testutil"
|
||||
vapi "github.com/hashicorp/vault/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
integration = flag.Bool("integration", false, "run integration tests")
|
||||
minVaultVer = version.Must(version.NewVersion("0.6.2"))
|
||||
)
|
||||
|
||||
// syncVault discovers available versions of Vault, downloads the binaries,
|
||||
// returns a map of version to binary path as well as a sorted list of
|
||||
// versions.
|
||||
func syncVault(t *testing.T) ([]*version.Version, map[string]string) {
|
||||
|
||||
binDir := filepath.Join(os.TempDir(), "vault-bins/")
|
||||
|
||||
urls := vaultVersions(t)
|
||||
|
||||
sorted, versions, err := pruneVersions(urls)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the binaries we need to download
|
||||
missing, err := missingVault(binDir, versions)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create the directory for the binaries
|
||||
require.NoError(t, createBinDir(binDir))
|
||||
|
||||
// Download in parallel
|
||||
start := time.Now()
|
||||
errCh := make(chan error, len(missing))
|
||||
for ver, url := range missing {
|
||||
go func(dst, url string) {
|
||||
errCh <- getVault(dst, url)
|
||||
}(filepath.Join(binDir, ver), url)
|
||||
}
|
||||
for i := 0; i < len(missing); i++ {
|
||||
select {
|
||||
case err := <-errCh:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(5 * time.Minute):
|
||||
require.Fail(t, "timed out downloading Vault binaries")
|
||||
}
|
||||
}
|
||||
if n := len(missing); n > 0 {
|
||||
t.Logf("Downloaded %d versions of Vault in %s", n, time.Now().Sub(start))
|
||||
}
|
||||
|
||||
binaries := make(map[string]string, len(versions))
|
||||
for ver := range versions {
|
||||
binaries[ver] = filepath.Join(binDir, ver)
|
||||
}
|
||||
return sorted, binaries
|
||||
}
|
||||
|
||||
// vaultVersions discovers available Vault versions from releases.hashicorp.com
|
||||
// and returns a map of version to url.
|
||||
func vaultVersions(t *testing.T) map[string]string {
|
||||
resp, err := http.Get("https://releases.hashicorp.com/vault/index.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
respJson := struct {
|
||||
Versions map[string]struct {
|
||||
Builds []struct {
|
||||
Version string `json:"version"`
|
||||
Os string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
URL string `json:"url"`
|
||||
} `json:"builds"`
|
||||
}
|
||||
}{}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&respJson))
|
||||
require.NoError(t, resp.Body.Close())
|
||||
|
||||
versions := map[string]string{}
|
||||
for vk, vv := range respJson.Versions {
|
||||
gover, err := version.NewVersion(vk)
|
||||
if err != nil {
|
||||
t.Logf("error parsing Vault version %q -> %v", vk, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip ancient versions
|
||||
if gover.LessThan(minVaultVer) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip prerelease and enterprise versions
|
||||
if gover.Prerelease() != "" || gover.Metadata() != "" {
|
||||
continue
|
||||
}
|
||||
|
||||
url := ""
|
||||
for _, b := range vv.Builds {
|
||||
buildver, err := version.NewVersion(b.Version)
|
||||
if err != nil {
|
||||
t.Logf("error parsing Vault build version %q -> %v", b.Version, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if buildver.Prerelease() != "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if buildver.Metadata() != "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if b.Os != runtime.GOOS {
|
||||
continue
|
||||
}
|
||||
|
||||
if b.Arch != runtime.GOARCH {
|
||||
continue
|
||||
}
|
||||
|
||||
// Match!
|
||||
url = b.URL
|
||||
break
|
||||
}
|
||||
|
||||
if url != "" {
|
||||
versions[vk] = url
|
||||
}
|
||||
}
|
||||
|
||||
return versions
|
||||
}
|
||||
|
||||
// pruneVersions only takes the latest Z for each X.Y.Z release. Returns a
|
||||
// sorted list and map of kept versions.
|
||||
func pruneVersions(all map[string]string) ([]*version.Version, map[string]string, error) {
|
||||
if len(all) == 0 {
|
||||
return nil, nil, fmt.Errorf("0 Vault versions")
|
||||
}
|
||||
|
||||
sorted := make([]*version.Version, 0, len(all))
|
||||
|
||||
for k := range all {
|
||||
sorted = append(sorted, version.Must(version.NewVersion(k)))
|
||||
}
|
||||
|
||||
sort.Sort(version.Collection(sorted))
|
||||
|
||||
keep := make([]*version.Version, 0, len(all))
|
||||
|
||||
for _, v := range sorted {
|
||||
segments := v.Segments()
|
||||
if len(segments) < 3 {
|
||||
// Drop malformed versions
|
||||
continue
|
||||
}
|
||||
|
||||
if len(keep) == 0 {
|
||||
keep = append(keep, v)
|
||||
continue
|
||||
}
|
||||
|
||||
last := keep[len(keep)-1].Segments()
|
||||
|
||||
if segments[0] == last[0] && segments[1] == last[1] {
|
||||
// current X.Y == last X.Y, replace last with current
|
||||
keep[len(keep)-1] = v
|
||||
} else {
|
||||
// current X.Y != last X.Y, append
|
||||
keep = append(keep, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new map of canonicalized versions to urls
|
||||
urls := make(map[string]string, len(keep))
|
||||
for _, v := range keep {
|
||||
origURL := all[v.Original()]
|
||||
if origURL == "" {
|
||||
return nil, nil, fmt.Errorf("missing version %s", v.Original())
|
||||
}
|
||||
urls[v.String()] = origURL
|
||||
}
|
||||
|
||||
return keep, urls, nil
|
||||
}
|
||||
|
||||
// createBinDir creates the binary directory
|
||||
func createBinDir(binDir string) error {
|
||||
// Check if the directory exists, otherwise create it
|
||||
f, err := os.Stat(binDir)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to stat directory: %v", err)
|
||||
}
|
||||
|
||||
if f != nil && f.IsDir() {
|
||||
return nil
|
||||
} else if f != nil {
|
||||
if err := os.RemoveAll(binDir); err != nil {
|
||||
return fmt.Errorf("failed to remove file at directory path: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create the directory
|
||||
if err := os.Mkdir(binDir, 075); err != nil {
|
||||
return fmt.Errorf("failed to make directory: %v", err)
|
||||
}
|
||||
if err := os.Chmod(binDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to chmod: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// missingVault returns the binaries that must be downloaded. versions key must
|
||||
// be the Vault version.
|
||||
func missingVault(binDir string, versions map[string]string) (map[string]string, error) {
|
||||
files, err := os.ReadDir(binDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to stat directory: %v", err)
|
||||
}
|
||||
|
||||
// Copy versions so we don't mutate it
|
||||
missingSet := make(map[string]string, len(versions))
|
||||
for k, v := range versions {
|
||||
missingSet[k] = v
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
delete(missingSet, f.Name())
|
||||
}
|
||||
|
||||
return missingSet, nil
|
||||
}
|
||||
|
||||
// getVault downloads the given Vault binary
|
||||
func getVault(dst, url string) error {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Wrap in an in-mem buffer
|
||||
b := bytes.NewBuffer(nil)
|
||||
if _, err := io.Copy(b, resp.Body); err != nil {
|
||||
return fmt.Errorf("error reading response body: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
zreader, err := zip.NewReader(bytes.NewReader(b.Bytes()), resp.ContentLength)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if l := len(zreader.File); l != 1 {
|
||||
return fmt.Errorf("unexpected number of files in zip: %v", l)
|
||||
}
|
||||
|
||||
// Copy the file to its destination
|
||||
out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
zfile, err := zreader.File[0].Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open zip file: %v", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(out, zfile); err != nil {
|
||||
return fmt.Errorf("failed to decompress file to destination: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestVaultCompatibility tests compatibility across Vault versions
|
||||
func TestVaultCompatibility(t *testing.T) {
|
||||
if !*integration {
|
||||
t.Skip("skipping test in non-integration mode: add -integration flag to run")
|
||||
}
|
||||
|
||||
sorted, vaultBinaries := syncVault(t)
|
||||
|
||||
for _, v := range sorted {
|
||||
ver := v.String()
|
||||
bin := vaultBinaries[ver]
|
||||
require.NotZerof(t, bin, "missing version: %s", ver)
|
||||
t.Run(ver, func(t *testing.T) {
|
||||
testVaultCompatibility(t, bin, ver)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// testVaultCompatibility tests compatibility with the given vault binary
|
||||
func testVaultCompatibility(t *testing.T, vault string, version string) {
|
||||
require := require.New(t)
|
||||
|
||||
// Create a Vault server
|
||||
v := testutil.NewTestVaultFromPath(t, vault)
|
||||
defer v.Stop()
|
||||
|
||||
token := setupVault(t, v.Client, version)
|
||||
|
||||
// Create a Nomad agent using the created vault
|
||||
nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) {
|
||||
if c.Vault == nil {
|
||||
c.Vault = &config.VaultConfig{}
|
||||
}
|
||||
c.Vault.Enabled = pointer.Of(true)
|
||||
c.Vault.Token = token
|
||||
c.Vault.Role = "nomad-cluster"
|
||||
c.Vault.AllowUnauthenticated = pointer.Of(true)
|
||||
c.Vault.Addr = v.HTTPAddr
|
||||
})
|
||||
defer nomad.Shutdown()
|
||||
|
||||
// Submit the Nomad job that requests a Vault token and cats that the Vault
|
||||
// token is there
|
||||
c := nomad.Client()
|
||||
j := c.Jobs()
|
||||
_, _, err := j.Register(job, nil)
|
||||
require.NoError(err)
|
||||
|
||||
// Wait for there to be an allocation terminated successfully
|
||||
//var allocID string
|
||||
testutil.WaitForResult(func() (bool, error) {
|
||||
// Get the allocations for the job
|
||||
allocs, _, err := j.Allocations(*job.ID, false, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
l := len(allocs)
|
||||
switch l {
|
||||
case 0:
|
||||
return false, fmt.Errorf("want one alloc; got zero")
|
||||
case 1:
|
||||
default:
|
||||
// exit early
|
||||
require.Fail("too many allocations; something failed")
|
||||
}
|
||||
alloc := allocs[0]
|
||||
//allocID = alloc.ID
|
||||
if alloc.ClientStatus == "complete" {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("client status %q", alloc.ClientStatus)
|
||||
}, func(err error) {
|
||||
require.NoError(err, "allocation did not finish")
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// setupVault takes the Vault client and creates the required policies and
|
||||
// roles. It returns the token that should be used by Nomad
|
||||
func setupVault(t *testing.T, client *vapi.Client, vaultVersion string) string {
|
||||
// Write the policy
|
||||
sys := client.Sys()
|
||||
|
||||
// pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint
|
||||
// perform this using a raw HTTP request
|
||||
newApi := version.Must(version.NewVersion("0.9.0"))
|
||||
testVersion := version.Must(version.NewVersion(vaultVersion))
|
||||
if testVersion.LessThan(newApi) {
|
||||
body := map[string]string{
|
||||
"rules": policy,
|
||||
}
|
||||
request := client.NewRequest("PUT", "/v1/sys/policy/nomad-server")
|
||||
if err := request.SetJSONBody(body); err != nil {
|
||||
require.NoError(t, err, "failed to set JSON body on legacy policy creation")
|
||||
}
|
||||
if _, err := client.RawRequest(request); err != nil {
|
||||
require.NoError(t, err, "failed to create legacy policy")
|
||||
}
|
||||
} else {
|
||||
if err := sys.PutPolicy("nomad-server", policy); err != nil {
|
||||
require.NoError(t, err, "failed to create policy")
|
||||
}
|
||||
}
|
||||
|
||||
// Build the role
|
||||
l := client.Logical()
|
||||
l.Write("auth/token/roles/nomad-cluster", role)
|
||||
|
||||
// Create a new token with the role
|
||||
a := client.Auth().Token()
|
||||
req := vapi.TokenCreateRequest{
|
||||
Policies: []string{"nomad-server"},
|
||||
Period: "72h",
|
||||
NoParent: true,
|
||||
}
|
||||
s, err := a.Create(&req)
|
||||
if err != nil {
|
||||
require.NoError(t, err, "failed to create child token")
|
||||
}
|
||||
|
||||
// Get the client token
|
||||
if s == nil || s.Auth == nil {
|
||||
require.NoError(t, err, "bad secret response")
|
||||
}
|
||||
|
||||
return s.Auth.ClientToken
|
||||
}
|
||||
|
|
@ -0,0 +1,267 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package vaultcompat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-cleanhttp"
|
||||
"github.com/hashicorp/go-set"
|
||||
"github.com/hashicorp/go-version"
|
||||
nomadapi "github.com/hashicorp/nomad/api"
|
||||
"github.com/hashicorp/nomad/testutil"
|
||||
vaultapi "github.com/hashicorp/vault/api"
|
||||
"github.com/shoenig/test/must"
|
||||
"github.com/shoenig/test/wait"
|
||||
)
|
||||
|
||||
const (
|
||||
binDir = "vault-bins"
|
||||
envGate = "NOMAD_E2E_VAULTCOMPAT"
|
||||
)
|
||||
|
||||
func TestVaultCompat(t *testing.T) {
|
||||
if os.Getenv(envGate) != "1" {
|
||||
t.Skip(envGate + " is not set; skipping")
|
||||
}
|
||||
t.Run("testVaultVersions", testVaultVersions)
|
||||
}
|
||||
|
||||
func testVaultVersions(t *testing.T) {
|
||||
versions := scanVaultVersions(t, getMinimumVersion(t))
|
||||
versions.ForEach(func(b build) bool {
|
||||
downloadVaultBuild(t, b)
|
||||
testVaultBuild(t, b)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func testVaultBuild(t *testing.T, b build) {
|
||||
t.Run("vault("+b.Version+")", func(t *testing.T) {
|
||||
vStop, vc := startVault(t, b)
|
||||
defer vStop()
|
||||
setupVault(t, vc)
|
||||
|
||||
nStop, nc := startNomad(t, vc)
|
||||
defer nStop()
|
||||
runCatJob(t, nc)
|
||||
|
||||
// give nomad and vault time to stop
|
||||
defer func() { time.Sleep(5 * time.Second) }()
|
||||
})
|
||||
}
|
||||
|
||||
func runCatJob(t *testing.T, nc *nomadapi.Client) {
|
||||
b, err := os.ReadFile("input/cat.hcl")
|
||||
must.NoError(t, err)
|
||||
|
||||
jobs := nc.Jobs()
|
||||
job, err := jobs.ParseHCL(string(b), true)
|
||||
must.NoError(t, err, must.Sprint("failed to parse job HCL"))
|
||||
|
||||
_, _, err = jobs.Register(job, nil)
|
||||
must.NoError(t, err, must.Sprint("failed to register job"))
|
||||
|
||||
must.Wait(t, wait.InitialSuccess(
|
||||
wait.ErrorFunc(func() error {
|
||||
allocs, _, err := jobs.Allocations(*job.ID, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n := len(allocs); n != 1 {
|
||||
return fmt.Errorf("expected 1 alloc, got %d", n)
|
||||
}
|
||||
if s := allocs[0].ClientStatus; s != "complete" {
|
||||
return fmt.Errorf("expected alloc status complete, got %s", s)
|
||||
}
|
||||
return nil
|
||||
}),
|
||||
wait.Timeout(20*time.Second),
|
||||
wait.Gap(1*time.Second),
|
||||
))
|
||||
|
||||
t.Log("success running cat job")
|
||||
|
||||
_, _, err = jobs.Deregister(*job.Name, true, nil)
|
||||
must.NoError(t, err, must.Sprint("faild to deregister job"))
|
||||
}
|
||||
|
||||
func startVault(t *testing.T, b build) (func(), *vaultapi.Client) {
|
||||
path := filepath.Join(os.TempDir(), binDir, b.Version, "vault")
|
||||
vlt := testutil.NewTestVaultFromPath(t, path)
|
||||
return vlt.Stop, vlt.Client
|
||||
}
|
||||
|
||||
func setupVault(t *testing.T, vc *vaultapi.Client) {
|
||||
policy, err := os.ReadFile("input/policy.hcl")
|
||||
must.NoError(t, err)
|
||||
|
||||
sys := vc.Sys()
|
||||
must.NoError(t, sys.PutPolicy("nomad-server", string(policy)))
|
||||
|
||||
log := vc.Logical()
|
||||
log.Write("auth/token/roles/nomad-cluster", role)
|
||||
|
||||
token := vc.Auth().Token()
|
||||
secret, err := token.Create(&vaultapi.TokenCreateRequest{
|
||||
Policies: []string{"nomad-server"},
|
||||
Period: "72h",
|
||||
NoParent: true,
|
||||
})
|
||||
must.NoError(t, err, must.Sprint("failed to create vault token"))
|
||||
must.NotNil(t, secret)
|
||||
must.NotNil(t, secret.Auth)
|
||||
}
|
||||
|
||||
func startNomad(t *testing.T, vc *vaultapi.Client) (func(), *nomadapi.Client) {
|
||||
ts := testutil.NewTestServer(t, func(c *testutil.TestServerConfig) {
|
||||
c.Vault = &testutil.VaultConfig{
|
||||
Enabled: true,
|
||||
Address: vc.Address(),
|
||||
Token: vc.Token(),
|
||||
Role: "nomad-cluster",
|
||||
AllowUnauthenticated: true,
|
||||
}
|
||||
c.DevMode = true
|
||||
c.Client = &testutil.ClientConfig{
|
||||
Enabled: true,
|
||||
}
|
||||
c.LogLevel = "off"
|
||||
})
|
||||
nc, err := nomadapi.NewClient(&nomadapi.Config{
|
||||
Address: "http://" + ts.HTTPAddr,
|
||||
})
|
||||
must.NoError(t, err, must.Sprint("unable to create nomad api client"))
|
||||
return ts.Stop, nc
|
||||
}
|
||||
|
||||
func downloadVaultBuild(t *testing.T, b build) {
|
||||
path := filepath.Join(os.TempDir(), binDir, b.Version)
|
||||
must.NoError(t, os.MkdirAll(path, 0755))
|
||||
|
||||
if _, err := os.Stat(filepath.Join(path, "vault")); !os.IsNotExist(err) {
|
||||
t.Log("download: already have vault at", path)
|
||||
return
|
||||
}
|
||||
|
||||
t.Log("download: installing vault at", path)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "hc-install", "install", "-version", b.Version, "-path", path, "vault")
|
||||
bs, err := cmd.CombinedOutput()
|
||||
must.NoError(t, err, must.Sprintf("failed to download vault %s: %s", b.Version, string(bs)))
|
||||
}
|
||||
|
||||
func getMinimumVersion(t *testing.T) *version.Version {
|
||||
v, err := version.NewVersion("1.1.0")
|
||||
must.NoError(t, err)
|
||||
return v
|
||||
}
|
||||
|
||||
type build struct {
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
func (b build) String() string { return b.Version }
|
||||
|
||||
func (b build) compare(o build) int {
|
||||
B := version.Must(version.NewVersion(b.Version))
|
||||
O := version.Must(version.NewVersion(o.Version))
|
||||
return B.Compare(O)
|
||||
}
|
||||
|
||||
type vaultJSON struct {
|
||||
Versions map[string]struct {
|
||||
Builds []build `json:"builds"`
|
||||
}
|
||||
}
|
||||
|
||||
func usable(v, minimum *version.Version) bool {
|
||||
switch {
|
||||
case v.Prerelease() != "":
|
||||
return false
|
||||
case v.Metadata() != "":
|
||||
return false
|
||||
case v.LessThan(minimum):
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func keep(b build) bool {
|
||||
switch {
|
||||
case b.OS != runtime.GOOS:
|
||||
return false
|
||||
case b.Arch != runtime.GOARCH:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// A tracker keeps track of the set of patch versions for each minor version.
|
||||
// The patch versions are stored in a treeset so we can grab the highest patch
|
||||
// version of each minor version at the end.
|
||||
type tracker map[int]*set.TreeSet[build, set.Compare[build]]
|
||||
|
||||
func (t tracker) add(v *version.Version, b build) {
|
||||
y := v.Segments()[1] // minor version
|
||||
|
||||
// create the treeset for this minor version if needed
|
||||
if _, exists := t[y]; !exists {
|
||||
cmp := func(g, h build) int { return g.compare(h) }
|
||||
t[y] = set.NewTreeSet[build, set.Compare[build]](cmp)
|
||||
}
|
||||
|
||||
// insert the patch version into the set of patch versions for this minor version
|
||||
t[y].Insert(b)
|
||||
}
|
||||
|
||||
func scanVaultVersions(t *testing.T, minimum *version.Version) *set.Set[build] {
|
||||
httpClient := cleanhttp.DefaultClient()
|
||||
httpClient.Timeout = 1 * time.Minute
|
||||
response, err := httpClient.Get("https://releases.hashicorp.com/vault/index.json")
|
||||
must.NoError(t, err, must.Sprint("unable to download vault versions index"))
|
||||
var payload vaultJSON
|
||||
must.NoError(t, json.NewDecoder(response.Body).Decode(&payload))
|
||||
must.Close(t, response.Body)
|
||||
|
||||
// sort the versions for the Y in each vault version X.Y.Z
|
||||
// this only works for vault 1.Y.Z which is fine for now
|
||||
track := make(tracker)
|
||||
|
||||
for s, obj := range payload.Versions {
|
||||
v, err := version.NewVersion(s)
|
||||
must.NoError(t, err, must.Sprint("unable to parse vault version"))
|
||||
if !usable(v, minimum) {
|
||||
continue
|
||||
}
|
||||
for _, build := range obj.Builds {
|
||||
if keep(build) {
|
||||
track.add(v, build)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// take the latest patch version for each minor version
|
||||
result := set.New[build](len(track))
|
||||
for _, tree := range track {
|
||||
max := tree.Max()
|
||||
result.Insert(max)
|
||||
}
|
||||
return result
|
||||
}
|
||||
4
go.mod
4
go.mod
|
|
@ -64,7 +64,7 @@ require (
|
|||
github.com/hashicorp/go-plugin v1.4.10
|
||||
github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.4
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2
|
||||
github.com/hashicorp/go-set v0.1.9
|
||||
github.com/hashicorp/go-set v0.1.13
|
||||
github.com/hashicorp/go-sockaddr v1.0.2
|
||||
github.com/hashicorp/go-syslog v1.0.0
|
||||
github.com/hashicorp/go-uuid v1.0.3
|
||||
|
|
@ -113,7 +113,7 @@ require (
|
|||
github.com/shirou/gopsutil/v3 v3.23.4
|
||||
github.com/shoenig/go-landlock v0.1.5
|
||||
github.com/shoenig/go-m1cpu v0.1.6
|
||||
github.com/shoenig/test v0.6.6
|
||||
github.com/shoenig/test v0.6.7
|
||||
github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
|
||||
|
|
|
|||
8
go.sum
8
go.sum
|
|
@ -877,8 +877,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25L
|
|||
github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.1/go.mod h1:l8slYwnJA26yBz+ErHpp2IRCLr0vuOMGBORIz4rRiAs=
|
||||
github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.2 h1:phcbL8urUzF/kxA/Oj6awENaRwfWsjP59GW7u2qlDyY=
|
||||
github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.2/go.mod h1:l8slYwnJA26yBz+ErHpp2IRCLr0vuOMGBORIz4rRiAs=
|
||||
github.com/hashicorp/go-set v0.1.9 h1:XuQSsDfOAvgRjoKWG2qg8NxVEQJMXGdrZh8BgX6O8n4=
|
||||
github.com/hashicorp/go-set v0.1.9/go.mod h1:/IR7VHUqnKI+QfKkaMjZ575bf65Y8DzHRKnOobRpNcQ=
|
||||
github.com/hashicorp/go-set v0.1.13 h1:k1B5goY3c7OKEzpK+gwAhJexxzAJwDN8kId8YvWrihA=
|
||||
github.com/hashicorp/go-set v0.1.13/go.mod h1:0/D+R4MFUzJ6XmvjU7liXtznF1eQDxh84GJlhXw+lvo=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
|
||||
|
|
@ -1292,8 +1292,8 @@ github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZ
|
|||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
|
||||
github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/shoenig/test v0.6.6 h1:Oe8TPH9wAbv++YPNDKJWUnI8Q4PPWCx3UbOfH+FxiMU=
|
||||
github.com/shoenig/test v0.6.6/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/shoenig/test v0.6.7 h1:k92ohN9VyRfZn0ezNfwamtIBT/5byyfLVktRmL/Jmek=
|
||||
github.com/shoenig/test v0.6.7/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ type VaultConfig struct {
|
|||
Address string `json:"address"`
|
||||
AllowUnauthenticated bool `json:"allow_unauthenticated"`
|
||||
Token string `json:"token"`
|
||||
Role string `json:"role"`
|
||||
}
|
||||
|
||||
// ACLConfig is used to configure ACLs
|
||||
|
|
|
|||
Loading…
Reference in New Issue