From 8e6e53cf63c9adb4ae17e43c59c45a6882631121 Mon Sep 17 00:00:00 2001 From: Alexander Scheel Date: Wed, 2 Nov 2022 13:33:17 -0400 Subject: [PATCH] Use hashicorp mirror for container pulls (#17778) When running the test suite in CI (where requests are centralized from relatively few IPs), we'd occasionally hit Dockerhub's rate limits. Luckily Hashicorp runs a (limited) public mirror of the containers we need, so we can switch to them here in the tests. For consistency between developer and CI, we've opted to have the tests always pull from the Hashicorp mirror, rather than updating the CI runner to prefer the mirror. We exclude nomad and influxdb as we don't presently mirror these repos. Signed-off-by: Alexander Scheel Signed-off-by: Alexander Scheel --- builtin/credential/radius/backend_test.go | 2 +- builtin/logical/pki/zlint_test.go | 2 +- builtin/logical/rabbitmq/backend_test.go | 2 +- builtin/logical/ssh/backend_test.go | 2 +- command/server/server_seal_transit_acc_test.go | 5 +++-- helper/testhelpers/cassandra/cassandrahelper.go | 7 ++++--- helper/testhelpers/consul/consulhelper.go | 2 +- helper/testhelpers/fakegcsserver/fake-gcs-server.go | 2 +- helper/testhelpers/ldap/ldaphelper.go | 2 +- helper/testhelpers/minio/miniohelper.go | 2 +- helper/testhelpers/mongodb/mongodbhelper.go | 7 ++++--- helper/testhelpers/mysql/mysqlhelper.go | 9 +++++---- helper/testhelpers/postgresql/postgresqlhelper.go | 6 +++--- physical/aerospike/aerospike_test.go | 2 +- physical/cockroachdb/cockroachdb_test.go | 2 +- physical/couchdb/couchdb_test.go | 3 ++- physical/dynamodb/dynamodb_test.go | 2 +- 17 files changed, 32 insertions(+), 27 deletions(-) diff --git a/builtin/credential/radius/backend_test.go b/builtin/credential/radius/backend_test.go index ddac6e946..17cf54367 100644 --- a/builtin/credential/radius/backend_test.go +++ b/builtin/credential/radius/backend_test.go @@ -57,7 +57,7 @@ client 128.0.0.0/1 { ` containerfile := ` -FROM jumanjiman/radiusd:latest +FROM docker.mirror.hashicorp.services/jumanjiman/radiusd:latest COPY clients.conf /etc/raddb/clients.conf ` diff --git a/builtin/logical/pki/zlint_test.go b/builtin/logical/pki/zlint_test.go index 1e5b38ccd..119310c62 100644 --- a/builtin/logical/pki/zlint_test.go +++ b/builtin/logical/pki/zlint_test.go @@ -18,7 +18,7 @@ var ( func buildZLintContainer(t *testing.T) { containerfile := ` -FROM golang:latest +FROM docker.mirror.hashicorp.services/library/golang:latest RUN go install github.com/zmap/zlint/v3/cmd/zlint@latest ` diff --git a/builtin/logical/rabbitmq/backend_test.go b/builtin/logical/rabbitmq/backend_test.go index 89659796b..7df1384fe 100644 --- a/builtin/logical/rabbitmq/backend_test.go +++ b/builtin/logical/rabbitmq/backend_test.go @@ -36,7 +36,7 @@ func prepareRabbitMQTestContainer(t *testing.T) (func(), string) { } runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "rabbitmq", + ImageRepo: "docker.mirror.hashicorp.services/library/rabbitmq", ImageTag: "3-management", ContainerName: "rabbitmq", Ports: []string{"15672/tcp"}, diff --git a/builtin/logical/ssh/backend_test.go b/builtin/logical/ssh/backend_test.go index e6dc9aee1..4ad4a9f3c 100644 --- a/builtin/logical/ssh/backend_test.go +++ b/builtin/logical/ssh/backend_test.go @@ -137,7 +137,7 @@ func prepareTestContainer(t *testing.T, tag, caPublicKeyPEM string) (func(), str } runner, err := docker.NewServiceRunner(docker.RunOptions{ ContainerName: "openssh", - ImageRepo: "linuxserver/openssh-server", + ImageRepo: "docker.mirror.hashicorp.services/linuxserver/openssh-server", ImageTag: tag, Env: []string{ "DOCKER_MODS=linuxserver/mods:openssh-server-openssh-client", diff --git a/command/server/server_seal_transit_acc_test.go b/command/server/server_seal_transit_acc_test.go index 3f13aee93..7f357b7f2 100644 --- a/command/server/server_seal_transit_acc_test.go +++ b/command/server/server_seal_transit_acc_test.go @@ -131,8 +131,9 @@ func prepareTestContainer(t *testing.T) (func(), *DockerVaultConfig) { } runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "vault", - ImageTag: "latest", + ContainerName: "vault", + ImageRepo: "docker.mirror.hashicorp.services/hashicorp/vault", + ImageTag: "latest", Cmd: []string{ "server", "-log-level=trace", "-dev", fmt.Sprintf("-dev-root-token-id=%s", rootToken), "-dev-listen-address=0.0.0.0:8200", diff --git a/helper/testhelpers/cassandra/cassandrahelper.go b/helper/testhelpers/cassandra/cassandrahelper.go index 9d8ceeea8..899136f16 100644 --- a/helper/testhelpers/cassandra/cassandrahelper.go +++ b/helper/testhelpers/cassandra/cassandrahelper.go @@ -90,9 +90,10 @@ func PrepareTestContainer(t *testing.T, opts ...ContainerOpt) (Host, func()) { } containerCfg := &containerConfig{ - imageName: "cassandra", - version: "3.11", - env: []string{"CASSANDRA_BROADCAST_ADDRESS=127.0.0.1"}, + imageName: "docker.mirror.hashicorp.services/library/cassandra", + containerName: "cassandra", + version: "3.11", + env: []string{"CASSANDRA_BROADCAST_ADDRESS=127.0.0.1"}, } for _, opt := range opts { diff --git a/helper/testhelpers/consul/consulhelper.go b/helper/testhelpers/consul/consulhelper.go index e88ead2bf..7d7984b17 100644 --- a/helper/testhelpers/consul/consulhelper.go +++ b/helper/testhelpers/consul/consulhelper.go @@ -58,7 +58,7 @@ func PrepareTestContainer(t *testing.T, version string, isEnterprise bool, doBoo if isEnterprise { version += "-ent" name = "consul-enterprise" - repo = "hashicorp/consul-enterprise" + repo = "docker.mirror.hashicorp.services/hashicorp/consul-enterprise" license, hasLicense := os.LookupEnv("CONSUL_LICENSE") envVars = append(envVars, "CONSUL_LICENSE="+license) diff --git a/helper/testhelpers/fakegcsserver/fake-gcs-server.go b/helper/testhelpers/fakegcsserver/fake-gcs-server.go index 59c9001ad..ed83970d1 100644 --- a/helper/testhelpers/fakegcsserver/fake-gcs-server.go +++ b/helper/testhelpers/fakegcsserver/fake-gcs-server.go @@ -27,7 +27,7 @@ func PrepareTestContainer(t *testing.T, version string) (func(), docker.ServiceC } runner, err := docker.NewServiceRunner(docker.RunOptions{ ContainerName: "fake-gcs-server", - ImageRepo: "fsouza/fake-gcs-server", + ImageRepo: "docker.mirror.hashicorp.services/fsouza/fake-gcs-server", ImageTag: version, Cmd: []string{"-scheme", "http", "-public-host", "storage.gcs.127.0.0.1.nip.io:4443"}, Ports: []string{"4443/tcp"}, diff --git a/helper/testhelpers/ldap/ldaphelper.go b/helper/testhelpers/ldap/ldaphelper.go index 394ef34da..b248c0294 100644 --- a/helper/testhelpers/ldap/ldaphelper.go +++ b/helper/testhelpers/ldap/ldaphelper.go @@ -14,7 +14,7 @@ func PrepareTestContainer(t *testing.T, version string) (cleanup func(), cfg *ld runner, err := docker.NewServiceRunner(docker.RunOptions{ // Currently set to "michelvocks" until https://github.com/rroemhild/docker-test-openldap/pull/14 // has been merged. - ImageRepo: "michelvocks/docker-test-openldap", + ImageRepo: "docker.mirror.hashicorp.services/michelvocks/docker-test-openldap", ImageTag: version, ContainerName: "ldap", Ports: []string{"389/tcp"}, diff --git a/helper/testhelpers/minio/miniohelper.go b/helper/testhelpers/minio/miniohelper.go index c53758505..2969ce215 100644 --- a/helper/testhelpers/minio/miniohelper.go +++ b/helper/testhelpers/minio/miniohelper.go @@ -32,7 +32,7 @@ func PrepareTestContainer(t *testing.T, version string) (func(), *Config) { } runner, err := docker.NewServiceRunner(docker.RunOptions{ ContainerName: "minio", - ImageRepo: "minio/minio", + ImageRepo: "docker.mirror.hashicorp.services/minio/minio", ImageTag: version, Env: []string{ "MINIO_ACCESS_KEY=" + accessKeyID, diff --git a/helper/testhelpers/mongodb/mongodbhelper.go b/helper/testhelpers/mongodb/mongodbhelper.go index fabd8a680..c4288a4b5 100644 --- a/helper/testhelpers/mongodb/mongodbhelper.go +++ b/helper/testhelpers/mongodb/mongodbhelper.go @@ -27,9 +27,10 @@ func PrepareTestContainerWithDatabase(t *testing.T, version, dbName string) (fun } runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "mongo", - ImageTag: version, - Ports: []string{"27017/tcp"}, + ContainerName: "mongo", + ImageRepo: "docker.mirror.hashicorp.services/library/mongo", + ImageTag: version, + Ports: []string{"27017/tcp"}, }) if err != nil { t.Fatalf("could not start docker mongo: %s", err) diff --git a/helper/testhelpers/mysql/mysqlhelper.go b/helper/testhelpers/mysql/mysqlhelper.go index 145c91e25..82b47f07e 100644 --- a/helper/testhelpers/mysql/mysqlhelper.go +++ b/helper/testhelpers/mysql/mysqlhelper.go @@ -29,10 +29,11 @@ func PrepareTestContainer(t *testing.T, legacy bool, pw string) (func(), string) } runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "mysql", - ImageTag: imageVersion, - Ports: []string{"3306/tcp"}, - Env: []string{"MYSQL_ROOT_PASSWORD=" + pw}, + ContainerName: "mysql", + ImageRepo: "docker.mirror.hashicorp.services/library/mysql", + ImageTag: imageVersion, + Ports: []string{"3306/tcp"}, + Env: []string{"MYSQL_ROOT_PASSWORD=" + pw}, }) if err != nil { t.Fatalf("could not start docker mysql: %s", err) diff --git a/helper/testhelpers/postgresql/postgresqlhelper.go b/helper/testhelpers/postgresql/postgresqlhelper.go index 3a6c3eb07..17b2151ab 100644 --- a/helper/testhelpers/postgresql/postgresqlhelper.go +++ b/helper/testhelpers/postgresql/postgresqlhelper.go @@ -17,7 +17,7 @@ func PrepareTestContainer(t *testing.T, version string) (func(), string) { "POSTGRES_DB=database", } - _, cleanup, url, _ := prepareTestContainer(t, "postgres", "postgres", version, "secret", true, false, false, env) + _, cleanup, url, _ := prepareTestContainer(t, "postgres", "docker.mirror.hashicorp.services/postgres", version, "secret", true, false, false, env) return cleanup, url } @@ -28,7 +28,7 @@ func PrepareTestContainerWithPassword(t *testing.T, version, password string) (f "POSTGRES_DB=database", } - _, cleanup, url, _ := prepareTestContainer(t, "postgres", "postgres", version, password, true, false, false, env) + _, cleanup, url, _ := prepareTestContainer(t, "postgres", "docker.mirror.hashicorp.services/postgres", version, password, true, false, false, env) return cleanup, url } @@ -40,7 +40,7 @@ func PrepareTestContainerRepmgr(t *testing.T, name, version string, envVars []st "REPMGR_PASSWORD=repmgrpass", "POSTGRESQL_PASSWORD=secret") - return prepareTestContainer(t, name, "bitnami/postgresql-repmgr", version, "secret", false, true, true, env) + return prepareTestContainer(t, name, "docker.mirror.hashicorp.services/bitnami/postgresql-repmgr", version, "secret", false, true, true, env) } func prepareTestContainer(t *testing.T, name, repo, version, password string, diff --git a/physical/aerospike/aerospike_test.go b/physical/aerospike/aerospike_test.go index 653f00ff6..1a76656f2 100644 --- a/physical/aerospike/aerospike_test.go +++ b/physical/aerospike/aerospike_test.go @@ -41,7 +41,7 @@ type aerospikeConfig struct { func prepareAerospikeContainer(t *testing.T) (func(), *aerospikeConfig) { runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "aerospike/aerospike-server", + ImageRepo: "docker.mirror.hashicorp.services/aerospike/aerospike-server", ContainerName: "aerospikedb", ImageTag: "5.6.0.5", Ports: []string{"3000/tcp", "3001/tcp", "3002/tcp", "3003/tcp"}, diff --git a/physical/cockroachdb/cockroachdb_test.go b/physical/cockroachdb/cockroachdb_test.go index 69a3c5653..70abfda98 100644 --- a/physical/cockroachdb/cockroachdb_test.go +++ b/physical/cockroachdb/cockroachdb_test.go @@ -36,7 +36,7 @@ func prepareCockroachDBTestContainer(t *testing.T) (func(), *Config) { } runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "cockroachdb/cockroach", + ImageRepo: "docker.mirror.hashicorp.services/cockroachdb/cockroach", ImageTag: "release-1.0", ContainerName: "cockroachdb", Cmd: []string{"start", "--insecure"}, diff --git a/physical/couchdb/couchdb_test.go b/physical/couchdb/couchdb_test.go index 4205f8e64..abf11b7c1 100644 --- a/physical/couchdb/couchdb_test.go +++ b/physical/couchdb/couchdb_test.go @@ -86,7 +86,8 @@ func prepareCouchdbDBTestContainer(t *testing.T) (func(), *couchDB) { } runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "couchdb", + ContainerName: "couchdb", + ImageRepo: "docker.mirror.hashicorp.services/library/couchdb", ImageTag: "1.6", Ports: []string{"5984/tcp"}, DoNotAutoRemove: true, diff --git a/physical/dynamodb/dynamodb_test.go b/physical/dynamodb/dynamodb_test.go index ae6722e24..1058a6e21 100644 --- a/physical/dynamodb/dynamodb_test.go +++ b/physical/dynamodb/dynamodb_test.go @@ -381,7 +381,7 @@ func prepareDynamoDBTestContainer(t *testing.T) (func(), *Config) { } runner, err := docker.NewServiceRunner(docker.RunOptions{ - ImageRepo: "cnadiminti/dynamodb-local", + ImageRepo: "docker.mirror.hashicorp.services/cnadiminti/dynamodb-local", ImageTag: "latest", ContainerName: "dynamodb", Ports: []string{"8000/tcp"},