From 92046f7d081173b2996755a02b96f5e39bfaf41a Mon Sep 17 00:00:00 2001 From: Pratyoy Mukhopadhyay <35388175+pmmukh@users.noreply.github.com> Date: Mon, 27 Sep 2021 13:49:10 -0700 Subject: [PATCH] [VAULT-3248] Check api and sdk dirs in go_test (#12630) * Check api and sdk dirs in go_test * Update typo in script * Append package names if non empty * Don't fail command if no test packages found * Add comments, clean up echoes * Use pushd/popd, misc review fixes --- .circleci/config.yml | 196 +++++++++++++++++--------- .circleci/config/commands/go_test.yml | 49 ++++--- api/go.sum | 3 +- 3 files changed, 161 insertions(+), 87 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af89dd12e..a0f8fe678 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -995,21 +995,36 @@ jobs: USE_DOCKER=0 USE_DOCKER=1 - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - if [ $USE_DOCKER == 1 ]; then - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - else - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - fi + # Check all directories with a go.mod file + modules=("." "api" "sdk") + all_package_names="" + + for dir in "${modules[@]}" + do + pushd "$dir" + # Split Go tests by prior test times. If use_docker is true, only run + # tests that depend on docker, otherwise only those that don't. + # The appended true condition ensures the command will succeed if no packages are found + if [ $USE_DOCKER == 1 ]; then + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + else + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + fi + # Move back into root directory + popd + # Append the test packages into the global list, if any are found + if [ -n "$package_names" ]; then + all_package_names+=" ${package_names}" + fi + done # After running tests split step, we are now running the following steps # in multiple different containers, each getting a different subset of @@ -1085,7 +1100,7 @@ jobs: -timeout=60m \ -parallel=20 \ \ - ${package_names} + ${all_package_names} else GOARCH=amd64 \ GOCACHE=/tmp/go-cache \ @@ -1097,7 +1112,7 @@ jobs: -timeout=60m \ -parallel=20 \ \ - ${package_names} + ${all_package_names} fi environment: GOPRIVATE: github.com/hashicorp/* @@ -1251,21 +1266,36 @@ jobs: USE_DOCKER=0 - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - if [ $USE_DOCKER == 1 ]; then - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - else - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - fi + # Check all directories with a go.mod file + modules=("." "api" "sdk") + all_package_names="" + + for dir in "${modules[@]}" + do + pushd "$dir" + # Split Go tests by prior test times. If use_docker is true, only run + # tests that depend on docker, otherwise only those that don't. + # The appended true condition ensures the command will succeed if no packages are found + if [ $USE_DOCKER == 1 ]; then + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + else + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + fi + # Move back into root directory + popd + # Append the test packages into the global list, if any are found + if [ -n "$package_names" ]; then + all_package_names+=" ${package_names}" + fi + done # After running tests split step, we are now running the following steps # in multiple different containers, each getting a different subset of @@ -1341,7 +1371,7 @@ jobs: -timeout=60m \ -parallel=20 \ -race \ - ${package_names} + ${all_package_names} else GOARCH=amd64 \ GOCACHE=/tmp/go-cache \ @@ -1353,7 +1383,7 @@ jobs: -timeout=60m \ -parallel=20 \ -race \ - ${package_names} + ${all_package_names} fi environment: GOPRIVATE: github.com/hashicorp/* @@ -1758,21 +1788,36 @@ jobs: USE_DOCKER=0 - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - if [ $USE_DOCKER == 1 ]; then - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - else - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - fi + # Check all directories with a go.mod file + modules=("." "api" "sdk") + all_package_names="" + + for dir in "${modules[@]}" + do + pushd "$dir" + # Split Go tests by prior test times. If use_docker is true, only run + # tests that depend on docker, otherwise only those that don't. + # The appended true condition ensures the command will succeed if no packages are found + if [ $USE_DOCKER == 1 ]; then + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + else + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + fi + # Move back into root directory + popd + # Append the test packages into the global list, if any are found + if [ -n "$package_names" ]; then + all_package_names+=" ${package_names}" + fi + done # After running tests split step, we are now running the following steps # in multiple different containers, each getting a different subset of @@ -1848,7 +1893,7 @@ jobs: -timeout=60m \ -parallel=20 \ \ - ${package_names} + ${all_package_names} else GOARCH=amd64 \ GOCACHE=/tmp/go-cache \ @@ -1860,7 +1905,7 @@ jobs: -timeout=60m \ -parallel=20 \ \ - ${package_names} + ${all_package_names} fi environment: GOPRIVATE: github.com/hashicorp/* @@ -2409,21 +2454,36 @@ jobs: USE_DOCKER=0 USE_DOCKER=1 - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - if [ $USE_DOCKER == 1 ]; then - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - else - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - fi + # Check all directories with a go.mod file + modules=("." "api" "sdk") + all_package_names="" + + for dir in "${modules[@]}" + do + pushd "$dir" + # Split Go tests by prior test times. If use_docker is true, only run + # tests that depend on docker, otherwise only those that don't. + # The appended true condition ensures the command will succeed if no packages are found + if [ $USE_DOCKER == 1 ]; then + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + else + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + fi + # Move back into root directory + popd + # Append the test packages into the global list, if any are found + if [ -n "$package_names" ]; then + all_package_names+=" ${package_names}" + fi + done # After running tests split step, we are now running the following steps # in multiple different containers, each getting a different subset of @@ -2499,7 +2559,7 @@ jobs: -timeout=60m \ -parallel=20 \ -race \ - ${package_names} + ${all_package_names} else GOARCH=amd64 \ GOCACHE=/tmp/go-cache \ @@ -2511,7 +2571,7 @@ jobs: -timeout=60m \ -parallel=20 \ -race \ - ${package_names} + ${all_package_names} fi environment: GOPRIVATE: github.com/hashicorp/* diff --git a/.circleci/config/commands/go_test.yml b/.circleci/config/commands/go_test.yml index 54ef79437..f9bc3e631 100644 --- a/.circleci/config/commands/go_test.yml +++ b/.circleci/config/commands/go_test.yml @@ -58,21 +58,36 @@ steps: USE_DOCKER=1 <> - # Split Go tests by prior test times. If use_docker is true, only run - # tests that depend on docker, otherwise only those that don't. - if [ $USE_DOCKER == 1 ]; then - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - else - package_names=$(go list -test -json ./... | - jq -r 'select(.Deps != null) | - select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | - .ForTest | select(. != null)' | - sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) - fi + # Check all directories with a go.mod file + modules=("." "api" "sdk") + all_package_names="" + + for dir in "${modules[@]}" + do + pushd "$dir" + # Split Go tests by prior test times. If use_docker is true, only run + # tests that depend on docker, otherwise only those that don't. + # The appended true condition ensures the command will succeed if no packages are found + if [ $USE_DOCKER == 1 ]; then + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + else + package_names=$(go list -test -json ./... | + jq -r 'select(.Deps != null) | + select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) | + .ForTest | select(. != null)' | + sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname) || true + fi + # Move back into root directory + popd + # Append the test packages into the global list, if any are found + if [ -n "$package_names" ]; then + all_package_names+=" ${package_names}" + fi + done # After running tests split step, we are now running the following steps # in multiple different containers, each getting a different subset of @@ -148,7 +163,7 @@ steps: -timeout=60m \ -parallel=20 \ << parameters.extra_flags >> \ - ${package_names} + ${all_package_names} else GOARCH=<< parameters.arch >> \ GOCACHE=<< parameters.cache_dir >> \ @@ -160,7 +175,7 @@ steps: -timeout=60m \ -parallel=20 \ << parameters.extra_flags >> \ - ${package_names} + ${all_package_names} fi - when: diff --git a/api/go.sum b/api/go.sum index aa8293638..5731a49ef 100644 --- a/api/go.sum +++ b/api/go.sum @@ -57,8 +57,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=