Add more functionality related to verifying a build and publishing

This commit is contained in:
Matt Keeler 2018-06-13 15:10:02 -04:00
parent 351841c7b2
commit 237c78d4a4
7 changed files with 622 additions and 67 deletions

View File

@ -75,6 +75,9 @@ linux:
# dist builds binaries for all platforms and packages them for distribution
dist:
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh release -t '$(DIST_TAG)' -b '$(DIST_BUILD)' -S '$(DIST_SIGN)'
publish:
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh publish
cov:
gocov test $(GOFILES) | gocov-html > /tmp/coverage.html
@ -139,11 +142,15 @@ tools:
go get -u -v $(GOTOOLS)
version:
@echo -n "Version without release: "
@echo -n "Version: "
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh version
@echo -n "Version with release: "
@echo -n "Version + release: "
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh version -R
@echo -n "Version + git: "
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh version -G
@echo -n "Version + release + git: "
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh version -R -G
docker-images:
@$(MAKE) -C build-support/docker images

View File

@ -18,3 +18,22 @@ case $(uname) in
;;
esac
MAIN_GOPATH=$(cut -d: -f1 <<< "${GOPATH}")
# Build debugging output is off by default
if test -z "${BUILD_DEBUG}"
then
BUILD_DEBUG=0
fi
# default publish host is github.com - only really useful to use something else for testing
if test -z "${PUBLISH_GIT_HOST}"
then
PUBLISH_GIT_HOST=github.com
fi
# default publish repo is hashicorp/consul - useful to override for testing as well as in the enterprise repo
if test -z "${PUBLISH_GIT_REPO}"
then
PUBLISH_GIT_REPO=hashicorp/consul.git
fi

View File

@ -5,7 +5,7 @@ function err {
tput setaf 1
fi
echo $@ 1>&2
echo "$@" 1>&2
if test "${COLORIZE}" -eq 1
then
@ -20,7 +20,7 @@ function status {
tput setaf 4
fi
echo $@
echo "$@"
if test "${COLORIZE}" -eq 1
then
@ -35,7 +35,7 @@ function status_stage {
tput setaf 2
fi
echo $@
echo "$@"
if test "${COLORIZE}" -eq 1
then
@ -43,6 +43,21 @@ function status_stage {
fi
}
function debug {
if is_set "${BUILD_DEBUG}"
then
if test "${COLORIZE}" -eq 1
then
tput setaf 6
fi
echo "$@"
if test "${COLORIZE}" -eq 1
then
tput sgr0
fi
fi
}
function is_set {
# Arguments:
# $1 - string value to check its truthiness
@ -70,14 +85,15 @@ function have_gpg_key {
# 0 - success (we can use this key for signing)
# * - failure (key cannot be used)
gpg --list-secret-keys $1 >dev/null 2>&1
gpg --list-secret-keys $1 > /dev/null 2>&1
return $?
}
function parse_version {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - boolean value for whether to omit the release version from the version string
# $2 - boolean value for whether the release version should be parsed from the source
# $3 - boolean whether to use GIT_DESCRIBE and GIT_COMMIT environment variables
#
# Return:
# 0 - success (will write the version to stdout)
@ -86,9 +102,6 @@ function parse_version {
# Notes:
# If the GOTAGS environment variable is present then it is used to determine which
# version file to use for parsing.
# If the GIT_DESCRIBE environment variable is present then it is used as the version
# If the GIT_COMMIT environment variable is preset it will be added to the end of
# the version string.
local vfile="${1}/version/version.go"
@ -99,6 +112,28 @@ function parse_version {
return 1
fi
local include_release="$2"
local use_git_env="$3"
local git_version=""
local git_commit=""
if test -z "${include_release}"
then
include_release=true
fi
if test -z "${use_git_env}"
then
use_git_env=true
fi
if is_set "${use_git_env}"
then
git_version="${GIT_DESCRIBE}"
git_commit="${GIT_COMMIT}"
fi
# Get the main version out of the source file
version_main=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
release_main=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
@ -116,22 +151,22 @@ function parse_version {
done
done
version=
# override the version from source with the value of the GIT_DESCRIBE env var if present
if test -n "$GIT_DESCRIBE"
if test -n "${git_version}"
then
version=$GIT_DESCRIBE
version="${git_version}"
else
version="${version_main}"
fi
if ! is_set $2
if is_set "${include_release}"
then
# Get the release version out of the source file
release=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
release="${release_main}"
# When no GIT_DESCRIBE env var is present and no release is in the source then we
# are definitely in dev mode
if test -z "$GIT_DESCRIBE" -a -z "$release"
if test -z "${git_version}" -a -z "$release"
then
release="dev"
fi
@ -142,9 +177,9 @@ function parse_version {
version="${version}-${release}"
# add the git commit to the version
if test -n "$GIT_COMMIT"
if test -n "${git_commit}"
then
version="${version} (${GIT_COMMIT})"
version="${version} (${git_commit})"
fi
fi
fi
@ -158,6 +193,7 @@ function get_version {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - Whether the release version should be parsed from source (optional)
# $3 - Whether to use GIT_DESCRIBE and GIT_COMMIT environment variables
#
# Returns:
# 0 - success (the version is also echoed to stdout)
@ -173,7 +209,7 @@ function get_version {
if test -z "$vers"
then
# parse the OSS version from version.go
vers="$(parse_version ${1} ${2})"
vers="$(parse_version ${1} ${2} ${3})"
fi
if test -z "$vers"
@ -183,4 +219,224 @@ function get_version {
echo $vers
return 0
fi
}
function git_branch {
# Arguments:
# $1 - Path to the git repo (optional - assumes pwd is git repo otherwise)
#
# Returns:
# 0 - success
# * - failure
#
# Notes:
# Echos the current branch to stdout when successful
local gdir="$(pwd)"
if test -d "$1"
then
gdir="$1"
fi
pushd "${gdir}" > /dev/null
local ret=0
local head="$(git status -b --porcelain=v2 | awk '{if ($1 == "#" && $2 =="branch.head") { print $3 }}')" || ret=1
popd > /dev/null
test ${ret} -eq 0 && echo "$head"
return ${ret}
}
function git_upstream {
# Arguments:
# $1 - Path to the git repo (optional - assumes pwd is git repo otherwise)
#
# Returns:
# 0 - success
# * - failure
#
# Notes:
# Echos the current upstream branch to stdout when successful
local gdir="$(pwd)"
if test -d "$1"
then
gdir="$1"
fi
pushd "${gdir}" > /dev/null
local ret=0
local head="$(git status -b --porcelain=v2 | awk '{if ($1 == "#" && $2 =="branch.upstream") { print $3 }}')" || ret=1
popd > /dev/null
test ${ret} -eq 0 && echo "$head"
return ${ret}
}
function git_log_summary {
# Arguments:
# $1 - Path to the git repo (optional - assumes pwd is git repo otherwise)
#
# Returns:
# 0 - success
# * - failure
#
local gdir="$(pwd)"
if test -d "$1"
then
gdir="$1"
fi
pushd "${gdir}" > /dev/null
local ret=0
local head=$(git_branch) || ret=1
local upstream=$(git_upstream) || ret=1
local rev_range="${head}...${upstream}"
if test ${ret} -eq 0
then
status "Git Changes:"
git log --pretty=oneline ${rev_range} || ret=1
fi
return $ret
}
function normalize_git_url {
url="${1#https://}"
url="${url#git@}"
url="${url%.git}"
url="$(sed -e 's/\([^\/:]*\)[:\/]\(.*\)/\1:\2/' <<< "${url}")"
echo "$url"
return 0
}
function find_git_remote {
# Arguments:
# $1 - Path to the top level Consul source
#
# Returns:
# 0 - success
# * - error
#
# Note:
# The remote name to use for publishing will be echoed to stdout upon success
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. find_git_remote must be called with the path to the top level source as the first argument'"
return 1
fi
need_url=$(normalize_git_url "${PUBLISH_GIT_HOST}:${PUBLISH_GIT_REPO}")
pushd "$1" > /dev/null
local ret=1
for remote in $(git remote)
do
url=$(git remote get-url --push ${remote}) || continue
url=$(normalize_git_url "${url}")
if test "${url}" == "${need_url}"
then
echo "${remote}"
ret=0
break
fi
done
popd > /dev/null
return $ret
}
function confirm_git_push_changes {
# Arguments:
# $1 - Path to git repo
#
# Returns:
# 0 - success
# * - error
#
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. confirm_git_push_changes must be called with the path to a git repo as the first argument'"
return 1
fi
pushd "${1}" > /dev/null
declare -i ret=0
git_log_summary || ret=1
if test ${ret} -eq 0
then
# put a empty line between the git changes and the prompt
echo ""
local answer=""
while true
do
case "${answer}" in
[yY]* )
status "Changes Accepted"
ret=0
break
;;
[nN]* )
err "Changes Rejected"
ret=1
break
;;
* )
read -p "Are these changes correct? [y/n]: " answer
;;
esac
done
fi
popd > /dev/null
return $ret
}
function is_git_clean {
# Arguments:
# $1 - Path to git repo
# $2 - boolean whether the git status should be output when not clean
#
# Returns:
# 0 - success
# * - error
#
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. is_git_clean must be called with the path to a git repo as the first argument'"
return 1
fi
local output_status="$2"
pushd "${1}" > /dev/null
local ret=0
test -z "$(git status --porcelain=v2 2> /dev/null)" || ret=1
if is_set "${output_status}" && test "$ret" -ne 0
then
err "Git repo is not clean"
# --porcelain=v1 is the same as --short except uncolorized
git status --porcelain=v1
fi
popd > /dev/null
return ${ret}
}

View File

@ -77,7 +77,7 @@ function build_ui {
docker rm ${container_id} > /dev/null
fi
if test $ret -eq 0
if test ${ret} -eq 0
then
rm -rf ${1}/pkg/web_ui/v2
cp -r ${1}/ui-v2/dist ${1}/pkg/web_ui/v2
@ -124,7 +124,7 @@ function build_ui_legacy {
status "Running build in container" &&
docker start -i ${container_id} &&
status "Copying back artifacts" &&
docker cp ${container_id}:/consul-src/pkg/web_ui ${sdir}/pkg/web_ui/v1
docker cp ${container_id}:/consul-src/pkg/web_ui/v1/. ${sdir}/pkg/web_ui/v1
)
ret=$?
docker rm ${container_id} > /dev/null

View File

@ -33,12 +33,17 @@ function tag_release {
pushd "$1" > /dev/null
local ret=0
local branch_to_tag=$(git_branch) || ret=1
# perform an usngined release if requested (mainly for testing locally)
if is_set "$RELEASE_UNSIGNED"
if test ${ret} -ne 0
then
err "ERROR: Failed to determine git branch to tag"
elif is_set "$RELEASE_UNSIGNED"
then
(
git commit --allow-empty -a -m "Release v${2}" &&
git tag -a -m "Version ${2}" "v${2}" master
git tag -a -m "Version ${2}" "v${2}" "${branch_to_tag}"
)
ret=$?
# perform a signed release (official releases should do this)
@ -46,7 +51,7 @@ function tag_release {
then
(
git commit --allow-empty -a --gpg-sign=${gpg_key} -m "Release v${2}" &&
git tag -a -m "Version ${2}" -s -u ${gpg_key} "v${2}" master
git tag -a -m "Version ${2}" -s -u ${gpg_key} "v${2}" "${branch_to_tag}"
)
ret=$?
# unsigned release not requested and gpg key isn't useable
@ -72,11 +77,14 @@ function package_release {
err "ERROR: '$1' is not a directory. package_release must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="${1}"
local ret=0
local vers="${2}"
if test -z "${vers}"
then
vers=$(get_version $1 true)
vers=$(get_version "${sdir}" true false)
ret=$?
if test "$ret" -ne 0
then
@ -85,9 +93,6 @@ function package_release {
fi
fi
local sdir="$1"
local ret=0
rm -rf "${sdir}/pkg/dist" > /dev/null 2>&1
mkdir -p "${sdir}/pkg/dist" >/dev/null 2>&1
for platform in $(find "${sdir}/pkg/bin" -mindepth 1 -maxdepth 1 -type d)
@ -154,10 +159,101 @@ function sign_release {
gpg_key=$2
fi
gpg --default-key "${gpg_key}" --detach-sig "$1"
gpg --default-key "${gpg_key}" --detach-sig --yes -v "$1"
return $?
}
function check_release {
# Arguments:
# $1 - Path to the release files
# $2 - Version to expect
# $3 - boolean whether to expect the signature file
#
# Returns:
# 0 - success
# * - failure
declare -i ret=0
declare -a expected_files
expected_files+=("consul_${2}_SHA256SUMS")
echo "check sig: $3"
if is_set "$3"
then
expected_files+=("consul_${2}_SHA256SUMS.sig")
fi
expected_files+=("consul_${2}_darwin_386.zip")
expected_files+=("consul_${2}_darwin_amd64.zip")
expected_files+=("consul_${2}_freebsd_386.zip")
expected_files+=("consul_${2}_freebsd_amd64.zip")
expected_files+=("consul_${2}_freebsd_arm.zip")
expected_files+=("consul_${2}_linux_386.zip")
expected_files+=("consul_${2}_linux_amd64.zip")
expected_files+=("consul_${2}_linux_arm.zip")
expected_files+=("consul_${2}_linux_arm64.zip")
expected_files+=("consul_${2}_solaris_amd64.zip")
expected_files+=("consul_${2}_windows_386.zip")
expected_files+=("consul_${2}_windows_amd64.zip")
declare -a found_files
status_stage "==> Verifying release contents - ${2}"
debug "Expecting Files:"
for fname in "${expected_files[@]}"
do
debug " $fname"
done
pushd "$1" > /dev/null
for actual_fname in $(ls)
do
local found=0
for i in "${!expected_files[@]}"
do
local expected_fname="${expected_files[i]}"
if test "${expected_fname}" == "${actual_fname}"
then
# remove from the expected_files array
unset 'expected_files[i]'
# append to the list of found files
found_files+=("${expected_fname}")
# mark it as found so we dont error
found=1
break
fi
done
if test $found -ne 1
then
err "ERROR: Release build has an extra file: ${actual_fname}"
ret=1
fi
done
popd > /dev/null
for fname in "${expected_files[@]}"
do
err "ERROR: Release build is missing a file: $fname"
ret=1
done
if test $ret -eq 0
then
status "Release build contents:"
for fname in "${found_files[@]}"
do
echo " $fname"
done
fi
return $ret
}
function build_consul_release {
build_consul "$1" "" "$2"
}
@ -192,13 +288,33 @@ function build_release {
local do_sha256="$4"
local gpg_key="$5"
local vers=$(get_version ${sdir} true)
if test -z "${gpg_key}"
then
gpg_key=${HASHICORP_GPG_KEY}
fi
local vers="$(get_version ${sdir} true false)"
if test $? -ne 0
then
err "Please specify a version (couldn't find one based on build tags)."
return 1
fi
if ! is_git_clean "${sdir}" true && ! is_set "${ALLOW_DIRTY_GIT}"
then
err "ERROR: Refusing to build because Git is dirty. Set ALLOW_DIRTY_GIT=1 in the environment to proceed anyways"
return 1
fi
if ! is_set "${RELEASE_UNSIGNED}"
then
if ! have_gpg_key "${gpg_key}"
then
err "ERROR: Aborting build because no useable GPG key is present. Set RELEASE_UNSIGNED=1 to bypass this check"
return 1
fi
fi
# Make sure we arent in dev mode
unset CONSUL_DEV
@ -295,5 +411,6 @@ function build_release {
fi
fi
return 0
}
check_release "${sdir}/pkg/dist" "${vers}" "${do_sha256}"
return $?
}

View File

@ -0,0 +1,135 @@
function hashicorp_release {
# Arguments:
# $1 - Path to directory containing all of the release artifacts
#
# Returns:
# 0 - success
# * - failure
#
# Notes:
# Requires the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables
# to be set
status "Uploading files"
hc-releases upload "${1}" || return 1
status "Publishing the release"
hc-releases publish || return 1
return 0
}
function push_git_release {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - Tag to push
#
# Returns:
# 0 - success
# * - error
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. push_git_release must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
local ret=0
# find the correct remote corresponding to the desired repo (basically prevent pushing enterprise to oss or oss to enterprise)
local remote=$(find_git_remote "${sdir}") || return 1
local head=$(git_branch "${sdir}") || return 1
local upstream=$(git_upstream "${sdir}") || return 1
status "Using git remote: ${remote}"
# upstream branch for this branch does not track the remote we need to push to
if test "${upstream#${remote}}" == "${upstream}"
then
err "ERROR: Upstream branch '${upstream}' does not track the correct remote '${remote}'"
return 1
fi
pushd "${sdir}" > /dev/null
status "Pushing local branch ${head} to ${upstream}"
if ! git push "${remote}"
then
err "ERROR: Failed to push to remote: ${remote}"
ret=1
fi
status "Pushing tag ${2} to ${remote}"
if test "${ret}" -eq 0 && ! git push "${remote}" "${2}"
then
err "ERROR: Failed to push tag ${2} to ${remote}"
ret = 1
fi
popd > /dev/null
return $ret
}
function publish_release {
# Arguments:
# $1 - Path to top level Consul source that contains the built release
# $2 - boolean whether to publish to git upstream
# $3 - boolean whether to publish to releases.hashicorp.com
#
# Returns:
# 0 - success
# * - error
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. publish_release must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
local pub_git="$2"
local pub_hc_releases="$3"
if test -z "${pub_git}"
then
pub_git=1
fi
if test -z "${pub_hc_releases}"
then
pub_hc_releases=1
fi
local vers="$(get_version ${sdir} true false)"
if test $? -ne 0
then
err "Please specify a version (couldn't parse one from the source)."
return 1
fi
status_page "==> Verifying release files"
check_release "${sdir}/pkg/dist" "${vers}" true
status_stage "==> Confirming Git is clean"
is_git_clean "$1" true || return 1
status_stage "==> Confirming Git Changes"
confirm_git_push_changes "$1" || return 1
if is_set "${pub_git}"
then
status_stage "==> Pushing to Git"
push_git_release "$1" "v${vers}" || return 1
fi
if is_set "${pub_hc_releases}"
then
status_stage "==> Publishing to releases.hashicorp.com"
hashicorp_release "${sdir}/pkg/dist" || return 1
fi
return 0
}

View File

@ -103,6 +103,11 @@ Subcommands:
-a | --build-arch ARCH Space separated string of architectures to build
publish: Publishes a release build.
-s | --source DIR Path to the source to build.
Defaults to "${SOURCE_DIR}"
release: Performs a release build.
Options:
@ -163,6 +168,7 @@ function main {
declare build_os
declare build_arch
declare -i vers_release
declare -i vers_git
declare -i use_refresh=1
declare -i default_refresh=0
@ -178,13 +184,18 @@ function main {
declare -i use_xc=0
declare default_build_os=""
declare default_build_arch=""
declare -i use_vers_rel
declare -i default_vers_rel=1
declare -i use_version_args
declare -i default_vers_rel=0
declare -i default_vers_git=0
declare command="$1"
shift
case "${command}" in
assetfs )
use_image=1
default_image="${GO_BUILD_CONTAINER_DEFAULT}"
;;
consul )
use_image=1
default_image="${GO_BUILD_CONTAINER_DEFAULT}"
@ -192,6 +203,13 @@ function main {
consul-local )
use_xc=1
;;
publish )
use_refresh=0
;;
release )
use_rel=1
use_refresh=0
;;
ui )
use_image=1
default_image="${UI_BUILD_CONTAINER_DEFAULT}"
@ -202,15 +220,7 @@ function main {
;;
version )
use_refresh=0
use_vers_rel=1
;;
assetfs )
use_image=1
default_image="${GO_BUILD_CONTAINER_DEFAULT}"
;;
release )
use_rel=1
use_refresh=0
use_version_args=1
;;
-h | --help)
usage
@ -232,6 +242,7 @@ function main {
declare -i have_build_os_arg=0
declare -i have_build_arch_arg=0
declare -i have_vers_rel_arg=0
declare -i have_vers_git_arg=0
while test $# -gt 0
do
@ -251,9 +262,15 @@ function main {
shift 2
;;
-R | --release )
option_check "${use_vers_rel}" "${have_vers_rel_arg}" "${command}" "-R/--release" || return 1
option_check "${use_version_args}" "${have_vers_rel_arg}" "${command}" "-R/--release" || return 1
have_vers_rel_arg=1
vers_release=0
vers_release=1
shift
;;
-G | --git )
option_check "${use_version_args}" "${have_vers_git_arg}" "${command}" "-G/--git" || return 1
have_vers_git_arg=1
vers_git=1
shift
;;
-r | --refresh)
@ -313,8 +330,19 @@ function main {
test $have_build_os_arg -ne 1 && build_os="${default_build_os}"
test $have_build_arch_arg -ne 1 && build_arch="${default_build_os}"
test $have_vers_rel_arg -ne 1 && vers_release="${default_vers_rel}"
test $have_vers_git_arg -ne 1 && vers_git="${default_vers_git}"
case "${command}" in
assetfs )
if is_set "${refresh_docker}"
then
status_stage "==> Refreshing Consul build container image"
export GO_BUILD_TAG="${image}"
refresh_docker_images ${sdir} go-build-image || return 1
fi
status_stage "==> Build Static Assets"
build_assetfs "${sdir}" "${image}" || return 1
;;
consul )
if is_set "${refresh_docker}"
then
@ -328,6 +356,16 @@ function main {
consul-local )
build_consul_local "${sdir}" "${build_os}" "${build_arch}" "" || return 1
;;
publish )
publish_release "${sdir}" true true || return 1
;;
release )
if is_set "${refresh_docker}"
then
refresh_docker_images ${sdir} || return 1
fi
build_release "${sdir}" "${rel_tag}" "${rel_build}" "${rel_sign}" "${rel_gpg_key}" || return 1
;;
ui )
if is_set "${refresh_docker}"
@ -350,27 +388,10 @@ function main {
build_ui_legacy "${sdir}" "${image}" || return 1
;;
version )
parse_version "${sdir}" "${vers_release}"|| return 1
;;
assetfs )
if is_set "${refresh_docker}"
then
status_stage "==> Refreshing Consul build container image"
export GO_BUILD_TAG="${image}"
refresh_docker_images ${sdir} go-build-image || return 1
fi
status_stage "==> Build Static Assets"
build_assetfs "${sdir}" "${image}" || return 1
;;
release )
if is_set "${refresh_docker}"
then
refresh_docker_images ${sdir} || return 1
fi
build_release "${sdir}" "${rel_tag}" "${rel_build}" "${rel_sign}" "${rel_gpg_key}" || return 1
parse_version "${sdir}" "${vers_release}" "${vers_git}" || return 1
;;
*)
err "Unkown subcommand: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'assetfs', version' and 'release'"
err "Unkown subcommand: '$1' - possible values are 'assetfs', consul', 'consul-local' 'publish', 'release', 'ui', 'ui-legacy' and 'version'"
return 1
;;
esac