luxolus
/
open-consul
2
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Wiki Activity

Redo the build system 

Improvements:
   - More modular
   - Building within docker doesn’t use volumes so can be run on a remote docker host
   - Build containers include only minimal context so they only rarely need to be rebuilt and most of the time can be used from the cache.
   - 3 build containers instead of 1. One based off of the upstream golang containers for building go stuff with all our required GOTOOLS installed. One like the old container based off ubuntu bionic for building the old UI (didn’t bother creating a much better container as this shouldn’t be needed once we completely remove the legacy UI). One for building the new UI. Its alpine based with all the node, ember, yarn stuff installed.
   - Top level makefile has the ability to do a container based build without running make dist
   - Can build for arbitrary platforms at the top level using: make consul-docker XC_OS=… XC_ARCH=…
   - overridable functionality to allow for customizations to the enterprise build (like to generate multiple binaries)
   - unified how we compile our go. always use gox even for dev-builds or rather always use the tooling around our scripts which will make sure things get copied to the correct places throughout the filesystem.
This commit is contained in:
Matt Keeler 2018-06-12 16:55:52 -04:00
parent d587a60662
commit 351841c7b2
15 changed files with 1273 additions and 810 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
GNUmakefile
View File

@ -33,6 +33,10 @@ UI_BUILD_TAG?=consul-build-ui
UI_LEGACY_BUILD_TAG?=consul-build-ui-legacy UI_LEGACY_BUILD_TAG?=consul-build-ui-legacy
BUILD_CONTAINER_NAME?=consul-builder BUILD_CONTAINER_NAME?=consul-builder
DIST_TAG?=1
DIST_BUILD?=1
DIST_SIGN?=1
export GO_BUILD_TAG export GO_BUILD_TAG
export UI_BUILD_TAG export UI_BUILD_TAG
export UI_LEGACY_BUILD_TAG export UI_LEGACY_BUILD_TAG
@ -47,18 +51,13 @@ export GOLDFLAGS
all: bin all: bin
bin: tools bin: tools
@mkdir -p bin/ @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul-local
@GOTAGS='$(GOTAGS)' sh -c "'$(CURDIR)/scripts/build.sh'"
# dev creates binaries for testing locally - these are put into ./bin and $GOPATH # dev creates binaries for testing locally - these are put into ./bin and $GOPATH
dev: changelogfmt vendorfmt dev-build dev: changelogfmt vendorfmt dev-build
dev-build: dev-build:
@echo "--> Building consul" @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul-local -o '$(GOOS)' -a '$(GOARCH)'
mkdir -p pkg/$(GOOS)_$(GOARCH)/ bin/
go install -ldflags '$(GOLDFLAGS)' -tags '$(GOTAGS)'
cp $(GOPATH)/bin/consul bin/
cp $(GOPATH)/bin/consul pkg/$(GOOS)_$(GOARCH)
vendorfmt: vendorfmt:
@echo "--> Formatting vendor/vendor.json" @echo "--> Formatting vendor/vendor.json"
@ -71,12 +70,11 @@ changelogfmt:
# linux builds a linux package independent of the source platform # linux builds a linux package independent of the source platform
linux: linux:
mkdir -p pkg/linux_amd64/ @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul-local -o linux -a amd64
GOOS=linux GOARCH=amd64 go build -ldflags '$(GOLDFLAGS)' -tags '$(GOTAGS)' -o pkg/linux_amd64/consul
# dist builds binaries for all platforms and packages them for distribution # dist builds binaries for all platforms and packages them for distribution
dist: dist:
@GOTAGS='$(GOTAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'" @$(SHELL) $(CURDIR)/build-support/scripts/build.sh release -t '$(DIST_TAG)' -b '$(DIST_BUILD)' -S '$(DIST_SIGN)'
cov: cov:
gocov test $(GOFILES) | gocov-html > /tmp/coverage.html gocov test $(GOFILES) | gocov-html > /tmp/coverage.html
@ -128,8 +126,7 @@ vet:
# Build the static web ui and build static assets inside a Docker container, the # Build the static web ui and build static assets inside a Docker container, the
# same way a release build works. This implicitly does a "make static-assets" at # same way a release build works. This implicitly does a "make static-assets" at
# the end. # the end.
ui: ui: ui-legacy-docker ui-docker static-assets
@sh -c "'$(CURDIR)/scripts/ui.sh'"
# If you've run "make ui" manually then this will get called for you. This is # If you've run "make ui" manually then this will get called for you. This is
# also run as part of the release build script when it verifies that there are no # also run as part of the release build script when it verifies that there are no
@ -141,6 +138,12 @@ static-assets:
tools: tools:
go get -u -v $(GOTOOLS) go get -u -v $(GOTOOLS)
version:
@echo -n "Version without release: "
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh version
@echo -n "Version with release: "
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh version -R
docker-images: docker-images:
@$(MAKE) -C build-support/docker images @$(MAKE) -C build-support/docker images
@ -156,7 +159,7 @@ ui-legacy-build-image:
static-assets-docker: go-build-image static-assets-docker: go-build-image
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh assetfs @$(SHELL) $(CURDIR)/build-support/scripts/build.sh assetfs
go-docker: go-build-image consul-docker: go-build-image
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul
ui-docker: ui-build-image ui-docker: ui-build-image
@ -165,7 +168,6 @@ ui-docker: ui-build-image
ui-legacy-docker: ui-legacy-build-image ui-legacy-docker: ui-legacy-build-image
@$(SHELL) $(CURDIR)/build-support/scripts/build.sh ui-legacy @$(SHELL) $(CURDIR)/build-support/scripts/build.sh ui-legacy
release-docker: ui-docker ui-legacy-docker static-assets-docker go-docker
.PHONY: all ci bin dev dist cov test cover format vet ui static-assets tools vendorfmt .PHONY: all ci bin dev dist cov test cover format vet ui static-assets tools vendorfmt
.PHONY: docker-images go-build-iamge ui-build-image ui-legacy-build-image static-assets-docker go-docker ui-docker ui-legacy-docker release-docker .PHONY: docker-images go-build-image ui-build-image ui-legacy-build-image static-assets-docker consul-docker ui-docker ui-legacy-docker version

20
build-support/functions/00-vars.sh Normal file
View File

@ -0,0 +1,20 @@
# GPG Key ID to use for publically released builds
HASHICORP_GPG_KEY="348FFC4C"
# Default Image Names
UI_BUILD_CONTAINER_DEFAULT="consul-build-ui"
UI_LEGACY_BUILD_CONTAINER_DEFAULT="consul-build-ui-legacy"
GO_BUILD_CONTAINER_DEFAULT="consul-build-go"
# Whether to colorize shell output
COLORIZE=1
# determine GOPATH and the first GOPATH to use for intalling binaries
GOPATH=${GOPATH:-$(go env GOPATH)}
case $(uname) in
CYGWIN*)
GOPATH="$(cygpath $GOPATH)"
;;
esac
MAIN_GOPATH=$(cut -d: -f1 <<< "${GOPATH}")

186
build-support/functions/01-util.sh Normal file
View File

@ -0,0 +1,186 @@
function err {
if test "${COLORIZE}" -eq 1
then
tput bold
tput setaf 1
fi
echo $@ 1>&2
if test "${COLORIZE}" -eq 1
then
tput sgr0
fi
}
function status {
if test "${COLORIZE}" -eq 1
then
tput bold
tput setaf 4
fi
echo $@
if test "${COLORIZE}" -eq 1
then
tput sgr0
fi
}
function status_stage {
if test "${COLORIZE}" -eq 1
then
tput bold
tput setaf 2
fi
echo $@
if test "${COLORIZE}" -eq 1
then
tput sgr0
fi
}
function is_set {
# Arguments:
# $1 - string value to check its truthiness
#
# Return:
# 0 - is truthy (backwards I know but allows syntax like `if is_set <var>` to work)
# 1 - is not truthy
local val=$(tr '[:upper:]' '[:lower:]' <<< "$1")
case $val in
1 | t | true | y | yes)
return 0
;;
*)
return 1
;;
esac
}
function have_gpg_key {
# Arguments:
# $1 - GPG Key id to check if we have installed
#
# Return:
# 0 - success (we can use this key for signing)
# * - failure (key cannot be used)
gpg --list-secret-keys $1 >dev/null 2>&1
return $?
}
function parse_version {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - boolean value for whether to omit the release version from the version string
#
# Return:
# 0 - success (will write the version to stdout)
# * - error (no version output)
#
# Notes:
# If the GOTAGS environment variable is present then it is used to determine which
# version file to use for parsing.
# If the GIT_DESCRIBE environment variable is present then it is used as the version
# If the GIT_COMMIT environment variable is preset it will be added to the end of
# the version string.
local vfile="${1}/version/version.go"
# ensure the version file exists
if ! test -f "${vfile}"
then
err "Error - File not found: ${vfile}"
return 1
fi
# Get the main version out of the source file
version_main=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
release_main=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
# try to determine the version if we have build tags
for tag in "$GOTAGS"
do
for vfile in $(ls "${1}/version/version_*.go" 2> /dev/null| sort)
do
if grep -q "// +build $tag" $file
then
version_main=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
release_main=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
fi
done
done
version=
# override the version from source with the value of the GIT_DESCRIBE env var if present
if test -n "$GIT_DESCRIBE"
then
version=$GIT_DESCRIBE
fi
if ! is_set $2
then
# Get the release version out of the source file
release=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
# When no GIT_DESCRIBE env var is present and no release is in the source then we
# are definitely in dev mode
if test -z "$GIT_DESCRIBE" -a -z "$release"
then
release="dev"
fi
# Add the release to the version
if test -n "$release"
then
version="${version}-${release}"
# add the git commit to the version
if test -n "$GIT_COMMIT"
then
version="${version} (${GIT_COMMIT})"
fi
fi
fi
# Output the version
echo "$version" | tr -d "'"
return 0
}
function get_version {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - Whether the release version should be parsed from source (optional)
#
# Returns:
# 0 - success (the version is also echoed to stdout)
# 1 - error
#
# Notes:
# If a VERSION environment variable is present it will override any parsing of the version from the source
# In addition to processing the main version.go, version_*.go files will be processed if they have
# a Go build tag that matches the one in the GOTAGS environment variable. This tag processing is
# primitive though and will not match complex build tags in the files with negation etc.
local vers="$VERSION"
if test -z "$vers"
then
# parse the OSS version from version.go
vers="$(parse_version ${1} ${2})"
fi
if test -z "$vers"
then
return 1
else
echo $vers
return 0
fi
}

382
build-support/functions/02-build.sh Normal file
View File

@ -0,0 +1,382 @@
function refresh_docker_images {
# Arguments:
# $1 - Path to top level Consul source
# $2 - Which make target to invoke (optional)
#
# Return:
# 0 - success
# * - failure
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. refresh_docker_images must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
local targets="$2"
test -n "${targets}" || targets="images"
make -C "${sdir}/build-support/docker" $targets
return $?
}
function build_ui {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
#
# Notes:
# Use the GIT_COMMIT environment variable to pass off to the build
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. build_ui must be called with the path to the top level source as the first argument'"
return 1
fi
local image_name=${UI_BUILD_CONTAINER_DEFAULT}
if test -n "$2"
then
image_name="$2"
fi
local sdir="$1"
local ui_dir="${1}/ui-v2"
# parse the version
version=$(parse_version "${sdir}")
local commit_hash="${GIT_COMMIT}"
if test -z "${commit_hash}"
then
commit_hash=$(git rev-parse --short HEAD)
fi
# make sure we run within the ui dir
pushd ${ui_dir} > /dev/null
status "Creating the UI Build Container with image: ${image_name}"
local container_id=$(docker create -it -e "CONSUL_GIT_SHA=${commit_hash}" -e "CONSUL_VERSION=${version}" ${image_name})
local ret=$?
if test $ret -eq 0
then
status "Copying the source from '${ui_dir}' to /consul-src within the container"
(
docker cp . ${container_id}:/consul-src &&
status "Running build in container" && docker start -i ${container_id} &&
rm -rf ${1}/ui-v2/dist &&
status "Copying back artifacts" && docker cp ${container_id}:/consul-src/dist ${1}/ui-v2/dist
)
ret=$?
docker rm ${container_id} > /dev/null
fi
if test $ret -eq 0
then
rm -rf ${1}/pkg/web_ui/v2
cp -r ${1}/ui-v2/dist ${1}/pkg/web_ui/v2
fi
popd > /dev/null
return $ret
}
function build_ui_legacy {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. build_ui_legacy must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
local ui_legacy_dir="${sdir}/ui"
local image_name=${UI_LEGACY_BUILD_CONTAINER_DEFAULT}
if test -n "$2"
then
image_name="$2"
fi
pushd ${ui_legacy_dir} > /dev/null
status "Creating the Legacy UI Build Container with image: ${image_name}"
rm -r ${sdir}/pkg/web_ui/v1 >/dev/null 2>&1
mkdir -p ${sdir}/pkg/web_ui/v1
local container_id=$(docker create -it ${image_name})
local ret=$?
if test $ret -eq 0
then
status "Copying the source from '${ui_legacy_dir}' to /consul-src/ui within the container"
(
docker cp . ${container_id}:/consul-src/ui &&
status "Running build in container" &&
docker start -i ${container_id} &&
status "Copying back artifacts" &&
docker cp ${container_id}:/consul-src/pkg/web_ui ${sdir}/pkg/web_ui/v1
)
ret=$?
docker rm ${container_id} > /dev/null
fi
popd > /dev/null
return $ret
}
function build_assetfs {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
#
# Note:
# The GIT_COMMIT, GIT_DIRTY and GIT_DESCRIBE environment variables will be used if present
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. build_assetfs must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
local image_name=${GO_BUILD_CONTAINER_DEFAULT}
if test -n "$2"
then
image_name="$2"
fi
pushd ${sdir} > /dev/null
status "Creating the Go Build Container with image: ${image_name}"
local container_id=$(docker create -it -e GIT_COMMIT=${GIT_COMMIT} -e GIT_DIRTY=${GIT_DIRTY} -e GIT_DESCRIBE=${GIT_DESCRIBE} ${image_name} make static-assets ASSETFS_PATH=bindata_assetfs.go)
local ret=$?
if test $ret -eq 0
then
status "Copying the sources from '${sdir}/(pkg|GNUmakefile)' to /go/src/github.com/hashicorp/consul/pkg"
(
tar -c pkg/web_ui GNUmakefile | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul &&
status "Running build in container" && docker start -i ${container_id} &&
status "Copying back artifacts" && docker cp ${container_id}:/go/src/github.com/hashicorp/consul/bindata_assetfs.go ${sdir}/agent/bindata_assetfs.go
)
ret=$?
docker rm ${container_id} > /dev/null
fi
popd >/dev/null
return $ret
}
function build_consul_post {
# Arguments
# $1 - Path to the top level Consul source
# $2 - build suffix (Optional)
#
# Returns:
# 0 - success
# * - error
#
# Notes:
# pkg/bin is where to place binary packages
# pkg.bin.new is where the just built binaries are located
# bin is where to place the local systems versions
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. build_consul_post must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
pushd "${sdir}" > /dev/null
# recreate the pkg dir
rm -r pkg/bin/* 2> /dev/null
mkdir -p pkg/bin 2> /dev/null
# move all files in pkg.new into pkg
cp -r pkg.bin.new/* pkg/bin/
rm -r pkg.bin.new
DEV_PLATFORM="./pkg/bin/$(go env GOOS)_$(go env GOARCH)${2}"
for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f)
do
# recreate the bin dir
rm -r bin/* 2> /dev/null
mkdir -p bin 2> /dev/null
cp ${F} bin/
cp ${F} ${MAIN_GOPATH}/bin
done
popd > /dev/null
return 0
}
function build_consul {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - build suffix (optional - must specify if needing to specify the docker image)
# $3 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
#
# Note:
# The GOLDFLAGS and GOTAGS environment variables will be used if set
# If the CONSUL_DEV environment var is truthy only the local platform/architecture is built.
# If the XC_OS or the XC_ARCH environment vars are present then only those platforms/architectures
# will be built. Otherwise all supported platform/architectures are built
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. build_consul must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
local build_suffix="$2"
local image_name=${GO_BUILD_CONTAINER_DEFAULT}
if test -n "$3"
then
image_name="$3"
fi
pushd ${sdir} > /dev/null
status "Creating the Go Build Container with image: ${image_name}"
if is_set "${CONSUL_DEV}"
then
if test -z "${XC_OS}"
then
XC_OS=$(go env GOOS)
fi
if test -z "${XC_ARCH}"
then
XC_ARCH=$(go env GOARCH)
fi
fi
XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"}
local container_id=$(docker create -it -e CGO_ENABLED=0 ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/bin/{{.OS}}_{{.Arch}}${build_suffix}/consul" -tags="${GOTAGS}")
ret=$?
if test $ret -eq 0
then
status "Copying the source from '${sdir}' to /go/src/github.com/hashicorp/consul/pkg"
(
tar -c $(ls | grep -v "ui\|ui-v2\|website\|bin\|.git") | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul &&
status "Running build in container" &&
docker start -i ${container_id} &&
status "Copying back artifacts" &&
docker cp ${container_id}:/go/src/github.com/hashicorp/consul/pkg/bin pkg.bin.new
)
ret=$?
docker rm ${container_id} > /dev/null
if test $ret -eq 0
then
build_consul_post "${sdir}" "${build_suffix}"
ret=$?
else
rm -r pkg.bin.new 2> /dev/null
fi
fi
popd > /dev/null
return $ret
}
function build_consul_local {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - Space separated string of OSes to build. If empty will use env vars for determination.
# $3 - Space separated string of architectures to build. If empty will use env vars for determination.
# $4 - build suffix (optional)
#
# Returns:
# 0 - success
# * - error
#
# Note:
# The GOLDFLAGS and GOTAGS environment variables will be used if set
# If the CONSUL_DEV environment var is truthy only the local platform/architecture is built.
# If the XC_OS or the XC_ARCH environment vars are present then only those platforms/architectures
# will be built. Otherwise all supported platform/architectures are built
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. build_consul must be called with the path to the top level source as the first argument'"
return 1
fi
local sdir="$1"
local build_os="$2"
local build_arch="$3"
local build_suffix="$4"
pushd ${sdir} > /dev/null
if is_set "${CONSUL_DEV}"
then
if test -z "${XC_OS}"
then
XC_OS=$(go env GOOS)
fi
if test -z "${XC_ARCH}"
then
XC_ARCH=$(go env GOARCH)
fi
fi
XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"}
if test -z "${build_os}"
then
build_os="${XC_OS}"
fi
if test -z "${build_arch}"
then
build_arch="${XC_ARCH}"
fi
status_stage "==> Building Consul - OSes: ${build_os}, Architectures: ${build_arch}"
mkdir pkg.bin.new 2> /dev/null
CGO_ENABLED=0 gox \
-os="${build_os}" \
-arch="${build_arch}" \
-osarch="!darwin/arm !darwin/arm64" \
-ldflags="${GOLDFLAGS}" \
-output "pkg.bin.new/{{.OS}}_{{.Arch}}${build_suffix}/consul" \
-tags="${GOTAGS}" \
.
if test $? -ne 0
then
err "ERROR: Failed to build Consul"
rm -r pkg.bin.new
return 1
fi
build_consul_post "${sdir}" "${build_suffix}"
if test $? -ne 0
then
err "ERROR: Failed postprocessing Consul binaries"
return 1
fi
return 0
}

299
build-support/functions/03-release.sh Normal file
View File

@ -0,0 +1,299 @@
function tag_release {
# Arguments:
# $1 - Path to top level consul source
# $2 - Version string to use for tagging the release
# $3 - Alternative GPG key id used for signing the release commit (optional)
#
# Returns:
# 0 - success
# * - error
#
# Notes:
# If the RELEASE_UNSIGNED environment variable is set then no gpg signing will occur
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. tag_release must be called with the path to the top level source as the first argument'"
return 1
fi
if test -z "$2"
then
err "ERROR: tag_release must be called with a version number as the second argument"
return 1
fi
# determine whether the gpg key to use is being overridden
local gpg_key=${HASHICORP_GPG_KEY}
if test -n "$3"
then
gpg_key=$3
fi
pushd "$1" > /dev/null
local ret=0
# perform an usngined release if requested (mainly for testing locally)
if is_set "$RELEASE_UNSIGNED"
then
(
git commit --allow-empty -a -m "Release v${2}" &&
git tag -a -m "Version ${2}" "v${2}" master
)
ret=$?
# perform a signed release (official releases should do this)
elif have_gpg_key ${gpg_key}
then
(
git commit --allow-empty -a --gpg-sign=${gpg_key} -m "Release v${2}" &&
git tag -a -m "Version ${2}" -s -u ${gpg_key} "v${2}" master
)
ret=$?
# unsigned release not requested and gpg key isn't useable
else
err "ERROR: GPG key ${gpg_key} is not in the local keychain - to continue set RELEASE_UNSIGNED=1 in the env"
ret=1
fi
popd > /dev/null
return $ret
}
function package_release {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - Version to use in the names of the zip files (optional)
#
# Returns:
# 0 - success
# * - error
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. package_release must be called with the path to the top level source as the first argument'"
return 1
fi
local vers="${2}"
if test -z "${vers}"
then
vers=$(get_version $1 true)
ret=$?
if test "$ret" -ne 0
then
err "ERROR: failed to determine the version."
return $ret
fi
fi
local sdir="$1"
local ret=0
rm -rf "${sdir}/pkg/dist" > /dev/null 2>&1
mkdir -p "${sdir}/pkg/dist" >/dev/null 2>&1
for platform in $(find "${sdir}/pkg/bin" -mindepth 1 -maxdepth 1 -type d)
do
local os_arch=$(basename $platform)
local dest="${sdir}/pkg/dist/consul_${vers}_${os_arch}.zip"
status "Compressing ${os_arch} directory into ${dest}"
pushd "${platform}" > /dev/null
zip "${sdir}/pkg/dist/consul_${vers}_${os_arch}.zip" ./*
ret=$?
popd > /dev/null
if test "$ret" -ne 0
then
break
fi
done
return $ret
}
function shasum_release {
# Arguments:
# $1 - Path to directory containing the files to shasum
# $2 - File to output sha sums to
#
# Returns:
# 0 - success
# * - failure
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory and shasum_release requires passing a directory as the first argument"
return 1
fi
if test -z "$2"
then
err "ERROR: shasum_release requires a second argument to be the filename to output the shasums to but none was given"
return 1
fi
pushd $1 > /dev/null
shasum -a256 * > "$2"
ret=$?
popd >/dev/null
return $ret
}
function sign_release {
# Arguments:
# $1 - File to sign
# $2 - Alternative GPG key to use for signing
#
# Returns:
# 0 - success
# * - failure
# determine whether the gpg key to use is being overridden
local gpg_key=${HASHICORP_GPG_KEY}
if test -n "$2"
then
gpg_key=$2
fi
gpg --default-key "${gpg_key}" --detach-sig "$1"
return $?
}
function build_consul_release {
build_consul "$1" "" "$2"
}
function build_release {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - boolean whether to tag the release yet
# $3 - boolean whether to build the binaries
# $4 - boolean whether to generate the sha256 sums
# $5 - alternative gpg key to use for signing operations (optional)
#
# Returns:
# 0 - success
# * - error
if ! test -d "$1"
then
err "ERROR: '$1' is not a directory. build_release must be called with the path to the top level source as the first argument'"
return 1
fi
if test -z "$2" -o -z "$3" -o -z "$4"
then
err "ERROR: build_release requires 4 arguments to be specified: <path to consul source> <tag release bool?> <build binaries bool?> <shasum 256 bool?>"
return 1
fi
local sdir="$1"
local do_tag="$2"
local do_build="$3"
local do_sha256="$4"
local gpg_key="$5"
local vers=$(get_version ${sdir} true)
if test $? -ne 0
then
err "Please specify a version (couldn't find one based on build tags)."
return 1
fi
# Make sure we arent in dev mode
unset CONSUL_DEV
if is_set "${do_build}"
then
status_stage "==> Refreshing Docker Build Images"
refresh_docker_images "${sdir}"
if test $? -ne 0
then
err "ERROR: Failed to refresh docker images"
return 1
fi
status_stage "==> Building Legacy UI for version ${vers}"
build_ui_legacy "${sdir}" "${UI_LEGACY_BUILD_TAG}"
if test $? -ne 0
then
err "ERROR: Failed to build the legacy ui"
return 1
fi
status_stage "==> Building UI for version ${vers}"
build_ui "${sdir}" "${UI_BUILD_TAG}"
if test $? -ne 0
then
err "ERROR: Failed to build the ui"
return 1
fi
status_stage "==> Building Static Assets for version ${vers}"
build_assetfs "${sdir}" "${GO_BUILD_TAG}"
if test $? -ne 0
then
err "ERROR: Failed to build the static assets"
return 1
fi
if is_set "${do_tag}"
then
git add "${sdir}/agent/bindata_assetfs.go"
if test $? -ne 0
then
err "ERROR: Failed to git add the assetfs file"
return 1
fi
fi
fi
if is_set "${do_tag}"
then
status_stage "==> Tagging version ${vers}"
tag_release "${sdir}" "${vers}" "${gpg_key}"
if test $? -ne 0
then
err "ERROR: Failed to tag the release"
return 1
fi
fi
if is_set "${do_build}"
then
status_stage "==> Building Consul for version ${vers}"
build_consul_release "${sdir}" "${GO_BUILD_TAG}"
if test $? -ne 0
then
err "ERROR: Failed to build the Consul binaries"
return 1
fi
status_stage "==> Packaging up release binaries"
package_release "${sdir}" "${vers}"
if test $? -ne 0
then
err "ERROR: Failed to package the release binaries"
return 1
fi
fi
status_stage "==> Generating SHA 256 Hashes for Binaries"
shasum_release "${sdir}/pkg/dist" "consul_${vers}_SHA256SUMS"
if test $? -ne 0
then
err "ERROR: Failed to generate SHA 256 hashes for the release"
return 1
fi
if is_set "${do_sha256}"
then
sign_release "${sdir}/pkg/dist/consul_${vers}_SHA256SUMS" "${gpg_key}"
if test $? -ne 0
then
err "ERROR: Failed to sign the SHA 256 hashes file"
return 1
fi
fi
return 0
}

368
build-support/scripts/build.sh
View File

@ -1,39 +1,381 @@
#!/bin/bash #!/bin/bash
SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})"
pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null
SCRIPT_DIR=$(pwd) SCRIPT_DIR=$(pwd)
pushd ../.. > /dev/null pushd ../.. > /dev/null
SOURCE_DIR=$(pwd) SOURCE_DIR=$(pwd)
popd > /dev/null popd > /dev/null
pushd ../functions > /dev/null
FN_DIR=$(pwd)
popd > /dev/null
popd > /dev/null popd > /dev/null
source "${SCRIPT_DIR}/functions.sh" source "${SCRIPT_DIR}/functions.sh"
function can_parse_option {
local allowed="$1"
local command="$2"
local options="$3"
if test ${allowed} -ne 1
then
err "ERROR: subcommand ${command} does not support the ${options} options"
return 1
fi
return 0
}
function check_duplicate {
local is_dup="$1"
local command="$2"
local options="$3"
if test ${is_dup} -ne 0
then
err "ERROR: options ${options} may not be given more than once to the subcommand ${command}"
return 1
fi
return 0
}
function option_check {
can_parse_option "$1" "$3" "$4" && check_duplicate "$2" "$3" "$4"
return $?
}
function get_option_value {
# Arguments:
# $1 - bool whether the option should be allowed
# $2 - bool whether the option has been specified already
# $3 - the option value
# $4 - the command being executed
# $5 - the option names to use for logging
#
# Returns:
# 0 - success
# * - failure
option_check "$1" "$2" "$4" "$5" || return 1
if test -z "$3"
then
err "ERROR: options ${5} for subcommand ${4} require an argument but none was provided"
return 1
fi
echo "$3"
return 0
}
function usage {
cat <<-EOF
Usage: ${SCRIPT_NAME} <subcommand> [<options ...>]
Subcommands:
assetfs: Builds the bindata_assetfs.go file from previously build UI artifacts
Options:
-i | --image IMAGE Alternative Docker image to run the build within.
Defaults to ${GO_BUILD_CONTAINER_DEFAULT}
-s | --source DIR Path to source to build.
Defaults to "${SOURCE_DIR}"
-r | --refresh Enables refreshing the docker image prior to building.
consul: Builds the main Consul binary. This assumes the assetfs is up to date:
Options:
-i | --image IMAGE Alternative Docker image to run the build within.
Defaults to ${GO_BUILD_CONTAINER_DEFAULT}
-s | --source DIR Path to source to build.
Defaults to "${SOURCE_DIR}"
-r | --refresh Enables refreshing the docker image prior to building.
consul-local: Builds the main Consul binary on the local system (no docker)
-s | --source DIR Path to source to build.
Defaults to "${SOURCE_DIR}"
-o | --build-os OS Space separated string of OSes to build
-a | --build-arch ARCH Space separated string of architectures to build
release: Performs a release build.
Options:
-s | --source DIR Path to source to build.
Defaults to "${SOURCE_DIR}"
-t | --tag BOOL Whether to add a release commit and tag the build
Defaults to 1.
-b | --build BOOL Whether to perform the build of the ui's, assetfs and
binaries. Defaults to 1.
-S | --sign BOOL Whether to sign the generated SHA256SUMS file.
Defaults to 1.
-g | --gpg-key KEY Alternative GPG key to use for signing operations.
Defaults to ${HASHICORP_GPG_KEY}
ui: Builds the latest UI.
Options:
-i | --image IMAGE Alternative Docker image to run the build within.
Defaults to ${UI_BUILD_CONTAINER_DEFAULT}
-s | --source DIR Path to source to build.
Defaults to "${SOURCE_DIR}"
-r | --refresh Enables refreshing the docker image prior to building.
ui-legacy: Builds the legacy UI
Options:
-i | --image IMAGE Alternative Docker image to run the build within.
Defaults to ${UI_LEGACY_BUILD_CONTAINER_DEFAULT}
-s | --source DIR Path to source to build.
Defaults to "${SOURCE_DIR}"
-r | --refresh Enables refreshing the docker image prior to building.
version: Prints out the version parsed from source.
Options:
-s | --source DIR Path to source to build.
Defaults to "${SOURCE_DIR}"
EOF
}
function main { function main {
case "$1" in declare build_fn
consul ) declare sdir
build_consul "${SOURCE_DIR}" "${GO_BUILD_TAG}" declare image
return $? declare -i refresh_docker=0
declare -i rel_tag
declare -i rel_build
declare -i rel_sign
declare rel_gpg_key=""
declare build_os
declare build_arch
declare -i vers_release
declare -i use_refresh=1
declare -i default_refresh=0
declare -i use_sdir=1
declare default_sdir="${SOURCE_DIR}"
declare -i use_image=0
declare default_image=""
declare -i use_rel=0
declare -i default_rel_tag=1
declare -i default_rel_build=1
declare -i default_rel_sign=1
declare default_rel_gpg_key="${HASHICORP_GPG_KEY}"
declare -i use_xc=0
declare default_build_os=""
declare default_build_arch=""
declare -i use_vers_rel
declare -i default_vers_rel=1
declare command="$1"
shift
case "${command}" in
consul )
use_image=1
default_image="${GO_BUILD_CONTAINER_DEFAULT}"
;;
consul-local )
use_xc=1
;; ;;
ui ) ui )
build_ui "${SOURCE_DIR}" "${UI_BUILD_TAG}" use_image=1
return $? default_image="${UI_BUILD_CONTAINER_DEFAULT}"
;; ;;
ui-legacy ) ui-legacy )
build_ui_legacy "${SOURCE_DIR}" "${UI_LEGACY_BUILD_TAG}" use_image=1
return $? default_image="${UI_LEGACY_BUILD_CONTAINER_DEFAULT}"
;; ;;
version ) version )
parse_version "${SOURCE_DIR}" use_refresh=0
return $? use_vers_rel=1
;; ;;
assetfs ) assetfs )
build_assetfs "${SOURCE_DIR}" "${GO_BUILD_TAG}" use_image=1
return $? default_image="${GO_BUILD_CONTAINER_DEFAULT}"
;;
release )
use_rel=1
use_refresh=0
;;
-h | --help)
usage
return 0
;; ;;
*) *)
echo "Unkown build: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'version' and 'assetfs'" 1>&2 err "Unkown subcommand: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'assetfs', version' and 'release'"
return 1 return 1
;;
esac
declare -i have_image_arg=0
declare -i have_sdir_arg=0
declare -i have_rel_tag_arg=0
declare -i have_rel_build_arg=0
declare -i have_rel_sign_arg=0
declare -i have_rel_gpg_key_arg=0
declare -i have_refresh_arg=0
declare -i have_build_os_arg=0
declare -i have_build_arch_arg=0
declare -i have_vers_rel_arg=0
while test $# -gt 0
do
case $1 in
-h | --help )
usage
return 0
;;
-o | --build-os )
build_os=$(get_option_value "${use_xc}" "${have_build_os_arg}" "$2" "${command}" "-o/--xc-os") || return 1
have_build_os_arg=1
shift 2
;;
-a | --build-arch)
build_arch=$(get_option_value "${use_xc}" "${have_build_arch_arg}" "$2" "${command}" "-o/--xc-arch") || return 1
have_build_arch_arg=1
shift 2
;;
-R | --release )
option_check "${use_vers_rel}" "${have_vers_rel_arg}" "${command}" "-R/--release" || return 1
have_vers_rel_arg=1
vers_release=0
shift
;;
-r | --refresh)
option_check "${use_refresh}" "${have_refresh_arg}" "${command}" "-r/--refresh" || return 1
have_refresh_arg=1
refresh_docker=1
shift
;;
-i | --image )
image=$(get_option_value "${use_image}" "${have_image_arg}" "$2" "${command}" "-i/--image") || return 1
have_image_arg=1
shift 2
;;
-s | --source )
sdir=$(get_option_value "${use_sdir}" "${have_sdir_arg}" "$2" "${command}" "-s/--source") || return 1
if ! test -d "${sdir}"
then
err "ERROR: -s/--source is not a path to a top level directory"
return 1
fi
have_sdir_arg=1
shift 2
;;
-t | --tag )
rel_tag=$(get_option_value "${use_rel}" "${have_rel_tag_arg}" "$2" "${command}" "-t/--tag") || return 1
have_rel_tag_arg=1
shift 2
;;
-b | --build )
rel_build=$(get_option_value "${use_rel}" "${have_rel_build_arg}" "$2" "${command}" "-b/--build") || return 1
have_rel_build_arg=1
shift 2
;;
-S | --sign )
rel_sign=$(get_option_value "${use_rel}" "${have_rel_sign_arg}" "$2" "${command}" "-S/--sign") || return 1
have_rel_sign_arg=1
shift 2
;;
-g | --gpg-key )
rel_gpg_key=$(get_option_value "${use_rel}" "${have_rel_gpg_key_arg}" "$2" "${command}" "-g/--gpg-key") || return 1
shift 2
;;
*)
err "ERROR: Unknown option '$1' for subcommand ${command}"
return 1
;;
esac
done
test $have_image_arg -ne 1 && image="${default_image}"
test $have_sdir_arg -ne 1 && sdir="${default_sdir}"
test $have_rel_tag_arg -ne 1 && rel_tag="${default_rel_tag}"
test $have_rel_build_arg -ne 1 && rel_build="${default_rel_build}"
test $have_rel_sign_arg -ne 1 && rel_sign="${default_rel_sign}"
test $have_rel_gpg_key_arg -ne 1 && rel_gpg_key="${default_rel_gpg_key}"
test $have_refresh_arg -ne 1 && refresh_docker="${default_refresh}"
test $have_build_os_arg -ne 1 && build_os="${default_build_os}"
test $have_build_arch_arg -ne 1 && build_arch="${default_build_os}"
test $have_vers_rel_arg -ne 1 && vers_release="${default_vers_rel}"
case "${command}" in
consul )
if is_set "${refresh_docker}"
then
status_stage "==> Refreshing Consul build container image"
export GO_BUILD_TAG=${image}
refresh_docker_images ${sdir} go-build-image || return 1
fi
status_stage "==> Building Consul"
build_consul "${sdir}" "" "${image}" || return 1
;;
consul-local )
build_consul_local "${sdir}" "${build_os}" "${build_arch}" "" || return 1
;;
ui )
if is_set "${refresh_docker}"
then
status_stage "==> Refreshing UI build container image"
export UI_BUILD_TAG=${image}
refresh_docker_images ${sdir} ui-build-image || return 1
fi
status_stage "==> Building UI"
build_ui "${sdir}" "${image}" || return 1
;;
ui-legacy )
if is_set "${refresh_docker}"
then
status_stage "==> Refreshing Legacy UI build container image"
export UI_LEGACY_BUILD_TAG=${image}
refresh_docker_images ${sdir} ui-legacy-build-image || return 1
fi
status_stage "==> Building Legacy UI"
build_ui_legacy "${sdir}" "${image}" || return 1
;;
version )
parse_version "${sdir}" "${vers_release}"|| return 1
;;
assetfs )
if is_set "${refresh_docker}"
then
status_stage "==> Refreshing Consul build container image"
export GO_BUILD_TAG="${image}"
refresh_docker_images ${sdir} go-build-image || return 1
fi
status_stage "==> Build Static Assets"
build_assetfs "${sdir}" "${image}" || return 1
;;
release )
if is_set "${refresh_docker}"
then
refresh_docker_images ${sdir} || return 1
fi
build_release "${sdir}" "${rel_tag}" "${rel_build}" "${rel_sign}" "${rel_gpg_key}" || return 1
;;
*)
err "Unkown subcommand: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'assetfs', version' and 'release'"
return 1
;;
esac esac
return 0
} }
main $@ main $@

475
build-support/scripts/functions.sh
View File

@ -1,464 +1,17 @@
# GPG Key ID to use for publically released builds #
HASHICORP_GPG_KEY="348FFC4C" # NOTE: This file is meant to be sourced from other bash scripts/shells
#
# It provides all the scripting around building Consul and the release process
UI_BUILD_CONTAINER_DEFAULT="consul-build-ui" pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null
UI_LEGACY_BUILD_CONTAINER_DEFAULT="consul-build-ui-legacy" pushd ../functions > /dev/null
FUNC_DIR=$(pwd)
popd > /dev/null
popd > /dev/null
function is_set { func_sources=$(find ${FUNC_DIR} -type f -mindepth 1 -maxdepth 1 -name "*.sh" | sort -n)
# Arguments:
# $1 - string value to check its truthiness
#
# Return:
# 0 - is truthy (backwards I know but allows syntax like `if is_set <var>` to work)
# 1 - is not truthy
local val=$(tr '[:upper:]' '[:lower:]' <<< "$1")
case $val in
1 | t | true | y | yes)
return 0
;;
*)
return 1
;;
esac
}
function have_gpg_key { for src in $func_sources
# Arguments: do
# $1 - GPG Key id to check if we have installed source $src
# done
# Return:
# 0 - success (we can use this key for signing)
# * - failure (key cannot be used)
gpg --list-secret-keys $1 >dev/null 2>&1
return $?
}
function parse_version {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - boolean value for whether to omit the release version from the version string
#
# Return:
# 0 - success (will write the version to stdout)
# * - error (no version output)
#
# Notes:
# If the GIT_DESCRIBE environment variable is present then it is used as the version
# If the GIT_COMMIT environment variable is preset it will be added to the end of
# the version string.
local vfile="${1}/version/version.go"
# ensure the version file exists
if ! test -f "${vfile}"
then
echo "Error - File not found: ${vfile}" 1>&2
return 1
fi
# Get the main version out of the source file
version=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
# override the version from source with the value of the GIT_DESCRIBE env var if present
if test -n "$GIT_DESCRIBE"
then
version=$GIT_DESCRIBE
fi
if ! is_set $2
then
# Get the release version out of the source file
release=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile})
# When no GIT_DESCRIBE env var is present and no release is in the source then we
# are definitely in dev mode
if test -z "$GIT_DESCRIBE" -a -z "$release"
then
release="dev"
fi
# Add the release to the version
if test -n "$release"
then
version="${version}-${release}"
# add the git commit to the version
if test -n "$GIT_COMMIT"
then
version="${version} (${GIT_COMMIT})"
fi
fi
fi
# Output the version
echo "$version" | tr -d "'"
return 0
}
function get_version {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - Whether the release version should be parsed from source (optional)
#
# Returns:
# 0 - success (the version is also echoed to stdout)
# 1 - error
#
# Notes:
# If a VERSION environment variable is present it will override any parsing of the version from the source
# In addition to processing the main version.go, version_*.go files will be processed if they have
# a Go build tag that matches the one in the GOTAGS environment variable. This tag processing is
# primitive though and will not match complex build tags in the files with negation etc.
local vers="$VERSION"
if test -z "$vers"
then
# parse the OSS version from version.go
vers="$(parse_version ${1} ${2})"
# try to determine the version if we have build tags
for tag in "$GOTAGS"
do
for file in $(ls ${1}/version/version_*.go | sort)
do
if grep -q "// +build $tag" $file
then
vers=$(awk -F\" '/Version =/ {print $2; exit}' < $file )
fi
done
done
fi
if test -z "$vers"
then
return 1
else
echo $vers
return 0
fi
}
function tag_release {
# Arguments:
# $1 - Version string to use for tagging the release
# $2 - Alternative GPG key id used for signing the release commit (optional)
#
# Returns:
# 0 - success
# * - error
#
# Notes:
# If the RELEASE_UNSIGNED environment variable is set then no gpg signing will occur
if ! test -d "$1"
then
echo "ERROR: '$1' is not a directory. tag_release must be called with the path to the top level source as the first argument'" 1>&2
return 1
fi
if test -z "$2"
then
echo "ERROR: tag_release must be called with a version number as the second argument" 1>&2
return 1
fi
# determine whether the gpg key to use is being overridden
local gpg_key=${HASHICORP_GPG_KEY}
if test -n "$3"
then
gpg_key=$3
fi
pushd "$1" > /dev/null
local ret=0
# perform an usngined release if requested (mainly for testing locally)
if is_set "$RELEASE_UNSIGNED"
then
(
git commit --allow-empty -a -m "Release v${2}" &&
git tag -a -m "Version ${2}" "v${2}" master
)
ret=$?
# perform a signed release (official releases should do this)
elif have_gpg_key ${gpg_key}
then
(
git commit --allow-empty -a --gpg-sign=${gpg_key} -m "Release v${2}" &&
git tag -a -m "Version ${2}" -s -u ${gpg_key} "v${2}" master
)
ret=$?
# unsigned release not requested and gpg key isn't useable
else
echo "ERROR: GPG key ${gpg_key} is not in the local keychain - to continue set RELEASE_UNSIGNED=1 in the env"
ret=1
fi
popd > /dev/null
return $ret
}
function build_ui {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
#
# Notes:
# Use the GIT_COMMIT environment variable to pass off to the build
if ! test -d "$1"
then
echo "ERROR: '$1' is not a directory. build_ui must be called with the path to the top level source as the first argument'" 1>&2
return 1
fi
local image_name=${UI_BUILD_CONTAINER_DEFAULT}
if test -n "$2"
then
image_name="$2"
fi
local sdir="$1"
local ui_dir="${1}/ui-v2"
# parse the version
version=$(parse_version "${sdir}")
# make sure we run within the ui dir
pushd ${ui_dir} > /dev/null
echo "Creating the UI Build Container"
local container_id=$(docker create -it -e "CONSUL_GIT_SHA=${GIT_COMMIT}" -e "CONSUL_VERSION=${version}" ${image_name})
local ret=$?
if test $ret -eq 0
then
echo "Copying the source from '${ui_dir}' to /consul-src within the container"
(
docker cp . ${container_id}:/consul-src &&
echo "Running build in container" && docker start -i ${container_id} &&
rm -rf ${1}/ui-v2/dist &&
echo "Copying back artifacts" && docker cp ${container_id}:/consul-src/dist ${1}/ui-v2/dist
)
ret=$?
docker rm ${container_id} > /dev/null
fi
if test $ret -eq 0
then
rm -rf ${1}/pkg/web_ui/v2
cp -r ${1}/ui-v2/dist ${1}/pkg/web_ui/v2
fi
popd > /dev/null
return $ret
}
function build_ui_legacy {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
if ! test -d "$1"
then
echo "ERROR: '$1' is not a directory. build_ui_legacy must be called with the path to the top level source as the first argument'" 1>&2
return 1
fi
local sdir="$1"
local ui_legacy_dir="${sdir}/ui"
local image_name=${UI_LEGACY_BUILD_CONTAINER_DEFAULT}
if test -n "$2"
then
image_name="$2"
fi
pushd ${ui_legacy_dir} > /dev/null
echo "Creating the Legacy UI Build Container"
rm -r ${sdir}/pkg/web_ui/v1 >/dev/null 2>&1
mkdir -p ${sdir}/pkg/web_ui/v1
local container_id=$(docker create -it ${image_name})
local ret=$?
if test $ret -eq 0
then
echo "Copying the source from '${ui_legacy_dir}' to /consul-src/ui within the container"
(
docker cp . ${container_id}:/consul-src/ui &&
echo "Running build in container" &&
docker start -i ${container_id} &&
echo "Copying back artifacts" &&
docker cp ${container_id}:/consul-src/pkg/web_ui ${sdir}/pkg/web_ui/v1
)
ret=$?
docker rm ${container_id} > /dev/null
fi
popd > /dev/null
return $ret
}
function build_assetfs {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
#
# Note:
# The GIT_COMMIT, GIT_DIRTY and GIT_DESCRIBE environment variables will be used if present
if ! test -d "$1"
then
echo "ERROR: '$1' is not a directory. build_assetfs must be called with the path to the top level source as the first argument'" 1>&2
return 1
fi
local sdir="$1"
local image_name=${GO_BUILD_CONTAINER_DEFAULT}
if test -n "$2"
then
image_name="$2"
fi
pushd ${sdir} > /dev/null
echo "Creating the Go Build Container"
local container_id=$(docker create -it -e GIT_COMMIT=${GIT_COMMIT} -e GIT_DIRTY=${GIT_DIRTY} -e GIT_DESCRIBE=${GIT_DESCRIBE} ${image_name} make static-assets ASSETFS_PATH=bindata_assetfs.go)
local ret=$?
if test $ret -eq 0
then
echo "Copying the sources from '${sdir}/(pkg|GNUmakefile)' to /go/src/github.com/hashicorp/consul/pkg"
(
tar -c pkg/web_ui GNUmakefile | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul &&
echo "Running build in container" && docker start -i ${container_id} &&
echo "Copying back artifacts" && docker cp ${container_id}:/go/src/github.com/hashicorp/consul/bindata_assetfs.go ${sdir}/agent/bindata_assetfs.go
)
ret=$?
docker rm ${container_id} > /dev/null
fi
popd >/dev/null
return $ret
}
function build_consul {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - The docker image to run the build within (optional)
#
# Returns:
# 0 - success
# * - error
#
# Note:
# The GOLDFLAGS and GOTAGS environment variables will be used if set
# If the CONSUL_DEV environment var is truthy only the local platform/architecture is built.
# If the XC_OS or the XC_ARCH environment vars are present then only those platforms/architectures
# will be built. Otherwise all supported platform/architectures are built
if ! test -d "$1"
then
echo "ERROR: '$1' is not a directory. build_consul must be called with the path to the top level source as the first argument'" 1>&2
return 1
fi
local sdir="$1"
local image_name=${GO_BUILD_CONTAINER_DEFAULT}
if test -n "$2"
then
image_name="$2"
fi
pushd ${sdir} > /dev/null
echo "Creating the Go Build Container"
if is_set "${CONSUL_DEV}"
then
XC_OS=$(go_env GOOS)
XC_ARCH=$(go env GOARCH)
else
XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"}
fi
local container_id=$(docker create -it ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/{{.OS}}_{{.Arch}}/consul" -tags="${GOTAGS}")
ret=$?
if test $ret -eq 0
then
echo "Copying the source from '${sdir}' to /go/src/github.com/hashicorp/consul/pkg"
(
tar -c $(ls | grep -v "ui\|ui-v2\|website\|bin\|.git") | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul &&
echo "Running build in container" &&
docker start -i ${container_id} &&
echo "Copying back artifacts" &&
docker cp ${container_id}:/go/src/github.com/hashicorp/consul/pkg/ pkg.new
)
ret=$?
docker rm ${container_id} > /dev/null
DEV_PLATFORM="./pkg.new/$(go env GOOS)_$(go env GOARCH)"
for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f)
do
cp ${F} bin/
cp ${F} ${GOPATH}/bin
done
cp -r pkg.new/* pkg/
rm -r pkg.new
fi
popd > /dev/null
return $ret
}
function package_release {
# Arguments:
# $1 - Path to the top level Consul source
# $2 - Version to use in the names of the zip files (optional)
#
# Returns:
# 0 - success
# * - error
if ! test -d "$1"
then
echo "ERROR: '$1' is not a directory. package_release must be called with the path to the top level source as the first argument'" 1>&2
return 1
fi
local vers="${2}"
if test -z "${vers}"
then
vers=$(get_version $1 false)
ret=$?
if test "$ret" -ne 0
then
echo "ERROR: failed to determine the version." 1>&2
return $ret
fi
fi
local sdir="$1"
local ret=0
for platform in $(find "${sdir}/pkg" -mindepth 1 -maxdepth 1 -type d)
do
local os_arch=$(basename $platform)
pushd "${platform}" > /dev/null
zip "${sdir}/pkg/dist/consul_${vers}_${os_arch}.zip" ./*
ret=$?
popd > /dev/null
if test "$ret" -ne 0
then
break
fi
done
return $ret
}

77
scripts/build.sh
View File

@ -1,77 +0,0 @@
#!/usr/bin/env bash
#
# This script builds the application from source for multiple platforms.
set -e
export CGO_ENABLED=0
# Get the parent directory of where this script is.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
# Change into that directory
cd "$DIR"
# Determine the arch/os combos we're building for
XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"}
XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"}
# Delete the old dir
echo "==> Removing old directory..."
rm -f bin/*
rm -rf pkg/*
mkdir -p bin/
# If it's dev mode, only build for ourself
if [ "${CONSUL_DEV}x" != "x" ]; then
XC_OS=$(go env GOOS)
XC_ARCH=$(go env GOARCH)
fi
# Build!
echo "==> Building..."
"`which gox`" \
-os="${XC_OS}" \
-arch="${XC_ARCH}" \
-osarch="!darwin/arm !darwin/arm64" \
-ldflags "${GOLDFLAGS}" \
-output "pkg/{{.OS}}_{{.Arch}}/consul" \
-tags="${GOTAGS}" \
.
# Move all the compiled things to the $GOPATH/bin
GOPATH=${GOPATH:-$(go env GOPATH)}
case $(uname) in
CYGWIN*)
GOPATH="$(cygpath $GOPATH)"
;;
esac
OLDIFS=$IFS
IFS=: MAIN_GOPATH=($GOPATH)
IFS=$OLDIFS
# Copy our OS/Arch to the bin/ directory
DEV_PLATFORM="./pkg/$(go env GOOS)_$(go env GOARCH)"
for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f); do
cp ${F} bin/
cp ${F} ${MAIN_GOPATH}/bin/
done
if [ "${CONSUL_DEV}x" = "x" ]; then
# Zip and copy to the dist dir
echo "==> Packaging..."
for PLATFORM in $(find ./pkg -mindepth 1 -maxdepth 1 -type d); do
OSARCH=$(basename ${PLATFORM})
echo "--> ${OSARCH}"
pushd $PLATFORM >/dev/null 2>&1
zip ../${OSARCH}.zip ./*
popd >/dev/null 2>&1
done
fi
# Done!
echo
echo "==> Results:"
ls -hl bin/

34
scripts/consul-builder/Dockerfile
View File

@ -1,34 +0,0 @@
FROM ubuntu:bionic
ENV GOVERSION 1.10.1
RUN apt-get update -y && \
apt-get install --no-install-recommends -y -q \
build-essential \
ca-certificates \
curl \
git \
ruby \
ruby-dev \
zip \
zlib1g-dev \
nodejs \
npm && \
gem install bundler && \
npm install --global yarn && \
npm install --global ember-cli
RUN mkdir /goroot && \
mkdir /gopath && \
curl https://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \
tar xzf - -C /goroot --strip-components=1
# We want to ensure that release builds never have any cgo dependencies so we
# switch that off at the highest level.
ENV CGO_ENABLED 0
ENV GOPATH /gopath
ENV GOROOT /goroot
ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
RUN mkdir -p $GOPATH/src/github.com/hashicorp/consul
WORKDIR $GOPATH/src/github.com/hashicorp/consul

63
scripts/dist.sh
View File

@ -1,63 +0,0 @@
#!/usr/bin/env bash
set -e
# Get the version from the environment, or try to figure it out from the build tags.
# We process the files in the same order Go does to find the last matching tag.
if [ -z $VERSION ]; then
# get the OSS version from version.go
VERSION=$(awk -F\" '/Version =/ { print $2; exit }' <version/version.go)
# if we have build tags then try to determine the version
for tag in "$GOTAGS"; do
for file in $(ls version/version_*.go | sort); do
if grep -q "// +build $tag" $file; then
VERSION=$(awk -F\" '/Version =/ { print $2; exit }' <$file)
fi
done
done
fi
if [ -z $VERSION ]; then
echo "Please specify a version (couldn't find one based on build tags)."
exit 1
fi
echo "==> Building version $VERSION..."
# Get the parent directory of where this script is.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
# Change into that dir because we expect that.
cd $DIR
# Generate the tag.
if [ -z $NOTAG ]; then
echo "==> Tagging..."
git commit --allow-empty -a --gpg-sign=348FFC4C -m "Release v$VERSION"
git tag -a -m "Version $VERSION" -s -u 348FFC4C "v${VERSION}" master
fi
# Do a hermetic build inside a Docker container.
if [ -z $NOBUILD ]; then
docker build -t hashicorp/consul-builder scripts/consul-builder/
docker run --rm -e "GOTAGS=$GOTAGS" -v "$(pwd)":/gopath/src/github.com/hashicorp/consul hashicorp/consul-builder ./scripts/dist_build.sh
fi
# Zip all the files.
rm -rf ./pkg/dist
mkdir -p ./pkg/dist
for FILENAME in $(find ./pkg -mindepth 1 -maxdepth 1 -type f); do
FILENAME=$(basename $FILENAME)
cp ./pkg/${FILENAME} ./pkg/dist/consul_${VERSION}_${FILENAME}
done
# Make the checksums.
pushd ./pkg/dist
shasum -a256 * > ./consul_${VERSION}_SHA256SUMS
if [ -z $NOSIGN ]; then
echo "==> Signing..."
gpg --default-key 348FFC4C --detach-sig ./consul_${VERSION}_SHA256SUMS
fi
popd
exit 0

46
scripts/dist_build.sh
View File

@ -1,46 +0,0 @@
#!/usr/bin/env bash
set -e
# Get the parent directory of where this script is.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
# Change into that dir because we expect that.
cd $DIR
# Make sure build tools are available.
make tools
# # Build the standalone version of the web assets for the sanity check.
# pushd ui
# bundle
# make dist
# popd
# pushd ui-v2
# yarn install
# make dist
# popd
# # Fixup the timestamps to match what's checked in. This will allow us to cleanly
# # verify that the checked-in content is up to date without spurious diffs of the
# # file mod times.
# pushd pkg
# cat ../agent/bindata_assetfs.go | ../scripts/fixup_times.sh
# popd
# # Regenerate the built-in web assets. If there are any diffs after doing this
# # then we know something is up.
# make static-assets
# if ! git diff --quiet agent/bindata_assetfs.go; then
# echo "Checked-in web assets are out of date, build aborted"
# exit 1
# fi
# Now we are ready to do a clean build of everything. We no longer distribute the
# web UI so it's ok that gets blown away as part of this.
rm -rf pkg
make all
exit 0

10
scripts/fixup_times.sh
View File

@ -1,10 +0,0 @@
#!/usr/bin/env bash
set -e
regex='bindataFileInfo.*name: \"(.+)\".*time.Unix.(.+),'
while read line; do
if [[ $line =~ $regex ]]; then
file=${BASH_REMATCH[1]}
ts=${BASH_REMATCH[2]}
touch --date @$ts $file
fi
done

18
scripts/ui.sh
View File

@ -1,18 +0,0 @@
#!/usr/bin/env bash
set -e
# Get the parent directory of where this script is.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
# Change into that dir because we expect that.
cd $DIR
# Do a hermetic build inside a Docker container.
if [ -z $NOBUILD ]; then
docker build -t hashicorp/consul-builder scripts/consul-builder/
docker run --rm -v "$(pwd)":/gopath/src/github.com/hashicorp/consul hashicorp/consul-builder ./scripts/ui_build.sh
fi
exit 0

31
scripts/ui_build.sh
View File

@ -1,31 +0,0 @@
#!/usr/bin/env bash
set -e
# Get the parent directory of where this script is.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
# Change into that dir because we expect that.
cd $DIR
# Make sure build tools are available.
make tools
# Build the web assets.
echo "Building the V1 UI"
pushd ui
bundle
make dist
popd
echo "Building the V2 UI"
pushd ui-v2
yarn install
make dist
popd
# Make the static assets using the container version of the builder
make static-assets
exit 0

42
scripts/vagrant-linux-priv-go.sh
View File

@ -1,42 +0,0 @@
#!/usr/bin/env bash
function install_go() {
local go_version=1.9.1
local download=
download="https://storage.googleapis.com/golang/go${go_version}.linux-amd64.tar.gz"
if [ -d /usr/local/go ] ; then
return
fi
wget -q -O /tmp/go.tar.gz ${download}
tar -C /tmp -xf /tmp/go.tar.gz
sudo mv /tmp/go /usr/local
sudo chown -R root:root /usr/local/go
}
install_go
# Ensure that the GOPATH tree is owned by vagrant:vagrant
mkdir -p /opt/gopath
chown -R vagrant:vagrant /opt/gopath
# Ensure Go is on PATH
if [ ! -e /usr/bin/go ] ; then
ln -s /usr/local/go/bin/go /usr/bin/go
fi
if [ ! -e /usr/bin/gofmt ] ; then
ln -s /usr/local/go/bin/gofmt /usr/bin/gofmt
fi
# Ensure new sessions know about GOPATH
if [ ! -f /etc/profile.d/gopath.sh ] ; then
cat <<EOT > /etc/profile.d/gopath.sh
export GOPATH="/opt/gopath"
export PATH="/opt/gopath/bin:\$PATH"
EOT
chmod 755 /etc/profile.d/gopath.sh
fi