diff --git a/GNUmakefile b/GNUmakefile index a02db4066..ba7b16d06 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -33,6 +33,10 @@ UI_BUILD_TAG?=consul-build-ui UI_LEGACY_BUILD_TAG?=consul-build-ui-legacy BUILD_CONTAINER_NAME?=consul-builder +DIST_TAG?=1 +DIST_BUILD?=1 +DIST_SIGN?=1 + export GO_BUILD_TAG export UI_BUILD_TAG export UI_LEGACY_BUILD_TAG @@ -47,18 +51,13 @@ export GOLDFLAGS all: bin bin: tools - @mkdir -p bin/ - @GOTAGS='$(GOTAGS)' sh -c "'$(CURDIR)/scripts/build.sh'" + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul-local # dev creates binaries for testing locally - these are put into ./bin and $GOPATH dev: changelogfmt vendorfmt dev-build dev-build: - @echo "--> Building consul" - mkdir -p pkg/$(GOOS)_$(GOARCH)/ bin/ - go install -ldflags '$(GOLDFLAGS)' -tags '$(GOTAGS)' - cp $(GOPATH)/bin/consul bin/ - cp $(GOPATH)/bin/consul pkg/$(GOOS)_$(GOARCH) + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul-local -o '$(GOOS)' -a '$(GOARCH)' vendorfmt: @echo "--> Formatting vendor/vendor.json" @@ -71,12 +70,11 @@ changelogfmt: # linux builds a linux package independent of the source platform linux: - mkdir -p pkg/linux_amd64/ - GOOS=linux GOARCH=amd64 go build -ldflags '$(GOLDFLAGS)' -tags '$(GOTAGS)' -o pkg/linux_amd64/consul + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul-local -o linux -a amd64 # dist builds binaries for all platforms and packages them for distribution dist: - @GOTAGS='$(GOTAGS)' sh -c "'$(CURDIR)/scripts/dist.sh'" + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh release -t '$(DIST_TAG)' -b '$(DIST_BUILD)' -S '$(DIST_SIGN)' cov: gocov test $(GOFILES) | gocov-html > /tmp/coverage.html @@ -128,8 +126,7 @@ vet: # Build the static web ui and build static assets inside a Docker container, the # same way a release build works. This implicitly does a "make static-assets" at # the end. -ui: - @sh -c "'$(CURDIR)/scripts/ui.sh'" +ui: ui-legacy-docker ui-docker static-assets # If you've run "make ui" manually then this will get called for you. This is # also run as part of the release build script when it verifies that there are no @@ -141,6 +138,12 @@ static-assets: tools: go get -u -v $(GOTOOLS) +version: + @echo -n "Version without release: " + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh version + @echo -n "Version with release: " + @$(SHELL) $(CURDIR)/build-support/scripts/build.sh version -R + docker-images: @$(MAKE) -C build-support/docker images @@ -156,7 +159,7 @@ ui-legacy-build-image: static-assets-docker: go-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build.sh assetfs -go-docker: go-build-image +consul-docker: go-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build.sh consul ui-docker: ui-build-image @@ -165,7 +168,6 @@ ui-docker: ui-build-image ui-legacy-docker: ui-legacy-build-image @$(SHELL) $(CURDIR)/build-support/scripts/build.sh ui-legacy -release-docker: ui-docker ui-legacy-docker static-assets-docker go-docker .PHONY: all ci bin dev dist cov test cover format vet ui static-assets tools vendorfmt -.PHONY: docker-images go-build-iamge ui-build-image ui-legacy-build-image static-assets-docker go-docker ui-docker ui-legacy-docker release-docker +.PHONY: docker-images go-build-image ui-build-image ui-legacy-build-image static-assets-docker consul-docker ui-docker ui-legacy-docker version diff --git a/build-support/functions/00-vars.sh b/build-support/functions/00-vars.sh new file mode 100644 index 000000000..2b6c53afa --- /dev/null +++ b/build-support/functions/00-vars.sh @@ -0,0 +1,20 @@ +# GPG Key ID to use for publically released builds +HASHICORP_GPG_KEY="348FFC4C" + +# Default Image Names +UI_BUILD_CONTAINER_DEFAULT="consul-build-ui" +UI_LEGACY_BUILD_CONTAINER_DEFAULT="consul-build-ui-legacy" +GO_BUILD_CONTAINER_DEFAULT="consul-build-go" + +# Whether to colorize shell output +COLORIZE=1 + + +# determine GOPATH and the first GOPATH to use for intalling binaries +GOPATH=${GOPATH:-$(go env GOPATH)} +case $(uname) in + CYGWIN*) + GOPATH="$(cygpath $GOPATH)" + ;; +esac +MAIN_GOPATH=$(cut -d: -f1 <<< "${GOPATH}") diff --git a/build-support/functions/01-util.sh b/build-support/functions/01-util.sh new file mode 100644 index 000000000..34a79740b --- /dev/null +++ b/build-support/functions/01-util.sh @@ -0,0 +1,186 @@ +function err { + if test "${COLORIZE}" -eq 1 + then + tput bold + tput setaf 1 + fi + + echo $@ 1>&2 + + if test "${COLORIZE}" -eq 1 + then + tput sgr0 + fi +} + +function status { + if test "${COLORIZE}" -eq 1 + then + tput bold + tput setaf 4 + fi + + echo $@ + + if test "${COLORIZE}" -eq 1 + then + tput sgr0 + fi +} + +function status_stage { + if test "${COLORIZE}" -eq 1 + then + tput bold + tput setaf 2 + fi + + echo $@ + + if test "${COLORIZE}" -eq 1 + then + tput sgr0 + fi +} + +function is_set { + # Arguments: + # $1 - string value to check its truthiness + # + # Return: + # 0 - is truthy (backwards I know but allows syntax like `if is_set ` to work) + # 1 - is not truthy + + local val=$(tr '[:upper:]' '[:lower:]' <<< "$1") + case $val in + 1 | t | true | y | yes) + return 0 + ;; + *) + return 1 + ;; + esac +} + +function have_gpg_key { + # Arguments: + # $1 - GPG Key id to check if we have installed + # + # Return: + # 0 - success (we can use this key for signing) + # * - failure (key cannot be used) + + gpg --list-secret-keys $1 >dev/null 2>&1 + return $? +} + +function parse_version { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - boolean value for whether to omit the release version from the version string + # + # Return: + # 0 - success (will write the version to stdout) + # * - error (no version output) + # + # Notes: + # If the GOTAGS environment variable is present then it is used to determine which + # version file to use for parsing. + # If the GIT_DESCRIBE environment variable is present then it is used as the version + # If the GIT_COMMIT environment variable is preset it will be added to the end of + # the version string. + + local vfile="${1}/version/version.go" + + # ensure the version file exists + if ! test -f "${vfile}" + then + err "Error - File not found: ${vfile}" + return 1 + fi + + # Get the main version out of the source file + version_main=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) + release_main=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) + + # try to determine the version if we have build tags + for tag in "$GOTAGS" + do + for vfile in $(ls "${1}/version/version_*.go" 2> /dev/null| sort) + do + if grep -q "// +build $tag" $file + then + version_main=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) + release_main=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) + fi + done + done + + version= + + # override the version from source with the value of the GIT_DESCRIBE env var if present + if test -n "$GIT_DESCRIBE" + then + version=$GIT_DESCRIBE + fi + + if ! is_set $2 + then + # Get the release version out of the source file + release=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) + + # When no GIT_DESCRIBE env var is present and no release is in the source then we + # are definitely in dev mode + if test -z "$GIT_DESCRIBE" -a -z "$release" + then + release="dev" + fi + + # Add the release to the version + if test -n "$release" + then + version="${version}-${release}" + + # add the git commit to the version + if test -n "$GIT_COMMIT" + then + version="${version} (${GIT_COMMIT})" + fi + fi + fi + + # Output the version + echo "$version" | tr -d "'" + return 0 +} + +function get_version { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - Whether the release version should be parsed from source (optional) + # + # Returns: + # 0 - success (the version is also echoed to stdout) + # 1 - error + # + # Notes: + # If a VERSION environment variable is present it will override any parsing of the version from the source + # In addition to processing the main version.go, version_*.go files will be processed if they have + # a Go build tag that matches the one in the GOTAGS environment variable. This tag processing is + # primitive though and will not match complex build tags in the files with negation etc. + + local vers="$VERSION" + if test -z "$vers" + then + # parse the OSS version from version.go + vers="$(parse_version ${1} ${2})" + fi + + if test -z "$vers" + then + return 1 + else + echo $vers + return 0 + fi +} \ No newline at end of file diff --git a/build-support/functions/02-build.sh b/build-support/functions/02-build.sh new file mode 100644 index 000000000..166a297e7 --- /dev/null +++ b/build-support/functions/02-build.sh @@ -0,0 +1,382 @@ +function refresh_docker_images { + # Arguments: + # $1 - Path to top level Consul source + # $2 - Which make target to invoke (optional) + # + # Return: + # 0 - success + # * - failure + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. refresh_docker_images must be called with the path to the top level source as the first argument'" + return 1 + fi + + local sdir="$1" + local targets="$2" + + test -n "${targets}" || targets="images" + + make -C "${sdir}/build-support/docker" $targets + return $? +} + +function build_ui { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + # + # Notes: + # Use the GIT_COMMIT environment variable to pass off to the build + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. build_ui must be called with the path to the top level source as the first argument'" + return 1 + fi + + local image_name=${UI_BUILD_CONTAINER_DEFAULT} + if test -n "$2" + then + image_name="$2" + fi + + local sdir="$1" + local ui_dir="${1}/ui-v2" + + # parse the version + version=$(parse_version "${sdir}") + + local commit_hash="${GIT_COMMIT}" + if test -z "${commit_hash}" + then + commit_hash=$(git rev-parse --short HEAD) + fi + + # make sure we run within the ui dir + pushd ${ui_dir} > /dev/null + + status "Creating the UI Build Container with image: ${image_name}" + local container_id=$(docker create -it -e "CONSUL_GIT_SHA=${commit_hash}" -e "CONSUL_VERSION=${version}" ${image_name}) + local ret=$? + if test $ret -eq 0 + then + status "Copying the source from '${ui_dir}' to /consul-src within the container" + ( + docker cp . ${container_id}:/consul-src && + status "Running build in container" && docker start -i ${container_id} && + rm -rf ${1}/ui-v2/dist && + status "Copying back artifacts" && docker cp ${container_id}:/consul-src/dist ${1}/ui-v2/dist + ) + ret=$? + docker rm ${container_id} > /dev/null + fi + + if test $ret -eq 0 + then + rm -rf ${1}/pkg/web_ui/v2 + cp -r ${1}/ui-v2/dist ${1}/pkg/web_ui/v2 + fi + popd > /dev/null + return $ret +} + +function build_ui_legacy { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. build_ui_legacy must be called with the path to the top level source as the first argument'" + return 1 + fi + + local sdir="$1" + local ui_legacy_dir="${sdir}/ui" + + local image_name=${UI_LEGACY_BUILD_CONTAINER_DEFAULT} + if test -n "$2" + then + image_name="$2" + fi + + pushd ${ui_legacy_dir} > /dev/null + status "Creating the Legacy UI Build Container with image: ${image_name}" + rm -r ${sdir}/pkg/web_ui/v1 >/dev/null 2>&1 + mkdir -p ${sdir}/pkg/web_ui/v1 + local container_id=$(docker create -it ${image_name}) + local ret=$? + if test $ret -eq 0 + then + status "Copying the source from '${ui_legacy_dir}' to /consul-src/ui within the container" + ( + docker cp . ${container_id}:/consul-src/ui && + status "Running build in container" && + docker start -i ${container_id} && + status "Copying back artifacts" && + docker cp ${container_id}:/consul-src/pkg/web_ui ${sdir}/pkg/web_ui/v1 + ) + ret=$? + docker rm ${container_id} > /dev/null + fi + popd > /dev/null + return $ret +} + +function build_assetfs { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + # + # Note: + # The GIT_COMMIT, GIT_DIRTY and GIT_DESCRIBE environment variables will be used if present + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. build_assetfs must be called with the path to the top level source as the first argument'" + return 1 + fi + + local sdir="$1" + local image_name=${GO_BUILD_CONTAINER_DEFAULT} + if test -n "$2" + then + image_name="$2" + fi + + pushd ${sdir} > /dev/null + status "Creating the Go Build Container with image: ${image_name}" + local container_id=$(docker create -it -e GIT_COMMIT=${GIT_COMMIT} -e GIT_DIRTY=${GIT_DIRTY} -e GIT_DESCRIBE=${GIT_DESCRIBE} ${image_name} make static-assets ASSETFS_PATH=bindata_assetfs.go) + local ret=$? + if test $ret -eq 0 + then + status "Copying the sources from '${sdir}/(pkg|GNUmakefile)' to /go/src/github.com/hashicorp/consul/pkg" + ( + tar -c pkg/web_ui GNUmakefile | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul && + status "Running build in container" && docker start -i ${container_id} && + status "Copying back artifacts" && docker cp ${container_id}:/go/src/github.com/hashicorp/consul/bindata_assetfs.go ${sdir}/agent/bindata_assetfs.go + ) + ret=$? + docker rm ${container_id} > /dev/null + fi + popd >/dev/null + return $ret +} + +function build_consul_post { + # Arguments + # $1 - Path to the top level Consul source + # $2 - build suffix (Optional) + # + # Returns: + # 0 - success + # * - error + # + # Notes: + # pkg/bin is where to place binary packages + # pkg.bin.new is where the just built binaries are located + # bin is where to place the local systems versions + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. build_consul_post must be called with the path to the top level source as the first argument'" + return 1 + fi + + local sdir="$1" + + pushd "${sdir}" > /dev/null + + # recreate the pkg dir + rm -r pkg/bin/* 2> /dev/null + mkdir -p pkg/bin 2> /dev/null + + # move all files in pkg.new into pkg + cp -r pkg.bin.new/* pkg/bin/ + rm -r pkg.bin.new + + DEV_PLATFORM="./pkg/bin/$(go env GOOS)_$(go env GOARCH)${2}" + for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f) + do + # recreate the bin dir + rm -r bin/* 2> /dev/null + mkdir -p bin 2> /dev/null + + cp ${F} bin/ + cp ${F} ${MAIN_GOPATH}/bin + done + + popd > /dev/null + + return 0 +} + +function build_consul { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - build suffix (optional - must specify if needing to specify the docker image) + # $3 - The docker image to run the build within (optional) + # + # Returns: + # 0 - success + # * - error + # + # Note: + # The GOLDFLAGS and GOTAGS environment variables will be used if set + # If the CONSUL_DEV environment var is truthy only the local platform/architecture is built. + # If the XC_OS or the XC_ARCH environment vars are present then only those platforms/architectures + # will be built. Otherwise all supported platform/architectures are built + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. build_consul must be called with the path to the top level source as the first argument'" + return 1 + fi + + local sdir="$1" + local build_suffix="$2" + local image_name=${GO_BUILD_CONTAINER_DEFAULT} + if test -n "$3" + then + image_name="$3" + fi + + pushd ${sdir} > /dev/null + status "Creating the Go Build Container with image: ${image_name}" + if is_set "${CONSUL_DEV}" + then + if test -z "${XC_OS}" + then + XC_OS=$(go env GOOS) + fi + + if test -z "${XC_ARCH}" + then + XC_ARCH=$(go env GOARCH) + fi + fi + XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} + XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"} + + local container_id=$(docker create -it -e CGO_ENABLED=0 ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/bin/{{.OS}}_{{.Arch}}${build_suffix}/consul" -tags="${GOTAGS}") + ret=$? + + if test $ret -eq 0 + then + status "Copying the source from '${sdir}' to /go/src/github.com/hashicorp/consul/pkg" + ( + tar -c $(ls | grep -v "ui\|ui-v2\|website\|bin\|.git") | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul && + status "Running build in container" && + docker start -i ${container_id} && + status "Copying back artifacts" && + docker cp ${container_id}:/go/src/github.com/hashicorp/consul/pkg/bin pkg.bin.new + ) + ret=$? + docker rm ${container_id} > /dev/null + + if test $ret -eq 0 + then + build_consul_post "${sdir}" "${build_suffix}" + ret=$? + else + rm -r pkg.bin.new 2> /dev/null + fi + fi + popd > /dev/null + return $ret +} + +function build_consul_local { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - Space separated string of OSes to build. If empty will use env vars for determination. + # $3 - Space separated string of architectures to build. If empty will use env vars for determination. + # $4 - build suffix (optional) + # + # Returns: + # 0 - success + # * - error + # + # Note: + # The GOLDFLAGS and GOTAGS environment variables will be used if set + # If the CONSUL_DEV environment var is truthy only the local platform/architecture is built. + # If the XC_OS or the XC_ARCH environment vars are present then only those platforms/architectures + # will be built. Otherwise all supported platform/architectures are built + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. build_consul must be called with the path to the top level source as the first argument'" + return 1 + fi + + local sdir="$1" + local build_os="$2" + local build_arch="$3" + local build_suffix="$4" + + pushd ${sdir} > /dev/null + if is_set "${CONSUL_DEV}" + then + if test -z "${XC_OS}" + then + XC_OS=$(go env GOOS) + fi + + if test -z "${XC_ARCH}" + then + XC_ARCH=$(go env GOARCH) + fi + fi + XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} + XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"} + + if test -z "${build_os}" + then + build_os="${XC_OS}" + fi + + if test -z "${build_arch}" + then + build_arch="${XC_ARCH}" + fi + + status_stage "==> Building Consul - OSes: ${build_os}, Architectures: ${build_arch}" + mkdir pkg.bin.new 2> /dev/null + CGO_ENABLED=0 gox \ + -os="${build_os}" \ + -arch="${build_arch}" \ + -osarch="!darwin/arm !darwin/arm64" \ + -ldflags="${GOLDFLAGS}" \ + -output "pkg.bin.new/{{.OS}}_{{.Arch}}${build_suffix}/consul" \ + -tags="${GOTAGS}" \ + . + + if test $? -ne 0 + then + err "ERROR: Failed to build Consul" + rm -r pkg.bin.new + return 1 + fi + + build_consul_post "${sdir}" "${build_suffix}" + if test $? -ne 0 + then + err "ERROR: Failed postprocessing Consul binaries" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/build-support/functions/03-release.sh b/build-support/functions/03-release.sh new file mode 100644 index 000000000..c58281c9c --- /dev/null +++ b/build-support/functions/03-release.sh @@ -0,0 +1,299 @@ +function tag_release { + # Arguments: + # $1 - Path to top level consul source + # $2 - Version string to use for tagging the release + # $3 - Alternative GPG key id used for signing the release commit (optional) + # + # Returns: + # 0 - success + # * - error + # + # Notes: + # If the RELEASE_UNSIGNED environment variable is set then no gpg signing will occur + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. tag_release must be called with the path to the top level source as the first argument'" + return 1 + fi + + if test -z "$2" + then + err "ERROR: tag_release must be called with a version number as the second argument" + return 1 + fi + + # determine whether the gpg key to use is being overridden + local gpg_key=${HASHICORP_GPG_KEY} + if test -n "$3" + then + gpg_key=$3 + fi + + pushd "$1" > /dev/null + local ret=0 + + # perform an usngined release if requested (mainly for testing locally) + if is_set "$RELEASE_UNSIGNED" + then + ( + git commit --allow-empty -a -m "Release v${2}" && + git tag -a -m "Version ${2}" "v${2}" master + ) + ret=$? + # perform a signed release (official releases should do this) + elif have_gpg_key ${gpg_key} + then + ( + git commit --allow-empty -a --gpg-sign=${gpg_key} -m "Release v${2}" && + git tag -a -m "Version ${2}" -s -u ${gpg_key} "v${2}" master + ) + ret=$? + # unsigned release not requested and gpg key isn't useable + else + err "ERROR: GPG key ${gpg_key} is not in the local keychain - to continue set RELEASE_UNSIGNED=1 in the env" + ret=1 + fi + popd > /dev/null + return $ret +} + +function package_release { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - Version to use in the names of the zip files (optional) + # + # Returns: + # 0 - success + # * - error + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. package_release must be called with the path to the top level source as the first argument'" + return 1 + fi + + local vers="${2}" + if test -z "${vers}" + then + vers=$(get_version $1 true) + ret=$? + if test "$ret" -ne 0 + then + err "ERROR: failed to determine the version." + return $ret + fi + fi + + local sdir="$1" + local ret=0 + + rm -rf "${sdir}/pkg/dist" > /dev/null 2>&1 + mkdir -p "${sdir}/pkg/dist" >/dev/null 2>&1 + for platform in $(find "${sdir}/pkg/bin" -mindepth 1 -maxdepth 1 -type d) + do + local os_arch=$(basename $platform) + local dest="${sdir}/pkg/dist/consul_${vers}_${os_arch}.zip" + status "Compressing ${os_arch} directory into ${dest}" + pushd "${platform}" > /dev/null + zip "${sdir}/pkg/dist/consul_${vers}_${os_arch}.zip" ./* + ret=$? + popd > /dev/null + + if test "$ret" -ne 0 + then + break + fi + done + + return $ret +} + +function shasum_release { + # Arguments: + # $1 - Path to directory containing the files to shasum + # $2 - File to output sha sums to + # + # Returns: + # 0 - success + # * - failure + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory and shasum_release requires passing a directory as the first argument" + return 1 + fi + + if test -z "$2" + then + err "ERROR: shasum_release requires a second argument to be the filename to output the shasums to but none was given" + return 1 + fi + + pushd $1 > /dev/null + shasum -a256 * > "$2" + ret=$? + popd >/dev/null + + return $ret +} + +function sign_release { + # Arguments: + # $1 - File to sign + # $2 - Alternative GPG key to use for signing + # + # Returns: + # 0 - success + # * - failure + + # determine whether the gpg key to use is being overridden + local gpg_key=${HASHICORP_GPG_KEY} + if test -n "$2" + then + gpg_key=$2 + fi + + gpg --default-key "${gpg_key}" --detach-sig "$1" + return $? +} + +function build_consul_release { + build_consul "$1" "" "$2" +} + +function build_release { + # Arguments: + # $1 - Path to the top level Consul source + # $2 - boolean whether to tag the release yet + # $3 - boolean whether to build the binaries + # $4 - boolean whether to generate the sha256 sums + # $5 - alternative gpg key to use for signing operations (optional) + # + # Returns: + # 0 - success + # * - error + + if ! test -d "$1" + then + err "ERROR: '$1' is not a directory. build_release must be called with the path to the top level source as the first argument'" + return 1 + fi + + if test -z "$2" -o -z "$3" -o -z "$4" + then + err "ERROR: build_release requires 4 arguments to be specified: " + return 1 + fi + + local sdir="$1" + local do_tag="$2" + local do_build="$3" + local do_sha256="$4" + local gpg_key="$5" + + local vers=$(get_version ${sdir} true) + if test $? -ne 0 + then + err "Please specify a version (couldn't find one based on build tags)." + return 1 + fi + + # Make sure we arent in dev mode + unset CONSUL_DEV + + if is_set "${do_build}" + then + status_stage "==> Refreshing Docker Build Images" + refresh_docker_images "${sdir}" + if test $? -ne 0 + then + err "ERROR: Failed to refresh docker images" + return 1 + fi + + status_stage "==> Building Legacy UI for version ${vers}" + build_ui_legacy "${sdir}" "${UI_LEGACY_BUILD_TAG}" + if test $? -ne 0 + then + err "ERROR: Failed to build the legacy ui" + return 1 + fi + + status_stage "==> Building UI for version ${vers}" + build_ui "${sdir}" "${UI_BUILD_TAG}" + if test $? -ne 0 + then + err "ERROR: Failed to build the ui" + return 1 + fi + + status_stage "==> Building Static Assets for version ${vers}" + build_assetfs "${sdir}" "${GO_BUILD_TAG}" + if test $? -ne 0 + then + err "ERROR: Failed to build the static assets" + return 1 + fi + + if is_set "${do_tag}" + then + git add "${sdir}/agent/bindata_assetfs.go" + if test $? -ne 0 + then + err "ERROR: Failed to git add the assetfs file" + return 1 + fi + fi + fi + + if is_set "${do_tag}" + then + status_stage "==> Tagging version ${vers}" + tag_release "${sdir}" "${vers}" "${gpg_key}" + if test $? -ne 0 + then + err "ERROR: Failed to tag the release" + return 1 + fi + fi + + if is_set "${do_build}" + then + status_stage "==> Building Consul for version ${vers}" + build_consul_release "${sdir}" "${GO_BUILD_TAG}" + if test $? -ne 0 + then + err "ERROR: Failed to build the Consul binaries" + return 1 + fi + + status_stage "==> Packaging up release binaries" + package_release "${sdir}" "${vers}" + if test $? -ne 0 + then + err "ERROR: Failed to package the release binaries" + return 1 + fi + fi + + status_stage "==> Generating SHA 256 Hashes for Binaries" + shasum_release "${sdir}/pkg/dist" "consul_${vers}_SHA256SUMS" + if test $? -ne 0 + then + err "ERROR: Failed to generate SHA 256 hashes for the release" + return 1 + fi + + if is_set "${do_sha256}" + then + sign_release "${sdir}/pkg/dist/consul_${vers}_SHA256SUMS" "${gpg_key}" + if test $? -ne 0 + then + err "ERROR: Failed to sign the SHA 256 hashes file" + return 1 + fi + fi + + return 0 +} \ No newline at end of file diff --git a/build-support/scripts/build.sh b/build-support/scripts/build.sh index f1c387b22..10fc797c9 100755 --- a/build-support/scripts/build.sh +++ b/build-support/scripts/build.sh @@ -1,39 +1,381 @@ #!/bin/bash +SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})" pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null SCRIPT_DIR=$(pwd) pushd ../.. > /dev/null SOURCE_DIR=$(pwd) popd > /dev/null +pushd ../functions > /dev/null +FN_DIR=$(pwd) +popd > /dev/null popd > /dev/null source "${SCRIPT_DIR}/functions.sh" +function can_parse_option { + local allowed="$1" + local command="$2" + local options="$3" + + if test ${allowed} -ne 1 + then + err "ERROR: subcommand ${command} does not support the ${options} options" + return 1 + fi + return 0 +} + +function check_duplicate { + local is_dup="$1" + local command="$2" + local options="$3" + + if test ${is_dup} -ne 0 + then + err "ERROR: options ${options} may not be given more than once to the subcommand ${command}" + return 1 + fi + return 0 +} + +function option_check { + can_parse_option "$1" "$3" "$4" && check_duplicate "$2" "$3" "$4" + return $? +} + +function get_option_value { + # Arguments: + # $1 - bool whether the option should be allowed + # $2 - bool whether the option has been specified already + # $3 - the option value + # $4 - the command being executed + # $5 - the option names to use for logging + # + # Returns: + # 0 - success + # * - failure + + option_check "$1" "$2" "$4" "$5" || return 1 + + if test -z "$3" + then + err "ERROR: options ${5} for subcommand ${4} require an argument but none was provided" + return 1 + fi + + echo "$3" + return 0 +} + +function usage { +cat <<-EOF +Usage: ${SCRIPT_NAME} [] + +Subcommands: + assetfs: Builds the bindata_assetfs.go file from previously build UI artifacts + + Options: + -i | --image IMAGE Alternative Docker image to run the build within. + Defaults to ${GO_BUILD_CONTAINER_DEFAULT} + + -s | --source DIR Path to source to build. + Defaults to "${SOURCE_DIR}" + + -r | --refresh Enables refreshing the docker image prior to building. + + consul: Builds the main Consul binary. This assumes the assetfs is up to date: + + Options: + -i | --image IMAGE Alternative Docker image to run the build within. + Defaults to ${GO_BUILD_CONTAINER_DEFAULT} + + -s | --source DIR Path to source to build. + Defaults to "${SOURCE_DIR}" + + -r | --refresh Enables refreshing the docker image prior to building. + + consul-local: Builds the main Consul binary on the local system (no docker) + + -s | --source DIR Path to source to build. + Defaults to "${SOURCE_DIR}" + + -o | --build-os OS Space separated string of OSes to build + + -a | --build-arch ARCH Space separated string of architectures to build + + release: Performs a release build. + + Options: + -s | --source DIR Path to source to build. + Defaults to "${SOURCE_DIR}" + + -t | --tag BOOL Whether to add a release commit and tag the build + Defaults to 1. + + -b | --build BOOL Whether to perform the build of the ui's, assetfs and + binaries. Defaults to 1. + + -S | --sign BOOL Whether to sign the generated SHA256SUMS file. + Defaults to 1. + + -g | --gpg-key KEY Alternative GPG key to use for signing operations. + Defaults to ${HASHICORP_GPG_KEY} + + ui: Builds the latest UI. + + Options: + -i | --image IMAGE Alternative Docker image to run the build within. + Defaults to ${UI_BUILD_CONTAINER_DEFAULT} + + -s | --source DIR Path to source to build. + Defaults to "${SOURCE_DIR}" + + -r | --refresh Enables refreshing the docker image prior to building. + + ui-legacy: Builds the legacy UI + + Options: + -i | --image IMAGE Alternative Docker image to run the build within. + Defaults to ${UI_LEGACY_BUILD_CONTAINER_DEFAULT} + + -s | --source DIR Path to source to build. + Defaults to "${SOURCE_DIR}" + + -r | --refresh Enables refreshing the docker image prior to building. + + version: Prints out the version parsed from source. + + Options: + -s | --source DIR Path to source to build. + Defaults to "${SOURCE_DIR}" +EOF +} + function main { - case "$1" in - consul ) - build_consul "${SOURCE_DIR}" "${GO_BUILD_TAG}" - return $? + declare build_fn + declare sdir + declare image + declare -i refresh_docker=0 + declare -i rel_tag + declare -i rel_build + declare -i rel_sign + declare rel_gpg_key="" + declare build_os + declare build_arch + declare -i vers_release + + declare -i use_refresh=1 + declare -i default_refresh=0 + declare -i use_sdir=1 + declare default_sdir="${SOURCE_DIR}" + declare -i use_image=0 + declare default_image="" + declare -i use_rel=0 + declare -i default_rel_tag=1 + declare -i default_rel_build=1 + declare -i default_rel_sign=1 + declare default_rel_gpg_key="${HASHICORP_GPG_KEY}" + declare -i use_xc=0 + declare default_build_os="" + declare default_build_arch="" + declare -i use_vers_rel + declare -i default_vers_rel=1 + + declare command="$1" + shift + + case "${command}" in + consul ) + use_image=1 + default_image="${GO_BUILD_CONTAINER_DEFAULT}" + ;; + consul-local ) + use_xc=1 ;; ui ) - build_ui "${SOURCE_DIR}" "${UI_BUILD_TAG}" - return $? + use_image=1 + default_image="${UI_BUILD_CONTAINER_DEFAULT}" ;; ui-legacy ) - build_ui_legacy "${SOURCE_DIR}" "${UI_LEGACY_BUILD_TAG}" - return $? + use_image=1 + default_image="${UI_LEGACY_BUILD_CONTAINER_DEFAULT}" ;; version ) - parse_version "${SOURCE_DIR}" - return $? + use_refresh=0 + use_vers_rel=1 ;; assetfs ) - build_assetfs "${SOURCE_DIR}" "${GO_BUILD_TAG}" - return $? + use_image=1 + default_image="${GO_BUILD_CONTAINER_DEFAULT}" + ;; + release ) + use_rel=1 + use_refresh=0 + ;; + -h | --help) + usage + return 0 ;; *) - echo "Unkown build: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'version' and 'assetfs'" 1>&2 + err "Unkown subcommand: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'assetfs', version' and 'release'" return 1 + ;; + esac + + declare -i have_image_arg=0 + declare -i have_sdir_arg=0 + declare -i have_rel_tag_arg=0 + declare -i have_rel_build_arg=0 + declare -i have_rel_sign_arg=0 + declare -i have_rel_gpg_key_arg=0 + declare -i have_refresh_arg=0 + declare -i have_build_os_arg=0 + declare -i have_build_arch_arg=0 + declare -i have_vers_rel_arg=0 + + while test $# -gt 0 + do + case $1 in + -h | --help ) + usage + return 0 + ;; + -o | --build-os ) + build_os=$(get_option_value "${use_xc}" "${have_build_os_arg}" "$2" "${command}" "-o/--xc-os") || return 1 + have_build_os_arg=1 + shift 2 + ;; + -a | --build-arch) + build_arch=$(get_option_value "${use_xc}" "${have_build_arch_arg}" "$2" "${command}" "-o/--xc-arch") || return 1 + have_build_arch_arg=1 + shift 2 + ;; + -R | --release ) + option_check "${use_vers_rel}" "${have_vers_rel_arg}" "${command}" "-R/--release" || return 1 + have_vers_rel_arg=1 + vers_release=0 + shift + ;; + -r | --refresh) + option_check "${use_refresh}" "${have_refresh_arg}" "${command}" "-r/--refresh" || return 1 + have_refresh_arg=1 + refresh_docker=1 + shift + ;; + -i | --image ) + image=$(get_option_value "${use_image}" "${have_image_arg}" "$2" "${command}" "-i/--image") || return 1 + have_image_arg=1 + shift 2 + ;; + -s | --source ) + sdir=$(get_option_value "${use_sdir}" "${have_sdir_arg}" "$2" "${command}" "-s/--source") || return 1 + if ! test -d "${sdir}" + then + err "ERROR: -s/--source is not a path to a top level directory" + return 1 + fi + have_sdir_arg=1 + shift 2 + ;; + -t | --tag ) + rel_tag=$(get_option_value "${use_rel}" "${have_rel_tag_arg}" "$2" "${command}" "-t/--tag") || return 1 + have_rel_tag_arg=1 + shift 2 + ;; + -b | --build ) + rel_build=$(get_option_value "${use_rel}" "${have_rel_build_arg}" "$2" "${command}" "-b/--build") || return 1 + have_rel_build_arg=1 + shift 2 + ;; + -S | --sign ) + rel_sign=$(get_option_value "${use_rel}" "${have_rel_sign_arg}" "$2" "${command}" "-S/--sign") || return 1 + have_rel_sign_arg=1 + shift 2 + ;; + -g | --gpg-key ) + rel_gpg_key=$(get_option_value "${use_rel}" "${have_rel_gpg_key_arg}" "$2" "${command}" "-g/--gpg-key") || return 1 + shift 2 + ;; + *) + err "ERROR: Unknown option '$1' for subcommand ${command}" + return 1 + ;; + esac + done + + test $have_image_arg -ne 1 && image="${default_image}" + test $have_sdir_arg -ne 1 && sdir="${default_sdir}" + test $have_rel_tag_arg -ne 1 && rel_tag="${default_rel_tag}" + test $have_rel_build_arg -ne 1 && rel_build="${default_rel_build}" + test $have_rel_sign_arg -ne 1 && rel_sign="${default_rel_sign}" + test $have_rel_gpg_key_arg -ne 1 && rel_gpg_key="${default_rel_gpg_key}" + test $have_refresh_arg -ne 1 && refresh_docker="${default_refresh}" + test $have_build_os_arg -ne 1 && build_os="${default_build_os}" + test $have_build_arch_arg -ne 1 && build_arch="${default_build_os}" + test $have_vers_rel_arg -ne 1 && vers_release="${default_vers_rel}" + + case "${command}" in + consul ) + if is_set "${refresh_docker}" + then + status_stage "==> Refreshing Consul build container image" + export GO_BUILD_TAG=${image} + refresh_docker_images ${sdir} go-build-image || return 1 + fi + status_stage "==> Building Consul" + build_consul "${sdir}" "" "${image}" || return 1 + ;; + consul-local ) + build_consul_local "${sdir}" "${build_os}" "${build_arch}" "" || return 1 + ;; + ui ) + + if is_set "${refresh_docker}" + then + status_stage "==> Refreshing UI build container image" + export UI_BUILD_TAG=${image} + refresh_docker_images ${sdir} ui-build-image || return 1 + fi + status_stage "==> Building UI" + build_ui "${sdir}" "${image}" || return 1 + ;; + ui-legacy ) + if is_set "${refresh_docker}" + then + status_stage "==> Refreshing Legacy UI build container image" + export UI_LEGACY_BUILD_TAG=${image} + refresh_docker_images ${sdir} ui-legacy-build-image || return 1 + fi + status_stage "==> Building Legacy UI" + build_ui_legacy "${sdir}" "${image}" || return 1 + ;; + version ) + parse_version "${sdir}" "${vers_release}"|| return 1 + ;; + assetfs ) + if is_set "${refresh_docker}" + then + status_stage "==> Refreshing Consul build container image" + export GO_BUILD_TAG="${image}" + refresh_docker_images ${sdir} go-build-image || return 1 + fi + status_stage "==> Build Static Assets" + build_assetfs "${sdir}" "${image}" || return 1 + ;; + release ) + if is_set "${refresh_docker}" + then + refresh_docker_images ${sdir} || return 1 + fi + build_release "${sdir}" "${rel_tag}" "${rel_build}" "${rel_sign}" "${rel_gpg_key}" || return 1 + ;; + *) + err "Unkown subcommand: '$1' - possible values are 'consul', 'ui', 'ui-legacy', 'assetfs', version' and 'release'" + return 1 + ;; esac + + return 0 } main $@ diff --git a/build-support/scripts/functions.sh b/build-support/scripts/functions.sh index 2b44d2401..ec473662c 100644 --- a/build-support/scripts/functions.sh +++ b/build-support/scripts/functions.sh @@ -1,464 +1,17 @@ -# GPG Key ID to use for publically released builds -HASHICORP_GPG_KEY="348FFC4C" +# +# NOTE: This file is meant to be sourced from other bash scripts/shells +# +# It provides all the scripting around building Consul and the release process -UI_BUILD_CONTAINER_DEFAULT="consul-build-ui" -UI_LEGACY_BUILD_CONTAINER_DEFAULT="consul-build-ui-legacy" +pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null +pushd ../functions > /dev/null +FUNC_DIR=$(pwd) +popd > /dev/null +popd > /dev/null -function is_set { - # Arguments: - # $1 - string value to check its truthiness - # - # Return: - # 0 - is truthy (backwards I know but allows syntax like `if is_set ` to work) - # 1 - is not truthy - - local val=$(tr '[:upper:]' '[:lower:]' <<< "$1") - case $val in - 1 | t | true | y | yes) - return 0 - ;; - *) - return 1 - ;; - esac -} +func_sources=$(find ${FUNC_DIR} -type f -mindepth 1 -maxdepth 1 -name "*.sh" | sort -n) -function have_gpg_key { - # Arguments: - # $1 - GPG Key id to check if we have installed - # - # Return: - # 0 - success (we can use this key for signing) - # * - failure (key cannot be used) - - gpg --list-secret-keys $1 >dev/null 2>&1 - return $? -} - -function parse_version { - # Arguments: - # $1 - Path to the top level Consul source - # $2 - boolean value for whether to omit the release version from the version string - # - # Return: - # 0 - success (will write the version to stdout) - # * - error (no version output) - # - # Notes: - # If the GIT_DESCRIBE environment variable is present then it is used as the version - # If the GIT_COMMIT environment variable is preset it will be added to the end of - # the version string. - - local vfile="${1}/version/version.go" - - # ensure the version file exists - if ! test -f "${vfile}" - then - echo "Error - File not found: ${vfile}" 1>&2 - return 1 - fi - - # Get the main version out of the source file - version=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) - - # override the version from source with the value of the GIT_DESCRIBE env var if present - if test -n "$GIT_DESCRIBE" - then - version=$GIT_DESCRIBE - fi - - if ! is_set $2 - then - # Get the release version out of the source file - release=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' < ${vfile}) - - # When no GIT_DESCRIBE env var is present and no release is in the source then we - # are definitely in dev mode - if test -z "$GIT_DESCRIBE" -a -z "$release" - then - release="dev" - fi - - # Add the release to the version - if test -n "$release" - then - version="${version}-${release}" - - # add the git commit to the version - if test -n "$GIT_COMMIT" - then - version="${version} (${GIT_COMMIT})" - fi - fi - fi - - # Output the version - echo "$version" | tr -d "'" - return 0 -} - -function get_version { - # Arguments: - # $1 - Path to the top level Consul source - # $2 - Whether the release version should be parsed from source (optional) - # - # Returns: - # 0 - success (the version is also echoed to stdout) - # 1 - error - # - # Notes: - # If a VERSION environment variable is present it will override any parsing of the version from the source - # In addition to processing the main version.go, version_*.go files will be processed if they have - # a Go build tag that matches the one in the GOTAGS environment variable. This tag processing is - # primitive though and will not match complex build tags in the files with negation etc. - - local vers="$VERSION" - if test -z "$vers" - then - # parse the OSS version from version.go - vers="$(parse_version ${1} ${2})" - - # try to determine the version if we have build tags - for tag in "$GOTAGS" - do - for file in $(ls ${1}/version/version_*.go | sort) - do - if grep -q "// +build $tag" $file - then - vers=$(awk -F\" '/Version =/ {print $2; exit}' < $file ) - fi - done - done - fi - - if test -z "$vers" - then - return 1 - else - echo $vers - return 0 - fi -} - -function tag_release { - # Arguments: - # $1 - Version string to use for tagging the release - # $2 - Alternative GPG key id used for signing the release commit (optional) - # - # Returns: - # 0 - success - # * - error - # - # Notes: - # If the RELEASE_UNSIGNED environment variable is set then no gpg signing will occur - - if ! test -d "$1" - then - echo "ERROR: '$1' is not a directory. tag_release must be called with the path to the top level source as the first argument'" 1>&2 - return 1 - fi - - if test -z "$2" - then - echo "ERROR: tag_release must be called with a version number as the second argument" 1>&2 - return 1 - fi - - # determine whether the gpg key to use is being overridden - local gpg_key=${HASHICORP_GPG_KEY} - if test -n "$3" - then - gpg_key=$3 - fi - - pushd "$1" > /dev/null - local ret=0 - - # perform an usngined release if requested (mainly for testing locally) - if is_set "$RELEASE_UNSIGNED" - then - ( - git commit --allow-empty -a -m "Release v${2}" && - git tag -a -m "Version ${2}" "v${2}" master - ) - ret=$? - # perform a signed release (official releases should do this) - elif have_gpg_key ${gpg_key} - then - ( - git commit --allow-empty -a --gpg-sign=${gpg_key} -m "Release v${2}" && - git tag -a -m "Version ${2}" -s -u ${gpg_key} "v${2}" master - ) - ret=$? - # unsigned release not requested and gpg key isn't useable - else - echo "ERROR: GPG key ${gpg_key} is not in the local keychain - to continue set RELEASE_UNSIGNED=1 in the env" - ret=1 - fi - popd > /dev/null - return $ret -} - -function build_ui { - # Arguments: - # $1 - Path to the top level Consul source - # $2 - The docker image to run the build within (optional) - # - # Returns: - # 0 - success - # * - error - # - # Notes: - # Use the GIT_COMMIT environment variable to pass off to the build - - if ! test -d "$1" - then - echo "ERROR: '$1' is not a directory. build_ui must be called with the path to the top level source as the first argument'" 1>&2 - return 1 - fi - - local image_name=${UI_BUILD_CONTAINER_DEFAULT} - if test -n "$2" - then - image_name="$2" - fi - - local sdir="$1" - local ui_dir="${1}/ui-v2" - - # parse the version - version=$(parse_version "${sdir}") - - # make sure we run within the ui dir - pushd ${ui_dir} > /dev/null - - echo "Creating the UI Build Container" - local container_id=$(docker create -it -e "CONSUL_GIT_SHA=${GIT_COMMIT}" -e "CONSUL_VERSION=${version}" ${image_name}) - local ret=$? - if test $ret -eq 0 - then - echo "Copying the source from '${ui_dir}' to /consul-src within the container" - ( - docker cp . ${container_id}:/consul-src && - echo "Running build in container" && docker start -i ${container_id} && - rm -rf ${1}/ui-v2/dist && - echo "Copying back artifacts" && docker cp ${container_id}:/consul-src/dist ${1}/ui-v2/dist - ) - ret=$? - docker rm ${container_id} > /dev/null - fi - - if test $ret -eq 0 - then - rm -rf ${1}/pkg/web_ui/v2 - cp -r ${1}/ui-v2/dist ${1}/pkg/web_ui/v2 - fi - popd > /dev/null - return $ret -} - -function build_ui_legacy { - # Arguments: - # $1 - Path to the top level Consul source - # $2 - The docker image to run the build within (optional) - # - # Returns: - # 0 - success - # * - error - - if ! test -d "$1" - then - echo "ERROR: '$1' is not a directory. build_ui_legacy must be called with the path to the top level source as the first argument'" 1>&2 - return 1 - fi - - local sdir="$1" - local ui_legacy_dir="${sdir}/ui" - - local image_name=${UI_LEGACY_BUILD_CONTAINER_DEFAULT} - if test -n "$2" - then - image_name="$2" - fi - - pushd ${ui_legacy_dir} > /dev/null - echo "Creating the Legacy UI Build Container" - rm -r ${sdir}/pkg/web_ui/v1 >/dev/null 2>&1 - mkdir -p ${sdir}/pkg/web_ui/v1 - local container_id=$(docker create -it ${image_name}) - local ret=$? - if test $ret -eq 0 - then - echo "Copying the source from '${ui_legacy_dir}' to /consul-src/ui within the container" - ( - docker cp . ${container_id}:/consul-src/ui && - echo "Running build in container" && - docker start -i ${container_id} && - echo "Copying back artifacts" && - docker cp ${container_id}:/consul-src/pkg/web_ui ${sdir}/pkg/web_ui/v1 - ) - ret=$? - docker rm ${container_id} > /dev/null - fi - popd > /dev/null - return $ret -} - -function build_assetfs { - # Arguments: - # $1 - Path to the top level Consul source - # $2 - The docker image to run the build within (optional) - # - # Returns: - # 0 - success - # * - error - # - # Note: - # The GIT_COMMIT, GIT_DIRTY and GIT_DESCRIBE environment variables will be used if present - - if ! test -d "$1" - then - echo "ERROR: '$1' is not a directory. build_assetfs must be called with the path to the top level source as the first argument'" 1>&2 - return 1 - fi - - local sdir="$1" - local image_name=${GO_BUILD_CONTAINER_DEFAULT} - if test -n "$2" - then - image_name="$2" - fi - - pushd ${sdir} > /dev/null - echo "Creating the Go Build Container" - local container_id=$(docker create -it -e GIT_COMMIT=${GIT_COMMIT} -e GIT_DIRTY=${GIT_DIRTY} -e GIT_DESCRIBE=${GIT_DESCRIBE} ${image_name} make static-assets ASSETFS_PATH=bindata_assetfs.go) - local ret=$? - if test $ret -eq 0 - then - echo "Copying the sources from '${sdir}/(pkg|GNUmakefile)' to /go/src/github.com/hashicorp/consul/pkg" - ( - tar -c pkg/web_ui GNUmakefile | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul && - echo "Running build in container" && docker start -i ${container_id} && - echo "Copying back artifacts" && docker cp ${container_id}:/go/src/github.com/hashicorp/consul/bindata_assetfs.go ${sdir}/agent/bindata_assetfs.go - ) - ret=$? - docker rm ${container_id} > /dev/null - fi - popd >/dev/null - return $ret -} - -function build_consul { - # Arguments: - # $1 - Path to the top level Consul source - # $2 - The docker image to run the build within (optional) - # - # Returns: - # 0 - success - # * - error - # - # Note: - # The GOLDFLAGS and GOTAGS environment variables will be used if set - # If the CONSUL_DEV environment var is truthy only the local platform/architecture is built. - # If the XC_OS or the XC_ARCH environment vars are present then only those platforms/architectures - # will be built. Otherwise all supported platform/architectures are built - - if ! test -d "$1" - then - echo "ERROR: '$1' is not a directory. build_consul must be called with the path to the top level source as the first argument'" 1>&2 - return 1 - fi - - local sdir="$1" - local image_name=${GO_BUILD_CONTAINER_DEFAULT} - if test -n "$2" - then - image_name="$2" - fi - - pushd ${sdir} > /dev/null - echo "Creating the Go Build Container" - if is_set "${CONSUL_DEV}" - then - XC_OS=$(go_env GOOS) - XC_ARCH=$(go env GOARCH) - else - XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} - XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"} - fi - - local container_id=$(docker create -it ${image_name} gox -os="${XC_OS}" -arch="${XC_ARCH}" -osarch="!darwin/arm !darwin/arm64" -ldflags "${GOLDFLAGS}" -output "pkg/{{.OS}}_{{.Arch}}/consul" -tags="${GOTAGS}") - ret=$? - - if test $ret -eq 0 - then - echo "Copying the source from '${sdir}' to /go/src/github.com/hashicorp/consul/pkg" - ( - tar -c $(ls | grep -v "ui\|ui-v2\|website\|bin\|.git") | docker cp - ${container_id}:/go/src/github.com/hashicorp/consul && - echo "Running build in container" && - docker start -i ${container_id} && - echo "Copying back artifacts" && - docker cp ${container_id}:/go/src/github.com/hashicorp/consul/pkg/ pkg.new - ) - ret=$? - docker rm ${container_id} > /dev/null - - DEV_PLATFORM="./pkg.new/$(go env GOOS)_$(go env GOARCH)" - for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f) - do - cp ${F} bin/ - cp ${F} ${GOPATH}/bin - done - - cp -r pkg.new/* pkg/ - rm -r pkg.new - fi - popd > /dev/null - return $ret -} - -function package_release { - # Arguments: - # $1 - Path to the top level Consul source - # $2 - Version to use in the names of the zip files (optional) - # - # Returns: - # 0 - success - # * - error - - if ! test -d "$1" - then - echo "ERROR: '$1' is not a directory. package_release must be called with the path to the top level source as the first argument'" 1>&2 - return 1 - fi - - local vers="${2}" - if test -z "${vers}" - then - vers=$(get_version $1 false) - ret=$? - if test "$ret" -ne 0 - then - echo "ERROR: failed to determine the version." 1>&2 - return $ret - fi - fi - - local sdir="$1" - local ret=0 - for platform in $(find "${sdir}/pkg" -mindepth 1 -maxdepth 1 -type d) - do - local os_arch=$(basename $platform) - pushd "${platform}" > /dev/null - zip "${sdir}/pkg/dist/consul_${vers}_${os_arch}.zip" ./* - ret=$? - popd > /dev/null - - if test "$ret" -ne 0 - then - break - fi - done - - return $ret -} \ No newline at end of file +for src in $func_sources +do + source $src +done \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100755 index 038b210fd..000000000 --- a/scripts/build.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -# -# This script builds the application from source for multiple platforms. -set -e - -export CGO_ENABLED=0 - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that directory -cd "$DIR" - -# Determine the arch/os combos we're building for -XC_ARCH=${XC_ARCH:-"386 amd64 arm arm64"} -XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} - -# Delete the old dir -echo "==> Removing old directory..." -rm -f bin/* -rm -rf pkg/* -mkdir -p bin/ - -# If it's dev mode, only build for ourself -if [ "${CONSUL_DEV}x" != "x" ]; then - XC_OS=$(go env GOOS) - XC_ARCH=$(go env GOARCH) -fi - -# Build! -echo "==> Building..." -"`which gox`" \ - -os="${XC_OS}" \ - -arch="${XC_ARCH}" \ - -osarch="!darwin/arm !darwin/arm64" \ - -ldflags "${GOLDFLAGS}" \ - -output "pkg/{{.OS}}_{{.Arch}}/consul" \ - -tags="${GOTAGS}" \ - . - -# Move all the compiled things to the $GOPATH/bin -GOPATH=${GOPATH:-$(go env GOPATH)} -case $(uname) in - CYGWIN*) - GOPATH="$(cygpath $GOPATH)" - ;; -esac -OLDIFS=$IFS -IFS=: MAIN_GOPATH=($GOPATH) -IFS=$OLDIFS - -# Copy our OS/Arch to the bin/ directory -DEV_PLATFORM="./pkg/$(go env GOOS)_$(go env GOARCH)" -for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f); do - cp ${F} bin/ - cp ${F} ${MAIN_GOPATH}/bin/ -done - -if [ "${CONSUL_DEV}x" = "x" ]; then - # Zip and copy to the dist dir - echo "==> Packaging..." - for PLATFORM in $(find ./pkg -mindepth 1 -maxdepth 1 -type d); do - OSARCH=$(basename ${PLATFORM}) - echo "--> ${OSARCH}" - - pushd $PLATFORM >/dev/null 2>&1 - zip ../${OSARCH}.zip ./* - popd >/dev/null 2>&1 - done -fi - -# Done! -echo -echo "==> Results:" -ls -hl bin/ diff --git a/scripts/consul-builder/Dockerfile b/scripts/consul-builder/Dockerfile deleted file mode 100644 index 24a6a21df..000000000 --- a/scripts/consul-builder/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM ubuntu:bionic - -ENV GOVERSION 1.10.1 - -RUN apt-get update -y && \ - apt-get install --no-install-recommends -y -q \ - build-essential \ - ca-certificates \ - curl \ - git \ - ruby \ - ruby-dev \ - zip \ - zlib1g-dev \ - nodejs \ - npm && \ - gem install bundler && \ - npm install --global yarn && \ - npm install --global ember-cli - -RUN mkdir /goroot && \ - mkdir /gopath && \ - curl https://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \ - tar xzf - -C /goroot --strip-components=1 - -# We want to ensure that release builds never have any cgo dependencies so we -# switch that off at the highest level. -ENV CGO_ENABLED 0 -ENV GOPATH /gopath -ENV GOROOT /goroot -ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH - -RUN mkdir -p $GOPATH/src/github.com/hashicorp/consul -WORKDIR $GOPATH/src/github.com/hashicorp/consul diff --git a/scripts/dist.sh b/scripts/dist.sh deleted file mode 100755 index 81e8ac935..000000000 --- a/scripts/dist.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the version from the environment, or try to figure it out from the build tags. -# We process the files in the same order Go does to find the last matching tag. -if [ -z $VERSION ]; then - # get the OSS version from version.go - VERSION=$(awk -F\" '/Version =/ { print $2; exit }' Building version $VERSION..." - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd $DIR - -# Generate the tag. -if [ -z $NOTAG ]; then - echo "==> Tagging..." - git commit --allow-empty -a --gpg-sign=348FFC4C -m "Release v$VERSION" - git tag -a -m "Version $VERSION" -s -u 348FFC4C "v${VERSION}" master -fi - -# Do a hermetic build inside a Docker container. -if [ -z $NOBUILD ]; then - docker build -t hashicorp/consul-builder scripts/consul-builder/ - docker run --rm -e "GOTAGS=$GOTAGS" -v "$(pwd)":/gopath/src/github.com/hashicorp/consul hashicorp/consul-builder ./scripts/dist_build.sh -fi - -# Zip all the files. -rm -rf ./pkg/dist -mkdir -p ./pkg/dist -for FILENAME in $(find ./pkg -mindepth 1 -maxdepth 1 -type f); do - FILENAME=$(basename $FILENAME) - cp ./pkg/${FILENAME} ./pkg/dist/consul_${VERSION}_${FILENAME} -done - -# Make the checksums. -pushd ./pkg/dist -shasum -a256 * > ./consul_${VERSION}_SHA256SUMS -if [ -z $NOSIGN ]; then - echo "==> Signing..." - gpg --default-key 348FFC4C --detach-sig ./consul_${VERSION}_SHA256SUMS -fi -popd - -exit 0 diff --git a/scripts/dist_build.sh b/scripts/dist_build.sh deleted file mode 100755 index 586eaafdf..000000000 --- a/scripts/dist_build.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd $DIR - -# Make sure build tools are available. -make tools - -# # Build the standalone version of the web assets for the sanity check. -# pushd ui -# bundle -# make dist -# popd - -# pushd ui-v2 -# yarn install -# make dist -# popd - -# # Fixup the timestamps to match what's checked in. This will allow us to cleanly -# # verify that the checked-in content is up to date without spurious diffs of the -# # file mod times. -# pushd pkg -# cat ../agent/bindata_assetfs.go | ../scripts/fixup_times.sh -# popd - -# # Regenerate the built-in web assets. If there are any diffs after doing this -# # then we know something is up. -# make static-assets -# if ! git diff --quiet agent/bindata_assetfs.go; then -# echo "Checked-in web assets are out of date, build aborted" -# exit 1 -# fi - -# Now we are ready to do a clean build of everything. We no longer distribute the -# web UI so it's ok that gets blown away as part of this. -rm -rf pkg -make all - -exit 0 diff --git a/scripts/fixup_times.sh b/scripts/fixup_times.sh deleted file mode 100755 index 43994a0e6..000000000 --- a/scripts/fixup_times.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -e -regex='bindataFileInfo.*name: \"(.+)\".*time.Unix.(.+),' -while read line; do - if [[ $line =~ $regex ]]; then - file=${BASH_REMATCH[1]} - ts=${BASH_REMATCH[2]} - touch --date @$ts $file - fi -done diff --git a/scripts/ui.sh b/scripts/ui.sh deleted file mode 100755 index 321ed6c1e..000000000 --- a/scripts/ui.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd $DIR - -# Do a hermetic build inside a Docker container. -if [ -z $NOBUILD ]; then - docker build -t hashicorp/consul-builder scripts/consul-builder/ - docker run --rm -v "$(pwd)":/gopath/src/github.com/hashicorp/consul hashicorp/consul-builder ./scripts/ui_build.sh -fi - -exit 0 diff --git a/scripts/ui_build.sh b/scripts/ui_build.sh deleted file mode 100755 index c578d396c..000000000 --- a/scripts/ui_build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -# Change into that dir because we expect that. -cd $DIR - -# Make sure build tools are available. -make tools - -# Build the web assets. -echo "Building the V1 UI" -pushd ui -bundle -make dist -popd - -echo "Building the V2 UI" -pushd ui-v2 -yarn install -make dist -popd - -# Make the static assets using the container version of the builder -make static-assets - -exit 0 diff --git a/scripts/vagrant-linux-priv-go.sh b/scripts/vagrant-linux-priv-go.sh deleted file mode 100755 index 9ead97c31..000000000 --- a/scripts/vagrant-linux-priv-go.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -function install_go() { - local go_version=1.9.1 - local download= - - download="https://storage.googleapis.com/golang/go${go_version}.linux-amd64.tar.gz" - - if [ -d /usr/local/go ] ; then - return - fi - - wget -q -O /tmp/go.tar.gz ${download} - - tar -C /tmp -xf /tmp/go.tar.gz - sudo mv /tmp/go /usr/local - sudo chown -R root:root /usr/local/go -} - -install_go - -# Ensure that the GOPATH tree is owned by vagrant:vagrant -mkdir -p /opt/gopath -chown -R vagrant:vagrant /opt/gopath - -# Ensure Go is on PATH -if [ ! -e /usr/bin/go ] ; then - ln -s /usr/local/go/bin/go /usr/bin/go -fi -if [ ! -e /usr/bin/gofmt ] ; then - ln -s /usr/local/go/bin/gofmt /usr/bin/gofmt -fi - - -# Ensure new sessions know about GOPATH -if [ ! -f /etc/profile.d/gopath.sh ] ; then - cat < /etc/profile.d/gopath.sh -export GOPATH="/opt/gopath" -export PATH="/opt/gopath/bin:\$PATH" -EOT - chmod 755 /etc/profile.d/gopath.sh -fi