2017-10-31 17:38:29 +00:00
SHELL = bash
2017-08-18 05:29:26 +00:00
PROJECT_ROOT := $( patsubst %/,%,$( dir $( abspath $( lastword $( MAKEFILE_LIST) ) ) ) )
2020-05-15 16:19:59 +00:00
THIS_OS := $( shell uname | cut -d- -f1)
2017-08-18 05:29:26 +00:00
GIT_COMMIT := $( shell git rev-parse HEAD)
GIT_DIRTY := $( if $( shell git status --porcelain) ,+CHANGES)
2017-11-03 23:01:01 +00:00
GO_LDFLAGS := " -X github.com/hashicorp/nomad/version.GitCommit= $( GIT_COMMIT) $( GIT_DIRTY) "
2019-09-06 14:05:02 +00:00
GO_TAGS ?= codegen_generated
2017-08-18 05:29:26 +00:00
2020-05-15 16:19:59 +00:00
GO_TEST_CMD = $( if $( shell command -v gotestsum 2>/dev/null) ,gotestsum --,go test )
2018-12-15 19:00:27 +00:00
2019-01-23 20:39:34 +00:00
i f e q ( $( origin GOTEST_PKGS_EXCLUDE ) , u n d e f i n e d )
GOTEST_PKGS ?= "./..."
e l s e
2019-09-03 15:04:27 +00:00
GOTEST_PKGS = $( shell go list ./... | sed 's/github.com\/hashicorp\/nomad/./' | egrep -v " ^( $( GOTEST_PKGS_EXCLUDE) )(/.*)? $$ " )
2019-01-23 20:39:34 +00:00
e n d i f
2017-08-18 05:29:26 +00:00
default : help
2020-05-15 16:19:59 +00:00
i f e q ( , $( findstring $ ( THIS_OS ) ,Darwin Linux FreeBSD Windows MSYS_NT ) )
$( error Building Nomad is currently only supported on Darwin and Linux ; not $ ( THIS_OS ) )
2017-08-18 05:29:26 +00:00
e n d i f
2019-01-08 01:39:29 +00:00
# On Linux we build for Linux and Windows
2017-08-18 05:29:26 +00:00
i f e q ( L i n u x , $( THIS_OS ) )
2017-08-22 00:09:17 +00:00
2019-05-10 14:14:18 +00:00
i f e q ( $( CI ) , t r u e )
$( info Running in a CI environment, verbose mode is disabled)
e l s e
VERBOSE = "true"
e n d i f
2017-12-06 19:30:31 +00:00
2017-08-18 05:29:26 +00:00
ALL_TARGETS += linux_386 \
linux_amd64 \
linux_arm \
linux_arm64 \
windows_386 \
windows_amd64
e n d i f
2017-09-06 22:46:31 +00:00
# On MacOS, we only build for MacOS
2017-08-18 05:29:26 +00:00
i f e q ( D a r w i n , $( THIS_OS ) )
ALL_TARGETS += darwin_amd64
2020-05-30 14:29:47 +00:00
# Copy CGO files for darwin into place
2017-08-18 05:29:26 +00:00
e n d i f
2017-09-06 22:46:31 +00:00
# On FreeBSD, we only build for FreeBSD
i f e q ( F r e e B S D , $( THIS_OS ) )
ALL_TARGETS += freebsd_amd64
e n d i f
2019-08-13 14:12:57 +00:00
# include per-user customization after all variables are defined
- i n c l u d e G N U M a k e f i l e . l o c a l
2017-08-18 05:29:26 +00:00
pkg/darwin_amd64/nomad : $( SOURCE_FILES ) ## Build Nomad for darwin/amd64
2017-09-19 14:47:10 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2017-08-18 05:29:26 +00:00
@CGO_ENABLED= 1 GOOS = darwin GOARCH = amd64 \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-08-18 05:29:26 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
2017-09-06 22:46:31 +00:00
pkg/freebsd_amd64/nomad : $( SOURCE_FILES ) ## Build Nomad for freebsd/amd64
@echo " ==> Building $@ ... "
@CGO_ENABLED= 1 GOOS = freebsd GOARCH = amd64 \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-09-06 22:46:31 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
2017-08-18 05:29:26 +00:00
pkg/linux_386/nomad : $( SOURCE_FILES ) ## Build Nomad for linux/386
2017-09-19 14:47:10 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2017-08-18 05:29:26 +00:00
@CGO_ENABLED= 1 GOOS = linux GOARCH = 386 \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-08-18 05:29:26 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
pkg/linux_amd64/nomad : $( SOURCE_FILES ) ## Build Nomad for linux/amd64
2017-09-19 14:47:10 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2017-08-18 05:29:26 +00:00
@CGO_ENABLED= 1 GOOS = linux GOARCH = amd64 \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-08-18 05:29:26 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
pkg/linux_arm/nomad : $( SOURCE_FILES ) ## Build Nomad for linux/arm
2017-09-19 14:47:10 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2017-08-18 05:29:26 +00:00
@CGO_ENABLED= 1 GOOS = linux GOARCH = arm CC = arm-linux-gnueabihf-gcc-5 \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-08-18 05:29:26 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
pkg/linux_arm64/nomad : $( SOURCE_FILES ) ## Build Nomad for linux/arm64
2017-09-19 14:47:10 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2017-08-18 05:29:26 +00:00
@CGO_ENABLED= 1 GOOS = linux GOARCH = arm64 CC = aarch64-linux-gnu-gcc-5 \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-08-18 05:29:26 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
# If CGO support for Windows is ever required, set the following variables
# in the environment for `go build` for both the windows/amd64 and the
# windows/386 targets:
# CC=i686-w64-mingw32-gcc
# CXX=i686-w64-mingw32-g++
pkg/windows_386/nomad : $( SOURCE_FILES ) ## Build Nomad for windows/386
2017-09-19 14:47:10 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2017-08-18 05:29:26 +00:00
@CGO_ENABLED= 1 GOOS = windows GOARCH = 386 \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-08-18 05:29:26 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
2017-08-29 19:15:55 +00:00
-o " $@ .exe "
2017-08-18 05:29:26 +00:00
pkg/windows_amd64/nomad : $( SOURCE_FILES ) ## Build Nomad for windows/amd64
2017-09-19 14:47:10 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2017-09-06 00:23:21 +00:00
@CGO_ENABLED= 1 GOOS = windows GOARCH = amd64 \
2017-08-18 05:29:26 +00:00
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2017-08-18 05:29:26 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
2017-08-29 19:15:55 +00:00
-o " $@ .exe "
2017-08-18 05:29:26 +00:00
2020-03-03 12:25:21 +00:00
pkg/linux_ppc64le/nomad : $( SOURCE_FILES ) ## Build Nomad for linux/arm64
2020-03-03 13:16:45 +00:00
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
2020-03-12 13:48:29 +00:00
@CGO_ENABLED= 1 GOOS = linux GOARCH = ppc64le \
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2020-03-12 13:48:29 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
pkg/linux_s390x/nomad : $( SOURCE_FILES ) ## Build Nomad for linux/arm64
@echo " ==> Building $@ with tags $( GO_TAGS) ... "
@CGO_ENABLED= 1 GOOS = linux GOARCH = s390x \
2020-03-03 13:16:45 +00:00
go build \
2020-04-07 17:32:07 +00:00
-trimpath \
2020-03-03 13:16:45 +00:00
-ldflags $( GO_LDFLAGS) \
-tags " $( GO_TAGS) " \
-o " $@ "
2020-03-03 12:25:21 +00:00
2017-08-18 05:29:26 +00:00
# Define package targets for each of the build targets we actually have on this system
d e f i n e m a k e P a c k a g e T a r g e t
pkg/$(1).zip : pkg /$( 1) /nomad
@echo " ==> Packaging for $( 1) ... "
2017-08-29 05:29:49 +00:00
@zip -j pkg/$( 1) .zip pkg/$( 1) /*
2017-08-18 05:29:26 +00:00
e n d e f
# Reify the package targets
$( foreach t ,$ ( ALL_TARGETS ) ,$ ( eval $ ( call makePackageTarget ,$ ( t ) ) ) )
.PHONY : bootstrap
2019-04-17 22:28:17 +00:00
bootstrap : deps lint -deps git -hooks # Install all dependencies
2017-08-18 05:29:26 +00:00
.PHONY : deps
2017-10-24 17:49:53 +00:00
deps : ## Install build and development dependencies
2020-05-30 14:29:47 +00:00
## Keep versions in sync with tools/go.mod for now (see https://github.com/golang/go/issues/30515)
2017-08-18 05:29:26 +00:00
@echo "==> Updating build dependencies..."
2020-06-04 22:25:54 +00:00
GO111MODULE = on cd tools && go get github.com/hashicorp/go-bindata/go-bindata@bf7910af899725e4938903fb32048c7c0b15f12e
GO111MODULE = on cd tools && go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@234c15e7648ff35458026de92b34c637bae5e6f7
2020-05-30 14:29:47 +00:00
GO111MODULE = on cd tools && go get github.com/a8m/tree/cmd/tree
GO111MODULE = on cd tools && go get gotest.tools/gotestsum@v0.4.2
GO111MODULE = on cd tools && go get github.com/hashicorp/hcl/v2/cmd/hclfmt@v2.5.1
GO111MODULE = on cd tools && go get github.com/golang/protobuf/protoc-gen-go@v1.3.4
GO111MODULE = on cd tools && go get github.com/hashicorp/go-msgpack/codec/codecgen@v1.1.5
2017-08-18 05:29:26 +00:00
2017-10-24 17:49:53 +00:00
.PHONY : lint -deps
lint-deps : ## Install linter dependencies
2020-05-30 14:29:47 +00:00
## Keep versions in sync with tools/go.mod (see https://github.com/golang/go/issues/30515)
2017-10-24 17:49:53 +00:00
@echo "==> Updating linter dependencies..."
2020-05-30 14:29:47 +00:00
GO111MODULE = on cd tools && go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.24.0
GO111MODULE = on cd tools && go get github.com/client9/misspell/cmd/misspell@v0.3.4
2020-07-01 14:32:45 +00:00
GO111MODULE = on cd tools && go get github.com/hashicorp/go-hclog/hclogvet@v0.1.3
2017-10-24 17:49:53 +00:00
2019-04-17 22:28:17 +00:00
.PHONY : git -hooks
git-dir = $( shell git rev-parse --git-dir)
git-hooks : $( git -dir ) /hooks /pre -push
$(git-dir)/hooks/% : dev /hooks /%
cp $^ $@
chmod 755 $@
2017-08-18 05:29:26 +00:00
.PHONY : check
check : ## Lint the source code
@echo "==> Linting source code..."
2019-08-16 13:11:31 +00:00
@golangci-lint run -j 1
2020-08-27 20:21:07 +00:00
2020-07-01 14:08:37 +00:00
@echo "==> Linting hclog statements..."
@hclogvet .
2019-12-06 01:27:14 +00:00
2017-09-27 18:14:37 +00:00
@echo "==> Spell checking website..."
2020-04-23 14:38:08 +00:00
@misspell -error -source= text website/pages/
2017-08-18 05:29:26 +00:00
2019-03-14 14:56:27 +00:00
@echo "==> Check proto files are in-sync..."
@$( MAKE) proto
2020-04-23 18:32:44 +00:00
@if ( git status -s | grep -q .pb.go) ; then echo the following proto files are out of sync; git status -s | grep .pb.go; exit 1; fi
@echo "==> Check format of jobspecs and HCL files..."
@$( MAKE) hclfmt
@if ( git status -s | grep -q -e '\.hcl$$' -e '\.nomad$$' ) ; then echo the following HCL files are out of sync; git status -s | grep -e '\.hcl$$' -e '\.nomad$$' ; exit 1; fi
2019-03-14 14:56:27 +00:00
2019-04-28 17:31:43 +00:00
@echo "==> Check API package is isolated from rest"
2020-05-30 14:29:47 +00:00
@cd ./api && if go list --test -f '{{ join .Deps "\n" }}' . | grep github.com/hashicorp/nomad/ | grep -v -e /vendor/ -e /nomad/api/ -e nomad/api.test; then echo " /api package depends the ^^ above internal nomad packages. Remove such dependency" ; exit 1; fi
2020-03-09 19:06:47 +00:00
2020-08-27 20:21:07 +00:00
@echo "==> Checking Go mod.."
@GO111MODULE= on go mod tidy
@if ( git status --porcelain | grep -Eq "go\.(mod|sum)" ) ; then \
echo go.mod or go.sum needs updating; \
git --no-pager diff go.mod; \
git --no-pager diff go.sum; \
exit 1; fi
2020-08-31 11:59:16 +00:00
@echo "==> Check raft util msg type mapping are in-sync..."
@go generate ./helper/raftutil/
@if ( git status -s ./helper/raftutil| grep -q .go) ; then echo "raftutil helper message type mapping is out of sync. Run go generate ./... and push." ; exit 1; fi
2017-09-09 00:42:42 +00:00
.PHONY : checkscripts
checkscripts : ## Lint shell scripts
@echo "==> Linting scripts..."
2019-03-18 12:45:25 +00:00
@find scripts -type f -name '*.sh' | xargs shellcheck
2017-09-09 00:42:42 +00:00
2018-11-07 19:51:03 +00:00
.PHONY : generate -all
2019-08-14 14:23:33 +00:00
generate-all : generate -structs proto generate -examples
2018-11-07 19:51:03 +00:00
.PHONY : generate -structs
generate-structs : LOCAL_PACKAGES = $( shell go list ./... | grep -v '/vendor /')
generate-structs : ## Update generated code
2019-01-30 16:00:17 +00:00
@echo "--> Running go generate..."
2017-08-18 05:29:26 +00:00
@go generate $( LOCAL_PACKAGES)
2018-09-28 17:09:01 +00:00
.PHONY : proto
proto :
2019-01-30 16:00:17 +00:00
@echo "--> Generating proto bindings..."
2020-04-22 20:29:20 +00:00
@for file in $$ ( git ls-files "*.proto" | grep -E -v -- "vendor\/.*.proto|demo\/.*.proto" ) ; do \
2018-09-28 17:09:01 +00:00
protoc -I . -I ../../.. --go_out= plugins = grpc:. $$ file; \
done
2019-08-14 14:23:33 +00:00
.PHONY : generate -examples
generate-examples : command /job_init .bindata_assetfs .go
command/job_init.bindata_assetfs.go : command /assets /*
go-bindata-assetfs -pkg command -o command/job_init.bindata_assetfs.go ./command/assets/...
2018-09-28 17:09:01 +00:00
2019-08-05 12:07:43 +00:00
.PHONY : changelogfmt
2017-12-12 21:52:58 +00:00
changelogfmt :
@echo "--> Making [GH-xxxx] references clickable..."
@sed -E 's|([^\[])\[GH-([0-9]+)\]|\1[[GH-\2](https://github.com/hashicorp/nomad/issues/\2)]|g' CHANGELOG.md > changelog.tmp && mv changelog.tmp CHANGELOG.md
2019-08-05 12:07:43 +00:00
## We skip the terraform directory as there are templated hcl configurations
## that do not successfully compile without rendering
.PHONY : hclfmt
hclfmt :
@echo "--> Formatting HCL"
2019-12-02 15:52:42 +00:00
@find . -path ./terraform -prune -o -name 'upstart.nomad' -prune -o \( -name '*.nomad' -o -name '*.hcl' \) -exec \
s h - c 'hclfmt -w {} || echo in path {}' ';'
2017-12-12 16:30:36 +00:00
2020-05-30 14:29:47 +00:00
.PHONY : tidy
tidy :
@echo "--> Tidy up submodules"
@cd tools && go mod tidy
@cd api && go mod tidy
@echo "--> Tidy nomad module"
@go mod tidy
.PHONY : sync
sync : tidy
@echo "--> Sync vendor directory"
@go mod vendor
2017-08-18 05:29:26 +00:00
.PHONY : dev
dev : GOOS =$( shell go env GOOS )
dev : GOARCH =$( shell go env GOARCH )
dev : GOPATH =$( shell go env GOPATH )
2019-01-08 01:39:29 +00:00
dev : DEV_TARGET =pkg /$( GOOS ) _ $( GOARCH ) /nomad
2020-06-03 14:35:09 +00:00
dev : changelogfmt hclfmt ## Build for the current development platform
2017-08-18 05:29:26 +00:00
@echo "==> Removing old development build..."
@rm -f $( PROJECT_ROOT) /$( DEV_TARGET)
@rm -f $( PROJECT_ROOT) /bin/nomad
@rm -f $( GOPATH) /bin/nomad
@$( MAKE) --no-print-directory \
$( DEV_TARGET) \
2019-08-02 15:55:52 +00:00
GO_TAGS = " $( GO_TAGS) $( NOMAD_UI_TAG) "
2017-08-18 05:29:26 +00:00
@mkdir -p $( PROJECT_ROOT) /bin
@mkdir -p $( GOPATH) /bin
@cp $( PROJECT_ROOT) /$( DEV_TARGET) $( PROJECT_ROOT) /bin/
@cp $( PROJECT_ROOT) /$( DEV_TARGET) $( GOPATH) /bin
2017-10-26 17:08:19 +00:00
.PHONY : prerelease
dev: avoid codecgen code in downstream projects
This is an attempt to ease dependency management for external driver
plugins, by avoiding requiring them to compile ugorji/go generated
files. Plugin developers reported some pain with the brittleness of
ugorji/go dependency in particular, specially when using go mod, the
default go mod manager in golang 1.13.
Context
--------
Nomad uses msgpack to persist and serialize internal structs, using
ugorji/go library. As an optimization, we use ugorji/go code generation
to speedup process and aovid the relection-based slow path.
We commit these generated files in repository when we cut and tag the
release to ease reproducability and debugging old releases. Thus,
downstream projects that depend on release tag, indirectly depends on
ugorji/go generated code.
Sadly, the generated code is brittle and specific to the version of
ugorji/go being used. When go mod picks another version of ugorji/go
then nomad (go mod by default uses release according to semver),
downstream projects face compilation errors.
Interestingly, downstream projects don't commonly serialize nomad
internal structs. Drivers and device plugins use grpc instead of
msgpack for the most part. In the few cases where they use msgpag (e.g.
decoding task config), they do without codegen path as they run on
driver specific structs not the nomad internal structs. Also, the
ugorji/go serialization through reflection is generally backward
compatible (mod some ugorji/go regression bugs that get introduced every
now and then :( ).
Proposal
---------
The proposal here is to keep committing ugorji/go codec generated files
for releases but to use a go tag for them.
All nomad development through the makefile, including releasing, CI and
dev flow, has the tag enabled.
Downstream plugin projects, by default, will skip these files and life
proceed as normal for them.
The downside is that nomad developers who use generated code but avoid
using make must start passing additional go tag argument. Though this
is not a blessed configuration.
2019-09-06 12:43:26 +00:00
prerelease : GO_TAGS =ui codegen_generated release
2019-03-21 18:57:03 +00:00
prerelease : generate -all ember -dist static -assets ## Generate all the static assets for a Nomad release
2017-10-26 17:08:19 +00:00
2017-08-18 05:29:26 +00:00
.PHONY : release
dev: avoid codecgen code in downstream projects
This is an attempt to ease dependency management for external driver
plugins, by avoiding requiring them to compile ugorji/go generated
files. Plugin developers reported some pain with the brittleness of
ugorji/go dependency in particular, specially when using go mod, the
default go mod manager in golang 1.13.
Context
--------
Nomad uses msgpack to persist and serialize internal structs, using
ugorji/go library. As an optimization, we use ugorji/go code generation
to speedup process and aovid the relection-based slow path.
We commit these generated files in repository when we cut and tag the
release to ease reproducability and debugging old releases. Thus,
downstream projects that depend on release tag, indirectly depends on
ugorji/go generated code.
Sadly, the generated code is brittle and specific to the version of
ugorji/go being used. When go mod picks another version of ugorji/go
then nomad (go mod by default uses release according to semver),
downstream projects face compilation errors.
Interestingly, downstream projects don't commonly serialize nomad
internal structs. Drivers and device plugins use grpc instead of
msgpack for the most part. In the few cases where they use msgpag (e.g.
decoding task config), they do without codegen path as they run on
driver specific structs not the nomad internal structs. Also, the
ugorji/go serialization through reflection is generally backward
compatible (mod some ugorji/go regression bugs that get introduced every
now and then :( ).
Proposal
---------
The proposal here is to keep committing ugorji/go codec generated files
for releases but to use a go tag for them.
All nomad development through the makefile, including releasing, CI and
dev flow, has the tag enabled.
Downstream plugin projects, by default, will skip these files and life
proceed as normal for them.
The downside is that nomad developers who use generated code but avoid
using make must start passing additional go tag argument. Though this
is not a blessed configuration.
2019-09-06 12:43:26 +00:00
release : GO_TAGS =ui codegen_generated release
2017-10-26 17:08:19 +00:00
release : clean $( foreach t ,$ ( ALL_TARGETS ) ,pkg /$ ( t ) .zip ) ## Build all release packages which can be built on this platform.
2017-08-18 05:29:26 +00:00
@echo "==> Results:"
@tree --dirsfirst $( PROJECT_ROOT) /pkg
.PHONY : test
2017-09-19 14:47:10 +00:00
test : ## Run the Nomad test suite and/or the Nomad UI test suite
@if [ ! $( SKIP_NOMAD_TESTS) ] ; then \
make test-nomad; \
fi
2018-10-08 18:44:23 +00:00
@if [ $( RUN_WEBSITE_TESTS) ] ; then \
make test-website; \
fi
2017-09-19 14:47:10 +00:00
@if [ $( RUN_UI_TESTS) ] ; then \
make test-ui; \
fi
2018-09-19 17:21:57 +00:00
@if [ $( RUN_E2E_TESTS) ] ; then \
make e2e-test; \
fi
2017-09-19 14:47:10 +00:00
.PHONY : test -nomad
2017-10-24 16:35:51 +00:00
test-nomad : dev ## Run Nomad test suites
2017-08-18 05:29:26 +00:00
@echo "==> Running Nomad test suites:"
2018-12-15 19:00:27 +00:00
$( if $( ENABLE_RACE) ,GORACE= " strip_path_prefix= $( GOPATH) /src " ) $( GO_TEST_CMD) \
2018-08-27 21:15:56 +00:00
$( if $( ENABLE_RACE) ,-race) $( if $( VERBOSE) ,-v) \
-cover \
2018-12-08 19:50:47 +00:00
-timeout= 15m \
dev: avoid codecgen code in downstream projects
This is an attempt to ease dependency management for external driver
plugins, by avoiding requiring them to compile ugorji/go generated
files. Plugin developers reported some pain with the brittleness of
ugorji/go dependency in particular, specially when using go mod, the
default go mod manager in golang 1.13.
Context
--------
Nomad uses msgpack to persist and serialize internal structs, using
ugorji/go library. As an optimization, we use ugorji/go code generation
to speedup process and aovid the relection-based slow path.
We commit these generated files in repository when we cut and tag the
release to ease reproducability and debugging old releases. Thus,
downstream projects that depend on release tag, indirectly depends on
ugorji/go generated code.
Sadly, the generated code is brittle and specific to the version of
ugorji/go being used. When go mod picks another version of ugorji/go
then nomad (go mod by default uses release according to semver),
downstream projects face compilation errors.
Interestingly, downstream projects don't commonly serialize nomad
internal structs. Drivers and device plugins use grpc instead of
msgpack for the most part. In the few cases where they use msgpag (e.g.
decoding task config), they do without codegen path as they run on
driver specific structs not the nomad internal structs. Also, the
ugorji/go serialization through reflection is generally backward
compatible (mod some ugorji/go regression bugs that get introduced every
now and then :( ).
Proposal
---------
The proposal here is to keep committing ugorji/go codec generated files
for releases but to use a go tag for them.
All nomad development through the makefile, including releasing, CI and
dev flow, has the tag enabled.
Downstream plugin projects, by default, will skip these files and life
proceed as normal for them.
The downside is that nomad developers who use generated code but avoid
using make must start passing additional go tag argument. Though this
is not a blessed configuration.
2019-09-06 12:43:26 +00:00
-tags " $( GO_TAGS) " \
2019-05-10 14:14:18 +00:00
$( GOTEST_PKGS) $( if $( VERBOSE) , >test.log ; echo $$ ? > exit-code)
@if [ $( VERBOSE) ] ; then \
bash -C " $( PROJECT_ROOT) /scripts/test_check.sh " ; \
fi
2017-08-18 05:29:26 +00:00
2020-05-30 14:29:47 +00:00
.PHONY : test -nomad -module
test-nomad-module : dev ## Run Nomad test suites on a sub-module
@echo "==> Running Nomad test suites on sub-module:"
@cd $( GOTEST_MOD) && $( if $( ENABLE_RACE) ,GORACE= " strip_path_prefix= $( GOPATH) /src " ) $( GO_TEST_CMD) \
$( if $( ENABLE_RACE) ,-race) $( if $( VERBOSE) ,-v) \
-cover \
-timeout= 15m \
-tags " $( GO_TAGS) " \
./... $( if $( VERBOSE) , >test.log ; echo $$ ? > exit-code)
@if [ $( VERBOSE) ] ; then \
bash -C " $( PROJECT_ROOT) /scripts/test_check.sh " ; \
fi
2018-09-19 01:46:33 +00:00
.PHONY : e 2e -test
2018-09-19 17:38:20 +00:00
e2e-test : dev ## Run the Nomad e2e test suite
2018-09-19 01:46:33 +00:00
@echo "==> Running Nomad E2E test suites:"
2020-10-14 18:53:21 +00:00
go test \
$( if $( ENABLE_RACE) ,-race) $( if $( VERBOSE) ,-v) \
-timeout= 900s \
-tags " $( GO_TAGS) " \
github.com/hashicorp/nomad/e2e
.PHONY : integration -test
integration-test : dev ## Run Nomad integration tests
@echo "==> Running Nomad integration test suites:"
2018-09-19 01:46:33 +00:00
go test \
$( if $( ENABLE_RACE) ,-race) $( if $( VERBOSE) ,-v) \
-cover \
-timeout= 900s \
dev: avoid codecgen code in downstream projects
This is an attempt to ease dependency management for external driver
plugins, by avoiding requiring them to compile ugorji/go generated
files. Plugin developers reported some pain with the brittleness of
ugorji/go dependency in particular, specially when using go mod, the
default go mod manager in golang 1.13.
Context
--------
Nomad uses msgpack to persist and serialize internal structs, using
ugorji/go library. As an optimization, we use ugorji/go code generation
to speedup process and aovid the relection-based slow path.
We commit these generated files in repository when we cut and tag the
release to ease reproducability and debugging old releases. Thus,
downstream projects that depend on release tag, indirectly depends on
ugorji/go generated code.
Sadly, the generated code is brittle and specific to the version of
ugorji/go being used. When go mod picks another version of ugorji/go
then nomad (go mod by default uses release according to semver),
downstream projects face compilation errors.
Interestingly, downstream projects don't commonly serialize nomad
internal structs. Drivers and device plugins use grpc instead of
msgpack for the most part. In the few cases where they use msgpag (e.g.
decoding task config), they do without codegen path as they run on
driver specific structs not the nomad internal structs. Also, the
ugorji/go serialization through reflection is generally backward
compatible (mod some ugorji/go regression bugs that get introduced every
now and then :( ).
Proposal
---------
The proposal here is to keep committing ugorji/go codec generated files
for releases but to use a go tag for them.
All nomad development through the makefile, including releasing, CI and
dev flow, has the tag enabled.
Downstream plugin projects, by default, will skip these files and life
proceed as normal for them.
The downside is that nomad developers who use generated code but avoid
using make must start passing additional go tag argument. Though this
is not a blessed configuration.
2019-09-06 12:43:26 +00:00
-tags " $( GO_TAGS) " \
2020-10-14 12:43:28 +00:00
github.com/hashicorp/nomad/e2e/vaultcompat/ \
2018-09-25 18:29:09 +00:00
-integration
2018-09-19 01:46:33 +00:00
2017-08-18 05:29:26 +00:00
.PHONY : clean
clean : GOPATH =$( shell go env GOPATH )
clean : ## Remove build artifacts
@echo "==> Cleaning build artifacts..."
@rm -rf " $( PROJECT_ROOT) /bin/ "
@rm -rf " $( PROJECT_ROOT) /pkg/ "
@rm -f " $( GOPATH) /bin/nomad "
2017-09-07 18:49:22 +00:00
.PHONY : testcluster
testcluster : ## Bring up a Linux test cluster using Vagrant. Set PROVIDER if necessary.
vagrant up nomad-server01 \
nomad-server02 \
nomad-server03 \
nomad-client01 \
nomad-client02 \
nomad-client03 \
$( if $( PROVIDER) ,--provider $( PROVIDER) )
2017-09-19 14:47:10 +00:00
.PHONY : static -assets
static-assets : ## Compile the static routes to serve alongside the API
@echo "--> Generating static assets"
2018-03-19 22:49:25 +00:00
@go-bindata-assetfs -pkg agent -prefix ui -modtime 1480000000 -tags ui -o bindata_assetfs.go ./ui/dist/...
2017-09-19 14:47:10 +00:00
@mv bindata_assetfs.go command/agent
.PHONY : test -ui
2017-11-13 19:57:17 +00:00
test-ui : ## Run Nomad UI test suite
2017-09-19 14:47:10 +00:00
@echo "--> Installing JavaScript assets"
2017-12-14 01:04:15 +00:00
@cd ui && npm rebuild node-sass
2017-09-19 14:47:10 +00:00
@cd ui && yarn install
@echo "--> Running ember tests"
@cd ui && npm test
.PHONY : ember -dist
ember-dist : ## Build the static UI assets from source
@echo "--> Installing JavaScript assets"
2018-03-20 18:30:09 +00:00
@cd ui && yarn install --silent
2017-09-19 14:47:10 +00:00
@cd ui && npm rebuild node-sass
@echo "--> Building Ember application"
@cd ui && npm run build
.PHONY : dev -ui
dev-ui : ember -dist static -assets
@$( MAKE) NOMAD_UI_TAG = "ui" dev ## Build a dev binary with the UI baked in
2017-08-18 05:29:26 +00:00
HELP_FORMAT = " \033[36m%-25s\033[0m %s\n"
.PHONY : help
help : ## Display this usage information
@echo "Valid targets:"
@grep -E '^[^ ]+:.*?## .*$$' $( MAKEFILE_LIST) | \
sort | \
awk ' BEGIN { FS = ":.*?## " } ; \
{ printf $( HELP_FORMAT) , $$ 1, $$ 2} '
2017-11-13 19:57:17 +00:00
@echo ""
@echo "This host will build the following targets if 'make release' is invoked:"
2017-08-18 05:29:26 +00:00
@echo $( ALL_TARGETS) | sed 's/^/ /'
2019-06-20 00:20:13 +00:00
2019-08-12 16:39:54 +00:00
.PHONY : ui -screenshots
2019-06-20 00:20:13 +00:00
ui-screenshots :
@echo "==> Collecting UI screenshots..."
2020-05-30 14:29:47 +00:00
# Build the screenshots image if it doesn't exist yet
2019-06-20 00:20:13 +00:00
@if [ [ " $$ (docker images -q nomad-ui-screenshots 2> /dev/null) " = = "" ] ] ; then \
docker build --tag= "nomad-ui-screenshots" ./scripts/screenshots; \
fi
@docker run \
--rm \
--volume " $( shell pwd ) /scripts/screenshots/screenshots:/screenshots " \
nomad-ui-screenshots
2019-08-12 16:39:54 +00:00
.PHONY : ui -screenshots -local
2019-06-20 00:20:13 +00:00
ui-screenshots-local :
@echo "==> Collecting UI screenshots (local)..."
2019-08-02 15:55:52 +00:00
@cd scripts/screenshots/src && SCREENSHOTS_DIR = "../screenshots" node index.js
