ci: make ember-ci-test (browserstack) (#6751)

* ci: make ember-ci-test (browserstack) VLTES-28

* Update .circleci/config.yml

Co-Authored-By: Josh Freda <jfreda@users.noreply.github.com>

* Update .circleci/config.yml

Co-Authored-By: Josh Freda <jfreda@users.noreply.github.com>

* ci: rename test-ember -> test-ui-browserstack

* Makefile: ember-ci-test -> test-ui-browserstack

- Use the same name for this test everywhere to reduce cognitive load.

* browserstack: exit non-zero on failure to connect

* .travis.yml: ember-ci-test -> test-ui-browserstack

* browserstack: add vault bin to the path

* Makefile: browserstack: fail early w/clear msgs

This might save someone time later. The same checks could be applied
elsewhere too trivially.

.circleci/config.yml

@@ -117,6 +117,25 @@ jobs:
       - store_test_results:
           path: ui/test-results
 
+  test-ui-browserstack:
+    docker:
+      - image: *NODE_IMAGE
+    steps:
+      - checkout
+      - restore_cache:
+          key: *YARN_LOCK_CACHE_KEY
+      - attach_workspace:
+          at: .
+      - run:
+          name: Run BrowserStack tests
+          command: |
+            set -eux -o pipefail
+
+            # Add ./bin to the PATH so vault binary can be run by Ember tests
+            export PATH="${PWD}"/bin:${PATH}
+
+            make test-ui-browserstack
+
   test-go:
     machine: true
     environment:
@@ -203,3 +222,7 @@ workflows:
       - test-go:
           requires:
             - build-go-dev
+      - test-ui-browserstack:
+          requires:
+            - install-ui-dependencies
+            - build-go-dev

.travis.yml

@@ -39,7 +39,7 @@ branches:
 
 env:
   - TEST_COMMAND='make dev test-ember'
-  - TEST_COMMAND='make dev ember-ci-test'
+  - TEST_COMMAND='make dev test-ui-browserstack'
   - TEST_COMMAND='travis_wait 75 make testtravis'
   - TEST_COMMAND='travis_wait 75 make testracetravis'
   - GO111MODULE=on

Makefile

@@ -128,7 +128,20 @@ test-ember:
 	@echo "--> Running ember tests"
 	@cd ui && yarn run test-oss
 
-ember-ci-test:
+ember-ci-test: # Deprecated, to be removed soon.
+	@echo "ember-ci-test is deprecated in favour of test-ui-browserstack"
+	@exit 1
+
+check-vault-in-path:
+	@VAULT_BIN=$$(command -v vault) || { echo "vault command not found"; exit 1; }; \
+		[ -x "$$VAULT_BIN" ] || { echo "$$VAULT_BIN not executable"; exit 1; }; \
+		printf "Using Vault at %s:\n\$$ vault version\n%s\n" "$$VAULT_BIN" "$$(vault version)"
+
+check-browserstack-creds:
+	@[ -n "$$BROWSERSTACK_ACCESS_KEY" ] || { echo "BROWSERSTACK_ACCESS_KEY not set"; exit 1; }
+	@[ -n "$$BROWSERSTACK_USERNAME" ] || { echo "BROWSERSTACK_USERNAME not set"; exit 1; }
+
+test-ui-browserstack: check-vault-in-path check-browserstack-creds
 	@echo "--> Installing JavaScript assets"
 	@cd ui && yarn --ignore-optional
 	@echo "--> Running ember tests in Browserstack"
@@ -204,6 +217,6 @@ hana-database-plugin:
 mongodb-database-plugin:
 	@CGO_ENABLED=0 go build -o bin/mongodb-database-plugin ./plugins/database/mongodb/mongodb-database-plugin
 
-.PHONY: bin default prep test vet bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin static-assets ember-dist ember-dist-dev static-dist static-dist-dev assetcheck
+.PHONY: bin default prep test vet bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin static-assets ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path check-browserstack-creds test-ui-browserstack
 
 .NOTPARALLEL: ember-dist ember-dist-dev static-assets

ui/scripts/run-browserstack-tests.js

@@ -16,8 +16,8 @@ function run(command, args = []) {
 }
 
 (async function() {
-  await run('ember', ['browserstack:connect']);
   try {
+    await run('ember', ['browserstack:connect']);
     try {
       await run('ember', ['test', '-f=secrets/secret/create', '-c', 'testem.browserstack.js']);