diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 71d8d2a2..4c6b3bb6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -129,15 +129,10 @@ jobs:
               run: |
                   direnv exec . engage > >(tee -a test_output.log)
 
-            - name: Sync Complement repository
-              uses: actions/checkout@v4
-              with:
-                repository: 'matrix-org/complement'
-                path: complement_src
-
             - name: Run Complement tests
               run: |
-                  direnv exec . bin/complement 'complement_src' 'complement_test_logs.jsonl' 'complement_test_results.jsonl'
+                  # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op
+                  direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log)
                   cp -v -f result complement_oci_image.tar.gz
 
             - name: Upload Complement OCI image
@@ -163,7 +158,7 @@ jobs:
 
             - name: Diff Complement results with checked-in repo results
               run: |
-                  diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log)
+                  diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log)
 
             - name: Update Job Summary
               if: success() || failure()
@@ -178,7 +173,7 @@ jobs:
 
                       echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY
                       echo '```diff' >> $GITHUB_STEP_SUMMARY
-                      tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >>     $GITHUB_STEP_SUMMARY
+                      tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >>     $GITHUB_STEP_SUMMARY
                       echo '```' >> $GITHUB_STEP_SUMMARY
                   fi
 
diff --git a/bin/complement b/bin/complement
index c679035a..a715b14b 100755
--- a/bin/complement
+++ b/bin/complement
@@ -7,7 +7,7 @@ set -euo pipefail
 # The `COMPLEMENT_SRC` environment variable is set in the Nix dev shell, which
 # points to a store path containing the Complement source code. It's likely you
 # want to just pass that as the first argument to use it here.
-COMPLEMENT_SRC="$1"
+COMPLEMENT_SRC="${COMPLEMENT_SRC:-$1}"
 
 # A `.jsonl` file to write test logs to
 LOG_FILE="$2"
@@ -17,12 +17,19 @@ RESULTS_FILE="$3"
 
 OCI_IMAGE="complement-conduit:main"
 
-# Complement tests that are skipped due to flakiness/reliability issues (likely
-# Complement itself induced based on various open issues)
-#
-# According to Go docs, these are separated by forward slashes and not pipes (why)
+# Complement tests that are skipped due to flakiness/reliability issues
 SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*'
 
+# $COMPLEMENT_SRC needs to be a directory to Complement source code
+if [ -f "$COMPLEMENT_SRC" ]; then
+    echo "\$COMPLEMENT_SRC must be a directory/path to Complement source code"
+    exit 1
+fi
+
+# quick test to make sure we can actually write to $LOG_FILE and $RESULTS_FILE
+touch $LOG_FILE && rm -v $LOG_FILE
+touch $RESULTS_FILE && rm -v $RESULTS_FILE
+
 toplevel="$(git rev-parse --show-toplevel)"
 
 pushd "$toplevel" > /dev/null