diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3529a879..1cda826e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -117,7 +117,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        version: ["1.67.0", beta, nightly]
+        version: ["1.67.0", beta, "nightly-2024-05-23"]
     steps:
       - uses: actions/checkout@v4
 
@@ -137,7 +137,7 @@ jobs:
         run: just no-default-features
 
       - name: just build-bench
-        if: matrix.version == 'nightly'
+        if: contains( matrix.version, 'nightly' )
         run: just build-bench
 
   ## Execute the clippy checks
diff --git a/justfile b/justfile
index c576193e..8f8fc746 100644
--- a/justfile
+++ b/justfile
@@ -3,6 +3,8 @@ export TARGET_DIR := join(justfile_directory(), "target")
 export TDNS_BIND_PATH := join(TARGET_DIR, "bind")
 export TEST_DATA := join(join(justfile_directory(), "tests"), "test-data")
 
+NIGHTLY_DATE := "2024-05-23"
+
 ## MSRV
 MSRV := env_var_or_default('MSRV', "")
 
@@ -71,7 +73,7 @@ compatibility: init-bind9
 
 # Build all bench marking tools, i.e. check that they work, but don't run
 build-bench:
-    cargo ws exec cargo +nightly bench --no-run
+    cargo ws exec cargo +nightly-{{NIGHTLY_DATE}} bench --no-run
 
 [private]
 clippy-inner feature='':