diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 197f045b7..2119ed619 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -92,7 +92,7 @@ jobs:
       - name: Compile docs and zip them up
         run: |
           poetry install
-          poetry run -- mkdocs build --no-directory-urls
+          poetry run -- mkdocs build -f mkdocs-offline.yml
           archive="jj-${{ github.event.release.tag_name }}-docs-html.tar.gz"
           tar czf "$archive" -C "rendered-docs" .
           echo "ASSET=$archive" >> $GITHUB_ENV
diff --git a/mkdocs-offline.yml b/mkdocs-offline.yml
new file mode 100644
index 000000000..e9c3ca177
--- /dev/null
+++ b/mkdocs-offline.yml
@@ -0,0 +1,10 @@
+# This config is good if you plan to use the rendered docs from
+# your file system. To use, run:
+#      poetry run -- mkdocs -f mkdocs-offline.yml
+INHERIT: 'mkdocs.yml'
+plugins:
+    - offline
+# Turns out the `offline` plugin forces the following
+# option no matter what, and therefore shouldn't be
+# used in the main config file.
+use_directory_urls: false
diff --git a/mkdocs.yml b/mkdocs.yml
index f9ebcb1e4..8601d129e 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -12,7 +12,6 @@ extra:
   version:
     provider: mike
 plugins:
-    - offline
     - search
     - redirects:
         redirect_maps: