diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 241b95a88..470bb5efe 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -2,6 +2,7 @@
 name = "jj-cli"
 description = "Jujutsu - an experimental version control system"
 default-run = "jj"
+autotests = false
 
 version = { workspace = true }
 edition = { workspace = true }
@@ -26,6 +27,9 @@ name = "fake-diff-editor"
 path = "testing/fake-diff-editor.rs"
 required-features = ["test-fakes"]
 
+[[test]]
+name = "runner"
+
 [build-dependencies]
 cargo_metadata = { workspace = true }
 
diff --git a/cli/tests/common/mod.rs b/cli/tests/common/mod.rs
index 4b768782e..6197fae50 100644
--- a/cli/tests/common/mod.rs
+++ b/cli/tests/common/mod.rs
@@ -19,7 +19,6 @@ use std::path::{Path, PathBuf};
 use itertools::Itertools as _;
 use regex::{Captures, Regex};
 use tempfile::TempDir;
-use testutils;
 
 pub struct TestEnvironment {
     _temp_dir: TempDir,
diff --git a/cli/tests/runner.rs b/cli/tests/runner.rs
new file mode 100644
index 000000000..dcd71c016
--- /dev/null
+++ b/cli/tests/runner.rs
@@ -0,0 +1,68 @@
+use std::path::PathBuf;
+
+mod common;
+
+#[test]
+fn test_no_forgotten_test_files() {
+    let test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests");
+    testutils::assert_no_forgotten_test_files(&test_dir);
+}
+
+mod test_abandon_command;
+mod test_alias;
+mod test_branch_command;
+mod test_builtin_aliases;
+mod test_cat_command;
+mod test_checkout;
+mod test_chmod_command;
+mod test_commit_command;
+mod test_commit_template;
+mod test_concurrent_operations;
+mod test_config_command;
+mod test_debug_command;
+mod test_describe_command;
+mod test_diff_command;
+mod test_diffedit_command;
+mod test_duplicate_command;
+mod test_edit_command;
+mod test_generate_md_cli_help;
+mod test_git_clone;
+mod test_git_colocated;
+mod test_git_fetch;
+mod test_git_import_export;
+mod test_git_init;
+mod test_git_push;
+mod test_git_remotes;
+mod test_git_submodule;
+mod test_gitignores;
+mod test_global_opts;
+mod test_immutable_commits;
+mod test_init_command;
+mod test_interdiff_command;
+mod test_log_command;
+mod test_move_command;
+mod test_new_command;
+mod test_next_prev_commands;
+mod test_obslog_command;
+mod test_operations;
+mod test_rebase_command;
+mod test_repo_change_report;
+mod test_resolve_command;
+mod test_restore_command;
+mod test_revset_output;
+mod test_root;
+mod test_shell_completion;
+mod test_show_command;
+mod test_sparse_command;
+mod test_split_command;
+mod test_squash_command;
+mod test_status_command;
+mod test_tag_command;
+mod test_templater;
+mod test_tree_level_conflicts;
+mod test_undo;
+mod test_unsquash_command;
+mod test_untrack_command;
+mod test_util_command;
+mod test_working_copy;
+mod test_workspaces;
diff --git a/cli/tests/test_abandon_command.rs b/cli/tests/test_abandon_command.rs
index 51d692f23..840899814 100644
--- a/cli/tests/test_abandon_command.rs
+++ b/cli/tests/test_abandon_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn create_commit(test_env: &TestEnvironment, repo_path: &Path, name: &str, parents: &[&str]) {
     if parents.is_empty() {
         test_env.jj_cmd_ok(repo_path, &["new", "root()", "-m", name]);
diff --git a/cli/tests/test_alias.rs b/cli/tests/test_alias.rs
index 74b727327..712e25f91 100644
--- a/cli/tests/test_alias.rs
+++ b/cli/tests/test_alias.rs
@@ -18,8 +18,6 @@ use itertools::Itertools as _;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_alias_basic() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_branch_command.rs b/cli/tests/test_branch_command.rs
index 9e30bc153..e957e6d66 100644
--- a/cli/tests/test_branch_command.rs
+++ b/cli/tests/test_branch_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_branch_multiple_names() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_builtin_aliases.rs b/cli/tests/test_builtin_aliases.rs
index 4f36ae550..b8c1c8477 100644
--- a/cli/tests/test_builtin_aliases.rs
+++ b/cli/tests/test_builtin_aliases.rs
@@ -14,9 +14,7 @@
 
 use std::path::PathBuf;
 
-use common::TestEnvironment;
-
-pub mod common;
+use crate::common::TestEnvironment;
 
 fn set_up(trunk_name: &str) -> (TestEnvironment, PathBuf) {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_cat_command.rs b/cli/tests/test_cat_command.rs
index 07ec1722d..8c093bbb6 100644
--- a/cli/tests/test_cat_command.rs
+++ b/cli/tests/test_cat_command.rs
@@ -14,8 +14,6 @@
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_cat() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_checkout.rs b/cli/tests/test_checkout.rs
index 1fa440cdd..b52e065a0 100644
--- a/cli/tests/test_checkout.rs
+++ b/cli/tests/test_checkout.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_checkout() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_chmod_command.rs b/cli/tests/test_chmod_command.rs
index eab255e2a..ac4fbeb26 100644
--- a/cli/tests/test_chmod_command.rs
+++ b/cli/tests/test_chmod_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn create_commit(
     test_env: &TestEnvironment,
     repo_path: &Path,
diff --git a/cli/tests/test_commit_command.rs b/cli/tests/test_commit_command.rs
index 5d1e2ecce..233859e99 100644
--- a/cli/tests/test_commit_command.rs
+++ b/cli/tests/test_commit_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_commit_with_description_from_cli() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_commit_template.rs b/cli/tests/test_commit_template.rs
index 6bb6a74be..663992315 100644
--- a/cli/tests/test_commit_template.rs
+++ b/cli/tests/test_commit_template.rs
@@ -12,10 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::TestEnvironment;
 use regex::Regex;
 
-pub mod common;
+use crate::common::TestEnvironment;
 
 #[test]
 fn test_log_parents() {
diff --git a/cli/tests/test_concurrent_operations.rs b/cli/tests/test_concurrent_operations.rs
index b247e2e3a..12136e3e8 100644
--- a/cli/tests/test_concurrent_operations.rs
+++ b/cli/tests/test_concurrent_operations.rs
@@ -18,8 +18,6 @@ use itertools::Itertools as _;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_concurrent_operation_divergence() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_config_command.rs b/cli/tests/test_config_command.rs
index 8d4540368..483c70ad7 100644
--- a/cli/tests/test_config_command.rs
+++ b/cli/tests/test_config_command.rs
@@ -18,8 +18,6 @@ use regex::Regex;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_config_list_single() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_debug_command.rs b/cli/tests/test_debug_command.rs
index e60ad43b3..636dc659a 100644
--- a/cli/tests/test_debug_command.rs
+++ b/cli/tests/test_debug_command.rs
@@ -17,8 +17,6 @@ use regex::Regex;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_debug_revset() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_describe_command.rs b/cli/tests/test_describe_command.rs
index 4effd8210..1a6fc1126 100644
--- a/cli/tests/test_describe_command.rs
+++ b/cli/tests/test_describe_command.rs
@@ -14,8 +14,6 @@
 
 use crate::common::{get_stderr_string, TestEnvironment};
 
-pub mod common;
-
 #[test]
 fn test_describe() {
     let mut test_env = TestEnvironment::default();
diff --git a/cli/tests/test_diff_command.rs b/cli/tests/test_diff_command.rs
index ad3beb89a..93063b0b0 100644
--- a/cli/tests/test_diff_command.rs
+++ b/cli/tests/test_diff_command.rs
@@ -12,10 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::TestEnvironment;
 use itertools::Itertools;
 
-pub mod common;
+use crate::common::{escaped_fake_diff_editor_path, TestEnvironment};
 
 #[test]
 fn test_diff_basic() {
@@ -720,7 +719,7 @@ fn test_diff_external_tool() {
     "###);
 
     // Inlined command arguments
-    let command = common::escaped_fake_diff_editor_path();
+    let command = escaped_fake_diff_editor_path();
     let config = format!(r#"--config-toml=ui.diff.tool=["{command}", "$right", "$left"]"#);
     insta::assert_snapshot!(test_env.jj_cmd_success(&repo_path, &["diff", &config]), @r###"
     file2
diff --git a/cli/tests/test_diffedit_command.rs b/cli/tests/test_diffedit_command.rs
index 31a64df6f..3c7b93790 100644
--- a/cli/tests/test_diffedit_command.rs
+++ b/cli/tests/test_diffedit_command.rs
@@ -14,8 +14,6 @@
 
 use crate::common::{escaped_fake_diff_editor_path, TestEnvironment};
 
-pub mod common;
-
 #[test]
 fn test_diffedit() {
     let mut test_env = TestEnvironment::default();
diff --git a/cli/tests/test_duplicate_command.rs b/cli/tests/test_duplicate_command.rs
index c37912e80..efce45ed5 100644
--- a/cli/tests/test_duplicate_command.rs
+++ b/cli/tests/test_duplicate_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn create_commit(test_env: &TestEnvironment, repo_path: &Path, name: &str, parents: &[&str]) {
     if parents.is_empty() {
         test_env.jj_cmd_ok(repo_path, &["new", "root()", "-m", name]);
diff --git a/cli/tests/test_edit_command.rs b/cli/tests/test_edit_command.rs
index 422e4b442..89adb32ef 100644
--- a/cli/tests/test_edit_command.rs
+++ b/cli/tests/test_edit_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_edit() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_generate_md_cli_help.rs b/cli/tests/test_generate_md_cli_help.rs
index ba7b46d86..4b69a8eb2 100644
--- a/cli/tests/test_generate_md_cli_help.rs
+++ b/cli/tests/test_generate_md_cli_help.rs
@@ -16,8 +16,6 @@ use insta::assert_snapshot;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 const PREAMBLE: &str = r#"
 !!! warning
 
diff --git a/cli/tests/test_git_clone.rs b/cli/tests/test_git_clone.rs
index 93fd3360b..d77a2fb2a 100644
--- a/cli/tests/test_git_clone.rs
+++ b/cli/tests/test_git_clone.rs
@@ -14,9 +14,7 @@
 
 use std::path::{self, Path, PathBuf};
 
-use crate::common::TestEnvironment;
-
-pub mod common;
+use crate::common::{get_stderr_string, get_stdout_string, TestEnvironment};
 
 fn set_up_non_empty_git_repo(git_repo: &git2::Repository) {
     let signature =
@@ -85,8 +83,8 @@ fn test_git_clone() {
         .jj_cmd(test_env.env_root(), &["git", "clone", "bad", "failed"])
         .assert()
         .code(1);
-    let stdout = test_env.normalize_output(&common::get_stdout_string(&assert));
-    let stderr = test_env.normalize_output(&common::get_stderr_string(&assert));
+    let stdout = test_env.normalize_output(&get_stdout_string(&assert));
+    let stderr = test_env.normalize_output(&get_stderr_string(&assert));
     insta::assert_snapshot!(stdout, @"");
     insta::assert_snapshot!(stderr, @r###"
     Fetching into new repo in "$TEST_ENV/failed"
@@ -100,8 +98,8 @@ fn test_git_clone() {
         .jj_cmd(test_env.env_root(), &["git", "clone", "bad", "failed"])
         .assert()
         .code(1);
-    let stdout = test_env.normalize_output(&common::get_stdout_string(&assert));
-    let stderr = test_env.normalize_output(&common::get_stderr_string(&assert));
+    let stdout = test_env.normalize_output(&get_stdout_string(&assert));
+    let stderr = test_env.normalize_output(&get_stderr_string(&assert));
     insta::assert_snapshot!(stdout, @"");
     insta::assert_snapshot!(stderr, @r###"
     Fetching into new repo in "$TEST_ENV/failed"
@@ -239,8 +237,8 @@ fn test_git_clone_colocate() {
         )
         .assert()
         .code(1);
-    let stdout = test_env.normalize_output(&common::get_stdout_string(&assert));
-    let stderr = test_env.normalize_output(&common::get_stderr_string(&assert));
+    let stdout = test_env.normalize_output(&get_stdout_string(&assert));
+    let stderr = test_env.normalize_output(&get_stderr_string(&assert));
     insta::assert_snapshot!(stdout, @"");
     insta::assert_snapshot!(stderr, @r###"
     Fetching into new repo in "$TEST_ENV/failed"
@@ -257,8 +255,8 @@ fn test_git_clone_colocate() {
         )
         .assert()
         .code(1);
-    let stdout = test_env.normalize_output(&common::get_stdout_string(&assert));
-    let stderr = test_env.normalize_output(&common::get_stderr_string(&assert));
+    let stdout = test_env.normalize_output(&get_stdout_string(&assert));
+    let stderr = test_env.normalize_output(&get_stderr_string(&assert));
     insta::assert_snapshot!(stdout, @"");
     insta::assert_snapshot!(stderr, @r###"
     Fetching into new repo in "$TEST_ENV/failed"
diff --git a/cli/tests/test_git_colocated.rs b/cli/tests/test_git_colocated.rs
index 7a1a9e8d9..d59bbf459 100644
--- a/cli/tests/test_git_colocated.rs
+++ b/cli/tests/test_git_colocated.rs
@@ -18,8 +18,6 @@ use git2::Oid;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_git_colocated() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_git_fetch.rs b/cli/tests/test_git_fetch.rs
index 073b7dc5f..7b45f8faf 100644
--- a/cli/tests/test_git_fetch.rs
+++ b/cli/tests/test_git_fetch.rs
@@ -15,8 +15,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 /// Creates a remote Git repo containing a branch with the same name
 fn init_git_remote(test_env: &TestEnvironment, remote: &str) {
     let git_repo_path = test_env.env_root().join(remote);
diff --git a/cli/tests/test_git_import_export.rs b/cli/tests/test_git_import_export.rs
index b64c275eb..74953d0f2 100644
--- a/cli/tests/test_git_import_export.rs
+++ b/cli/tests/test_git_import_export.rs
@@ -18,8 +18,6 @@ use jj_lib::backend::CommitId;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_resolution_of_git_tracking_branches() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_git_init.rs b/cli/tests/test_git_init.rs
index c9ae09cf2..08e85e8b5 100644
--- a/cli/tests/test_git_init.rs
+++ b/cli/tests/test_git_init.rs
@@ -19,8 +19,6 @@ use test_case::test_case;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn init_git_repo(git_repo_path: &Path, bare: bool) -> git2::Repository {
     init_git_repo_with_opts(git_repo_path, git2::RepositoryInitOptions::new().bare(bare))
 }
diff --git a/cli/tests/test_git_push.rs b/cli/tests/test_git_push.rs
index 86592e4d6..8fedaf6a4 100644
--- a/cli/tests/test_git_push.rs
+++ b/cli/tests/test_git_push.rs
@@ -16,8 +16,6 @@ use std::path::PathBuf;
 
 use crate::common::{get_stderr_string, get_stdout_string, TestEnvironment};
 
-pub mod common;
-
 fn set_up() -> (TestEnvironment, PathBuf) {
     let test_env = TestEnvironment::default();
     test_env.jj_cmd_ok(test_env.env_root(), &["init", "--git", "origin"]);
diff --git a/cli/tests/test_git_remotes.rs b/cli/tests/test_git_remotes.rs
index 7d4993b56..cf8dc5a3a 100644
--- a/cli/tests/test_git_remotes.rs
+++ b/cli/tests/test_git_remotes.rs
@@ -16,8 +16,6 @@ use std::fs;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_git_remotes() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_git_submodule.rs b/cli/tests/test_git_submodule.rs
index 252b0253e..a09da75ac 100644
--- a/cli/tests/test_git_submodule.rs
+++ b/cli/tests/test_git_submodule.rs
@@ -14,8 +14,6 @@
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_gitsubmodule_print_gitmodules() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_gitignores.rs b/cli/tests/test_gitignores.rs
index 4148792cf..486a8e688 100644
--- a/cli/tests/test_gitignores.rs
+++ b/cli/tests/test_gitignores.rs
@@ -16,8 +16,6 @@ use std::io::Write;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_gitignores() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_global_opts.rs b/cli/tests/test_global_opts.rs
index 1d0ab065a..4fde6eef1 100644
--- a/cli/tests/test_global_opts.rs
+++ b/cli/tests/test_global_opts.rs
@@ -16,8 +16,6 @@ use std::ffi::OsString;
 
 use crate::common::{get_stderr_string, TestEnvironment};
 
-pub mod common;
-
 #[test]
 fn test_non_utf8_arg() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_immutable_commits.rs b/cli/tests/test_immutable_commits.rs
index 8db6dc8c1..66dfb0e7d 100644
--- a/cli/tests/test_immutable_commits.rs
+++ b/cli/tests/test_immutable_commits.rs
@@ -14,8 +14,6 @@
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_rewrite_immutable_generic() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_init_command.rs b/cli/tests/test_init_command.rs
index 72d1c2329..fd8921eb0 100644
--- a/cli/tests/test_init_command.rs
+++ b/cli/tests/test_init_command.rs
@@ -18,8 +18,6 @@ use test_case::test_case;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn init_git_repo(git_repo_path: &Path, bare: bool) -> git2::Repository {
     init_git_repo_with_opts(git_repo_path, git2::RepositoryInitOptions::new().bare(bare))
 }
diff --git a/cli/tests/test_interdiff_command.rs b/cli/tests/test_interdiff_command.rs
index 3d2bf3009..67257be91 100644
--- a/cli/tests/test_interdiff_command.rs
+++ b/cli/tests/test_interdiff_command.rs
@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::TestEnvironment;
-
-pub mod common;
+use crate::common::TestEnvironment;
 
 #[test]
 fn test_interdiff_basic() {
diff --git a/cli/tests/test_log_command.rs b/cli/tests/test_log_command.rs
index ae66b395a..36ab07097 100644
--- a/cli/tests/test_log_command.rs
+++ b/cli/tests/test_log_command.rs
@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::{get_stdout_string, TestEnvironment};
-
-pub mod common;
+use crate::common::{get_stdout_string, TestEnvironment};
 
 #[test]
 fn test_log_with_empty_revision() {
diff --git a/cli/tests/test_move_command.rs b/cli/tests/test_move_command.rs
index 9d103d361..24a635658 100644
--- a/cli/tests/test_move_command.rs
+++ b/cli/tests/test_move_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_move() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_new_command.rs b/cli/tests/test_new_command.rs
index e269cc2c4..635ed6ff8 100644
--- a/cli/tests/test_new_command.rs
+++ b/cli/tests/test_new_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_new() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_next_prev_commands.rs b/cli/tests/test_next_prev_commands.rs
index 05d14f015..dbf1ad7a8 100644
--- a/cli/tests/test_next_prev_commands.rs
+++ b/cli/tests/test_next_prev_commands.rs
@@ -15,8 +15,6 @@
 
 use crate::common::{get_stderr_string, get_stdout_string, TestEnvironment};
 
-pub mod common;
-
 #[test]
 fn test_next_simple() {
     // Move from first => second.
diff --git a/cli/tests/test_obslog_command.rs b/cli/tests/test_obslog_command.rs
index e6a235a65..296918b05 100644
--- a/cli/tests/test_obslog_command.rs
+++ b/cli/tests/test_obslog_command.rs
@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::{get_stdout_string, TestEnvironment};
-
-pub mod common;
+use crate::common::{get_stdout_string, TestEnvironment};
 
 #[test]
 fn test_obslog_with_or_without_diff() {
diff --git a/cli/tests/test_operations.rs b/cli/tests/test_operations.rs
index f2a13a0f3..f77e691d8 100644
--- a/cli/tests/test_operations.rs
+++ b/cli/tests/test_operations.rs
@@ -19,8 +19,6 @@ use regex::Regex;
 
 use crate::common::{get_stdout_string, TestEnvironment};
 
-pub mod common;
-
 #[test]
 fn test_op_log() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_rebase_command.rs b/cli/tests/test_rebase_command.rs
index 38e79640a..bab009513 100644
--- a/cli/tests/test_rebase_command.rs
+++ b/cli/tests/test_rebase_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn create_commit(test_env: &TestEnvironment, repo_path: &Path, name: &str, parents: &[&str]) {
     if parents.is_empty() {
         test_env.jj_cmd_ok(repo_path, &["new", "root()", "-m", name]);
diff --git a/cli/tests/test_repo_change_report.rs b/cli/tests/test_repo_change_report.rs
index 6c0762822..d8ea91bd8 100644
--- a/cli/tests/test_repo_change_report.rs
+++ b/cli/tests/test_repo_change_report.rs
@@ -14,8 +14,6 @@
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_report_conflicts() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_resolve_command.rs b/cli/tests/test_resolve_command.rs
index 5698ad0e4..a95656cd7 100644
--- a/cli/tests/test_resolve_command.rs
+++ b/cli/tests/test_resolve_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn create_commit(
     test_env: &TestEnvironment,
     repo_path: &Path,
diff --git a/cli/tests/test_restore_command.rs b/cli/tests/test_restore_command.rs
index 6dd07dbc1..c3d8792ed 100644
--- a/cli/tests/test_restore_command.rs
+++ b/cli/tests/test_restore_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_restore() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_revset_output.rs b/cli/tests/test_revset_output.rs
index 80db52d81..2e0322e58 100644
--- a/cli/tests/test_revset_output.rs
+++ b/cli/tests/test_revset_output.rs
@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::TestEnvironment;
-
-pub mod common;
+use crate::common::TestEnvironment;
 
 #[test]
 fn test_syntax_error() {
diff --git a/cli/tests/test_root.rs b/cli/tests/test_root.rs
index 6870eae24..352d73c73 100644
--- a/cli/tests/test_root.rs
+++ b/cli/tests/test_root.rs
@@ -19,8 +19,6 @@ use testutils::{TestRepoBackend, TestWorkspace};
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test_case(TestRepoBackend::Local ; "local backend")]
 #[test_case(TestRepoBackend::Git ; "git backend")]
 fn test_root(backend: TestRepoBackend) {
diff --git a/cli/tests/test_shell_completion.rs b/cli/tests/test_shell_completion.rs
index eb96a0be7..1314bd807 100644
--- a/cli/tests/test_shell_completion.rs
+++ b/cli/tests/test_shell_completion.rs
@@ -16,8 +16,6 @@ use insta::assert_snapshot;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_deprecated_flags() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_show_command.rs b/cli/tests/test_show_command.rs
index 62fa85bbb..5a8570daf 100644
--- a/cli/tests/test_show_command.rs
+++ b/cli/tests/test_show_command.rs
@@ -12,11 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::TestEnvironment;
 use itertools::Itertools;
 use regex::Regex;
 
-pub mod common;
+use crate::common::TestEnvironment;
 
 #[test]
 fn test_show() {
diff --git a/cli/tests/test_sparse_command.rs b/cli/tests/test_sparse_command.rs
index 2b7e35d7e..b40fd66ef 100644
--- a/cli/tests/test_sparse_command.rs
+++ b/cli/tests/test_sparse_command.rs
@@ -16,8 +16,6 @@ use std::io::Write;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_sparse_manage_patterns() {
     let mut test_env = TestEnvironment::default();
diff --git a/cli/tests/test_split_command.rs b/cli/tests/test_split_command.rs
index 42ebc0e28..65b0c64e2 100644
--- a/cli/tests/test_split_command.rs
+++ b/cli/tests/test_split_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_split_by_paths() {
     let mut test_env = TestEnvironment::default();
diff --git a/cli/tests/test_squash_command.rs b/cli/tests/test_squash_command.rs
index 9ea7aab08..a37b09ed9 100644
--- a/cli/tests/test_squash_command.rs
+++ b/cli/tests/test_squash_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_squash() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_status_command.rs b/cli/tests/test_status_command.rs
index 5566e94c9..574dda071 100644
--- a/cli/tests/test_status_command.rs
+++ b/cli/tests/test_status_command.rs
@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::TestEnvironment;
-
-pub mod common;
+use crate::common::TestEnvironment;
 
 #[test]
 fn test_status_merge() {
diff --git a/cli/tests/test_tag_command.rs b/cli/tests/test_tag_command.rs
index 96d9f4e22..407c5cf95 100644
--- a/cli/tests/test_tag_command.rs
+++ b/cli/tests/test_tag_command.rs
@@ -14,8 +14,6 @@
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 fn set_up_tagged_git_repo(git_repo: &git2::Repository) {
     let signature =
         git2::Signature::new("Some One", "some.one@example.com", &git2::Time::new(0, 0)).unwrap();
diff --git a/cli/tests/test_templater.rs b/cli/tests/test_templater.rs
index 23916039e..22bfd011b 100644
--- a/cli/tests/test_templater.rs
+++ b/cli/tests/test_templater.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_templater_parse_error() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_tree_level_conflicts.rs b/cli/tests/test_tree_level_conflicts.rs
index fbde55f29..8480a7099 100644
--- a/cli/tests/test_tree_level_conflicts.rs
+++ b/cli/tests/test_tree_level_conflicts.rs
@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common::TestEnvironment;
-
-pub mod common;
+use crate::common::TestEnvironment;
 
 #[test]
 fn test_enable_tree_level_conflicts() {
diff --git a/cli/tests/test_undo.rs b/cli/tests/test_undo.rs
index 05ea89414..12e7b9a2b 100644
--- a/cli/tests/test_undo.rs
+++ b/cli/tests/test_undo.rs
@@ -15,8 +15,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_undo_rewrite_with_child() {
     // Test that if we undo an operation that rewrote some commit, any descendants
diff --git a/cli/tests/test_unsquash_command.rs b/cli/tests/test_unsquash_command.rs
index 1b00df58b..a6b160625 100644
--- a/cli/tests/test_unsquash_command.rs
+++ b/cli/tests/test_unsquash_command.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_unsquash() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_untrack_command.rs b/cli/tests/test_untrack_command.rs
index b94da7f6b..78804b59c 100644
--- a/cli/tests/test_untrack_command.rs
+++ b/cli/tests/test_untrack_command.rs
@@ -16,8 +16,6 @@ use std::path::PathBuf;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_untrack() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_util_command.rs b/cli/tests/test_util_command.rs
index 352050030..93b3c664a 100644
--- a/cli/tests/test_util_command.rs
+++ b/cli/tests/test_util_command.rs
@@ -16,8 +16,6 @@ use insta::assert_snapshot;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_util_config_schema() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_working_copy.rs b/cli/tests/test_working_copy.rs
index 8d313db26..e54437b28 100644
--- a/cli/tests/test_working_copy.rs
+++ b/cli/tests/test_working_copy.rs
@@ -14,8 +14,6 @@
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 #[test]
 fn test_snapshot_large_file() {
     let test_env = TestEnvironment::default();
diff --git a/cli/tests/test_workspaces.rs b/cli/tests/test_workspaces.rs
index de50925e6..d4fc31270 100644
--- a/cli/tests/test_workspaces.rs
+++ b/cli/tests/test_workspaces.rs
@@ -16,8 +16,6 @@ use std::path::Path;
 
 use crate::common::TestEnvironment;
 
-pub mod common;
-
 /// Test adding a second workspace
 #[test]
 fn test_workspaces_add_second_workspace() {
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index 7aad8b420..7a480cd05 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -1,6 +1,7 @@
 [package]
 name = "jj-lib"
 description = "Library for Jujutsu - an experimental version control system"
+autotests = false
 
 version = { workspace = true }
 edition = { workspace = true }
@@ -11,6 +12,9 @@ repository = { workspace = true }
 documentation = { workspace = true }
 readme = { workspace = true }
 
+[[test]]
+name = "runner"
+
 [[bench]]
 name = "diff_bench"
 harness = false
diff --git a/lib/tests/runner.rs b/lib/tests/runner.rs
new file mode 100644
index 000000000..e9d76cde4
--- /dev/null
+++ b/lib/tests/runner.rs
@@ -0,0 +1,33 @@
+use std::path::PathBuf;
+
+#[test]
+fn test_no_forgotten_test_files() {
+    let test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests");
+    testutils::assert_no_forgotten_test_files(&test_dir);
+}
+
+mod test_bad_locking;
+mod test_commit_builder;
+mod test_commit_concurrent;
+mod test_conflicts;
+mod test_default_revset_graph_iterator;
+mod test_diff_summary;
+mod test_git;
+mod test_git_backend;
+mod test_id_prefix;
+mod test_index;
+mod test_init;
+mod test_load_repo;
+mod test_local_working_copy;
+mod test_local_working_copy_concurrent;
+mod test_local_working_copy_sparse;
+mod test_merge_trees;
+mod test_merged_tree;
+mod test_mut_repo;
+mod test_operations;
+mod test_refs;
+mod test_revset;
+mod test_rewrite;
+mod test_signing;
+mod test_view;
+mod test_workspace;
diff --git a/lib/testutils/src/lib.rs b/lib/testutils/src/lib.rs
index 950c20c4d..fdd871aa5 100644
--- a/lib/testutils/src/lib.rs
+++ b/lib/testutils/src/lib.rs
@@ -505,3 +505,23 @@ pub fn assert_abandoned_with_parent(
     );
     new_parent_commit
 }
+
+pub fn assert_no_forgotten_test_files(test_dir: &Path) {
+    let runner_path = test_dir.join("runner.rs");
+    let runner = fs::read_to_string(&runner_path).unwrap();
+    let entries = fs::read_dir(test_dir).unwrap();
+    for entry in entries {
+        let path = entry.unwrap().path();
+        if let Some(ext) = path.extension() {
+            let name = path.file_stem().unwrap();
+            if ext == "rs" && name != "runner" {
+                let search = format!("mod {};", name.to_str().unwrap());
+                assert!(
+                    runner.contains(&search),
+                    "missing `{search}` declaration in {}",
+                    runner_path.display()
+                );
+            }
+        }
+    }
+}