diff --git a/Cargo.lock b/Cargo.lock
index ce7e025f5..1e6a4aa4d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -251,6 +251,15 @@ dependencies = [
  "itertools",
 ]
 
+[[package]]
+name = "criterion_bencher_compat"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdb92ac87446e7097e0f429c0bb509a1040bfae23bc0320acac30d9fec23d1ed"
+dependencies = [
+ "criterion",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.1"
@@ -558,6 +567,7 @@ dependencies = [
  "clap_complete",
  "config",
  "criterion",
+ "criterion_bencher_compat",
  "git2",
  "hex",
  "indoc",
diff --git a/Cargo.toml b/Cargo.toml
index 5cdc15916..e06983678 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,6 +16,10 @@ categories = ["command-line-utilities", "development-tools"]
 name = "jj"
 path = "src/main.rs"
 
+[[bench]]
+name = "diff_bench"
+harness = false
+
 [workspace]
 members = ["lib"]
 
@@ -42,3 +46,5 @@ thiserror = "1.0.30"
 [dev-dependencies]
 test-case = "1.2.3"
 regex = "1.5.4"
+criterion = "0.3.5"
+criterion_bencher_compat = "0.3.4"
diff --git a/lib/benches/diff_bench.rs b/benches/diff_bench.rs
similarity index 92%
rename from lib/benches/diff_bench.rs
rename to benches/diff_bench.rs
index 09c82f36a..3cd4217d1 100644
--- a/lib/benches/diff_bench.rs
+++ b/benches/diff_bench.rs
@@ -1,9 +1,7 @@
-#![feature(test)]
-
-extern crate test;
+use bencher::{benchmark_group, benchmark_main, Bencher};
+use criterion_bencher_compat as bencher;
 
 use jujutsu_lib::diff;
-use test::Bencher;
 
 fn unchanged_lines(count: usize) -> (String, String) {
     let mut lines = vec![];
@@ -33,43 +31,36 @@ fn reversed_lines(count: usize) -> (String, String) {
     (left_lines.join(""), right_lines.join(""))
 }
 
-#[bench]
 fn bench_diff_1k_unchanged_lines(b: &mut Bencher) {
     let (left, right) = unchanged_lines(1000);
     b.iter(|| diff::diff(left.as_bytes(), right.as_bytes()));
 }
 
-#[bench]
 fn bench_diff_10k_unchanged_lines(b: &mut Bencher) {
     let (left, right) = unchanged_lines(10000);
     b.iter(|| diff::diff(left.as_bytes(), right.as_bytes()));
 }
 
-#[bench]
 fn bench_diff_1k_modified_lines(b: &mut Bencher) {
     let (left, right) = modified_lines(1000);
     b.iter(|| diff::diff(left.as_bytes(), right.as_bytes()));
 }
 
-#[bench]
 fn bench_diff_10k_modified_lines(b: &mut Bencher) {
     let (left, right) = modified_lines(10000);
     b.iter(|| diff::diff(left.as_bytes(), right.as_bytes()));
 }
 
-#[bench]
 fn bench_diff_1k_lines_reversed(b: &mut Bencher) {
     let (left, right) = reversed_lines(1000);
     b.iter(|| diff::diff(left.as_bytes(), right.as_bytes()));
 }
 
-#[bench]
 fn bench_diff_10k_lines_reversed(b: &mut Bencher) {
     let (left, right) = reversed_lines(10000);
     b.iter(|| diff::diff(left.as_bytes(), right.as_bytes()));
 }
 
-#[bench]
 fn bench_diff_git_git_read_tree_c(b: &mut Bencher) {
     b.iter(|| {
         diff::diff(
@@ -214,3 +205,15 @@ int main(int argc, char **argv)
         )
     });
 }
+
+benchmark_group!(
+    benches,
+    bench_diff_1k_unchanged_lines,
+    bench_diff_10k_unchanged_lines,
+    bench_diff_1k_modified_lines,
+    bench_diff_10k_modified_lines,
+    bench_diff_1k_lines_reversed,
+    bench_diff_10k_lines_reversed,
+    bench_diff_git_git_read_tree_c,
+);
+benchmark_main!(benches);