test: add fuzzing tests for gc snapshot

Cargo.lock

@@ -687,6 +687,7 @@ dependencies = [
  "color-backtrace",
  "ctor 0.2.6",
  "dev-utils",
+ "ensure-cov",
  "enum-as-inner 0.5.1",
  "enum_dispatch",
  "fxhash",
@@ -1161,6 +1162,7 @@ dependencies = [
  "dev-utils",
  "dhat",
  "either",
+ "ensure-cov",
  "enum-as-inner 0.5.1",
  "enum_dispatch",
  "fxhash",

crates/examples/benches/bench_text.rs

@@ -43,7 +43,7 @@ fn bench_text(c: &mut Criterion) {
                 doc
             },
             |doc| {
-                doc.export_fast_snapshot();
+                doc.export(loro::ExportMode::Snapshot);
             },
             criterion::BatchSize::SmallInput,
         )
@@ -53,7 +53,7 @@ fn bench_text(c: &mut Criterion) {
         b.iter_batched(
             || apply_text_actions(&actions, 1),
             |doc| {
-                doc.export_fast_snapshot();
+                doc.export(loro::ExportMode::Snapshot);
             },
             criterion::BatchSize::SmallInput,
         )
@@ -68,7 +68,7 @@ fn bench_text(c: &mut Criterion) {
                 }
                 if doc_snapshot.get().is_none() {
                     let doc = doc.get().unwrap();
-                    let snapshot = doc.export_fast_snapshot();
+                    let snapshot = doc.export(loro::ExportMode::Snapshot);
                     println!("B4 fast_snapshot size: {:?}", ByteSize(snapshot.len()));
                     doc_snapshot.set(snapshot).unwrap();
                 }
@@ -87,7 +87,7 @@ fn bench_text(c: &mut Criterion) {
             || {
                 if doc_x100_snapshot.get().is_none() {
                     let doc = apply_text_actions(&actions, 100);
-                    let snapshot = doc.export_fast_snapshot();
+                    let snapshot = doc.export(loro::ExportMode::Snapshot);
                     println!("B4x100 fast_snapshot size: {:?}", ByteSize(snapshot.len()));
                     doc_x100_snapshot.set(snapshot).unwrap();
                 }
@@ -107,7 +107,7 @@ fn bench_text(c: &mut Criterion) {
             || {
                 if doc_x100_snapshot.get().is_none() {
                     let doc = apply_text_actions(&actions, 100);
-                    let snapshot = doc.export_fast_snapshot();
+                    let snapshot = doc.export(loro::ExportMode::Snapshot);
                     println!("B4x100 fast_snapshot size: {:?}", ByteSize(snapshot.len()));
                     doc_x100_snapshot.set(snapshot).unwrap();
                 }

crates/examples/examples/time_tracker.rs

@@ -51,7 +51,7 @@ pub fn main() {
 
     println!("total_time: {}", total_time);
     println!("mem: {}", get_mem_usage());
-    let snapshot = doc.export_fast_snapshot();
+    let snapshot = doc.export(loro::ExportMode::Snapshot);
     println!("Snapshot Size {}", ByteSize(snapshot.len()));
     println!("mem: {}", get_mem_usage());
     let gc_snapshot = doc.export(loro::ExportMode::GcSnapshot(&doc.oplog_frontiers()));

crates/examples/src/utils.rs

@@ -38,7 +38,7 @@ pub fn bench_fast_snapshot(doc: &LoroDoc) {
     {
         println!("======== New snapshot mode =========");
         let start = Instant::now();
-        let snapshot = doc.export_fast_snapshot();
+        let snapshot = doc.export(loro::ExportMode::Snapshot);
         let elapsed = start.elapsed();
         println!("Fast Snapshot size: {}", ByteSize(snapshot.len()));
         println!("Export fast snapshot time: {:?}", elapsed);
@@ -70,7 +70,7 @@ pub fn bench_fast_snapshot(doc: &LoroDoc) {
         );
 
         let start = Instant::now();
-        let _snapshot = new_doc.export_fast_snapshot();
+        let _snapshot = new_doc.export(loro::ExportMode::Snapshot);
         let elapsed = start.elapsed();
         println!(
             "Export fast snapshot time (from doc created by fast snapshot): {:?}",
@@ -118,7 +118,7 @@ pub fn bench_fast_snapshot(doc: &LoroDoc) {
         );
 
         let start = Instant::now();
-        let _snapshot = new_doc.export_fast_snapshot();
+        let _snapshot = new_doc.export(loro::ExportMode::Snapshot);
         let elapsed = start.elapsed();
         println!(
             "Export fast snapshot time (from doc created by fast snapshot): {:?}",

crates/fuzz/Cargo.toml

@@ -22,6 +22,7 @@ serde_json = "1"
 num_cpus = "1.16.0"
 rayon = "1.10.0"
 bytes = "1"
+ensure-cov = "0.1.0"
 
 [dev-dependencies]
 ctor = "0.2"

crates/fuzz/fuzz/Cargo.lock

@@ -208,6 +208,12 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
 
+[[package]]
+name = "ensure-cov"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33753185802e107b8fa907192af1f0eca13b1fb33327a59266d650fef29b2b4e"
+
 [[package]]
 name = "enum-as-inner"
 version = "0.5.1"
@@ -262,6 +268,7 @@ version = "0.1.0"
 dependencies = [
  "arbitrary",
  "bytes",
+ "ensure-cov",
  "enum-as-inner 0.5.1",
  "enum_dispatch",
  "fxhash",
@@ -498,6 +505,7 @@ dependencies = [
  "generic-btree",
  "loro-delta 0.16.2",
  "loro-internal 0.16.2",
+ "loro-kv-store",
  "tracing",
 ]
 
@@ -618,6 +626,7 @@ dependencies = [
  "arref",
  "bytes",
  "either",
+ "ensure-cov",
  "enum-as-inner 0.5.1",
  "enum_dispatch",
  "fxhash",
@@ -723,6 +732,7 @@ name = "loro-kv-store"
 version = "0.16.2"
 dependencies = [
  "bytes",
+ "ensure-cov",
  "fxhash",
  "loro-common 0.16.2",
  "lz4_flex",

crates/fuzz/fuzz/Cargo.toml

@@ -30,6 +30,12 @@ path = "fuzz_targets/all.rs"
 test = false
 doc = false
 
+[[bin]]
+name = "gc_fuzz"
+path = "fuzz_targets/gc_fuzz.rs"
+test = false
+doc = false
+
 [[bin]]
 name = "mov"
 path = "fuzz_targets/mov.rs"

crates/fuzz/fuzz/fuzz_targets/gc_fuzz.rs

@@ -0,0 +1,9 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+use fuzz::{test_multi_sites_with_gc, Action, FuzzTarget};
+
+fuzz_target!(|actions: Vec<Action>| {
+    test_multi_sites_with_gc(5, vec![FuzzTarget::All], &mut actions.clone());
+});

crates/fuzz/src/crdt_fuzzer.rs

@@ -1,7 +1,7 @@
 use std::{
     collections::VecDeque,
     fmt::{Debug, Display},
-    sync::{Arc, Mutex},
+    sync::{atomic::AtomicUsize, Arc, Mutex},
     thread,
     time::Instant,
 };
@@ -225,10 +225,14 @@ impl CRDTFuzzer {
                     }
                     2 => {
                         info_span!("FastSnapshot", from = i, to = j).in_scope(|| {
-                            b_doc.import(&a_doc.export_fast_snapshot()).unwrap();
+                            b_doc
+                                .import(&a_doc.export(loro::ExportMode::Snapshot))
+                                .unwrap();
                         });
                         info_span!("FastSnapshot", from = j, to = i).in_scope(|| {
-                            a_doc.import(&b_doc.export_fast_snapshot()).unwrap();
+                            a_doc
+                                .import(&b_doc.export(loro::ExportMode::Snapshot))
+                                .unwrap();
                         });
                     }
                     _ => {
@@ -269,7 +273,7 @@ impl CRDTFuzzer {
     }
 }
 
-#[derive(Eq, Hash, PartialEq)]
+#[derive(Eq, Hash, PartialEq, Clone)]
 pub enum FuzzTarget {
     Map,
     List,
@@ -340,6 +344,105 @@ pub fn test_multi_sites(site_num: u8, fuzz_targets: Vec<FuzzTarget>, actions: &m
     });
 }
 
+pub fn test_multi_sites_with_gc(
+    site_num: u8,
+    fuzz_targets: Vec<FuzzTarget>,
+    actions: &mut [Action],
+) {
+    ensure_cov::notify_cov("fuzz_gc");
+    let mut fuzzer = CRDTFuzzer::new(site_num, fuzz_targets);
+    let mut applied = Vec::new();
+    let target_gc_index = actions.len() / 2;
+    for (i, action) in actions.iter_mut().enumerate() {
+        fuzzer.pre_process(action);
+        if i <= target_gc_index {
+            match action {
+                Action::Handle { site, .. } => {
+                    if *site == 0 {
+                        *site = 1
+                    }
+                }
+                Action::Checkout { site, to } => {
+                    if *site == 0 {
+                        *site = 1;
+                    }
+                }
+                Action::Undo { site, op_len } => {
+                    if *site == 0 {
+                        *site = 1;
+                    }
+                }
+                Action::SyncAllUndo { site, op_len } => {
+                    if *site == 0 {
+                        *site = 1;
+                    }
+                }
+                Action::Sync { from, to } => {
+                    if *to == 0 {
+                        *to = 1;
+                        if *from == *to {
+                            *from = 2;
+                        }
+                    }
+                }
+                Action::SyncAll => {}
+            }
+        }
+
+        info_span!("ApplyAction", ?action).in_scope(|| {
+            applied.push(action.clone());
+            info!("OptionsTable \n{}", (&applied).table());
+            // info!("Apply Action {:?}", applied);
+            fuzzer.apply_action(action);
+        });
+
+        if i == target_gc_index {
+            info_span!("GC 1 => 0").in_scope(|| {
+                fuzzer.actors[1].loro.attach();
+                let f = fuzzer.actors[1].loro.oplog_frontiers();
+                if !f.is_empty() {
+                    let bytes = fuzzer.actors[1]
+                        .loro
+                        .export(loro::ExportMode::GcSnapshot(&f));
+                    fuzzer.actors[0].loro.import(&bytes).unwrap();
+                }
+            })
+        }
+    }
+
+    // println!("OpTable \n{}", (&applied).table());
+    info_span!("check synced").in_scope(|| {
+        fuzzer.check_equal();
+    });
+    info_span!("check tracker").in_scope(|| {
+        fuzzer.check_tracker();
+    });
+    info_span!("check history").in_scope(|| {
+        fuzzer.check_history();
+    });
+
+    static COUNT: AtomicUsize = AtomicUsize::new(0);
+    if COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % 1_000 == 0 {
+        let must_meet = [
+            "fuzz_gc",
+            "gc_snapshot::need_calc",
+            "gc_snapshot::dont_need_calc",
+            "loro_internal::history_cache::find_text_chunks_in",
+            "loro_internal::history_cache::find_list_chunks_in",
+            "loro_internal::import",
+            "loro_internal::import::snapshot",
+            "loro_internal::import::snapshot::gc",
+            "loro_internal::import::snapshot::normal",
+            "loro_internal::history_cache::init_cache_by_visit_all_change_slow::visit_gc",
+        ];
+        for v in must_meet {
+            if ensure_cov::get_cov_for(v) == 0 {
+                println!("[COV-FAILED] {}", v)
+            }
+        }
+    }
+}
+
 pub fn minify_error<T, F, N>(site_num: u8, f: F, normalize: N, actions: Vec<T>)
 where
     F: Fn(u8, &mut [T]) + Send + Sync + 'static,

crates/fuzz/src/lib.rs

@@ -4,7 +4,7 @@ pub mod container;
 pub mod crdt_fuzzer;
 mod macros;
 mod value;
-pub use crdt_fuzzer::{test_multi_sites, Action, FuzzTarget};
+pub use crdt_fuzzer::{test_multi_sites, test_multi_sites_with_gc, Action, FuzzTarget};
 mod mem_kv_fuzzer;
 pub use mem_kv_fuzzer::{
     minify_simple as kv_minify_simple, test_mem_kv_fuzzer, test_random_bytes_import,

crates/fuzz/tests/compatibility.rs

@@ -40,7 +40,8 @@ fn updates_with_commit_message_can_be_imported_to_016() {
     }
 
     let doc3 = loro::LoroDoc::new();
-    doc3.import(&doc1.export_fast_snapshot()).unwrap();
+    doc3.import(&doc1.export(loro::ExportMode::Snapshot))
+        .unwrap();
     let change_from_2 = doc3.get_change(ID::new(doc2.peer_id(), 0)).unwrap();
     assert_eq!(change_from_2.len, 3);
     assert_eq!(doc3.get_deep_value(), doc1.get_deep_value());

crates/fuzz/tests/test.rs

@@ -8,6 +8,7 @@ use fuzz::{
     },
     container::{MapAction, TextAction, TextActionInner, TreeAction, TreeActionInner},
     crdt_fuzzer::{minify_error, test_multi_sites, Action::*, FuzzTarget, FuzzValue::*},
+    test_multi_sites_with_gc,
 };
 use loro::{ContainerType::*, LoroCounter, LoroDoc};
 

crates/kv-store/tests/test.rs

@@ -61,7 +61,7 @@ fn add_flush_add_scan() {
     }
 
     let bytes = store.export_all();
-    let mut store = MemKvStore::default();
+    let mut store = MemKvStore::new(MemKvConfig::new());
     store.import_all(bytes).unwrap();
     let mut iter = store.scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded);
     assert_eq!(
@@ -83,12 +83,12 @@ fn large_value() {
     let large_value: Vec<u8> = (0..100_000).map(|_| rng.gen()).collect();
     let large_value = Bytes::from(large_value);
 
-    let mut store = MemKvStore::default();
+    let mut store = MemKvStore::new(MemKvConfig::new());
     store.set(key, large_value.clone());
 
     let bytes = store.export_all();
     ensure_cov::assert_cov("kv_store::block::LargeValueBlock::encode::compress_fallback");
-    let mut imported_store = MemKvStore::default();
+    let mut imported_store = MemKvStore::new(MemKvConfig::new());
     imported_store.import_all(bytes).unwrap();
 
     let retrieved_value = imported_store.get(key).unwrap();

crates/loro-internal/Cargo.toml

@@ -53,6 +53,7 @@ md5 = "0.7.0"
 arref = "0.1.0"
 tracing = { version = "0.1" }
 nonmax = "0.5.5"
+ensure-cov = "0.1.0"
 
 
 [dev-dependencies]

crates/loro-internal/src/encoding/fast_snapshot.rs

@@ -18,7 +18,6 @@ use std::io::{Read, Write};
 use crate::{oplog::ChangeStore, LoroDoc, OpLog, VersionVector};
 use bytes::{Buf, Bytes};
 use loro_common::{IdSpan, LoroError, LoroResult};
-use tracing::trace;
 
 use super::encode_reordered::import_changes_to_oplog;
 
@@ -103,11 +102,13 @@ pub(crate) fn decode_snapshot(doc: &LoroDoc, bytes: Bytes) -> LoroResult<()> {
     let need_calc = state_bytes.is_none();
     let state_frontiers;
     if gc_bytes.is_empty() {
+        ensure_cov::notify_cov("loro_internal::import::snapshot::normal");
         if let Some(bytes) = state_bytes {
             state.store.decode(bytes)?;
         }
         state_frontiers = oplog.frontiers().clone();
     } else {
+        ensure_cov::notify_cov("loro_internal::import::snapshot::gc");
         let gc_state_frontiers = state
             .store
             .decode_gc(gc_bytes.clone(), oplog.dag().trimmed_frontiers().clone())?;
@@ -121,8 +122,10 @@ pub(crate) fn decode_snapshot(doc: &LoroDoc, bytes: Bytes) -> LoroResult<()> {
         });
 
         if need_calc {
+            ensure_cov::notify_cov("gc_snapshot::need_calc");
             state_frontiers = gc_state_frontiers.unwrap();
         } else {
+            ensure_cov::notify_cov("gc_snapshot::dont_need_calc");
             state_frontiers = oplog.frontiers().clone();
         }
     }

crates/loro-internal/src/history_cache.rs

@@ -204,7 +204,11 @@ impl ContainerHistoryCache {
             configure: &Default::default(),
             peer: 0,
         };
+
         if let Some(state) = self.gc.as_ref() {
+            ensure_cov::notify_cov(
+                "loro_internal::history_cache::init_cache_by_visit_all_change_slow::visit_gc",
+            );
             let mut store = state.store.try_lock().unwrap();
             for (idx, c) in store.iter_all_containers_mut() {
                 match idx.get_type() {
@@ -309,6 +313,7 @@ impl ContainerHistoryCache {
         idx: ContainerIdx,
         target_span: loro_common::IdSpan,
     ) -> Vec<RichtextStateChunk> {
+        ensure_cov::notify_cov("loro_internal::history_cache::find_text_chunks_in");
         let Some(state) = self.gc.as_ref() else {
             return Vec::new();
         };
@@ -348,6 +353,7 @@ impl ContainerHistoryCache {
         idx: ContainerIdx,
         target_span: loro_common::IdSpan,
     ) -> Vec<SliceWithId> {
+        ensure_cov::notify_cov("loro_internal::history_cache::find_list_chunks_in");
         let Some(state) = self.gc.as_ref() else {
             return Vec::new();
         };

crates/loro-internal/src/loro.rs

@@ -489,6 +489,7 @@ impl LoroDoc {
 
     #[tracing::instrument(skip_all)]
     fn _import_with(&self, bytes: &[u8], origin: InternalString) -> Result<(), LoroError> {
+        ensure_cov::notify_cov("loro_internal::import");
         let parsed = parse_header_and_body(bytes)?;
         info!("Importing with mode={:?}", &parsed.mode);
         match parsed.mode {
@@ -510,6 +511,7 @@ impl LoroDoc {
             }
             EncodeMode::Snapshot => {
                 if self.can_reset_with_snapshot() {
+                    ensure_cov::notify_cov("loro_internal::import::snapshot");
                     tracing::info!("Init by snapshot {}", self.peer_id());
                     decode_snapshot(self, parsed.mode, parsed.body)?;
                 } else {
@@ -647,14 +649,6 @@ impl LoroDoc {
         ans
     }
 
-    #[instrument(skip_all)]
-    pub fn export_fast_snapshot(&self) -> Vec<u8> {
-        self.commit_then_stop();
-        let ans = export_fast_snapshot(self);
-        self.renew_txn_if_auto_commit();
-        ans
-    }
-
     /// Import the json schema updates.
     ///
     /// only supports backward compatibility but not forward compatibility.

crates/loro/src/lib.rs

@@ -672,11 +672,6 @@ impl LoroDoc {
         self.doc.compact_change_store()
     }
 
-    /// Export the fast snapshot of the document.
-    pub fn export_fast_snapshot(&self) -> Vec<u8> {
-        self.doc.export_fast_snapshot()
-    }
-
     /// Export the document in the given mode.
     pub fn export(&self, mode: ExportMode) -> Vec<u8> {
         self.doc.export(mode)

crates/loro/tests/commit_message_test.rs

@@ -105,7 +105,7 @@ fn test_commit_message_sync_via_fast_snapshot() {
     text1.insert(5, " world").unwrap();
     doc1.commit_with(CommitOptions::new().commit_msg("second edit"));
 
-    let snapshot = doc1.export_fast_snapshot();
+    let snapshot = doc1.export(loro::ExportMode::Snapshot);
     doc2.import(&snapshot).unwrap();
 
     // Verify the commit messages were preserved in the snapshot
@@ -120,7 +120,8 @@ fn test_commit_message_sync_via_fast_snapshot() {
     assert_eq!(text2.to_string(), "hello world");
     text2.delete(0, 10).unwrap();
     doc2.set_next_commit_message("From text2");
-    doc1.import(&doc2.export_fast_snapshot()).unwrap();
+    doc1.import(&doc2.export(loro::ExportMode::Snapshot))
+        .unwrap();
     let c = doc1.get_change(ID::new(doc2.peer_id(), 0)).unwrap();
     assert_eq!(c.message(), "From text2");
 }

crates/loro/tests/loro_rust_test.rs

@@ -906,8 +906,12 @@ fn fast_snapshot_for_updates() {
 
     doc_b.commit();
 
-    doc_b.import(&doc_a.export_fast_snapshot()).unwrap();
-    doc_a.import(&doc_b.export_fast_snapshot()).unwrap();
+    doc_b
+        .import(&doc_a.export(loro::ExportMode::Snapshot))
+        .unwrap();
+    doc_a
+        .import(&doc_b.export(loro::ExportMode::Snapshot))
+        .unwrap();
 
     assert_eq!(doc_a.get_deep_value(), doc_b.get_deep_value());
 }