From abec22cd223fac89b0d670543f0ba9cbbbb8b58e Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 12 Jul 2023 12:30:36 +0800 Subject: [PATCH] fix: text sync issues --- .vscode/settings.json | 4 +- crates/loro-internal/Cargo.toml | 2 +- crates/loro-internal/fuzz/Cargo.lock | 62 ++++--- crates/loro-internal/fuzz/Cargo.toml | 6 + .../fuzz/fuzz_targets/text_refactored.rs | 5 + .../src/container/text/tracker.rs | 9 +- crates/loro-internal/src/fuzz.rs | 151 +++++++++++++++++- crates/loro-internal/src/log_store.rs | 2 +- .../loro-internal/src/refactor/diff_calc.rs | 6 +- crates/loro-internal/src/refactor/handler.rs | 19 ++- crates/loro-internal/src/refactor/loro.rs | 6 +- crates/loro-internal/src/refactor/oplog.rs | 80 ++++++---- .../loro-internal/src/refactor/oplog/dag.rs | 8 +- crates/loro-internal/src/refactor/state.rs | 75 +++++++-- .../src/refactor/state/list_state.rs | 16 +- .../src/refactor/state/map_state.rs | 6 +- .../src/refactor/state/text_state.rs | 25 ++- crates/loro-internal/src/refactor/txn.rs | 21 ++- 18 files changed, 396 insertions(+), 107 deletions(-) create mode 100644 crates/loro-internal/fuzz/fuzz_targets/text_refactored.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index 960c47f7..fb923ab2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,9 +23,7 @@ "RUST_BACKTRACE": "full", "DEBUG": "*" }, - "rust-analyzer.cargo.features": [ - // "test_utils" - ], + "rust-analyzer.cargo.features": ["test_utils"], "editor.defaultFormatter": "rust-lang.rust-analyzer", "rust-analyzer.server.extraEnv": { "RUSTUP_TOOLCHAIN": "stable" }, "editor.formatOnSave": true, diff --git a/crates/loro-internal/Cargo.toml b/crates/loro-internal/Cargo.toml index 4ee0732c..02ffdfca 100644 --- a/crates/loro-internal/Cargo.toml +++ b/crates/loro-internal/Cargo.toml @@ -33,7 +33,7 @@ append-only-bytes = { version = "0.1.8", features = ["u32_range", "serde"] } itertools = "0.10.5" enum_dispatch = "0.3.11" im = "15.1.0" -jumprope = "1.1.2" +jumprope = { version = "1.1.2", features = ["wchar_conversion"] } generic-btree = "0.3.1" [dev-dependencies] diff --git a/crates/loro-internal/fuzz/Cargo.lock b/crates/loro-internal/fuzz/Cargo.lock index 844c1ab9..baed6a8c 100644 --- a/crates/loro-internal/fuzz/Cargo.lock +++ b/crates/loro-internal/fuzz/Cargo.lock @@ -16,9 +16,12 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "append-only-bytes" -version = "0.1.4" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65bb255c86dda8d496b2ec8ba501c603030252cd52bee77a8862f2e46c8837a" +checksum = "dd736657a12852ffb42ed309ac3409382d93f76f49ae0ad69fae4ca927e584d9" +dependencies = [ + "serde", +] [[package]] name = "arbitrary" @@ -142,7 +145,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 1.0.105", ] [[package]] @@ -153,7 +156,7 @@ checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" dependencies = [ "darling_core", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -170,7 +173,7 @@ checksum = "4903dff04948f22033ca30232ab8eca2c3fc4c913a8b6a34ee5199699814817f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -188,7 +191,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -200,7 +203,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -517,7 +520,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -601,7 +604,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.105", "version_check", ] @@ -618,18 +621,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.21" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] @@ -743,9 +746,9 @@ checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" -version = "1.0.149" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] @@ -772,18 +775,18 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.25", ] [[package]] @@ -897,6 +900,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tabled" version = "0.10.0" @@ -918,7 +932,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -938,7 +952,7 @@ checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -967,7 +981,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.105", ] [[package]] @@ -1036,7 +1050,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.105", "wasm-bindgen-shared", ] @@ -1058,7 +1072,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.105", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/crates/loro-internal/fuzz/Cargo.toml b/crates/loro-internal/fuzz/Cargo.toml index a475df6a..25f67f4c 100644 --- a/crates/loro-internal/fuzz/Cargo.toml +++ b/crates/loro-internal/fuzz/Cargo.toml @@ -71,3 +71,9 @@ name = "unicode" path = "fuzz_targets/unicode.rs" test = false doc = false + +[[bin]] +name = "text_refactored" +path = "fuzz_targets/text_refactored.rs" +test = false +doc = false diff --git a/crates/loro-internal/fuzz/fuzz_targets/text_refactored.rs b/crates/loro-internal/fuzz/fuzz_targets/text_refactored.rs new file mode 100644 index 00000000..f3ac2c6a --- /dev/null +++ b/crates/loro-internal/fuzz/fuzz_targets/text_refactored.rs @@ -0,0 +1,5 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use loro_internal::fuzz::{test_multi_sites_refactored, Action}; + +fuzz_target!(|actions: Vec| { test_multi_sites_refactored(8, &mut actions.clone()) }); diff --git a/crates/loro-internal/src/container/text/tracker.rs b/crates/loro-internal/src/container/text/tracker.rs index 11c80c27..772e06d2 100644 --- a/crates/loro-internal/src/container/text/tracker.rs +++ b/crates/loro-internal/src/container/text/tracker.rs @@ -1,4 +1,3 @@ -use debug_log::debug_dbg; use rle::{rle_tree::UnsafeCursor, HasLength, Sliceable}; use smallvec::SmallVec; @@ -383,6 +382,10 @@ impl Tracker { let text_content = content.as_list().expect("Content is not for list"); match text_content { InnerListOp::Insert { slice, pos } => { + if *pos > self.len() { + panic!("pos is out of range (pos={}, len={})", pos, self.len()); + } + let yspan = self.content .get_yspan_at_pos(id, *pos, slice.content_len(), slice.clone()); @@ -391,6 +394,10 @@ impl Tracker { }); } InnerListOp::Delete(span) => { + if span.end() as usize > self.len() { + panic!("pos is out of range"); + } + let mut spans = self .content .get_active_id_spans(span.start() as usize, span.atom_len()); diff --git a/crates/loro-internal/src/fuzz.rs b/crates/loro-internal/src/fuzz.rs index 8a0a871a..304244ca 100644 --- a/crates/loro-internal/src/fuzz.rs +++ b/crates/loro-internal/src/fuzz.rs @@ -6,7 +6,9 @@ use tabled::{TableIteratorExt, Tabled}; pub mod recursive; pub mod recursive_txn; -use crate::{array_mut_ref, id::PeerID, LoroCore, Transact, VersionVector}; +use crate::{ + array_mut_ref, id::PeerID, refactor::loro::LoroApp, LoroCore, Transact, VersionVector, +}; #[derive(arbitrary::Arbitrary, EnumAsInner, Clone, PartialEq, Eq, Debug)] pub enum Action { @@ -203,6 +205,70 @@ impl Actionable for Vec { } } +impl Actionable for Vec { + fn apply_action(&mut self, action: &Action) { + match action { + Action::Ins { content, pos, site } => { + let site = &mut self[*site as usize]; + let mut txn = site.txn().unwrap(); + let text = txn.get_text("text").unwrap(); + text.insert(&mut txn, *pos, &content.to_string()); + } + Action::Del { pos, len, site } => { + let site = &mut self[*site as usize]; + let mut txn = site.txn().unwrap(); + let text = txn.get_text("text").unwrap(); + text.delete(&mut txn, *pos, *len); + } + Action::Sync { from, to } => { + if from != to { + let (from, to) = arref::array_mut_ref!(self, [*from as usize, *to as usize]); + let to_vv = to.vv_cloned(); + to.import(&from.export_from(&to_vv)).unwrap(); + } + } + Action::SyncAll => { + for i in 1..self.len() { + let (a, b) = array_mut_ref!(self, [0, i]); + a.import(&b.export_from(&a.vv_cloned())).unwrap(); + } + for i in 1..self.len() { + let (a, b) = array_mut_ref!(self, [0, i]); + b.import(&a.export_from(&b.vv_cloned())).unwrap(); + } + } + } + } + + fn preprocess(&mut self, action: &mut Action) { + match action { + Action::Ins { pos, site, .. } => { + *site %= self.len() as u8; + let app_state = &mut self[*site as usize].app_state().lock().unwrap(); + let text = app_state.get_text("text").unwrap(); + change_pos_to_char_boundary(pos, text.len()); + } + Action::Del { pos, len, site } => { + *site %= self.len() as u8; + let app_state = &mut self[*site as usize].app_state().lock().unwrap(); + let text = app_state.get_text("text").unwrap(); + if text.is_empty() { + *len = 0; + *pos = 0; + return; + } + + change_delete_to_char_boundary(pos, len, text.len()); + } + Action::Sync { from, to } => { + *from %= self.len() as u8; + *to %= self.len() as u8; + } + Action::SyncAll => {} + } + } +} + pub fn change_delete_to_char_boundary(pos: &mut usize, len: &mut usize, str_len: usize) { *pos %= str_len + 1; *len = (*len).min(str_len - (*pos)); @@ -241,6 +307,37 @@ fn check_synced(sites: &mut [LoroCore]) { } } +fn check_synced_refactored(sites: &mut [LoroApp]) { + for i in 0..sites.len() - 1 { + for j in i + 1..sites.len() { + debug_log::group!("checking {} with {}", i, j); + let (a, b) = array_mut_ref!(sites, [i, j]); + { + debug_log::group!("Import {}", i); + a.import(&b.export_from(&a.vv_cloned())).unwrap(); + debug_log::group_end!(); + } + { + debug_log::group!("Import {}", j); + b.import(&a.export_from(&b.vv_cloned())).unwrap(); + debug_log::group_end!(); + } + check_eq_refactored(a, b); + debug_log::group_end!(); + } + } +} + +fn check_eq_refactored(site_a: &mut LoroApp, site_b: &mut LoroApp) { + let a = site_a.txn().unwrap(); + let text_a = a.get_text("text").unwrap(); + let b = site_b.txn().unwrap(); + let text_b = b.get_text("text").unwrap(); + let value_a = text_a.get_value(&a); + let value_b = text_b.get_value(&b); + assert_eq!(value_a, value_b); +} + pub fn test_single_client(mut actions: Vec) { let mut store = LoroCore::new(Default::default(), Some(1)); let mut text_container = store.get_text("haha"); @@ -442,6 +539,28 @@ pub fn normalize(site_num: u8, actions: &mut [Action]) -> Vec { applied } +pub fn test_multi_sites_refactored(site_num: u8, actions: &mut [Action]) { + let mut sites = Vec::new(); + for i in 0..site_num { + let loro = LoroApp::new(); + loro.set_peer_id(i as u64); + sites.push(loro); + } + + let mut applied = Vec::new(); + for action in actions.iter_mut() { + sites.preprocess(action); + applied.push(action.clone()); + debug_log!("\n{}", (&applied).table()); + sites.apply_action(action); + } + + debug_log::group!("CheckSynced"); + // println!("{}", actions.table()); + check_synced_refactored(&mut sites); + debug_log::group_end!(); +} + pub fn test_multi_sites(site_num: u8, actions: &mut [Action]) { let mut sites = Vec::new(); for i in 0..site_num { @@ -816,6 +935,36 @@ mod test { ) } + #[test] + fn fuzz_r() { + test_multi_sites_refactored( + 8, + &mut [ + Ins { + content: 9728, + pos: 0, + site: 57, + }, + Ins { + content: 205, + pos: 0, + site: 37, + }, + SyncAll, + Ins { + content: 52487, + pos: 5, + site: 54, + }, + ], + ); + } + + #[test] + fn mini_r() { + minify_error(2, vec![], test_multi_sites_refactored, normalize) + } + #[test] fn mini() { minify_error(8, vec![], test_multi_sites, normalize) diff --git a/crates/loro-internal/src/log_store.rs b/crates/loro-internal/src/log_store.rs index dfc4bb9c..9fd09494 100644 --- a/crates/loro-internal/src/log_store.rs +++ b/crates/loro-internal/src/log_store.rs @@ -6,7 +6,7 @@ mod import; mod iter; use crate::{version::Frontiers, LoroValue}; -pub(crate) use encoding::{decode_oplog, encode_oplog, encode_oplog_updates}; +pub(crate) use encoding::{decode_oplog, encode_oplog}; pub use encoding::{EncodeMode, LoroEncoder}; pub(crate) use import::ImportContext; use std::{ diff --git a/crates/loro-internal/src/refactor/diff_calc.rs b/crates/loro-internal/src/refactor/diff_calc.rs index b45bb78a..be3c881a 100644 --- a/crates/loro-internal/src/refactor/diff_calc.rs +++ b/crates/loro-internal/src/refactor/diff_calc.rs @@ -1,6 +1,5 @@ use std::{cmp::Ordering, collections::BinaryHeap}; -use debug_log::{debug_dbg, debug_log}; use enum_dispatch::enum_dispatch; use fxhash::{FxHashMap, FxHashSet}; @@ -45,7 +44,8 @@ impl DiffCalculator { ) -> Vec { let mut diffs = Vec::new(); let arena = &oplog.arena; - for (change, vv) in oplog.iter_causal(before, after) { + let (lca, iter) = oplog.iter_from_lca_causally(before, after); + for (change, vv) in iter { for op in change.ops.iter() { let container_id = arena.get_container_id(op.container).unwrap(); let calculator = self.calculators.entry(op.container).or_insert_with(|| { @@ -60,7 +60,7 @@ impl DiffCalculator { ContainerDiffCalculator::List(ListDiffCalculator::default()) } }; - new.start_tracking(oplog, before); + new.start_tracking(oplog, &lca); new }); diff --git a/crates/loro-internal/src/refactor/handler.rs b/crates/loro-internal/src/refactor/handler.rs index 27fa8a8a..1ec17092 100644 --- a/crates/loro-internal/src/refactor/handler.rs +++ b/crates/loro-internal/src/refactor/handler.rs @@ -1,4 +1,4 @@ -use std::borrow::Cow; +use std::{borrow::Cow, sync::Arc}; use crate::{ container::{ @@ -23,6 +23,10 @@ impl From for Text { impl Text { pub fn insert(&self, txn: &mut Transaction, pos: usize, s: &str) { + if s.is_empty() { + return; + } + txn.apply_local_op( self.container_idx, crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { @@ -33,6 +37,10 @@ impl Text { } pub fn delete(&self, txn: &mut Transaction, pos: usize, len: usize) { + if len == 0 { + return; + } + txn.apply_local_op( self.container_idx, crate::op::RawOpContent::List(ListOp::Delete(DeleteSpan { @@ -43,13 +51,16 @@ impl Text { } pub fn get_value(&self, txn: &Transaction) -> LoroValue { - txn.get_value_by_idx(self.container_idx) + LoroValue::String( + txn.get_value_by_idx(self.container_idx) + .into_string() + .unwrap_or_else(|_| Arc::new(String::new())), + ) } } #[cfg(test)] mod test { - use debug_log::debug_dbg; use crate::refactor::loro::LoroApp; @@ -92,7 +103,7 @@ mod test { txn.commit().unwrap(); loro.import(&loro2.export_from(&Default::default())) .unwrap(); - let mut txn = loro.txn().unwrap(); + let txn = loro.txn().unwrap(); let text = txn.get_text("hello").unwrap(); assert_eq!(&**text.get_value(&txn).as_string().unwrap(), "hello world"); } diff --git a/crates/loro-internal/src/refactor/loro.rs b/crates/loro-internal/src/refactor/loro.rs index 96843ca4..1b6856eb 100644 --- a/crates/loro-internal/src/refactor/loro.rs +++ b/crates/loro-internal/src/refactor/loro.rs @@ -1,7 +1,5 @@ use std::sync::{Arc, Mutex}; -use debug_log::debug_dbg; - use crate::{id::PeerID, LoroError, VersionVector}; use super::{ @@ -107,6 +105,10 @@ impl LoroApp { pub fn encode_snapshot(&self) -> Vec { unimplemented!(); } + + pub(crate) fn vv_cloned(&self) -> VersionVector { + self.oplog.lock().unwrap().vv().clone() + } } impl Default for LoroApp { diff --git a/crates/loro-internal/src/refactor/oplog.rs b/crates/loro-internal/src/refactor/oplog.rs index c8266950..3197bc2a 100644 --- a/crates/loro-internal/src/refactor/oplog.rs +++ b/crates/loro-internal/src/refactor/oplog.rs @@ -6,7 +6,6 @@ use std::rc::Rc; use debug_log::debug_dbg; use fxhash::FxHashMap; use rle::{HasLength, RleVec}; -use smallvec::SmallVec; // use tabled::measurment::Percent; use crate::change::{Change, Lamport, Timestamp}; @@ -33,7 +32,7 @@ pub struct OpLog { pub(crate) dag: AppDag, pub(crate) arena: SharedArena, pub(crate) changes: ClientChanges, - pub(crate) latest_lamport: Lamport, + pub(crate) latest_lamport: Lamport, //TODO use next lamport instead pub(crate) latest_timestamp: Timestamp, /// Pending changes that haven't been applied to the dag. /// A change can be imported only when all its deps are already imported. @@ -348,47 +347,64 @@ impl OpLog { decode_oplog(self, data) } - /// iterates over all changes that are causally after `from` and before `to` + /// iterates over all changes between LCA(common ancestors) to `to` causally + /// + /// This method assumes to > from /// /// it will include a version vector when the change is applied - pub(crate) fn iter_causal( + // TODO: refactor + pub(crate) fn iter_from_lca_causally( &self, from: &VersionVector, to: &VersionVector, - ) -> impl Iterator>)> { - let frontiers = from.to_frontiers(&self.dag); - let diff = from.diff(to).right; - let mut iter = self.dag.iter_causal(&frontiers, diff); + ) -> ( + VersionVector, + impl Iterator>)>, + ) { + debug_log::group!("iter_from_lca_causally"); + let from_frontiers = from.to_frontiers(&self.dag); + let to_frontiers = to.to_frontiers(&self.dag); + let common_ancestors = self + .dag + .find_common_ancestor(&from_frontiers, &to_frontiers); + let common_ancestors_vv = self.dag.frontiers_to_vv(&common_ancestors); + let diff = common_ancestors_vv.diff(to).right; + let mut iter = self.dag.iter_causal(&common_ancestors, diff); let mut node = iter.next(); let mut cur_cnt = 0; let vv = Rc::new(RefCell::new(VersionVector::default())); - std::iter::from_fn(move || { - if let Some(inner) = &node { - let mut inner_vv = vv.borrow_mut(); - inner_vv.clear(); - inner_vv.extend_to_include_vv(inner.data.vv.iter()); - let peer = inner.data.peer; - let cnt = inner.data.cnt.max(cur_cnt); - let end = inner.data.cnt + inner.data.len as Counter; - let change = self - .changes - .get(&peer) - .and_then(|x| x.get_by_atom_index(cnt).map(|x| x.element)) - .unwrap(); + ( + common_ancestors_vv, + std::iter::from_fn(move || { + if let Some(inner) = &node { + let mut inner_vv = vv.borrow_mut(); + inner_vv.clear(); + inner_vv.extend_to_include_vv(inner.data.vv.iter()); + let peer = inner.data.peer; + let cnt = inner.data.cnt.max(cur_cnt); + let end = inner.data.cnt + inner.data.len as Counter; + let change = self + .changes + .get(&peer) + .and_then(|x| x.get_by_atom_index(cnt).map(|x| x.element)) + .unwrap(); - if change.ctr_end() < end { - cur_cnt = change.ctr_end(); + if change.ctr_end() < end { + cur_cnt = change.ctr_end(); + } else { + node = iter.next(); + cur_cnt = 0; + } + + inner_vv.extend_to_include_end_id(change.id); + // debug_log::debug_dbg!(&change, &inner_vv); + Some((change, vv.clone())) } else { - node = iter.next(); - cur_cnt = 0; + debug_log::group_end!(); + None } - - inner_vv.extend_to_include_end_id(change.id); - Some((change, vv.clone())) - } else { - None - } - }) + }), + ) } } diff --git a/crates/loro-internal/src/refactor/oplog/dag.rs b/crates/loro-internal/src/refactor/oplog/dag.rs index 4853aa07..b1dd6d89 100644 --- a/crates/loro-internal/src/refactor/oplog/dag.rs +++ b/crates/loro-internal/src/refactor/oplog/dag.rs @@ -141,12 +141,14 @@ impl AppDag { let id = frontiers[0]; let Some(rle) = self.map.get(&id.peer) else { unreachable!() }; let Some(x) = rle.get_by_atom_index(id.counter) else { unreachable!() }; - x.element.vv.clone() + let mut vv = x.element.vv.clone(); + vv.extend_to_include_last_id(id); + vv }; for id in frontiers[1..].iter() { - let Some(rle) = self.map.get(&id.peer) else { continue }; - let Some(x) = rle.get_by_atom_index(id.counter) else { continue }; + let Some(rle) = self.map.get(&id.peer) else { unreachable!() }; + let Some(x) = rle.get_by_atom_index(id.counter) else { unreachable!() }; vv.extend_to_include_vv(x.element.vv.iter()); vv.extend_to_include_last_id(*id); } diff --git a/crates/loro-internal/src/refactor/state.rs b/crates/loro-internal/src/refactor/state.rs index f95e1202..cb22b6ce 100644 --- a/crates/loro-internal/src/refactor/state.rs +++ b/crates/loro-internal/src/refactor/state.rs @@ -1,3 +1,4 @@ +use enum_as_inner::EnumAsInner; use enum_dispatch::enum_dispatch; use fxhash::{FxHashMap, FxHashSet}; use ring::rand::SystemRandom; @@ -5,7 +6,7 @@ use ring::rand::SystemRandom; use crate::{ change::Lamport, configure::SecureRandomGenerator, - container::registry::ContainerIdx, + container::{registry::ContainerIdx, ContainerIdRaw}, event::Diff, id::{Counter, PeerID}, op::RawOp, @@ -17,9 +18,9 @@ mod list_state; mod map_state; mod text_state; -use list_state::List; -use map_state::Map; -use text_state::Text; +use list_state::ListState; +use map_state::MapState; +use text_state::TextState; use super::{arena::SharedArena, oplog::OpLog}; @@ -52,12 +53,27 @@ pub trait ContainerState: Clone { fn get_value(&self) -> LoroValue; } +#[allow(clippy::enum_variant_names)] #[enum_dispatch(ContainerState)] -#[derive(Clone)] +#[derive(EnumAsInner, Clone)] pub enum State { - List, - Map, - Text, + ListState, + MapState, + TextState, +} + +impl State { + pub fn new_list() -> Self { + Self::ListState(ListState::default()) + } + + pub fn new_map() -> Self { + Self::MapState(MapState::new()) + } + + pub fn new_text() -> Self { + Self::TextState(TextState::default()) + } } #[derive(Debug)] @@ -135,13 +151,20 @@ impl AppState { self.in_txn = false; } - pub(crate) fn commit_txn(&mut self, new_frontiers: Frontiers) { + pub(crate) fn commit_txn( + &mut self, + new_frontiers: Frontiers, + next_lamport: Lamport, + next_counter: Counter, + ) { for container_idx in std::mem::take(&mut self.changed_in_txn) { self.states.get_mut(&container_idx).unwrap().commit_txn(); } self.in_txn = false; self.frontiers = new_frontiers; + self.next_counter = next_counter; + self.next_lamport = next_lamport; } pub(super) fn get_state_mut(&mut self, idx: ContainerIdx) -> Option<&mut State> { @@ -149,7 +172,33 @@ impl AppState { } pub(crate) fn get_value_by_idx(&self, container_idx: ContainerIdx) -> LoroValue { - self.states.get(&container_idx).unwrap().get_value() + self.states + .get(&container_idx) + .map(|x| x.get_value()) + .unwrap_or(LoroValue::Null) + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_text>(&mut self, id: I) -> Option<&text_state::TextState> { + let id: ContainerIdRaw = id.into(); + let idx = match id { + ContainerIdRaw::Root { name } => Some(self.arena.register_container( + &crate::container::ContainerID::Root { + name, + container_type: crate::ContainerType::Text, + }, + )), + ContainerIdRaw::Normal { id: _ } => self + .arena + .id_to_idx(&id.with_type(crate::ContainerType::Text)), + }; + + let idx = idx.unwrap(); + self.states + .entry(idx) + .or_insert_with(State::new_text) + .as_text_state() } pub(super) fn is_in_txn(&self) -> bool { @@ -159,8 +208,8 @@ impl AppState { pub fn create_state(kind: ContainerType) -> State { match kind { - ContainerType::Text => State::Text(Text::new()), - ContainerType::Map => State::Map(Map::new()), - ContainerType::List => State::List(List::new()), + ContainerType::Text => State::TextState(TextState::new()), + ContainerType::Map => State::MapState(MapState::new()), + ContainerType::List => State::ListState(ListState::new()), } } diff --git a/crates/loro-internal/src/refactor/state/list_state.rs b/crates/loro-internal/src/refactor/state/list_state.rs index e0c4a6b5..fb80870a 100644 --- a/crates/loro-internal/src/refactor/state/list_state.rs +++ b/crates/loro-internal/src/refactor/state/list_state.rs @@ -19,14 +19,14 @@ use super::ContainerState; type ContainerMapping = Arc>>; -pub struct List { +pub struct ListState { list: BTree, in_txn: bool, undo_stack: Vec, child_container_to_leaf: Arc>>, } -impl Clone for List { +impl Clone for ListState { fn clone(&self) -> Self { Self { list: self.list.clone(), @@ -104,7 +104,7 @@ impl UseLengthFinder for ListImpl { } } -impl List { +impl ListState { pub fn new() -> Self { let mut tree = BTree::new(); let mapping: ContainerMapping = Arc::new(Mutex::new(Default::default())); @@ -215,7 +215,13 @@ impl List { } } -impl ContainerState for List { +impl Default for ListState { + fn default() -> Self { + Self::new() + } +} + +impl ContainerState for ListState { fn apply_diff(&mut self, diff: &Diff, arena: &SharedArena) { match diff { Diff::List(delta) => { @@ -325,7 +331,7 @@ mod test { #[test] fn test() { - let mut list = List::new(); + let mut list = ListState::new(); fn id(name: &str) -> ContainerID { ContainerID::new_root(name, crate::ContainerType::List) } diff --git a/crates/loro-internal/src/refactor/state/map_state.rs b/crates/loro-internal/src/refactor/state/map_state.rs index 1dba07d3..88dca4b6 100644 --- a/crates/loro-internal/src/refactor/state/map_state.rs +++ b/crates/loro-internal/src/refactor/state/map_state.rs @@ -13,13 +13,13 @@ use crate::{ use super::ContainerState; #[derive(Clone)] -pub struct Map { +pub struct MapState { map: FxHashMap, in_txn: bool, map_when_txn_start: FxHashMap>, } -impl ContainerState for Map { +impl ContainerState for MapState { fn apply_diff(&mut self, diff: &Diff, _arena: &SharedArena) { if let Diff::NewMap(delta) = diff { for (key, value) in delta.updated.iter() { @@ -78,7 +78,7 @@ impl ContainerState for Map { } } -impl Map { +impl MapState { pub fn new() -> Self { Self { map: FxHashMap::default(), diff --git a/crates/loro-internal/src/refactor/state/text_state.rs b/crates/loro-internal/src/refactor/state/text_state.rs index e42c63fd..9d661a9b 100644 --- a/crates/loro-internal/src/refactor/state/text_state.rs +++ b/crates/loro-internal/src/refactor/state/text_state.rs @@ -14,14 +14,14 @@ use crate::{ use super::ContainerState; #[derive(Default)] -pub struct Text { +pub struct TextState { pub(crate) rope: JumpRope, in_txn: bool, deleted_bytes: Vec, undo_stack: Vec, } -impl Clone for Text { +impl Clone for TextState { fn clone(&self) -> Self { Self { rope: self.rope.clone(), @@ -44,7 +44,7 @@ enum UndoItem { }, } -impl ContainerState for Text { +impl ContainerState for TextState { fn apply_diff(&mut self, diff: &Diff, arena: &SharedArena) { match diff { Diff::SeqRaw(delta) => { @@ -150,7 +150,7 @@ impl ContainerState for Text { } } -impl Text { +impl TextState { pub fn new() -> Self { Self { rope: JumpRope::new(), @@ -207,9 +207,22 @@ impl Text { }); } - fn len(&self) -> usize { + pub fn len_wchars(&self) -> usize { + self.rope.len_wchars() + } + + pub fn len_chars(&self) -> usize { + self.rope.len_chars() + } + + pub fn len(&self) -> usize { self.rope.len_bytes() } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } #[cfg(test)] @@ -218,7 +231,7 @@ mod test { #[test] fn abort_txn() { - let mut state = Text::new(); + let mut state = TextState::new(); state.insert(0, "haha"); state.start_txn(); state.insert(4, "1234"); diff --git a/crates/loro-internal/src/refactor/txn.rs b/crates/loro-internal/src/refactor/txn.rs index a0ef2fe1..2945ab2e 100644 --- a/crates/loro-internal/src/refactor/txn.rs +++ b/crates/loro-internal/src/refactor/txn.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use rle::RleVec; +use rle::{HasLength, RleVec}; use crate::{ change::{Change, Lamport}, @@ -60,6 +60,12 @@ impl Transaction { pub fn commit(&mut self) -> Result<(), LoroError> { let mut state = self.state.lock().unwrap(); + if self.local_ops.is_empty() { + state.abort_txn(); + self.finished = true; + return Ok(()); + } + let ops = std::mem::take(&mut self.local_ops); let mut oplog = self.oplog.lock().unwrap(); let deps = take(&mut self.frontiers); @@ -78,12 +84,17 @@ impl Transaction { self.abort(); return Err(err); } - state.commit_txn(Frontiers::from_id(last_id)); + state.commit_txn( + Frontiers::from_id(last_id), + self.next_lamport, + self.next_counter, + ); self.finished = true; Ok(()) } pub fn apply_local_op(&mut self, container: ContainerIdx, content: RawOpContent) { + let len = content.content_len(); let op = RawOp { id: ID { peer: self.peer, @@ -96,8 +107,8 @@ impl Transaction { self.push_local_op_to_log(&op); let mut state = self.state.lock().unwrap(); state.apply_local_op(op); - self.next_counter += 1; - self.next_lamport += 1; + self.next_counter += len as Counter; + self.next_lamport += len as Lamport; } fn push_local_op_to_log(&mut self, op: &RawOp) { @@ -107,7 +118,7 @@ impl Transaction { /// id can be a str, ContainerID, or ContainerIdRaw. /// if it's str it will use Root container, which will not be None - pub fn get_text>(&mut self, id: I) -> Option { + pub fn get_text>(&self, id: I) -> Option { let id: ContainerIdRaw = id.into(); let idx = match id { ContainerIdRaw::Root { name } => Some(self.arena.register_container(