mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-02-05 20:17:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

perf: refine state fast snapshot & fix a few tree event issues (#459)

Browse source 
* perf: refine state fast snapshot

* fix: tree apply diff err

* fix: get child index

* fix: use better tree event
This commit is contained in:
Zixuan Chen 2024-09-11 22:54:14 +08:00 committed by GitHub
parent f7bd2aefff
commit 21e3ffea45
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 348 additions and 245 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.vscode/settings.json vendored
View file

@ -50,7 +50,7 @@
],
"rust-analyzer.runnableEnv": {
"RUST_BACKTRACE": "full",
"DEBUG": "*"
// "DEBUG": "*"
},
"rust-analyzer.cargo.features": [
// "test_utils",

5
crates/fuzz/src/container/tree.rs
View file

@ -445,8 +445,9 @@ impl ApplyDiff for TreeTracker {
..
} => {
let Some(node) = self.find_node_by_id(target) else {
self.create_node(target, &parent.tree_id(), position.to_string(), index);
continue;
// self.create_node(target, &parent.tree_id(), position.to_string(), index);
// continue;
panic!("Expected move but the node needs to be created");
};
let mut node = if let Some(p) = node.parent {

347
crates/fuzz/tests/test.rs
View file

@ -8653,184 +8653,185 @@ fn fast_snapshot_2() {
)
}
#[test]
fn fast_snapshot_3() {
test_multi_sites(
5,
vec![FuzzTarget::All],
&mut [
Handle {
site: 4,
target: 251,
container: 251,
action: Generic(GenericAction {
value: Container(Unknown(86)),
bool: true,
key: 555819297,
pos: 18446744073709551615,
length: 1252228849668718591,
prop: 2449958197287707631,
}),
},
Handle {
site: 33,
target: 33,
container: 33,
action: Generic(GenericAction {
value: Container(Unknown(251)),
bool: true,
key: 4294967295,
pos: 9362721257822425599,
length: 18446713166840815823,
prop: 18446744073695002623,
}),
},
Handle {
site: 33,
target: 33,
container: 251,
action: Generic(GenericAction {
value: Container(Unknown(251)),
bool: true,
key: 2147220227,
pos: 18157383382357508095,
length: 18157383382357244923,
prop: 18384011580076532219,
}),
},
Handle {
site: 223,
target: 47,
container: 222,
action: Generic(GenericAction {
value: Container(Unknown(255)),
bool: true,
key: 4227595259,
pos: 18157383382357244923,
length: 2387225703656586235,
prop: 18446744073709551615,
}),
},
Handle {
site: 251,
target: 251,
container: 251,
action: Generic(GenericAction {
value: Container(Unknown(3)),
bool: true,
key: 4294934523,
pos: 18157383382357508095,
length: 18157383382357244923,
prop: 18384011580076532219,
}),
},
Handle {
site: 33,
target: 251,
container: 251,
action: Generic(GenericAction {
value: Container(Unknown(255)),
bool: true,
key: 4286577659,
pos: 18157383382357245951,
length: 18157383382357244923,
prop: 18446499024906297633,
}),
},
Handle {
site: 223,
target: 47,
container: 222,
action: Generic(GenericAction {
value: Container(Unknown(251)),
bool: true,
key: 4227595259,
pos: 18157383382357244923,
length: 2387225703656586235,
prop: 18446744073709551615,
}),
},
Handle {
site: 33,
target: 251,
container: 251,
action: Generic(GenericAction {
value: Container(Unknown(255)),
bool: true,
key: 4286577659,
pos: 18157383382357245951,
length: 18157383382357244923,
prop: 18446499024906297633,
}),
},
SyncAll,
Handle {
site: 223,
target: 222,
container: 221,
action: Generic(GenericAction {
value: Container(Unknown(4)),
bool: true,
key: 4294967295,
pos: 2387209068692373503,
length: 2594064589257963313,
prop: 18100529071917779530,
}),
},
Handle {
site: 33,
target: 251,
container: 251,
action: Generic(GenericAction {
value: Container(Unknown(255)),
bool: true,
key: 4286577659,
pos: 18157383382357245951,
length: 18157383382357244923,
prop: 18446499024906297633,
}),
},
Handle {
site: 239,
target: 227,
container: 227,
action: Generic(GenericAction {
value: Container(Text),
bool: true,
key: 555819352,
pos: 2387225703656530209,
length: 9332677834833697,
prop: 18446744073709494561,
}),
},
SyncAll,
Handle {
site: 33,
target: 251,
container: 251,
action: Generic(GenericAction {
value: Container(Text),
bool: false,
key: 4294907392,
pos: 1252228849668718591,
length: 18384256508149097455,
prop: 2387225707345346559,
}),
},
SyncAll,
Undo {
site: 123,
op_len: 2071690107,
},
],
)
}
#[test]
fn minify() {
minify_error(
5,
|n, actions| test_multi_sites(n, vec![FuzzTarget::All], actions),
|_, actions| actions.to_vec(),
vec![
Handle {
site: 187,
target: 122,
container: 36,
action: Generic(GenericAction {
value: Container(Unknown(255)),
bool: true,
key: 4287627263,
pos: 4902828863,
length: 9335720388467884032,
prop: 226866784668584321,
}),
},
Handle {
site: 27,
target: 27,
container: 27,
action: Generic(GenericAction {
value: I32(454761243),
bool: true,
key: 2812782503,
pos: 12080808863958804391,
length: 12080808863958804391,
prop: 12080808863958804391,
}),
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
Handle {
site: 27,
target: 27,
container: 49,
action: Generic(GenericAction {
value: I32(875640369),
bool: true,
key: 454761243,
pos: 1953184666628070298,
length: 144115188075855871,
prop: 4557431447142210354,
}),
},
Checkout {
site: 63,
to: 457129791,
},
Handle {
site: 93,
target: 52,
container: 27,
action: Generic(GenericAction {
value: I32(1061109567),
bool: true,
key: 1061109567,
pos: 1953184666628079423,
length: 1953184666628070235,
prop: 12041247832392499326,
}),
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 2805114791,
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
Handle {
site: 167,
target: 167,
container: 167,
action: Generic(GenericAction {
value: Container(Tree),
bool: true,
key: 2812782503,
pos: 12080808863958804391,
length: 12080808863958804391,
prop: 12080808863958804391,
}),
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
Handle {
site: 27,
target: 27,
container: 27,
action: Generic(GenericAction {
value: I32(1499027801),
bool: true,
key: 1499027801,
pos: 6438275382588823897,
length: 6438275382588823897,
prop: 6438275382588823897,
}),
},
Undo {
site: 89,
op_len: 1499027801,
},
Undo {
site: 89,
op_len: 1499027801,
},
Undo {
site: 89,
op_len: 1499027801,
},
Undo {
site: 89,
op_len: 1499027801,
},
Undo {
site: 89,
op_len: 1499027801,
},
Undo {
site: 89,
op_len: 2812782425,
},
SyncAllUndo {
site: 37,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 2812782503,
},
SyncAllUndo {
site: 167,
op_len: 454761383,
},
Handle {
site: 27,
target: 27,
container: 218,
action: Generic(GenericAction {
value: I32(454761243),
bool: true,
key: 454761243,
pos: 18446744073694550811,
length: 13924878376503476223,
prop: 17802464409370431,
}),
},
],
vec![],
)
}

26
crates/loro-internal/src/state/list_state.rs
View file

@ -552,8 +552,15 @@ mod snapshot {
use super::*;
impl FastStateSnapshot for ListState {
/// Encodes the ListState snapshot in a compact binary format:
/// 1. Encodes the list values using postcard serialization
/// 2. Encodes a table of unique peer IDs
/// 3. For each element, encodes its ID as:
/// - Index of the peer ID in the table (LEB128)
/// - Counter (LEB128)
/// - Lamport timestamp (LEB128)
fn encode_snapshot_fast<W: Write>(&mut self, mut w: W) {
let value = self.get_value();
let value = self.get_value().into_list().unwrap();
postcard::to_io(&value, &mut w).unwrap();
let mut peers: ValueRegister<PeerID> = ValueRegister::new();
let mut id_bytes = Vec::new();
@ -573,6 +580,14 @@ mod snapshot {
w.write_all(&id_bytes).unwrap();
}
fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> {
let (value, bytes) = postcard::take_from_bytes(bytes).map_err(|_| {
loro_common::LoroError::DecodeError(
"Decode list value failed".to_string().into_boxed_str(),
)
})?;
Ok((LoroValue::List(Arc::new(value)), bytes))
}
fn decode_snapshot_fast(
idx: ContainerIdx,
@ -611,14 +626,6 @@ mod snapshot {
Ok(ans)
}
fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> {
postcard::take_from_bytes(bytes).map_err(|_| {
loro_common::LoroError::DecodeError(
"Decode list value failed".to_string().into_boxed_str(),
)
})
}
}
}
@ -663,6 +670,7 @@ mod test {
list.insert(1, LoroValue::I64(2), IdFull::new(1, 1, 1));
let mut w = Vec::new();
list.encode_snapshot_fast(&mut w);
assert!(w.len() <= 28, "w.len() = {}", w.len());
let (v, left) = ListState::decode_value(&w).unwrap();
let mut new_list = ListState::decode_snapshot_fast(
ContainerIdx::from_index_and_type(0, loro_common::ContainerType::List),

15
crates/loro-internal/src/state/map_state.rs
View file

@ -294,8 +294,10 @@ impl MapState {
}
mod snapshot {
use fxhash::FxHashSet;
use loro_common::InternalString;
use std::sync::Arc;
use fxhash::{FxHashMap, FxHashSet};
use loro_common::{InternalString, LoroValue};
use serde_columnar::Itertools;
use crate::{
@ -313,7 +315,7 @@ mod snapshot {
// 3. leb128 peer_num + peers (in u64)
// 3. Groups of (leb128 peer_idx, leb128 lamport), each has a respective map entry
// from either 1 or 2 when they all sorted by the key strings
let value = self.get_value();
let value = self.get_value().into_map().unwrap();
postcard::to_io(&value, &mut w).unwrap();
let keys_with_none_value = self
@ -342,11 +344,13 @@ mod snapshot {
}
fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> {
postcard::take_from_bytes(bytes).map_err(|_| {
let (value, bytes) = postcard::take_from_bytes::<FxHashMap<String, LoroValue>>(bytes)
.map_err(|_| {
loro_common::LoroError::DecodeError(
"Decode map value failed".to_string().into_boxed_str(),
)
})
})?;
Ok((LoroValue::Map(Arc::new(value)), bytes))
}
fn decode_snapshot_fast(
@ -456,6 +460,7 @@ mod snapshot {
let mut bytes = Vec::new();
map.encode_snapshot_fast(&mut bytes);
assert!(bytes.len() <= 50);
let (value, bytes) = MapState::decode_value(&bytes).unwrap();
{

63
crates/loro-internal/src/state/movable_list_state.rs
View file

@ -18,7 +18,8 @@ use crate::{
handler::ValueOrHandler,
op::{ListSlice, Op, RawOp},
state::movable_list_state::inner::PushElemInfo,
txn::Transaction, DocState, ListDiff,
txn::Transaction,
DocState, ListDiff,
};
use self::{
@ -1547,7 +1548,7 @@ struct EncodedIdFull {
#[columnar(strategy = "DeltaRle")]
counter: i32,
#[columnar(strategy = "DeltaRle")]
lamport: u32,
lamport_sub_counter: i32,
}
#[columnar(ser, de)]
@ -1565,7 +1566,7 @@ struct EncodedFastSnapshot {
mod snapshot {
use std::io::Read;
use loro_common::{IdFull, IdLp, PeerID};
use loro_common::{IdFull, IdLp, LoroValue, PeerID};
use crate::{
encoding::value_register::ValueRegister,
@ -1578,8 +1579,23 @@ mod snapshot {
};
impl FastStateSnapshot for MovableListState {
/// Encodes the MovableListState into a compact binary format for fast snapshot storage and retrieval.
///
/// The encoding format consists of:
/// 1. The full value of the MovableListState, encoded using postcard serialization.
/// 2. A series of EncodedItemForFastSnapshot structs representing each visible list item:
/// - invisible_list_item: Count of invisible items before this item (RLE encoded)
/// - pos_id_eq_elem_id: Boolean indicating if position ID equals element ID (RLE encoded)
/// - elem_id_eq_last_set_id: Boolean indicating if element ID equals last set ID (RLE encoded)
/// 3. A series of EncodedIdFull structs for list item IDs:
/// - peer_idx: Index of the peer ID in a value register (delta-RLE encoded)
/// - counter: Operation counter (delta-RLE encoded)
/// - lamport: Lamport timestamp (delta-RLE encoded)
/// 4. EncodedId structs for element IDs (when different from position ID)
/// 5. EncodedId structs for last set IDs (when different from element ID)
/// 6. A list of unique peer IDs used in the encoding
fn encode_snapshot_fast<W: std::io::prelude::Write>(&mut self, mut w: W) {
let value = self.get_value();
let value = self.get_value().into_list().unwrap();
postcard::to_io(&value, &mut w).unwrap();
let mut peers: ValueRegister<PeerID> = ValueRegister::new();
let len = self.len();
@ -1597,20 +1613,20 @@ mod snapshot {
for item in self.list().iter() {
if let Some(elem_id) = item.pointed_by {
let elem = self.elements().get(&elem_id).unwrap();
let eq = elem_id.to_id() == item.id.idlp();
let elem_eq_list_item = elem_id.to_id() == item.id.idlp();
let elem_id_eq_last_set_id = elem.value_id.compact() == elem_id;
items.push(EncodedItemForFastSnapshot {
invisible_list_item: 0,
pos_id_eq_elem_id: eq,
pos_id_eq_elem_id: elem_eq_list_item,
elem_id_eq_last_set_id,
});
list_item_ids.push(super::EncodedIdFull {
peer_idx: peers.register(&item.id.peer),
counter: item.id.counter,
lamport: item.id.lamport,
lamport_sub_counter: (item.id.lamport as i32 - item.id.counter),
});
if !eq {
if !elem_eq_list_item {
elem_ids.push(super::EncodedId {
peer_idx: peers.register(&elem_id.peer),
lamport: elem_id.lamport.get(),
@ -1627,7 +1643,7 @@ mod snapshot {
list_item_ids.push(super::EncodedIdFull {
peer_idx: peers.register(&item.id.peer),
counter: item.id.counter,
lamport: item.id.lamport,
lamport_sub_counter: (item.id.lamport as i32 - item.id.counter),
});
}
}
@ -1649,11 +1665,13 @@ mod snapshot {
}
fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> {
postcard::take_from_bytes(bytes).map_err(|_| {
loro_common::LoroError::DecodeError(
"Decode list value failed".to_string().into_boxed_str(),
)
})
let (list_value, bytes) =
postcard::take_from_bytes::<Vec<LoroValue>>(bytes).map_err(|_| {
loro_common::LoroError::DecodeError(
"Decode list value failed".to_string().into_boxed_str(),
)
})?;
Ok((list_value.into(), bytes))
}
fn decode_snapshot_fast(
@ -1694,9 +1712,13 @@ mod snapshot {
let EncodedIdFull {
peer_idx,
counter,
lamport,
lamport_sub_counter,
} = list_item_id_iter.next().unwrap().unwrap();
let id_full = IdFull::new(peers[peer_idx], counter, lamport);
let id_full = IdFull::new(
peers[peer_idx],
counter,
(lamport_sub_counter + counter) as u32,
);
let elem_id = if pos_id_eq_elem_id {
id_full.idlp()
} else {
@ -1728,9 +1750,13 @@ mod snapshot {
let EncodedIdFull {
peer_idx,
counter,
lamport,
lamport_sub_counter,
} = list_item_id_iter.next().unwrap().unwrap();
let id_full = IdFull::new(peers[peer_idx], counter, lamport);
let id_full = IdFull::new(
peers[peer_idx],
counter,
(counter + lamport_sub_counter) as u32,
);
ans.inner.push_inner(id_full, None);
}
}
@ -1796,6 +1822,7 @@ mod snapshot {
let mut bytes = Vec::new();
list.encode_snapshot_fast(&mut bytes);
assert!(bytes.len() <= 117, "{}", bytes.len());
let (v, bytes) = MovableListState::decode_value(&bytes).unwrap();
assert_eq!(

65
crates/loro-internal/src/state/richtext_state.rs
View file

@ -939,7 +939,7 @@ mod snapshot {
#[columnar(strategy = "DeltaRle")]
counter: i32,
#[columnar(strategy = "DeltaRle")]
lamport: u32,
lamport_sub_counter: i32,
/// positive for text
/// 0 for mark start
/// -1 for mark end
@ -950,7 +950,7 @@ mod snapshot {
#[columnar(vec, ser, de, iterable)]
#[derive(Debug, Clone)]
struct EncodedMark {
key: InternalString,
key_idx: usize,
value: LoroValue,
info: u8,
}
@ -959,13 +959,27 @@ mod snapshot {
struct EncodedText {
#[columnar(class = "vec", iter = "EncodedTextSpan")]
spans: Vec<EncodedTextSpan>,
#[columnar(class = "vec", iter = "EncodedMark")]
keys: Vec<InternalString>,
marks: Vec<EncodedMark>,
}
impl FastStateSnapshot for RichtextState {
/// Encodes the RichtextState into a compact binary format for fast snapshot storage and retrieval.
///
/// The encoding format consists of:
/// 1. The full text content as a string, encoded using postcard serialization.
/// 2. A series of EncodedTextSpan structs representing text chunks and style markers:
/// - peer_idx: Index of the peer ID in a value register (delta-RLE encoded)
/// - counter: Operation counter (delta-RLE encoded)
/// - lamport_sub_counter: Lamport timestamp - counter (delta-RLE encoded)
/// - len: Length of text chunk or marker type (-1 for end, 0 for start, positive for text)
/// 3. A list of unique style keys as InternalString.
/// 4. A series of EncodedMark structs for style information:
/// - key_idx: Index of the style key in the keys list
/// - value: The style value
/// - info: Additional style information as a byte
fn encode_snapshot_fast<W: std::io::prelude::Write>(&mut self, mut w: W) {
let value = self.get_value();
let value = self.get_value().into_string().unwrap();
postcard::to_io(&value, &mut w).unwrap();
let mut spans = Vec::new();
let mut marks = Vec::new();
@ -985,6 +999,8 @@ mod snapshot {
}
}
let mut keys: ValueRegister<InternalString> = ValueRegister::new();
for chunk in iter {
match chunk {
RichtextStateChunk::Text(t) => {
@ -993,7 +1009,7 @@ mod snapshot {
spans.push(EncodedTextSpan {
peer_idx: peers.register(&id.peer),
counter: id.counter,
lamport: id.lamport,
lamport_sub_counter: id.lamport as i32 - id.counter as i32,
len: t.unicode_len(),
})
}
@ -1003,11 +1019,11 @@ mod snapshot {
spans.push(EncodedTextSpan {
peer_idx: peers.register(&id.peer),
counter: id.counter,
lamport: id.lamport,
lamport_sub_counter: id.lamport as i32 - id.counter as i32,
len: 0,
});
marks.push(EncodedMark {
key: style.key.clone(),
key_idx: keys.register(&style.key),
value: style.value.clone(),
info: style.info.to_byte(),
})
@ -1017,7 +1033,7 @@ mod snapshot {
spans.push(EncodedTextSpan {
peer_idx: peers.register(&id.peer),
counter: id.counter + 1,
lamport: id.lamport + 1,
lamport_sub_counter: id.lamport as i32 - id.counter as i32,
len: -1,
})
}
@ -1031,16 +1047,22 @@ mod snapshot {
w.write_all(&peer.to_le_bytes()).unwrap();
}
let bytes = serde_columnar::to_vec(&EncodedText { spans, marks }).unwrap();
let bytes = serde_columnar::to_vec(&EncodedText {
spans,
keys: keys.unwrap_vec(),
marks,
})
.unwrap();
w.write_all(&bytes).unwrap();
}
fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> {
postcard::take_from_bytes(bytes).map_err(|_| {
let (value, bytes) = postcard::take_from_bytes(bytes).map_err(|_| {
loro_common::LoroError::DecodeError(
"Decode list value failed".to_string().into_boxed_str(),
)
})
})?;
Ok((LoroValue::String(value), bytes))
}
fn decode_snapshot_fast(
@ -1064,6 +1086,7 @@ mod snapshot {
let string = string.into_string().unwrap();
let mut s = StrSlice::new_from_str(&string);
let iters = serde_columnar::from_bytes::<EncodedText>(bytes).unwrap();
let keys = iters.keys;
let span_iter = iters.spans.into_iter();
let mut mark_iter = iters.marks.into_iter();
let mut id_to_style = FxHashMap::default();
@ -1071,19 +1094,27 @@ mod snapshot {
let EncodedTextSpan {
peer_idx,
counter,
lamport,
lamport_sub_counter,
len,
} = span;
let id_full = IdFull::new(peers[peer_idx], counter, lamport);
let id_full = IdFull::new(
peers[peer_idx],
counter,
(lamport_sub_counter + counter as i32) as u32,
);
let chunk = match len {
0 => {
// Style Start
let EncodedMark { key, value, info } = mark_iter.next().unwrap();
let EncodedMark {
key_idx,
value,
info,
} = mark_iter.next().unwrap();
let style_op = Arc::new(StyleOp {
lamport,
lamport: (lamport_sub_counter + counter as i32) as u32,
peer: id_full.peer,
cnt: id_full.counter,
key,
key: keys[key_idx].clone(),
value,
info: TextStyleInfoFlag::from_byte(info),
});
@ -1133,6 +1164,8 @@ mod snapshot {
.get_text("text")
.unwrap()
.encode_snapshot_fast(&mut bytes);
assert!(bytes.len() <= 76, "w.len() = {}", bytes.len());
let delta = doc
.app_state()
.lock()

70
crates/loro-internal/src/state/tree_state.rs
View file

@ -979,9 +979,10 @@ impl ContainerState for TreeState {
}
TreeInternalDiff::Move { parent, position } => {
let old_parent = self.trees.get(&target).unwrap().parent;
let old_index = self.get_index_by_tree_id(&target).unwrap();
// If this is some, the node is still alive at the moment
let old_index = self.get_index_by_tree_id(&target);
if need_check {
let was_alive = !self.is_node_deleted(&target);
let was_alive = old_index.is_some();
if self
.mov(target, *parent, last_move_op, Some(position.clone()), true)
.is_ok()
@ -993,7 +994,7 @@ impl ContainerState for TreeState {
target,
action: TreeExternalDiff::Delete {
old_parent,
old_index,
old_index: old_index.unwrap(),
},
});
}
@ -1007,7 +1008,7 @@ impl ContainerState for TreeState {
index: self.get_index_by_tree_id(&target).unwrap(),
position: position.clone(),
old_parent,
old_index,
old_index: old_index.unwrap(),
},
});
} else {
@ -1027,16 +1028,30 @@ impl ContainerState for TreeState {
.unwrap();
let index = self.get_index_by_tree_id(&target).unwrap();
ans.push(TreeDiffItem {
target,
action: TreeExternalDiff::Move {
parent: *parent,
index,
position: position.clone(),
old_parent,
old_index,
},
});
match old_index {
Some(old_index) => {
ans.push(TreeDiffItem {
target,
action: TreeExternalDiff::Move {
parent: *parent,
index,
position: position.clone(),
old_parent,
old_index,
},
});
}
None => {
ans.push(TreeDiffItem {
target,
action: TreeExternalDiff::Create {
parent: *parent,
index,
position: position.clone(),
},
});
}
}
};
}
TreeInternalDiff::Delete { parent, position } => {
@ -1424,7 +1439,7 @@ mod snapshot {
use fractional_index::FractionalIndex;
use fxhash::FxHashMap;
use itertools::Itertools;
use loro_common::{IdFull, PeerID, TreeID};
use loro_common::{IdFull, Lamport, PeerID, TreeID};
use serde_columnar::columnar;
@ -1455,9 +1470,7 @@ mod snapshot {
#[columnar(strategy = "DeltaRle")]
last_set_counter: i32,
#[columnar(strategy = "DeltaRle")]
last_set_lamport: u32,
#[columnar(strategy = "DeltaRle")]
index: u32,
last_set_lamport_sub_counter: i32,
fractional_index_idx: usize,
}
@ -1506,8 +1519,7 @@ mod snapshot {
},
last_set_peer_idx: peers.register(&last_set_id.peer),
last_set_counter: last_set_id.counter,
last_set_lamport: last_set_id.lamport,
index: node.index as u32,
last_set_lamport_sub_counter: last_set_id.lamport as i32 - last_set_id.counter,
fractional_index_idx: position_register.register(&node.position),
})
}
@ -1526,6 +1538,19 @@ mod snapshot {
}
impl FastStateSnapshot for TreeState {
/// Encodes the TreeState into a compact binary format for efficient serialization.
///
/// The encoding schema:
/// 1. Encodes all nodes using a breadth-first search traversal, ensuring a consistent order.
/// 2. Uses a ValueRegister to deduplicate and index PeerIDs and TreeIDs, reducing redundancy.
/// - PeerIDs are stored once and referenced by index.
/// - TreeIDs are decomposed into peer index and counter for compact representation.
/// 3. Encodes fractional indexes using a PositionArena for space efficiency
/// 4. Utilizes delta encoding and run-length encoding for further size reduction:
/// - Delta encoding stores differences between consecutive values.
/// - Run-length encoding compresses sequences of repeated values.
/// 5. Stores parent relationships using indices, with special values for root and deleted nodes.
/// 6. Encodes last move operation details (peer_idx, counter[Delta], lamport clock[Delta]) for each node.
fn encode_snapshot_fast<W: std::io::prelude::Write>(&mut self, mut w: W) {
let all_nodes = self.bfs_all_nodes_for_fast_snapshot();
let (peers, encoded) = encode(self, all_nodes);
@ -1582,7 +1607,7 @@ mod snapshot {
IdFull::new(
peers[node.last_set_peer_idx],
node.last_set_counter,
node.last_set_lamport,
(node.last_set_lamport_sub_counter + node.last_set_counter) as Lamport,
),
Some(FractionalIndex::from_bytes(
fractional_indexes[node.fractional_index_idx].clone(),
@ -1614,6 +1639,7 @@ mod snapshot {
doc.set_peer_id(0).unwrap();
doc.start_auto_commit();
let tree = doc.get_tree("tree");
tree.enable_fractional_index(0);
let a = tree.create(TreeParentId::Root).unwrap();
let b = tree.create(TreeParentId::Root).unwrap();
let _c = tree.create(TreeParentId::Root).unwrap();
@ -1626,6 +1652,8 @@ mod snapshot {
tree_state.encode_snapshot_fast(&mut bytes);
(bytes, value)
};
assert!(bytes.len() == 55, "{}", bytes.len());
let mut new_tree_state = TreeState::decode_snapshot_fast(
ContainerIdx::from_index_and_type(0, loro_common::ContainerType::Tree),
(LoroValue::Null, &bytes),