fix: snapshot unknown and change value to i64

This commit is contained in:
leeeon233 2023-02-20 17:19:25 +08:00
parent b8e27dc011
commit af6342a52d
4 changed files with 478 additions and 396 deletions

View file

@ -380,13 +380,23 @@ impl Container for ListContainer {
InnerListOp::Insert { slice, pos } => {
if should_notify {
let mut delta = Delta::new();
let delta_vec = self.raw_data.slice(&slice.0).to_vec();
// unknown
let delta_vec = if slice.is_unknown() {
let mut ans = Vec::with_capacity(slice.atom_len());
for _ in 0..slice.content_len() {
ans.push(LoroValue::Null);
}
ans
} else {
self.raw_data.slice(&slice.0).to_vec()
};
delta.retain(*pos);
delta.insert(delta_vec);
context.push_diff(&self.id, Diff::List(delta));
}
self.update_hierarchy_on_insert(hierarchy, slice);
if !slice.is_unknown() {
self.update_hierarchy_on_insert(hierarchy, slice);
}
self.state.insert(*pos, slice.clone());
}
InnerListOp::Delete(span) => {
@ -729,7 +739,7 @@ impl ContainerWrapper for List {
#[cfg(test)]
mod test {
use crate::{LoroCore, LoroValue, PrelimContainer};
use crate::LoroCore;
#[test]
fn test_list_get() {

View file

@ -1286,45 +1286,27 @@ mod failed_tests {
test_multi_sites(
5,
&mut [
Map {
site: 1,
Text {
site: 0,
container_idx: 0,
key: 21,
value: Null,
pos: 0,
value: 32125,
is_del: false,
},
SyncAll,
Map {
site: 0,
Text {
site: 2,
container_idx: 0,
key: 0,
value: Container(C::Text),
pos: 1,
value: 1,
is_del: true,
},
List {
site: 4,
Text {
site: 2,
container_idx: 0,
key: 0,
value: I32(1),
},
Sync { from: 2, to: 4 },
Sync { from: 1, to: 2 },
Map {
site: 1,
container_idx: 0,
key: 21,
value: Null,
},
Sync { from: 1, to: 0 },
Map {
site: 1,
container_idx: 0,
key: 21,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 251,
value: Container(C::Text),
pos: 0,
value: 1,
is_del: true,
},
],
)
@ -1336,6 +1318,19 @@ mod failed_tests {
minify_error(
5,
vec![
Map {
site: 0,
container_idx: 255,
key: 255,
value: Null,
},
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
Map {
site: 21,
container_idx: 21,
@ -1364,7 +1359,73 @@ mod failed_tests {
site: 21,
container_idx: 21,
key: 21,
value: Container(C::Text),
value: I32(2105376125),
},
Text {
site: 125,
container_idx: 125,
pos: 125,
value: 32125,
is_del: true,
},
Text {
site: 125,
container_idx: 125,
pos: 125,
value: 32125,
is_del: true,
},
Text {
site: 125,
container_idx: 125,
pos: 125,
value: 32125,
is_del: false,
},
Text {
site: 125,
container_idx: 125,
pos: 125,
value: 32125,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
@ -1373,371 +1434,382 @@ mod failed_tests {
value: Container(C::List),
},
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 125,
value: I32(2105376125),
},
Text {
site: 125,
container_idx: 125,
pos: 125,
value: 32125,
is_del: true,
},
Text {
site: 125,
container_idx: 125,
pos: 125,
value: 27517,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 107,
value: 27499,
is_del: true,
},
Text {
site: 107,
container_idx: 107,
pos: 64,
value: 16448,
is_del: false,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 255,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Container(C::Text),
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
List {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 41,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Container(C::List),
},
Sync { from: 191, to: 191 },
Sync { from: 21, to: 21 },
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 20,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Container(C::List),
},
SyncAll,
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 255,
key: 255,
value: Null,
},
Map {
site: 255,
container_idx: 255,
key: 255,
value: Container(C::List),
},
Sync { from: 197, to: 64 },
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
Sync { from: 191, to: 191 },
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 5,
container_idx: 5,
key: 5,
value: Null,
},
Map {
site: 5,
container_idx: 5,
key: 5,
value: Null,
},
Map {
site: 5,
container_idx: 5,
key: 5,
value: Null,
},
Map {
site: 5,
container_idx: 5,
key: 5,
value: Null,
},
Map {
site: 5,
container_idx: 5,
key: 5,
value: Null,
},
Map {
site: 5,
container_idx: 5,
key: 5,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 61,
container_idx: 61,
key: 61,
value: Null,
},
List {
site: 61,
container_idx: 61,
key: 61,
value: Null,
},
List {
site: 61,
container_idx: 61,
key: 61,
value: Null,
},
List {
site: 61,
container_idx: 61,
key: 61,
value: Null,
},
List {
site: 61,
container_idx: 61,
key: 61,
value: Null,
},
Sync { from: 191, to: 46 },
Sync { from: 191, to: 191 },
Sync { from: 191, to: 191 },
Sync { from: 191, to: 191 },
Sync { from: 191, to: 65 },
Sync { from: 191, to: 191 },
Sync { from: 191, to: 191 },
Sync { from: 191, to: 191 },
Sync { from: 191, to: 191 },
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
value: Null,
},
Map {
site: 4,
container_idx: 58,
key: 21,
value: Null,
},
Map {
site: 64,
container_idx: 64,
key: 64,
value: Null,
},
List {
site: 64,
container_idx: 1,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 5,
container_idx: 15,
key: 255,
value: Container(C::Text),
},
Map {
site: 21,
container_idx: 21,
key: 21,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 21,
container_idx: 21,
key: 21,
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
List {
site: 64,
container_idx: 64,
key: 64,
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
List {
site: 64,
container_idx: 66,
key: 64,
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Sync { from: 64, to: 64 },
List {
site: 191,
container_idx: 191,
key: 191,
value: Container(C::List),
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Map {
site: 0,
container_idx: 0,
key: 0,
value: Null,
},
Sync { from: 191, to: 191 },
Sync { from: 191, to: 191 },
],
test_multi_sites,
normalize,

View file

@ -185,9 +185,12 @@ impl Hierarchy {
.unwrap()
.try_lock()
.unwrap()
.index_of_child(node_id)
.unwrap();
path.push(index);
.index_of_child(node_id);
if let Some(index) = index {
path.push(index);
} else {
return None;
}
} else {
match node_id {
ContainerID::Root {

View file

@ -15,7 +15,6 @@ use crate::{
list::list_op::{DeleteSpan, InnerListOp},
map::{InnerMapSet, ValueSlot},
pool_mapping::StateContent,
registry::ContainerIdx,
Container, ContainerID,
},
dag::{remove_included_frontiers, Dag},
@ -24,7 +23,7 @@ use crate::{
id::{ClientID, Counter, ID},
log_store::{encoding::encode_changes::get_lamport_by_deps, ImportContext},
op::{InnerContent, Op},
span::{HasIdSpan, HasLamportSpan},
span::HasLamportSpan,
version::TotalOrderStamp,
ContainerType, InternalString, LogStore, LoroCore, LoroError, LoroValue, VersionVector,
};
@ -128,8 +127,8 @@ struct SnapshotOpEncoding {
/// key index or insert/delete pos
#[columnar(strategy = "DeltaRle")]
prop: usize,
// list range start or del len or map value index
value: u64,
// list range start or del len or map value index, maybe negative
value: i64,
// List: the length of content when inserting, -2 when the inserted content is unknown, and -1 when deleting.
// Map: always -1
#[columnar(strategy = "Rle")]
@ -159,22 +158,22 @@ fn convert_inner_content(
op_content: &InnerContent,
key_to_idx: &mut FxHashMap<InternalString, usize>,
keys: &mut Vec<InternalString>,
) -> (usize, u64, i64) {
) -> (usize, i64, i64) {
let (prop, value, is_del) = match &op_content {
InnerContent::List(list_op) => match list_op {
InnerListOp::Insert { slice, pos } => {
if slice.is_unknown() {
(*pos, slice.content_len() as u64, ENCODED_UNKNOWN_SLICE)
(*pos, slice.content_len() as i64, ENCODED_UNKNOWN_SLICE)
} else {
(
*pos,
slice.0.start as u64,
slice.0.start as i64,
(slice.0.end - slice.0.start) as i64,
)
}
}
InnerListOp::Delete(span) => {
(span.pos as usize, span.len as u64, ENCODED_DELETED_CONTENT)
(span.pos as usize, span.len as i64, ENCODED_DELETED_CONTENT)
}
},
InnerContent::Map(map_set) => {
@ -184,7 +183,7 @@ fn convert_inner_content(
keys.push(key.clone());
keys.len() - 1
}),
*value as u64,
*value as i64,
ENCODED_PLACEHOLDER,
)
}
@ -329,14 +328,18 @@ pub(super) fn decode_snapshot(
// calc vv
let vv = calc_vv(&change_encodings, &ops, &clients, &idx_to_container_type);
// println!("remote vv {:?} self vv {:?}", vv, &store.vv);
let can_load = match vv.partial_cmp(&store.vv) {
Some(ord) => match ord {
std::cmp::Ordering::Less => {
// TODO warning
println!("[Warning] the vv of encoded snapshot is smaller than self, no change is applied");
debug_log::debug_log!("[Warning] the vv of encoded snapshot is smaller than self, no change is applied");
return Ok(vec![]);
}
std::cmp::Ordering::Equal => {
debug_log::debug_log!("vv is equal, no change is applied");
return Ok(vec![]);
}
std::cmp::Ordering::Equal => return Ok(vec![]),
std::cmp::Ordering::Greater => store.vv.is_empty(),
},
None => false,
@ -416,7 +419,7 @@ pub(super) fn decode_snapshot(
}
} else {
InnerListOp::Insert {
slice: (value as u32..(value as i64 + value2) as u32).into(),
slice: (value as u32..(value + value2) as u32).into(),
pos: prop,
}
}
@ -498,6 +501,7 @@ pub(super) fn decode_snapshot(
}
if can_load {
// println!("can load");
let mut import_context = load_snapshot(
store,
hierarchy,
@ -521,6 +525,7 @@ pub(super) fn decode_snapshot(
&clients,
);
let diff_changes = new_store.export(&store.vv);
// println!("diff change {:?}", diff_changes);
Ok(store.import(hierarchy, diff_changes))
}
}
@ -608,11 +613,11 @@ fn calc_vv(
_ => {
let is_del = value2 == ENCODED_DELETED_CONTENT;
if is_del {
value
value.unsigned_abs()
} else {
let is_unknown = value2 == ENCODED_UNKNOWN_SLICE;
if is_unknown {
value
value as u64
} else {
value2 as u64
}
@ -635,19 +640,11 @@ mod test {
#[test]
fn cannot_load() {
let mut loro = LoroCore::new(Default::default(), Some(1));
let mut list = loro.get_list("list");
list.insert(&loro, 0, ContainerType::Text).unwrap();
let mut map = loro.get_map("map");
map.insert(&loro, "0", ContainerType::List).unwrap();
map.delete(&loro, "0").unwrap();
let mut loro2 = LoroCore::new(Default::default(), Some(2));
loro2.import(loro.export(loro2.vv_cloned()));
let mut list2 = loro2.get_list("list");
list2.delete(&loro2, 0, 1).unwrap();
loro.import(loro2.export(loro.vv_cloned()));
let mut list = loro.get_list("list");
list.insert(&loro, 0, "abc").unwrap();
loro2.decode(&loro.encode_all()).unwrap();
loro.decode(&loro2.encode_all()).unwrap();
loro2.decode(&loro.encode_all()).unwrap();
}
}