diff --git a/Cargo.lock b/Cargo.lock index 5a4b9a72..a808b6cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,9 +46,9 @@ checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "append-only-bytes" @@ -638,17 +638,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "fractional_index" -version = "0.1.0" -source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99" -dependencies = [ - "imbl", - "rand", - "serde", - "smallvec", -] - [[package]] name = "fractional_index" version = "2.0.1" @@ -672,9 +661,10 @@ dependencies = [ "fxhash", "itertools 0.12.1", "loro 0.16.2", - "loro 0.5.1", - "loro-common 0.5.1", + "loro 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", "rand", + "serde_json", "tabled 0.10.0", "tracing", "tracing-chrome", @@ -946,9 +936,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.63" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -990,19 +980,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "loro" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99" -dependencies = [ - "either", - "enum-as-inner 0.6.0", - "generic-btree", - "loro-delta 0.5.1", - "loro-internal 0.5.1", - "tracing", -] - [[package]] name = "loro" version = "0.16.2" @@ -1020,19 +997,16 @@ dependencies = [ ] [[package]] -name = "loro-common" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99" +name = "loro" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ - "arbitrary", + "either", "enum-as-inner 0.6.0", - "fxhash", - "loro-rle 0.5.1", - "nonmax", - "serde", - "serde_columnar", - "string_cache", - "thiserror", + "generic-btree", + "loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "tracing", ] [[package]] @@ -1053,15 +1027,19 @@ dependencies = [ ] [[package]] -name = "loro-delta" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99" +name = "loro-common" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ - "arrayvec", - "enum-as-inner 0.5.1", - "generic-btree", - "heapless 0.8.0", - "tracing", + "arbitrary", + "enum-as-inner 0.6.0", + "fxhash", + "loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "nonmax", + "serde", + "serde_columnar", + "string_cache", + "thiserror", ] [[package]] @@ -1081,37 +1059,14 @@ dependencies = [ ] [[package]] -name = "loro-internal" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99" +name = "loro-delta" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ - "append-only-bytes", - "arref", - "either", + "arrayvec", "enum-as-inner 0.5.1", - "enum_dispatch", - "fractional_index 0.1.0", - "fxhash", "generic-btree", - "getrandom", - "im", - "itertools 0.12.1", - "leb128", - "loro-common 0.5.1", - "loro-delta 0.5.1", - "loro-rle 0.5.1", - "md5", - "num", - "num-derive", - "num-traits", - "once_cell", - "postcard", - "rand", - "serde", - "serde_columnar", - "serde_json", - "smallvec", - "thiserror", + "heapless 0.8.0", "tracing", ] @@ -1142,7 +1097,7 @@ dependencies = [ "loro-common 0.16.2", "loro-delta 0.16.2", "loro-rle 0.16.2", - "loro_fractional_index", + "loro_fractional_index 0.16.2", "md5", "miniz_oxide 0.7.1", "num", @@ -1154,7 +1109,7 @@ dependencies = [ "proptest-derive", "rand", "serde", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.5.0", "serde_columnar", "serde_json", "smallvec", @@ -1167,16 +1122,38 @@ dependencies = [ ] [[package]] -name = "loro-rle" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99" +name = "loro-internal" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ "append-only-bytes", "arref", - "enum-as-inner 0.6.0", + "either", + "enum-as-inner 0.5.1", + "enum_dispatch", "fxhash", + "generic-btree", + "getrandom", + "im", + "itertools 0.12.1", + "leb128", + "loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "md5", "num", + "num-derive", + "num-traits", + "once_cell", + "postcard", + "rand", + "serde", + "serde_columnar", + "serde_json", "smallvec", + "thiserror", + "tracing", ] [[package]] @@ -1196,6 +1173,19 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "loro-rle" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" +dependencies = [ + "append-only-bytes", + "arref", + "enum-as-inner 0.6.0", + "fxhash", + "num", + "smallvec", +] + [[package]] name = "loro-thunderdome" version = "0.6.2" @@ -1212,7 +1202,8 @@ dependencies = [ "loro-internal 0.16.2", "loro-rle 0.16.2", "serde", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.6.5", + "serde_json", "tracing", "tracing-wasm", "wasm-bindgen", @@ -1224,7 +1215,18 @@ name = "loro_fractional_index" version = "0.16.2" dependencies = [ "criterion 0.5.1", - "fractional_index 2.0.1", + "fractional_index", + "imbl", + "rand", + "serde", + "smallvec", +] + +[[package]] +name = "loro_fractional_index" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" +dependencies = [ "imbl", "rand", "serde", @@ -1844,6 +1846,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_columnar" version = "0.3.4" diff --git a/Cargo.toml b/Cargo.toml index 6c76cc0a..82022815 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,14 @@ resolver = "2" [workspace.dependencies] enum_dispatch = "0.3.11" -debug-log = { version = "0.3.1", features = [] } enum-as-inner = "0.5.1" fxhash = "0.2.1" tracing = { version = "0.1", features = [ "max_level_debug", "release_max_level_warn", ] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" serde_columnar = { version = "0.3.4" } itertools = "0.12.1" +smallvec = { version = "1.8.0", features = ["serde"] } diff --git a/crates/bench-utils/Cargo.toml b/crates/bench-utils/Cargo.toml index c8283f84..b20b83ae 100644 --- a/crates/bench-utils/Cargo.toml +++ b/crates/bench-utils/Cargo.toml @@ -13,4 +13,4 @@ loro-common = { path = "../loro-common" } enum-as-inner = "0.5.1" flate2 = "1.0.25" rand = "0.8.5" -serde_json = "1.0.89" +serde_json = { workspace = true } diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index c2936279..58378c07 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -11,7 +11,7 @@ bench-utils = { path = "../bench-utils" } loro = { path = "../loro" } tabled = "0.15.0" arbitrary = { version = "1.3.0", features = ["derive"] } -serde_json = "1.0.111" +serde_json = { workspace = true } tracing = "0.1.40" criterion = "0.4.0" diff --git a/crates/fractional_index/Cargo.toml b/crates/fractional_index/Cargo.toml index 505ed868..b7533e39 100644 --- a/crates/fractional_index/Cargo.toml +++ b/crates/fractional_index/Cargo.toml @@ -13,8 +13,8 @@ keywords = ["crdt", "local-first"] [dependencies] imbl = "^3.0" -smallvec = "^1.13" -serde = { version = "^1.0", features = ["derive", "rc"], optional = true } +smallvec = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"], optional = true } rand = { version = "^0.8" } [dev-dependencies] diff --git a/crates/fractional_index/src/lib.rs b/crates/fractional_index/src/lib.rs index e56b63ab..84cc525d 100644 --- a/crates/fractional_index/src/lib.rs +++ b/crates/fractional_index/src/lib.rs @@ -1,4 +1,7 @@ -use std::{fmt::Display, sync::Arc}; +use std::{ + fmt::{Display, Write}, + sync::Arc, +}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -27,6 +30,16 @@ impl FractionalIndex { FractionalIndex(Arc::new(bytes)) } + pub fn from_hex_string>(str: T) -> Self { + let s = str.as_ref(); + let mut bytes = Vec::with_capacity(s.len() / 2); + for i in 0..s.len() / 2 { + let byte = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).unwrap(); + bytes.push(byte); + } + FractionalIndex::from_bytes(bytes) + } + pub fn as_bytes(&self) -> &[u8] { &self.0 } @@ -61,7 +74,7 @@ pub(crate) fn new_after(bytes: &[u8]) -> Vec { } pub(crate) fn new_between(left: &[u8], right: &[u8], extra_capacity: usize) -> Option> { - let shorter_len = left.len().min(right.len()) - 1; + let shorter_len = left.len().min(right.len()); for i in 0..shorter_len { if left[i] < right[i] - 1 { let mut ans: Vec = left[0..=i].into(); @@ -183,19 +196,9 @@ impl Display for FractionalIndex { } } -const HEX_CHARS: &[u8] = b"0123456789abcdef"; - -pub fn byte_to_hex(byte: u8) -> String { - let mut s = String::new(); - s.push(HEX_CHARS[(byte >> 4) as usize] as char); - s.push(HEX_CHARS[(byte & 0xf) as usize] as char); - s -} - pub fn bytes_to_hex(bytes: &[u8]) -> String { - let mut s = String::with_capacity(bytes.len() * 2); - for byte in bytes { - s.push_str(&byte_to_hex(*byte)); - } - s + bytes.iter().fold(String::new(), |mut output, b| { + let _ = write!(output, "{b:02X}"); + output + }) } diff --git a/crates/fuzz/Cargo.toml b/crates/fuzz/Cargo.toml index 4ce63c76..4dab2e35 100644 --- a/crates/fuzz/Cargo.toml +++ b/crates/fuzz/Cargo.toml @@ -10,13 +10,15 @@ publish = false loro-without-counter = { path = "../loro", package = "loro" } loro = { git = "https://github.com/loro-dev/loro.git", features = [ "counter", -], rev = "cd04b27d65128420f6daaf792be9ef511483de99" } +], rev = "83938290ab2666d85c0c72169127611585a05cf9" } loro-common = { git = "https://github.com/loro-dev/loro.git", features = [ "counter", -], rev = "cd04b27d65128420f6daaf792be9ef511483de99" } +], rev = "83938290ab2666d85c0c72169127611585a05cf9" } # loro = { path = "../loro", package = "loro", features = ["counter"] } -# loro-common = { path = "../loro-common", features = ["counter"] } -# loro-without-counter = { git = "https://github.com/loro-dev/loro.git", branch = "zxch3n/loro-560-undoredo", package = "loro" } +# loro-common = { path = "../loro-common", package = "loro-common", features = [ +# "counter", +# ] } +# loro-without-counter = { git = "https://github.com/loro-dev/loro.git", rev = "eb6daf4f064238cbc5c3d357615f5ed73767e98c", package = "loro" } fxhash = { workspace = true } enum_dispatch = { workspace = true } enum-as-inner = { workspace = true } @@ -33,3 +35,4 @@ dev-utils = { path = "../dev-utils" } tracing-subscriber = "0.3.18" tracing-chrome = "0.7.1" color-backtrace = "0.6.1" +serde_json = "1" diff --git a/crates/fuzz/fuzz/Cargo.lock b/crates/fuzz/fuzz/Cargo.lock index 7b35879f..7f514917 100644 --- a/crates/fuzz/fuzz/Cargo.lock +++ b/crates/fuzz/fuzz/Cargo.lock @@ -206,27 +206,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "fractional_index" -version = "0.1.0" -dependencies = [ - "imbl", - "rand", - "serde", - "smallvec", -] - -[[package]] -name = "fractional_index" -version = "0.1.0" -source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125" -dependencies = [ - "imbl", - "rand", - "serde", - "smallvec", -] - [[package]] name = "fuzz" version = "0.1.0" @@ -236,9 +215,9 @@ dependencies = [ "enum_dispatch", "fxhash", "itertools 0.12.1", - "loro 0.5.1", - "loro 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", - "loro-common 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", + "loro 0.16.2", + "loro 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", "rand", "tabled", "tracing", @@ -444,37 +423,37 @@ dependencies = [ [[package]] name = "loro" -version = "0.5.1" +version = "0.16.2" dependencies = [ "either", "enum-as-inner 0.6.0", "generic-btree", - "loro-delta 0.5.1", - "loro-internal 0.5.1", + "loro-delta 0.16.2", + "loro-internal 0.16.2", "tracing", ] [[package]] name = "loro" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ "either", "enum-as-inner 0.6.0", "generic-btree", - "loro-delta 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", - "loro-internal 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", + "loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", "tracing", ] [[package]] name = "loro-common" -version = "0.5.1" +version = "0.16.2" dependencies = [ "arbitrary", "enum-as-inner 0.6.0", "fxhash", - "loro-rle 0.5.1", + "loro-rle 0.16.2", "nonmax", "serde", "serde_columnar", @@ -484,13 +463,13 @@ dependencies = [ [[package]] name = "loro-common" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ "arbitrary", "enum-as-inner 0.6.0", "fxhash", - "loro-rle 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", + "loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", "nonmax", "serde", "serde_columnar", @@ -500,7 +479,7 @@ dependencies = [ [[package]] name = "loro-delta" -version = "0.5.1" +version = "0.16.2" dependencies = [ "arrayvec", "enum-as-inner 0.5.1", @@ -511,8 +490,8 @@ dependencies = [ [[package]] name = "loro-delta" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ "arrayvec", "enum-as-inner 0.5.1", @@ -523,23 +502,23 @@ dependencies = [ [[package]] name = "loro-internal" -version = "0.5.1" +version = "0.16.2" dependencies = [ "append-only-bytes", "arref", "either", "enum-as-inner 0.5.1", "enum_dispatch", - "fractional_index 0.1.0", "fxhash", "generic-btree", "getrandom", "im", "itertools 0.12.1", "leb128", - "loro-common 0.5.1", - "loro-delta 0.5.1", - "loro-rle 0.5.1", + "loro-common 0.16.2", + "loro-delta 0.16.2", + "loro-rle 0.16.2", + "loro_fractional_index 0.16.2", "md5", "num", "num-derive", @@ -557,24 +536,24 @@ dependencies = [ [[package]] name = "loro-internal" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ "append-only-bytes", "arref", "either", "enum-as-inner 0.5.1", "enum_dispatch", - "fractional_index 0.1.0 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", "fxhash", "generic-btree", "getrandom", "im", "itertools 0.12.1", "leb128", - "loro-common 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", - "loro-delta 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", - "loro-rle 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)", + "loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", + "loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)", "md5", "num", "num-derive", @@ -592,7 +571,7 @@ dependencies = [ [[package]] name = "loro-rle" -version = "0.5.1" +version = "0.16.2" dependencies = [ "append-only-bytes", "arref", @@ -604,8 +583,8 @@ dependencies = [ [[package]] name = "loro-rle" -version = "0.5.1" -source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" dependencies = [ "append-only-bytes", "arref", @@ -621,6 +600,27 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" +[[package]] +name = "loro_fractional_index" +version = "0.16.2" +dependencies = [ + "imbl", + "rand", + "serde", + "smallvec", +] + +[[package]] +name = "loro_fractional_index" +version = "0.16.2" +source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9" +dependencies = [ + "imbl", + "rand", + "serde", + "smallvec", +] + [[package]] name = "md5" version = "0.7.0" diff --git a/crates/fuzz/src/crdt_fuzzer.rs b/crates/fuzz/src/crdt_fuzzer.rs index d48d0c8d..0415f368 100644 --- a/crates/fuzz/src/crdt_fuzzer.rs +++ b/crates/fuzz/src/crdt_fuzzer.rs @@ -197,20 +197,33 @@ impl CRDTFuzzer { info_span!("Attach", peer = j).in_scope(|| { b_doc.attach(); }); - if (i + j) % 2 == 0 { - info_span!("Updates", from = j, to = i).in_scope(|| { - a_doc.import(&b_doc.export_from(&a_doc.oplog_vv())).unwrap(); - }); - info_span!("Updates", from = i, to = j).in_scope(|| { - b_doc.import(&a_doc.export_from(&b_doc.oplog_vv())).unwrap(); - }); - } else { - info_span!("Snapshot", from = i, to = j).in_scope(|| { - b_doc.import(&a_doc.export_snapshot()).unwrap(); - }); - info_span!("Snapshot", from = j, to = i).in_scope(|| { - a_doc.import(&b_doc.export_snapshot()).unwrap(); - }); + match (i + j) % 3 { + 0 => { + info_span!("Updates", from = j, to = i).in_scope(|| { + a_doc.import(&b_doc.export_from(&a_doc.oplog_vv())).unwrap(); + }); + info_span!("Updates", from = i, to = j).in_scope(|| { + b_doc.import(&a_doc.export_from(&b_doc.oplog_vv())).unwrap(); + }); + } + 1 => { + info_span!("Snapshot", from = i, to = j).in_scope(|| { + b_doc.import(&a_doc.export_snapshot()).unwrap(); + }); + info_span!("Snapshot", from = j, to = i).in_scope(|| { + a_doc.import(&b_doc.export_snapshot()).unwrap(); + }); + } + _ => { + info_span!("JsonFormat", from = i, to = j).in_scope(|| { + let a_json = a_doc.export_json_updates(&b_doc.oplog_vv()); + b_doc.import_json_updates(a_json).unwrap(); + }); + info_span!("JsonFormat", from = j, to = i).in_scope(|| { + let b_json = b_doc.export_json_updates(&a_doc.oplog_vv()); + a_doc.import_json_updates(b_json).unwrap(); + }); + } } a.check_eq(b); a.record_history(); diff --git a/crates/fuzz/tests/json.rs b/crates/fuzz/tests/json.rs new file mode 100644 index 00000000..99a2f8c0 --- /dev/null +++ b/crates/fuzz/tests/json.rs @@ -0,0 +1,91 @@ +use fuzz::{ + actions::{ActionWrapper::*, GenericAction}, + crdt_fuzzer::{test_multi_sites, Action::*, FuzzTarget, FuzzValue::*}, +}; +use loro::ContainerType::*; + +#[ctor::ctor] +fn init() { + dev_utils::setup_test_log(); +} + +#[test] +fn unknown_json() { + let doc = loro::LoroDoc::new(); + let doc_with_unknown = loro_without_counter::LoroDoc::new(); + let counter = doc.get_counter("counter"); + counter.increment(5).unwrap(); + counter.increment(1).unwrap(); + // json format with counter + let json = doc.export_json_updates(&Default::default()); + // Test1: old version import newer version json + if doc_with_unknown + .import_json_updates(serde_json::to_string(&json).unwrap()) + .is_ok() + { + panic!("json schema don't support forward compatibility"); + } + + let snapshot_with_counter = doc.export_snapshot(); + let doc3_without_counter = loro_without_counter::LoroDoc::new(); + // Test2: older version import newer version snapshot with counter + doc3_without_counter.import(&snapshot_with_counter).unwrap(); + let unknown_json_from_snapshot = doc3_without_counter.export_json_updates(&Default::default()); + // { + // "container": "cid:root-counter:Unknown(5)", + // "content": { + // "type": "unknown", + // "value_type": "unknown", + // "value": {"kind":16,"data":[]}, + // "prop": 5 + // }, + // "counter": 0 + // } + // Test3: older version export json with binary unknown + let _json_with_binary_unknown = doc3_without_counter.export_json_updates(&Default::default()); + let new_doc = loro::LoroDoc::new(); + // Test4: newer version import older version json with binary unknown + if new_doc + .import_json_updates(serde_json::to_string(&unknown_json_from_snapshot).unwrap()) + .is_ok() + { + panic!("json schema don't support forward compatibility"); + } +} + +#[test] +fn sub_container() { + test_multi_sites( + 5, + vec![FuzzTarget::All], + &mut [ + Handle { + site: 0, + target: 1, + container: 0, + action: Generic(GenericAction { + value: Container(Text), + bool: true, + key: 4293853225, + pos: 18446744073709551615, + length: 4625477192774582511, + prop: 18446744073428216116, + }), + }, + Sync { from: 0, to: 1 }, + Handle { + site: 0, + target: 0, + container: 0, + action: Generic(GenericAction { + value: I32(0), + bool: false, + key: 0, + pos: 0, + length: 0, + prop: 0, + }), + }, + ], + ) +} diff --git a/crates/loro-common/Cargo.toml b/crates/loro-common/Cargo.toml index 8db666b4..201b7226 100644 --- a/crates/loro-common/Cargo.toml +++ b/crates/loro-common/Cargo.toml @@ -15,7 +15,7 @@ keywords = ["crdt", "local-first"] [dependencies] rle = { path = "../rle", version = "0.16.2", package = "loro-rle" } -serde = { version = "1", features = ["derive"] } +serde = { workspace = true } thiserror = "1.0.43" wasm-bindgen = { version = "=0.2.92", optional = true } fxhash = "0.2.1" diff --git a/crates/loro-common/src/error.rs b/crates/loro-common/src/error.rs index acedf54d..622ca73e 100644 --- a/crates/loro-common/src/error.rs +++ b/crates/loro-common/src/error.rs @@ -62,6 +62,8 @@ pub enum LoroError { UndoInvalidIdSpan(ID), #[error("PeerID cannot be changed. Expected: {expected:?}, Actual: {actual:?}")] UndoWithDifferentPeerId { expected: PeerID, actual: PeerID }, + #[error("The input JSON schema is invalid")] + InvalidJsonSchema, } #[derive(Error, Debug)] diff --git a/crates/loro-common/src/id.rs b/crates/loro-common/src/id.rs index c54c1b60..58c53626 100644 --- a/crates/loro-common/src/id.rs +++ b/crates/loro-common/src/id.rs @@ -57,6 +57,32 @@ impl TryFrom<&str> for ID { } } +impl TryFrom<&str> for IdLp { + type Error = LoroError; + + fn try_from(value: &str) -> Result { + if value.split('@').count() != 2 || !value.starts_with('L') { + return Err(LoroError::DecodeError("Invalid ID format".into())); + } + + let mut iter = value[1..].split('@'); + let lamport = iter + .next() + .unwrap() + .parse::() + .map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?; + let client_id = iter + .next() + .unwrap() + .parse::() + .map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?; + Ok(IdLp { + peer: client_id, + lamport, + }) + } +} + impl PartialOrd for ID { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) diff --git a/crates/loro-common/src/lib.rs b/crates/loro-common/src/lib.rs index b38417dc..4c42b31d 100644 --- a/crates/loro-common/src/lib.rs +++ b/crates/loro-common/src/lib.rs @@ -386,9 +386,28 @@ mod container { "Text" | "text" => Ok(ContainerType::Text), "Tree" | "tree" => Ok(ContainerType::Tree), "MovableList" | "movableList" => Ok(ContainerType::MovableList), - _ => Err(LoroError::DecodeError( + a => { + if a.ends_with(')') { + let start = a.find('(').ok_or_else(|| { + LoroError::DecodeError( + format!("Invalid container type string \"{}\"", value).into(), + ) + })?; + let k = a[start+1..a.len() - 1].parse().map_err(|_| { + LoroError::DecodeError( format!("Unknown container type \"{}\". The valid options are Map|List|Text|Tree|MovableList.", value).into(), - )), + ) + })?; + match ContainerType::try_from_u8(k) { + Ok(k) => Ok(k), + Err(_) => Ok(ContainerType::Unknown(k)), + } + } else { + Err(LoroError::DecodeError( + format!("Unknown container type \"{}\". The valid options are Map|List|Text|Tree|MovableList.", value).into(), + )) + } + } } } } @@ -527,5 +546,6 @@ mod test { assert!(ContainerID::try_from("cid:@:Map").is_err()); assert!(ContainerID::try_from("cid:x@0:Map").is_err()); assert!(ContainerID::try_from("id:0@0:Map").is_err()); + assert!(ContainerID::try_from("cid:0@0:Unknown(6)").is_ok()); } } diff --git a/crates/loro-common/src/value.rs b/crates/loro-common/src/value.rs index 59e56c9d..c609d6a8 100644 --- a/crates/loro-common/src/value.rs +++ b/crates/loro-common/src/value.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, hash::Hash, ops::Index, sync::Arc}; use enum_as_inner::EnumAsInner; use fxhash::FxHashMap; -use serde::{de::VariantAccess, ser::SerializeStruct, Deserialize, Serialize}; +use serde::{de::VariantAccess, Deserialize, Serialize}; use crate::ContainerID; @@ -481,6 +481,8 @@ pub mod wasm { } } +const LORO_CONTAINER_ID_PREFIX: &str = "🦜:"; + impl Serialize for LoroValue { fn serialize(&self, serializer: S) -> Result where @@ -498,9 +500,7 @@ impl Serialize for LoroValue { LoroValue::List(l) => serializer.collect_seq(l.iter()), LoroValue::Map(m) => serializer.collect_map(m.iter()), LoroValue::Container(id) => { - let mut state = serializer.serialize_struct("Container", 1)?; - state.serialize_field("Container", id)?; - state.end() + serializer.serialize_str(&format!("{}{}", LORO_CONTAINER_ID_PREFIX, id)) } } } else { @@ -610,6 +610,12 @@ impl<'de> serde::de::Visitor<'de> for LoroValueVisitor { where E: serde::de::Error, { + if let Some(id) = v.strip_prefix(LORO_CONTAINER_ID_PREFIX) { + return Ok(LoroValue::Container( + ContainerID::try_from(id) + .map_err(|_| serde::de::Error::custom("Invalid container id"))?, + )); + } Ok(LoroValue::String(Arc::new(v.to_owned()))) } @@ -617,6 +623,13 @@ impl<'de> serde::de::Visitor<'de> for LoroValueVisitor { where E: serde::de::Error, { + if let Some(id) = v.strip_prefix(LORO_CONTAINER_ID_PREFIX) { + return Ok(LoroValue::Container( + ContainerID::try_from(id) + .map_err(|_| serde::de::Error::custom("Invalid container id"))?, + )); + } + Ok(LoroValue::String(v.into())) } @@ -652,9 +665,7 @@ impl<'de> serde::de::Visitor<'de> for LoroValueVisitor { A: serde::de::MapAccess<'de>, { let mut ans: FxHashMap = FxHashMap::default(); - let mut last_key = None; while let Some((key, value)) = map.next_entry::()? { - last_key.get_or_insert_with(|| key.clone()); ans.insert(key, value); } diff --git a/crates/loro-internal/Cargo.toml b/crates/loro-internal/Cargo.toml index 627b15ba..7a9610c6 100644 --- a/crates/loro-internal/Cargo.toml +++ b/crates/loro-internal/Cargo.toml @@ -14,16 +14,16 @@ keywords = ["crdt", "local-first"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +smallvec = { workspace = true } loro-delta = { path = "../delta", version = "0.16.2", package = "loro-delta" } rle = { path = "../rle", version = "0.16.2", package = "loro-rle" } loro-common = { path = "../loro-common", version = "0.16.2" } fractional_index = { path = "../fractional_index", features = [ "serde", ], version = "0.16.2", package = "loro_fractional_index" } -smallvec = { version = "1.8.0", features = ["serde"] } postcard = "1" fxhash = { workspace = true } -serde = { version = "1", features = ["derive"] } +serde = { workspace = true } thiserror = "1" enum-as-inner = { workspace = true } num = "0.4.0" @@ -33,7 +33,7 @@ tabled = { version = "0.10.0", optional = true } wasm-bindgen = { version = "=0.2.92", optional = true } serde-wasm-bindgen = { version = "0.5.0", optional = true } js-sys = { version = "0.3.60", optional = true } -serde_json = { version = "1" } +serde_json = { workspace = true } arref = "0.1.0" serde_columnar = { workspace = true } append-only-bytes = { version = "0.1.12", features = ["u32_range"] } diff --git a/crates/loro-internal/benches/encode.rs b/crates/loro-internal/benches/encode.rs index d1c4c55c..62e2a90b 100644 --- a/crates/loro-internal/benches/encode.rs +++ b/crates/loro-internal/benches/encode.rs @@ -107,6 +107,21 @@ mod run { store2.import(&buf).unwrap(); }) }); + + b.bench_function("B4_encode_json_update", |b| { + ensure_ran(); + b.iter(|| { + let _ = loro.export_json_updates(&Default::default()); + }) + }); + b.bench_function("B4_decode_json_update", |b| { + ensure_ran(); + let json = loro.export_json_updates(&Default::default()); + b.iter(|| { + let store2 = LoroDoc::default(); + store2.import_json_updates(json.clone()).unwrap(); + }) + }); } } diff --git a/crates/loro-internal/examples/encoding.rs b/crates/loro-internal/examples/encoding.rs index 0565d390..b4952d74 100644 --- a/crates/loro-internal/examples/encoding.rs +++ b/crates/loro-internal/examples/encoding.rs @@ -62,6 +62,15 @@ fn main() { output.len(), ); + let json_updates = + serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap(); + let output = miniz_oxide::deflate::compress_to_vec(json_updates.as_bytes(), 6); + println!( + "json updates size {} after compression {}", + json_updates.len(), + output.len(), + ); + // { // // Delta encoding diff --git a/crates/loro-internal/examples/encoding_refactored.rs b/crates/loro-internal/examples/encoding_refactored.rs index 4dee3117..30225aa4 100644 --- a/crates/loro-internal/examples/encoding_refactored.rs +++ b/crates/loro-internal/examples/encoding_refactored.rs @@ -3,8 +3,8 @@ use criterion::black_box; use loro_internal::loro::LoroDoc; fn main() { - // log_size(); - bench_decode(); + log_size(); + // bench_decode(); // bench_decode_updates(); } @@ -23,9 +23,12 @@ fn log_size() { txn.commit().unwrap(); let snapshot = loro.export_snapshot(); let updates = loro.export_from(&Default::default()); + let json_updates = + serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap(); println!("\n"); println!("Snapshot size={}", snapshot.len()); println!("Updates size={}", updates.len()); + println!("Json Updates size={}", json_updates.as_bytes().len()); println!("\n"); loro.diagnose_size(); } diff --git a/crates/loro-internal/src/container/tree/tree_op.rs b/crates/loro-internal/src/container/tree/tree_op.rs index 190ece40..79063a70 100644 --- a/crates/loro-internal/src/container/tree/tree_op.rs +++ b/crates/loro-internal/src/container/tree/tree_op.rs @@ -12,26 +12,45 @@ use crate::state::TreeParentId; /// - **Move**: move target tree node a child node of the specified parent node. /// - **Delete**: move target tree node to [`loro_common::DELETED_TREE_ROOT`]. /// - +/// #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct TreeOp { - pub(crate) target: TreeID, - pub(crate) parent: Option, - // If the op is `delete`, the position is None - pub(crate) position: Option, +pub enum TreeOp { + Create { + target: TreeID, + parent: Option, + position: FractionalIndex, + }, + Move { + target: TreeID, + parent: Option, + position: FractionalIndex, + }, + Delete { + target: TreeID, + }, } impl TreeOp { + pub(crate) fn target(&self) -> TreeID { + match self { + TreeOp::Create { target, .. } => *target, + TreeOp::Move { target, .. } => *target, + TreeOp::Delete { target, .. } => *target, + } + } pub(crate) fn parent_id(&self) -> TreeParentId { - match self.parent { - Some(parent) => { - if TreeID::is_deleted_root(&parent) { - TreeParentId::Deleted - } else { - TreeParentId::Node(parent) - } + match self { + TreeOp::Create { parent, .. } => TreeParentId::from(*parent), + TreeOp::Move { parent, .. } => TreeParentId::from(*parent), + TreeOp::Delete { .. } => TreeParentId::Deleted, + } + } + pub(crate) fn fractional_index(&self) -> Option { + match self { + TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => { + Some(position.clone()) } - None => TreeParentId::Root, + TreeOp::Delete { .. } => None, } } } diff --git a/crates/loro-internal/src/diff_calc/tree.rs b/crates/loro-internal/src/diff_calc/tree.rs index 6133d877..2f5b8440 100644 --- a/crates/loro-internal/src/diff_calc/tree.rs +++ b/crates/loro-internal/src/diff_calc/tree.rs @@ -126,9 +126,9 @@ impl TreeDiffCalculator { tracing::info!("forward ops {:?}", forward_ops); for (lamport, op) in forward_ops { let op = MoveLamportAndID { - target: op.value.target, + target: op.value.target(), parent: op.value.parent_id(), - position: op.value.position.clone(), + position: op.value.fractional_index(), id: op.id_start(), lamport, effected: false, @@ -239,9 +239,9 @@ impl TreeDiffCalculator { && to.includes_id(op.id_start()) { let op = MoveLamportAndID { - target: op.value.target, + target: op.value.target(), parent: op.value.parent_id(), - position: op.value.position.clone(), + position: op.value.fractional_index(), id: op.id_start(), lamport: *lamport, effected: false, diff --git a/crates/loro-internal/src/encoding.rs b/crates/loro-internal/src/encoding.rs index 898bfe3d..805e141c 100644 --- a/crates/loro-internal/src/encoding.rs +++ b/crates/loro-internal/src/encoding.rs @@ -1,5 +1,6 @@ mod arena; mod encode_reordered; +pub(crate) mod json_schema; mod value; pub(crate) use value::OwnedValue; diff --git a/crates/loro-internal/src/encoding/encode_reordered.rs b/crates/loro-internal/src/encoding/encode_reordered.rs index c656f9be..0dee9e08 100644 --- a/crates/loro-internal/src/encoding/encode_reordered.rs +++ b/crates/loro-internal/src/encoding/encode_reordered.rs @@ -207,7 +207,7 @@ pub fn decode_import_blob_meta(bytes: &[u8]) -> LoroResult { }) } -fn import_changes_to_oplog( +pub(crate) fn import_changes_to_oplog( changes: Vec, oplog: &mut OpLog, ) -> Result<(Vec, Vec), LoroError> { @@ -356,13 +356,7 @@ fn extract_ops( let peer = arenas.peer_ids[peer_idx as usize]; let cid = &containers[container_index as usize]; let kind = ValueKind::from_u8(value_type); - let value = Value::decode( - kind, - &mut value_reader, - arenas, - ID::new(peer, counter), - prop, - )?; + let value = Value::decode(kind, &mut value_reader, arenas, ID::new(peer, counter))?; let content = decode_op( cid, @@ -372,6 +366,7 @@ fn extract_ops( arenas, &positions, prop, + ID::new(peer, counter), )?; let container = shared_arena.register_container(cid); @@ -886,7 +881,7 @@ mod encode { use crate::{ arena::SharedArena, change::{Change, Lamport}, - container::idx::ContainerIdx, + container::{idx::ContainerIdx, tree::tree_op::TreeOp}, encoding::value::{EncodedTreeMove, MarkStart, Value, ValueKind, ValueWriter}, op::{FutureInnerContent, Op}, }; @@ -1178,18 +1173,16 @@ mod encode { let key = registers.key.register(&map.key); key as i32 } - crate::op::InnerContent::Tree(op) => { - if let Some(position) = &op.position { - if let either::Either::Left(position_register) = &mut registers.position { - position_register.insert(position.as_bytes()); - } else { + crate::op::InnerContent::Tree(op) => match op { + TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => { + let either::Either::Left(position_register) = &mut registers.position else { unreachable!() - } + }; + position_register.insert(position.as_bytes()); 0 - } else { - -1 } - } + TreeOp::Delete { .. } => 0, + }, crate::op::InnerContent::Future(f) => match f { #[cfg(feature = "counter")] FutureInnerContent::Counter(_) => 0, @@ -1286,6 +1279,7 @@ mod encode { } #[inline] +#[allow(clippy::too_many_arguments)] fn decode_op( cid: &ContainerID, value: Value<'_>, @@ -1294,6 +1288,7 @@ fn decode_op( arenas: &DecodedArenas<'_>, positions: &[Vec], prop: i32, + op_id: ID, ) -> LoroResult { let content = match cid.container_type() { ContainerType::Text => match value { @@ -1390,6 +1385,7 @@ fn decode_op( &arenas.peer_ids, positions, &arenas.tree_ids.tree_ids, + op_id, )?), _ => { unreachable!() @@ -1450,7 +1446,7 @@ fn decode_op( ContainerType::Counter => { crate::op::InnerContent::Future(FutureInnerContent::Counter(prop as i64)) } - + // NOTE: The future container type need also try to parse the unknown type ContainerType::Unknown(_) => crate::op::InnerContent::Future(FutureInnerContent::Unknown { prop, value: value.into_owned(), diff --git a/crates/loro-internal/src/encoding/json_schema.rs b/crates/loro-internal/src/encoding/json_schema.rs new file mode 100644 index 00000000..4cbfaa1e --- /dev/null +++ b/crates/loro-internal/src/encoding/json_schema.rs @@ -0,0 +1,1175 @@ +use std::{borrow::Cow, sync::Arc}; + +use loro_common::{ContainerID, ContainerType, IdLp, LoroResult, LoroValue, PeerID, TreeID, ID}; +use rle::{HasLength, Sliceable}; + +use crate::{ + arena::SharedArena, + change::Change, + container::{ + list::list_op::{DeleteSpan, DeleteSpanWithId, InnerListOp}, + map::MapSet, + richtext::TextStyleInfoFlag, + tree::tree_op::TreeOp, + }, + op::{FutureInnerContent, InnerContent, Op, SliceRange}, + version::Frontiers, + OpLog, VersionVector, +}; + +use super::encode_reordered::{import_changes_to_oplog, ValueRegister}; +use op::{JsonOpContent, JsonSchema}; + +const SCHEMA_VERSION: u8 = 1; + +pub(crate) fn export_json<'a, 'c: 'a>(oplog: &'c OpLog, vv: &VersionVector) -> JsonSchema { + let actual_start_vv: VersionVector = vv + .iter() + .filter_map(|(&peer, &end_counter)| { + if end_counter == 0 { + return None; + } + + let this_end = oplog.vv().get(&peer).cloned().unwrap_or(0); + if this_end <= end_counter { + return Some((peer, this_end)); + } + + Some((peer, end_counter)) + }) + .collect(); + + let frontiers = oplog.dag.vv_to_frontiers(&actual_start_vv); + + let mut peer_register = ValueRegister::::new(); + let diff_changes = init_encode(oplog, &actual_start_vv); + let changes = encode_changes(&diff_changes, &oplog.arena, &mut peer_register); + JsonSchema { + changes, + schema_version: SCHEMA_VERSION, + peers: peer_register.unwrap_vec(), + start_version: frontiers, + } +} + +pub(crate) fn import_json(oplog: &mut OpLog, json: JsonSchema) -> LoroResult<()> { + let changes = decode_changes(json, &oplog.arena); + let (latest_ids, pending_changes) = import_changes_to_oplog(changes, oplog)?; + if oplog.try_apply_pending(latest_ids).should_update && !oplog.batch_importing { + oplog.dag.refresh_frontiers(); + } + oplog.import_unknown_lamport_pending_changes(pending_changes)?; + Ok(()) +} + +fn init_encode<'s, 'a: 's>(oplog: &'a OpLog, vv: &'_ VersionVector) -> Vec> { + let self_vv = oplog.vv(); + let start_vv = vv.trim(oplog.vv()); + let mut diff_changes: Vec> = Vec::new(); + for change in oplog.iter_changes_peer_by_peer(&start_vv, self_vv) { + let start_cnt = start_vv.get(&change.id.peer).copied().unwrap_or(0); + if change.id.counter < start_cnt { + let offset = start_cnt - change.id.counter; + diff_changes.push(Cow::Owned(change.slice(offset as usize, change.atom_len()))); + } else { + diff_changes.push(Cow::Borrowed(change)); + } + } + diff_changes.sort_by_key(|x| x.lamport); + diff_changes +} + +fn register_id(id: &ID, peer_register: &mut ValueRegister) -> ID { + let peer = peer_register.register(&id.peer); + ID::new(peer as PeerID, id.counter) +} + +fn register_idlp(idlp: &IdLp, peer_register: &mut ValueRegister) -> IdLp { + IdLp { + peer: peer_register.register(&idlp.peer) as PeerID, + lamport: idlp.lamport, + } +} + +fn register_tree_id(tree: &TreeID, peer_register: &mut ValueRegister) -> TreeID { + TreeID { + peer: peer_register.register(&tree.peer) as PeerID, + counter: tree.counter, + } +} + +fn register_container_id( + container: ContainerID, + peer_register: &mut ValueRegister, +) -> ContainerID { + match container { + ContainerID::Normal { + peer, + counter, + container_type, + } => ContainerID::Normal { + peer: peer_register.register(&peer) as PeerID, + counter, + container_type, + }, + r => r, + } +} + +fn convert_container_id(container: ContainerID, peers: &[PeerID]) -> ContainerID { + match container { + ContainerID::Normal { + peer, + counter, + container_type, + } => ContainerID::Normal { + peer: peers[peer as usize], + counter, + container_type, + }, + r => r, + } +} + +fn convert_id(id: &ID, peers: &[PeerID]) -> ID { + ID { + peer: peers[id.peer as usize], + counter: id.counter, + } +} + +fn convert_idlp(idlp: &IdLp, peers: &[PeerID]) -> IdLp { + IdLp { + lamport: idlp.lamport, + peer: peers[idlp.peer as usize], + } +} + +fn convert_tree_id(tree: &TreeID, peers: &[PeerID]) -> TreeID { + TreeID { + peer: peers[tree.peer as usize], + counter: tree.counter, + } +} + +fn encode_changes( + diff_changes: &[Cow<'_, Change>], + arena: &SharedArena, + peer_register: &mut ValueRegister, +) -> Vec { + let mut changes = Vec::with_capacity(diff_changes.len()); + for change in diff_changes.iter() { + let mut ops = Vec::with_capacity(change.ops().len()); + for Op { + counter, + container, + content, + } in change.ops().iter() + { + let mut container = arena.get_container_id(*container).unwrap(); + if container.is_normal() { + container = register_container_id(container, peer_register); + } + let op = match container.container_type() { + ContainerType::List => match content { + InnerContent::List(list) => JsonOpContent::List(match list { + InnerListOp::Insert { slice, pos } => { + let mut value = + arena.get_values(slice.0.start as usize..slice.0.end as usize); + value.iter_mut().for_each(|x| { + if let LoroValue::Container(id) = x { + if id.is_normal() { + *id = register_container_id(id.clone(), peer_register); + } + } + }); + op::ListOp::Insert { + pos: *pos, + value: value.into(), + } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => op::ListOp::Delete { + pos: *pos, + len: *signed_len, + start_id: register_id(id_start, peer_register), + }, + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::MovableList => match content { + InnerContent::List(list) => JsonOpContent::MovableList(match list { + InnerListOp::Insert { slice, pos } => { + let mut value = + arena.get_values(slice.0.start as usize..slice.0.end as usize); + value.iter_mut().for_each(|x| { + if let LoroValue::Container(id) = x { + if id.is_normal() { + *id = register_container_id(id.clone(), peer_register); + } + } + }); + op::MovableListOp::Insert { + pos: *pos, + value: value.into(), + } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => op::MovableListOp::Delete { + pos: *pos, + len: *signed_len, + start_id: register_id(id_start, peer_register), + }, + InnerListOp::Move { from, from_id, to } => op::MovableListOp::Move { + from: *from, + to: *to, + elem_id: register_idlp(from_id, peer_register), + }, + InnerListOp::Set { elem_id, value } => { + let value = if let LoroValue::Container(id) = value { + if id.is_normal() { + LoroValue::Container(register_container_id( + id.clone(), + peer_register, + )) + } else { + value.clone() + } + } else { + value.clone() + }; + op::MovableListOp::Set { + elem_id: register_idlp(elem_id, peer_register), + value, + } + } + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::Text => match content { + InnerContent::List(list) => JsonOpContent::Text(match list { + InnerListOp::InsertText { + slice, + unicode_start: _, + unicode_len: _, + pos, + } => { + let text = String::from_utf8(slice.as_bytes().to_vec()).unwrap(); + op::TextOp::Insert { pos: *pos, text } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => op::TextOp::Delete { + pos: *pos, + len: *signed_len, + start_id: register_id(id_start, peer_register), + }, + InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => op::TextOp::Mark { + start: *start, + end: *end, + style_key: key.to_string(), + style_value: value.clone(), + info: info.to_byte(), + }, + InnerListOp::StyleEnd => op::TextOp::MarkEnd, + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::Map => match content { + InnerContent::Map(MapSet { key, value }) => { + JsonOpContent::Map(if let Some(v) = value { + let value = if let LoroValue::Container(id) = v { + if id.is_normal() { + LoroValue::Container(register_container_id( + id.clone(), + peer_register, + )) + } else { + v.clone() + } + } else { + v.clone() + }; + op::MapOp::Insert { + key: key.to_string(), + value, + } + } else { + op::MapOp::Delete { + key: key.to_string(), + } + }) + } + + _ => unreachable!(), + }, + + ContainerType::Tree => match content { + InnerContent::Tree(op) => JsonOpContent::Tree(match op { + TreeOp::Create { + target, + parent, + position, + } => op::TreeOp::Create { + target: register_tree_id(target, peer_register), + parent: parent.map(|p| register_tree_id(&p, peer_register)), + fractional_index: position.clone(), + }, + TreeOp::Move { + target, + parent, + position, + } => op::TreeOp::Move { + target: register_tree_id(target, peer_register), + parent: parent.map(|p| register_tree_id(&p, peer_register)), + fractional_index: position.clone(), + }, + TreeOp::Delete { target } => op::TreeOp::Delete { + target: register_tree_id(target, peer_register), + }, + }), + _ => unreachable!(), + }, + ContainerType::Unknown(_) => { + let InnerContent::Future(FutureInnerContent::Unknown { prop, value }) = content + else { + unreachable!(); + }; + JsonOpContent::Future(op::FutureOpWrapper { + prop: *prop, + value: op::FutureOp::Unknown(value.clone()), + }) + } + #[cfg(feature = "counter")] + ContainerType::Counter => { + let InnerContent::Future(f) = content else { + unreachable!() + }; + match f { + FutureInnerContent::Counter(x) => { + JsonOpContent::Future(op::FutureOpWrapper { + prop: *x as i32, + value: op::FutureOp::Counter(super::value::OwnedValue::Future( + super::value::OwnedFutureValue::Counter, + )), + }) + } + _ => unreachable!(), + } + } + }; + ops.push(op::JsonOp { + counter: *counter, + container, + content: op, + }); + } + let c = op::Change { + id: register_id(&change.id, peer_register), + ops, + deps: change + .deps + .iter() + .map(|id| register_id(id, peer_register)) + .collect(), + lamport: change.lamport, + timestamp: change.timestamp, + msg: None, + }; + changes.push(c); + } + changes +} + +fn decode_changes(json: JsonSchema, arena: &SharedArena) -> Vec { + let JsonSchema { peers, changes, .. } = json; + let mut ans = Vec::with_capacity(changes.len()); + for op::Change { + id, + timestamp, + deps, + lamport, + msg: _, + ops, + } in changes + { + let id = convert_id(&id, &peers); + let ops = ops + .into_iter() + .map(|op| decode_op(op, arena, &peers)) + .collect(); + let change = Change { + id, + timestamp, + deps: Frontiers::from_iter(deps.into_iter().map(|id| convert_id(&id, &peers))), + lamport, + ops, + has_dependents: false, + }; + ans.push(change); + } + ans +} + +fn decode_op(op: op::JsonOp, arena: &SharedArena, peers: &[PeerID]) -> Op { + let op::JsonOp { + counter, + container, + content, + } = op; + let container = convert_container_id(container, peers); + let idx = arena.register_container(&container); + let content = match container.container_type() { + ContainerType::Text => match content { + JsonOpContent::Text(text) => match text { + op::TextOp::Insert { pos, text } => { + let (slice, result) = arena.alloc_str_with_slice(&text); + InnerContent::List(InnerListOp::InsertText { + slice, + unicode_start: result.start as u32, + unicode_len: (result.end - result.start) as u32, + pos, + }) + } + op::TextOp::Delete { + pos, + len, + start_id: id_start, + } => { + let id_start = convert_id(&id_start, peers); + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { + pos, + signed_len: len, + }, + })) + } + op::TextOp::Mark { + start, + end, + style_key, + style_value, + info, + } => InnerContent::List(InnerListOp::StyleStart { + start, + end, + key: style_key.into(), + value: style_value, + info: TextStyleInfoFlag::from_byte(info), + }), + op::TextOp::MarkEnd => InnerContent::List(InnerListOp::StyleEnd), + }, + _ => unreachable!(), + }, + ContainerType::List => match content { + JsonOpContent::List(list) => match list { + op::ListOp::Insert { pos, value } => { + let mut values = value.into_list().unwrap(); + Arc::make_mut(&mut values).iter_mut().for_each(|v| { + if let LoroValue::Container(id) = v { + if id.is_normal() { + *id = convert_container_id(id.clone(), peers); + } + } + }); + let range = arena.alloc_values(values.iter().cloned()); + InnerContent::List(InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos, + }) + } + op::ListOp::Delete { pos, len, start_id } => { + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start: convert_id(&start_id, peers), + span: DeleteSpan { + pos, + signed_len: len, + }, + })) + } + }, + _ => unreachable!(), + }, + ContainerType::MovableList => match content { + JsonOpContent::MovableList(list) => match list { + op::MovableListOp::Insert { pos, value } => { + let mut values = value.into_list().unwrap(); + Arc::make_mut(&mut values).iter_mut().for_each(|v| { + if let LoroValue::Container(id) = v { + if id.is_normal() { + *id = convert_container_id(id.clone(), peers); + } + } + }); + let range = arena.alloc_values(values.iter().cloned()); + InnerContent::List(InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos, + }) + } + op::MovableListOp::Delete { pos, len, start_id } => { + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start: convert_id(&start_id, peers), + span: DeleteSpan { + pos, + signed_len: len, + }, + })) + } + op::MovableListOp::Move { + from, + elem_id: from_id, + to, + } => { + let from_id = convert_idlp(&from_id, peers); + InnerContent::List(InnerListOp::Move { from, from_id, to }) + } + op::MovableListOp::Set { elem_id, mut value } => { + let elem_id = convert_idlp(&elem_id, peers); + if let LoroValue::Container(id) = &mut value { + *id = convert_container_id(id.clone(), peers); + } + InnerContent::List(InnerListOp::Set { elem_id, value }) + } + }, + _ => unreachable!(), + }, + ContainerType::Map => match content { + JsonOpContent::Map(map) => match map { + op::MapOp::Insert { key, mut value } => { + if let LoroValue::Container(id) = &mut value { + *id = convert_container_id(id.clone(), peers); + } + InnerContent::Map(MapSet { + key: key.into(), + value: Some(value), + }) + } + op::MapOp::Delete { key } => InnerContent::Map(MapSet { + key: key.into(), + value: None, + }), + }, + _ => unreachable!(), + }, + ContainerType::Tree => match content { + JsonOpContent::Tree(tree) => match tree { + op::TreeOp::Create { + target, + parent, + fractional_index, + } => InnerContent::Tree(TreeOp::Create { + target: convert_tree_id(&target, peers), + parent: parent.map(|p| convert_tree_id(&p, peers)), + position: fractional_index, + }), + op::TreeOp::Move { + target, + parent, + fractional_index, + } => InnerContent::Tree(TreeOp::Move { + target: convert_tree_id(&target, peers), + parent: parent.map(|p| convert_tree_id(&p, peers)), + position: fractional_index, + }), + op::TreeOp::Delete { target } => InnerContent::Tree(TreeOp::Delete { + target: convert_tree_id(&target, peers), + }), + }, + _ => unreachable!(), + }, + ContainerType::Unknown(_) => match content { + JsonOpContent::Future(op::FutureOpWrapper { + prop, + value: op::FutureOp::Unknown(value), + }) => InnerContent::Future(FutureInnerContent::Unknown { prop, value }), + _ => unreachable!(), + }, + #[cfg(feature = "counter")] + ContainerType::Counter => { + let JsonOpContent::Future(op::FutureOpWrapper { prop, value }) = content else { + unreachable!() + }; + match value { + op::FutureOp::Counter(_) => { + InnerContent::Future(FutureInnerContent::Counter(prop as i64)) + } + op::FutureOp::Unknown(_) => { + InnerContent::Future(FutureInnerContent::Counter(prop as i64)) + } + } + } // Note: The Future Type need try to parse Op from the unknown content + }; + Op { + counter, + container: idx, + content, + } +} + +impl TryFrom<&str> for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: &str) -> Result { + serde_json::from_str(value) + } +} + +impl TryFrom<&String> for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: &String) -> Result { + serde_json::from_str(value) + } +} + +impl TryFrom for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: String) -> Result { + serde_json::from_str(&value) + } +} + +pub mod op { + + use fractional_index::FractionalIndex; + use loro_common::{ContainerID, IdLp, Lamport, LoroValue, PeerID, TreeID, ID}; + use serde::{Deserialize, Serialize}; + use smallvec::SmallVec; + + use crate::{encoding::OwnedValue, version::Frontiers}; + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct JsonSchema { + pub schema_version: u8, + #[serde(with = "self::serde_impl::frontiers")] + pub start_version: Frontiers, + #[serde(with = "self::serde_impl::peer_id")] + pub peers: Vec, + pub changes: Vec, + } + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct Change { + #[serde(with = "self::serde_impl::id")] + pub id: ID, + pub timestamp: i64, + #[serde(with = "self::serde_impl::deps")] + pub deps: SmallVec<[ID; 2]>, + pub lamport: Lamport, + pub msg: Option, + pub ops: Vec, + } + + #[derive(Debug, Clone)] + pub struct JsonOp { + pub content: JsonOpContent, + pub container: ContainerID, + pub counter: i32, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(untagged)] + pub enum JsonOpContent { + List(ListOp), + MovableList(MovableListOp), + Map(MapOp), + Text(TextOp), + Tree(TreeOp), + // #[serde(with = "self::serde_impl::future_op")] + Future(FutureOpWrapper), + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct FutureOpWrapper { + #[serde(flatten)] + pub value: FutureOp, + pub prop: i32, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum ListOp { + Insert { + pos: usize, + value: LoroValue, + }, + Delete { + pos: isize, + len: isize, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum MovableListOp { + Insert { + pos: usize, + value: LoroValue, + }, + Delete { + pos: isize, + len: isize, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + Move { + from: u32, + to: u32, + #[serde(with = "self::serde_impl::idlp")] + elem_id: IdLp, + }, + Set { + #[serde(with = "self::serde_impl::idlp")] + elem_id: IdLp, + value: LoroValue, + }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum MapOp { + Insert { key: String, value: LoroValue }, + Delete { key: String }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum TextOp { + Insert { + pos: u32, + text: String, + }, + Delete { + pos: isize, + len: isize, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + Mark { + start: u32, + end: u32, + style_key: String, + style_value: LoroValue, + info: u8, + }, + MarkEnd, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum TreeOp { + Create { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + #[serde(with = "self::serde_impl::option_tree_id")] + parent: Option, + #[serde(default, with = "self::serde_impl::fractional_index")] + fractional_index: FractionalIndex, + }, + Move { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + #[serde(with = "self::serde_impl::option_tree_id")] + parent: Option, + #[serde(default, with = "self::serde_impl::fractional_index")] + fractional_index: FractionalIndex, + }, + Delete { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum FutureOp { + #[cfg(feature = "counter")] + Counter(OwnedValue), + Unknown(OwnedValue), + } + + mod serde_impl { + + use loro_common::{ContainerID, ContainerType}; + use serde::{ + de::{MapAccess, Visitor}, + ser::SerializeStruct, + Deserialize, Deserializer, Serialize, Serializer, + }; + + impl Serialize for super::JsonOp { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut s = serializer.serialize_struct("Op", 3)?; + s.serialize_field("container", &self.container.to_string())?; + s.serialize_field("content", &self.content)?; + s.serialize_field("counter", &self.counter)?; + s.end() + } + } + + impl<'de> Deserialize<'de> for super::JsonOp { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct __Visitor; + + impl<'de> Visitor<'de> for __Visitor { + type Value = super::JsonOp; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct Op") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let (_key, container) = map.next_entry::()?.unwrap(); + let is_unknown = container.ends_with(')'); + let container = ContainerID::try_from(container.as_str()) + .map_err(|_| serde::de::Error::custom("invalid container id"))?; + let op = if is_unknown { + let (_key, op) = + map.next_entry::()?.unwrap(); + super::JsonOpContent::Future(op) + } else { + match container.container_type() { + ContainerType::List => { + let (_key, op) = + map.next_entry::()?.unwrap(); + super::JsonOpContent::List(op) + } + ContainerType::MovableList => { + let (_key, op) = + map.next_entry::()?.unwrap(); + super::JsonOpContent::MovableList(op) + } + ContainerType::Map => { + let (_key, op) = + map.next_entry::()?.unwrap(); + super::JsonOpContent::Map(op) + } + ContainerType::Text => { + let (_key, op) = + map.next_entry::()?.unwrap(); + super::JsonOpContent::Text(op) + } + ContainerType::Tree => { + let (_key, op) = + map.next_entry::()?.unwrap(); + super::JsonOpContent::Tree(op) + } + #[cfg(feature = "counter")] + ContainerType::Counter => { + let (_key, v) = map.next_entry::()?.unwrap(); + super::JsonOpContent::Future(super::FutureOpWrapper { + prop: v as i32, + value: super::FutureOp::Counter( + crate::encoding::value::OwnedValue::Future( + crate::encoding::value::OwnedFutureValue::Counter, + ), + ), + }) + } + _ => unreachable!(), + } + }; + let (_, counter) = map.next_entry::()?.unwrap(); + Ok(super::JsonOp { + container, + content: op, + counter, + }) + } + } + const FIELDS: &[&str] = &["container", "content", "counter"]; + deserializer.deserialize_struct("JsonOp", FIELDS, __Visitor) + } + } + + // pub mod future_op { + + // use serde::{Deserialize, Deserializer}; + + // use crate::encoding::json_schema::op::FutureOp; + + // impl<'de> Deserialize<'de> for FutureOp { + // fn deserialize(d: D) -> Result + // where + // D: Deserializer<'de>, + // { + // enum Field { + // #[cfg(feature = "counter")] + // Counter, + // Unknown, + // } + // struct FieldVisitor; + // impl<'de> serde::de::Visitor<'de> for FieldVisitor { + // type Value = Field; + // fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + // f.write_str("field identifier") + // } + // fn visit_str(self, value: &str) -> Result + // where + // E: serde::de::Error, + // { + // match value { + // #[cfg(feature = "counter")] + // "counter" => Ok(Field::Counter), + // _ => Ok(Field::Unknown), + // } + // } + // } + // impl<'de> Deserialize<'de> for Field { + // fn deserialize(deserializer: D) -> Result + // where + // D: Deserializer<'de>, + // { + // deserializer.deserialize_identifier(FieldVisitor) + // } + // } + // let (tag, content) = d.deserialize_any( + // serde::__private::de::TaggedContentVisitor::::new( + // "type", + // "internally tagged enum FutureOp", + // ), + // )?; + // let __deserializer = + // serde::__private::de::ContentDeserializer::::new(content); + // match tag { + // #[cfg(feature = "counter")] + // Field::Counter => { + // let v = serde::Deserialize::deserialize(__deserializer)?; + // Ok(FutureOp::Counter(v)) + // } + // _ => { + // let v = serde::Deserialize::deserialize(__deserializer)?; + // Ok(FutureOp::Unknown(v)) + // } + // } + // } + // } + // } + + pub mod id { + use loro_common::ID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &ID, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&id.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + // NOTE: https://github.com/serde-rs/serde/issues/2467 we use String here + let str: String = Deserialize::deserialize(d)?; + let id: ID = ID::try_from(str.as_str()).unwrap(); + Ok(id) + } + } + + pub mod frontiers { + use loro_common::ID; + use serde::{ser::SerializeMap, Deserializer, Serializer}; + + use crate::version::Frontiers; + + pub fn serialize(f: &Frontiers, s: S) -> Result + where + S: Serializer, + { + let mut map = s.serialize_map(Some(f.len()))?; + for id in f.iter() { + map.serialize_entry(&id.peer.to_string(), &id.counter)?; + } + map.end() + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + struct __Visitor; + impl<'de> serde::de::Visitor<'de> for __Visitor { + type Value = Frontiers; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a Frontiers") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut f = Frontiers::default(); + while let Some((k, v)) = map.next_entry::()? { + f.push(ID::new(k.parse().unwrap(), v)) + } + Ok(f) + } + } + d.deserialize_map(__Visitor) + } + } + + pub mod deps { + use loro_common::ID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(deps: &[ID], s: S) -> Result + where + S: Serializer, + { + s.collect_seq(deps.iter().map(|x| x.to_string())) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let deps: Vec = Deserialize::deserialize(d)?; + Ok(deps + .into_iter() + .map(|x| ID::try_from(x.as_str()).unwrap()) + .collect()) + } + } + + pub mod peer_id { + use loro_common::PeerID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(peers: &[PeerID], s: S) -> Result + where + S: Serializer, + { + s.collect_seq(peers.iter().map(|x| x.to_string())) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let peers: Vec = Deserialize::deserialize(d)?; + Ok(peers.into_iter().map(|x| x.parse().unwrap()).collect()) + } + } + + pub mod idlp { + use loro_common::IdLp; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(idlp: &IdLp, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&idlp.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + let id: IdLp = IdLp::try_from(str.as_str()).unwrap(); + Ok(id) + } + } + + pub mod tree_id { + use loro_common::TreeID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &TreeID, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&id.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + let id: TreeID = TreeID::try_from(str.as_str()).unwrap(); + Ok(id) + } + } + + pub mod option_tree_id { + use loro_common::TreeID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &Option, s: S) -> Result + where + S: Serializer, + { + match id { + Some(id) => s.serialize_str(&id.to_string()), + None => s.serialize_none(), + } + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let str: Option = Deserialize::deserialize(d)?; + match str { + Some(str) => { + let id: TreeID = TreeID::try_from(str.as_str()).unwrap(); + Ok(Some(id)) + } + None => Ok(None), + } + } + } + + pub mod fractional_index { + use fractional_index::FractionalIndex; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(fi: &FractionalIndex, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&fi.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + let fi = FractionalIndex::from_hex_string(str); + Ok(fi) + } + } + } +} diff --git a/crates/loro-internal/src/encoding/value.rs b/crates/loro-internal/src/encoding/value.rs index 05f43b80..66419c94 100644 --- a/crates/loro-internal/src/encoding/value.rs +++ b/crates/loro-internal/src/encoding/value.rs @@ -180,32 +180,20 @@ pub enum FutureValue<'a> { // The future value cannot depend on the arena for encoding. Unknown { kind: u8, - prop: i32, data: &'a [u8], }, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum OwnedFutureValue { - #[cfg(feature = "counter")] - Counter, - // The future value cannot depend on the arena for encoding. - Unknown { - kind: u8, - prop: i32, - data: Vec, - }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "value_type", content = "value", rename_all = "snake_case")] pub enum OwnedValue { Null, True, False, I64(i64), F64(f64), - Str(String), - Binary(Vec), + Str(Arc), + Binary(Arc>), ContainerIdx(usize), DeleteOnce, DeleteSeq, @@ -223,9 +211,22 @@ pub enum OwnedValue { lamport: Lamport, value: LoroValue, }, + #[serde(untagged)] Future(OwnedFutureValue), } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "value_type", content = "value")] +pub enum OwnedFutureValue { + #[cfg(feature = "counter")] + Counter, + // The future value cannot depend on the arena for encoding. + Unknown { + kind: u8, + data: Arc>, + }, +} + impl<'a> Value<'a> { pub fn from_owned(owned_value: &'a OwnedValue) -> Self { match owned_value { @@ -264,13 +265,10 @@ impl<'a> Value<'a> { OwnedValue::Future(value) => match value { #[cfg(feature = "counter")] OwnedFutureValue::Counter => Value::Future(FutureValue::Counter), - OwnedFutureValue::Unknown { kind, prop, data } => { - Value::Future(FutureValue::Unknown { - kind: *kind, - prop: *prop, - data: data.as_slice(), - }) - } + OwnedFutureValue::Unknown { kind, data } => Value::Future(FutureValue::Unknown { + kind: *kind, + data: data.as_slice(), + }), }, } } @@ -284,12 +282,12 @@ impl<'a> Value<'a> { Value::I64(x) => OwnedValue::I64(x), Value::ContainerIdx(x) => OwnedValue::ContainerIdx(x), Value::F64(x) => OwnedValue::F64(x), - Value::Str(x) => OwnedValue::Str(x.to_owned()), + Value::Str(x) => OwnedValue::Str(Arc::new(x.to_owned())), Value::DeleteSeq => OwnedValue::DeleteSeq, Value::DeltaInt(x) => OwnedValue::DeltaInt(x), Value::LoroValue(x) => OwnedValue::LoroValue(x), Value::MarkStart(x) => OwnedValue::MarkStart(x), - Value::Binary(x) => OwnedValue::Binary(x.to_owned()), + Value::Binary(x) => OwnedValue::Binary(Arc::new(x.to_owned())), Value::TreeMove(x) => OwnedValue::TreeMove(x), Value::ListMove { from, @@ -312,11 +310,10 @@ impl<'a> Value<'a> { Value::Future(value) => match value { #[cfg(feature = "counter")] FutureValue::Counter => OwnedValue::Future(OwnedFutureValue::Counter), - FutureValue::Unknown { kind, prop, data } => { + FutureValue::Unknown { kind, data } => { OwnedValue::Future(OwnedFutureValue::Unknown { kind, - prop, - data: data.to_owned(), + data: Arc::new(data.to_owned()), }) } }, @@ -326,17 +323,12 @@ impl<'a> Value<'a> { fn decode_without_arena<'r: 'a>( future_kind: FutureValueKind, value_reader: &'r mut ValueReader, - prop: i32, ) -> LoroResult { - let bytes_length = value_reader.read_usize()?; + let bytes = value_reader.read_binary()?; let value = match future_kind { #[cfg(feature = "counter")] FutureValueKind::Counter => FutureValue::Counter, - FutureValueKind::Unknown(kind) => FutureValue::Unknown { - kind, - prop, - data: value_reader.take_bytes(bytes_length), - }, + FutureValueKind::Unknown(kind) => FutureValue::Unknown { kind, data: bytes }, }; Ok(Value::Future(value)) } @@ -346,7 +338,6 @@ impl<'a> Value<'a> { value_reader: &'r mut ValueReader, arenas: &'a DecodedArenas<'a>, id: ID, - prop: i32, ) -> LoroResult { Ok(match kind { ValueKind::Null => Value::Null, @@ -388,7 +379,7 @@ impl<'a> Value<'a> { } } ValueKind::Future(future_kind) => { - Self::decode_without_arena(future_kind, value_reader, prop)? + Self::decode_without_arena(future_kind, value_reader)? } }) } @@ -397,18 +388,18 @@ impl<'a> Value<'a> { value: FutureValue, value_writer: &mut ValueWriter, ) -> (FutureValueKind, usize) { + // Note: we should encode FutureValue as binary data. + // [binary data length, binary data] + // when decoding, we will use reader.read_binary() to read the binary data. + // So such as FutureValue::Counter, we should write 0 as the length of binary data first. match value { #[cfg(feature = "counter")] FutureValue::Counter => { // write bytes length value_writer.write_u8(0); - (FutureValueKind::Counter, 0) + (FutureValueKind::Counter, 1) } - FutureValue::Unknown { - kind, - prop: _, - data, - } => ( + FutureValue::Unknown { kind, data } => ( FutureValueKind::Unknown(kind), value_writer.write_binary(data), ), @@ -476,7 +467,7 @@ pub struct MarkStart { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct EncodedTreeMove { - pub subject_idx: usize, + pub target_idx: usize, pub is_parent_null: bool, pub parent_idx: usize, pub position: usize, @@ -488,6 +479,7 @@ impl EncodedTreeMove { peer_ids: &[u64], positions: &[Vec], tree_ids: &[EncodedTreeID], + op_id: ID, ) -> LoroResult { let parent = if self.is_parent_null { None @@ -500,55 +492,91 @@ impl EncodedTreeMove { counter as Counter, )) }; - let position = if parent.is_some_and(|x| TreeID::is_deleted_root(&x)) { + let is_delete = parent.is_some_and(|p| TreeID::is_deleted_root(&p)); + let position = if is_delete { None } else { let bytes = &positions[self.position]; Some(FractionalIndex::from_bytes(bytes.clone())) }; - let EncodedTreeID { peer_idx, counter } = tree_ids[self.subject_idx]; - Ok(TreeOp { - target: TreeID::new( - *(peer_ids - .get(peer_idx) - .ok_or(LoroError::DecodeDataCorruptionError)?), - counter as Counter, - ), - parent, - position, - }) + let EncodedTreeID { peer_idx, counter } = tree_ids[self.target_idx]; + let target = TreeID::new( + *(peer_ids + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?), + counter as Counter, + ); + + let is_create = target.id() == op_id; + let op = if is_delete { + TreeOp::Delete { target } + } else if is_create { + TreeOp::Create { + target, + parent, + position: position.unwrap(), + } + } else { + TreeOp::Move { + target, + parent, + position: position.unwrap(), + } + }; + Ok(op) } pub fn from_tree_op<'p, 'a: 'p>(op: &'a TreeOp, registers: &mut EncodedRegisters) -> Self { - let position = if let Some(position) = &op.position { - let bytes = position.as_bytes(); - let either::Right(position_register) = &mut registers.position else { - unreachable!() - }; - position_register.get(&bytes).unwrap() - } else { - debug_assert!(op.parent.is_some_and(|x| TreeID::is_deleted_root(&x))); - // placeholder - 0 - }; + match op { + TreeOp::Create { + target, + parent, + position, + } + | TreeOp::Move { + target, + parent, + position, + } => { + let bytes = position.as_bytes(); + let either::Right(position_register) = &mut registers.position else { + unreachable!() + }; + let position = position_register.get(&bytes).unwrap(); + let target_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&target.peer), + counter: target.counter, + }); - let target_idx = registers.tree_id.register(&EncodedTreeID { - peer_idx: registers.peer.register(&op.target.peer), - counter: op.target.counter, - }); - - let parent_idx = op.parent.map(|x| { - registers.tree_id.register(&EncodedTreeID { - peer_idx: registers.peer.register(&x.peer), - counter: x.counter, - }) - }); - - EncodedTreeMove { - subject_idx: target_idx, - is_parent_null: op.parent.is_none(), - parent_idx: parent_idx.unwrap_or(0), - position, + let parent_idx = parent.map(|x| { + registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&x.peer), + counter: x.counter, + }) + }); + EncodedTreeMove { + target_idx, + is_parent_null: parent.is_none(), + parent_idx: parent_idx.unwrap_or(0), + position, + } + } + TreeOp::Delete { target } => { + let target_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&target.peer), + counter: target.counter, + }); + let parent_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&TreeID::delete_root().peer), + counter: TreeID::delete_root().counter, + }); + EncodedTreeMove { + target_idx, + is_parent_null: false, + parent_idx, + position: 0, + } + } } } } @@ -900,12 +928,6 @@ impl<'a> ValueReader<'a> { }) } - pub fn take_bytes(&mut self, len: usize) -> &'a [u8] { - let ans = &self.raw[..len]; - self.raw = &self.raw[len..]; - ans - } - pub fn read_tree_move(&mut self) -> LoroResult { let subject_idx = self.read_usize()?; let is_parent_null = self.read_u8()? != 0; @@ -915,7 +937,7 @@ impl<'a> ValueReader<'a> { parent_idx = self.read_usize()?; } Ok(EncodedTreeMove { - subject_idx, + target_idx: subject_idx, is_parent_null, parent_idx, position, @@ -1039,7 +1061,7 @@ impl ValueWriter { fn write_tree_move(&mut self, op: &EncodedTreeMove) -> usize { let len = self.buffer.len(); - self.write_usize(op.subject_idx); + self.write_usize(op.target_idx); self.write_u8(op.is_parent_null as u8); self.write_usize(op.position); if op.is_parent_null { diff --git a/crates/loro-internal/src/handler/tree.rs b/crates/loro-internal/src/handler/tree.rs index 53dfd092..853ba90a 100644 --- a/crates/loro-internal/src/handler/tree.rs +++ b/crates/loro-internal/src/handler/tree.rs @@ -300,11 +300,7 @@ impl TreeHandler { let inner = self.inner.try_attached_state()?; txn.apply_local_op( inner.container_idx, - crate::op::RawOpContent::Tree(TreeOp { - target, - parent: Some(TreeID::delete_root()), - position: None, - }), + crate::op::RawOpContent::Tree(TreeOp::Delete { target }), EventHint::Tree(TreeDiffItem { target, action: TreeExternalDiff::Delete { @@ -528,10 +524,10 @@ impl TreeHandler { ) -> LoroResult { txn.apply_local_op( inner.container_idx, - crate::op::RawOpContent::Tree(TreeOp { + crate::op::RawOpContent::Tree(TreeOp::Create { target: tree_id, parent, - position: Some(position.clone()), + position: position.clone(), }), EventHint::Tree(TreeDiffItem { target: tree_id, @@ -557,10 +553,10 @@ impl TreeHandler { ) -> LoroResult<()> { txn.apply_local_op( inner.container_idx, - crate::op::RawOpContent::Tree(TreeOp { + crate::op::RawOpContent::Tree(TreeOp::Move { target, parent, - position: Some(position.clone()), + position: position.clone(), }), EventHint::Tree(TreeDiffItem { target, diff --git a/crates/loro-internal/src/lib.rs b/crates/loro-internal/src/lib.rs index d52827ac..d9fef258 100644 --- a/crates/loro-internal/src/lib.rs +++ b/crates/loro-internal/src/lib.rs @@ -63,6 +63,7 @@ pub(crate) use id::{PeerID, ID}; pub(crate) use loro_common::InternalString; pub use container::ContainerType; +pub use encoding::json_schema::op::*; pub use loro_common::{loro_value, to_value}; #[cfg(feature = "wasm")] pub use value::wasm; diff --git a/crates/loro-internal/src/loro.rs b/crates/loro-internal/src/loro.rs index 2575ab47..c25f2cdd 100644 --- a/crates/loro-internal/src/loro.rs +++ b/crates/loro-internal/src/loro.rs @@ -28,7 +28,8 @@ use crate::{ cursor::{AbsolutePosition, CannotFindRelativePosition, Cursor, PosQueryResult}, dag::DagUtils, encoding::{ - decode_snapshot, export_snapshot, parse_header_and_body, EncodeMode, ParsedHeaderAndBody, + decode_snapshot, export_snapshot, json_schema::op::JsonSchema, parse_header_and_body, + EncodeMode, ParsedHeaderAndBody, }, event::{str_to_path, EventTriggerKind, Index}, handler::{Handler, MovableListHandler, TextHandler, TreeHandler, ValueOrHandler}, @@ -570,6 +571,30 @@ impl LoroDoc { ans } + /// Import the json schema updates. + /// + /// only supports backward compatibility but not forward compatibility. + pub fn import_json_updates>(&self, json: T) -> LoroResult<()> { + let json = json.try_into().map_err(|_| LoroError::InvalidJsonSchema)?; + self.commit_then_stop(); + self.update_oplog_and_apply_delta_to_state_if_needed( + |oplog| crate::encoding::json_schema::import_json(oplog, json), + Default::default(), + )?; + self.emit_events(); + self.renew_txn_if_auto_commit(); + Ok(()) + } + + pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema { + self.commit_then_stop(); + let oplog = self.oplog.lock().unwrap(); + let json = crate::encoding::json_schema::export_json(&oplog, vv); + drop(oplog); + self.renew_txn_if_auto_commit(); + json + } + /// Get the version vector of the current OpLog #[inline] pub fn oplog_vv(&self) -> VersionVector { diff --git a/crates/loro-internal/src/op/content.rs b/crates/loro-internal/src/op/content.rs index e8e3c96e..57f73b37 100644 --- a/crates/loro-internal/src/op/content.rs +++ b/crates/loro-internal/src/op/content.rs @@ -52,7 +52,7 @@ impl InnerContent { } } crate::op::InnerContent::Tree(t) => { - let id = t.target.associated_meta_container(); + let id = t.target().associated_meta_container(); f(&id); } crate::op::InnerContent::Future(f) => match &f { diff --git a/crates/loro-internal/src/parent.rs b/crates/loro-internal/src/parent.rs index 10549015..00c9e062 100644 --- a/crates/loro-internal/src/parent.rs +++ b/crates/loro-internal/src/parent.rs @@ -8,11 +8,7 @@ use loro_common::LoroValue; use crate::{ change::Change, - container::{ - list::list_op::{ListOp}, - map::MapSet, - tree::tree_op::TreeOp, - }, + container::{list::list_op::ListOp, map::MapSet}, op::{ListSlice, RawOp, RawOpContent}, DocState, OpLog, }; @@ -71,7 +67,8 @@ impl DocState { self.arena.set_parent(idx, Some(container)); } } - RawOpContent::Tree(TreeOp { target, .. }) => { + RawOpContent::Tree(tree) => { + let target = tree.target(); // create associated metadata container // TODO: maybe we could create map container only when setting metadata let container_id = target.associated_meta_container(); diff --git a/crates/loro-internal/src/state/tree_state.rs b/crates/loro-internal/src/state/tree_state.rs index 75a2d4b9..9761c3a6 100644 --- a/crates/loro-internal/src/state/tree_state.rs +++ b/crates/loro-internal/src/state/tree_state.rs @@ -945,15 +945,31 @@ impl ContainerState for TreeState { fn apply_local_op(&mut self, raw_op: &RawOp, _op: &Op) -> LoroResult<()> { match &raw_op.content { - crate::op::RawOpContent::Tree(tree) => { - let TreeOp { + crate::op::RawOpContent::Tree(tree) => match tree { + TreeOp::Create { target, parent, position, - } = tree; - let parent = TreeParentId::from(*parent); - self.mov(*target, parent, raw_op.id_full(), position.clone(), true) - } + } + | TreeOp::Move { + target, + parent, + position, + } => { + let parent = TreeParentId::from(*parent); + self.mov( + *target, + parent, + raw_op.id_full(), + Some(position.clone()), + true, + ) + } + TreeOp::Delete { target } => { + let parent = TreeParentId::Deleted; + self.mov(*target, parent, raw_op.id_full(), None, true) + } + }, _ => unreachable!(), } } @@ -1053,12 +1069,27 @@ impl ContainerState for TreeState { for op in ctx.ops { assert_eq!(op.op.atom_len(), 1); let content = op.op.content.as_tree().unwrap(); - let target = content.target; - let parent = content.parent; - let position = content.position.clone(); - let parent = TreeParentId::from(parent); - self.mov(target, parent, op.id_full(), position, false) - .unwrap(); + match content { + TreeOp::Create { + target, + parent, + position, + } + | TreeOp::Move { + target, + parent, + position, + } => { + let parent = TreeParentId::from(*parent); + self.mov(*target, parent, op.id_full(), Some(position.clone()), false) + .unwrap() + } + TreeOp::Delete { target } => { + let parent = TreeParentId::Deleted; + self.mov(*target, parent, op.id_full(), None, false) + .unwrap() + } + }; } } } diff --git a/crates/loro-internal/src/value.rs b/crates/loro-internal/src/value.rs index 06012883..85278099 100644 --- a/crates/loro-internal/src/value.rs +++ b/crates/loro-internal/src/value.rs @@ -760,19 +760,3 @@ pub mod wasm { } } } - -#[cfg(test)] -#[cfg(feature = "json")] -mod json_test { - use crate::{fx_map, value::ToJson, LoroValue}; - - #[test] - fn list() { - let list = LoroValue::List( - vec![12.into(), "123".into(), fx_map!("kk" => 123.into()).into()].into(), - ); - let json = list.to_json(); - println!("{}", json); - assert_eq!(LoroValue::from_json(&json), list); - } -} diff --git a/crates/loro-wasm/Cargo.toml b/crates/loro-wasm/Cargo.toml index 8d0ff03f..3342257e 100644 --- a/crates/loro-wasm/Cargo.toml +++ b/crates/loro-wasm/Cargo.toml @@ -12,14 +12,15 @@ crate-type = ["cdylib", "rlib"] js-sys = "0.3.60" loro-internal = { path = "../loro-internal", features = ["wasm"] } wasm-bindgen = "=0.2.92" -serde-wasm-bindgen = { version = "0.5.0" } +serde-wasm-bindgen = { version = "^0.6.5" } wasm-bindgen-derive = "0.2.1" console_error_panic_hook = { version = "0.1.6", optional = true } getrandom = { version = "0.2.10", features = ["js"] } -serde = { version = "1", features = ["derive"] } +serde = { workspace = true } rle = { path = "../rle", package = "loro-rle" } tracing-wasm = "0.2.1" tracing = { version = "0.1" } +serde_json = "1" [features] default = ["console_error_panic_hook"] diff --git a/crates/loro-wasm/src/lib.rs b/crates/loro-wasm/src/lib.rs index 4855fa85..7835f657 100644 --- a/crates/loro-wasm/src/lib.rs +++ b/crates/loro-wasm/src/lib.rs @@ -20,7 +20,7 @@ use loro_internal::{ obs::SubID, undo::{UndoItemMeta, UndoOrRedo}, version::Frontiers, - ContainerType, DiffEvent, HandlerTrait, LoroDoc, LoroValue, MovableListHandler, + ContainerType, DiffEvent, HandlerTrait, JsonSchema, LoroDoc, LoroValue, MovableListHandler, UndoManager as InnerUndoManager, VersionVector as InternalVersionVector, }; use rle::HasLength; @@ -159,6 +159,10 @@ extern "C" { pub type JsCursorQueryAns; #[wasm_bindgen(typescript_type = "UndoConfig")] pub type JsUndoConfig; + #[wasm_bindgen(typescript_type = "JsonSchema")] + pub type JsJsonSchema; + #[wasm_bindgen(typescript_type = "string | JsonSchema")] + pub type JsJsonSchemaOrString; } mod observer { @@ -873,6 +877,39 @@ impl Loro { } } + /// Export updates from the specific version to the current version with JSON format. + #[wasm_bindgen(js_name = "exportJsonUpdates")] + pub fn export_json_updates(&self, vv: Option) -> JsResult { + let mut json_vv = Default::default(); + if let Some(vv) = vv { + json_vv = vv.0; + } + let json_schema = self.0.export_json_updates(&json_vv); + let s = serde_wasm_bindgen::Serializer::new(); + let v = json_schema + .serialize(&s) + .map_err(std::convert::Into::::into)?; + Ok(v.into()) + } + + /// Import updates from the JSON format. + /// + /// only supports backward compatibility but not forward compatibility. + #[wasm_bindgen(js_name = "importJsonUpdates")] + pub fn import_json_updates(&self, json: JsJsonSchemaOrString) -> JsResult<()> { + let json: JsValue = json.into(); + if JsValue::is_string(&json) { + let json_str = json.as_string().unwrap(); + return self + .0 + .import_json_updates(json_str.as_str()) + .map_err(|e| e.into()); + } + let json_schema: JsonSchema = serde_wasm_bindgen::from_value(json)?; + self.0.import_json_updates(json_schema)?; + Ok(()) + } + /// Import a snapshot or a update to current doc. /// /// Note: @@ -2726,7 +2763,7 @@ impl LoroTreeNode { /// // / \ /// // node2 node /// ``` - #[wasm_bindgen(js_name = "createNode")] + #[wasm_bindgen(js_name = "createNode", skip_typescript)] pub fn create_node(&self, index: Option) -> JsResult { let id = if let Some(index) = index { self.tree.create_at(Some(self.id), index)? @@ -2864,7 +2901,7 @@ impl LoroTreeNode { /// /// The objects returned are new js objects each time because they need to cross /// the WASM boundary. - #[wasm_bindgen] + #[wasm_bindgen(skip_typescript)] pub fn children(&self) -> Array { let children = self.tree.children(Some(self.id)); let children = children.into_iter().map(|c| { @@ -2912,7 +2949,7 @@ impl LoroTree { /// // / \ /// // node root /// ``` - #[wasm_bindgen(js_name = "createNode")] + #[wasm_bindgen(js_name = "createNode", skip_typescript)] pub fn create_node( &mut self, parent: &JsParentTreeID, @@ -2983,7 +3020,7 @@ impl LoroTree { } /// Get LoroTreeNode by the TreeID. - #[wasm_bindgen(js_name = "getNodeByID")] + #[wasm_bindgen(js_name = "getNodeByID", skip_typescript)] pub fn get_node_by_id(&self, target: &JsTreeID) -> Option { let target: JsValue = target.into(); let target = TreeID::try_from(target).ok()?; @@ -4020,3 +4057,125 @@ interface LoroMovableList { export type Side = -1 | 0 | 1; "#; + +#[wasm_bindgen(typescript_custom_section)] +const JSON_SCHEMA_TYPES: &'static str = r#" +export type JsonOpID = `${number}@${PeerID}`; +export type JsonContainerID = `🦜:${ContainerID}` ; +export type JsonValue = + | JsonContainerID + | string + | number + | boolean + | null + | { [key: string]: JsonValue } + | Uint8Array + | JsonValue[]; + +export type JsonSchema = { + schema_version: number; + start_version: Map, + peers: PeerID[], + changes: JsonChange[] +}; + +export type JsonChange = { + id: JsonOpID + timestamp: number, + deps: JsonOpID[], + lamport: number, + msg: string | null, + ops: JsonOp[] +} + +export type JsonOp = { + container: ContainerID, + counter: number, + content: ListOp | TextOp | MapOp | TreeOp | MovableListOp | UnknownOp +} + +export type ListOp = { + type: "insert", + pos: number, + value: JsonValue +} | { + type: "delete", + pos: number, + len: number, + start_id: JsonOpID, +}; + +export type MovableListOp = { + type: "insert", + pos: number, + value: JsonValue +} | { + type: "delete", + pos: number, + len: number, + start_id: JsonOpID, +}| { + type: "move", + from: number, + to: number, + elem_id: JsonOpID, +}|{ + type: "set", + elem_id: JsonOpID, + value: JsonValue +} + +export type TextOp = { + type: "insert", + pos: number, + text: string +} | { + type: "delete", + pos: number, + len: number, + start_id: JsonOpID, +} | { + type: "mark", + start: number, + end: number, + style_key: string, + style_value: JsonValue, + info: number +}|{ + type: "mark_end" +}; + +export type MapOp = { + type: "insert", + key: string, + value: JsonValue +} | { + type: "delete", + key: string, +}; + +export type TreeOp = { + type: "create", + target: TreeID, + parent: TreeID | undefined, + fractional_index: string +}|{ + type: "move", + target: TreeID, + parent: TreeID | undefined, + fractional_index: string +}|{ + type: "delete", + target: TreeID +}; + +export type UnknownOp = { + type: "unknown" + prop: number, + value_type: "unknown", + value: { + kind: number, + data: Uint8Array + } +}; +"#; diff --git a/crates/loro/src/lib.rs b/crates/loro/src/lib.rs index 988bbc04..1f2202e7 100644 --- a/crates/loro/src/lib.rs +++ b/crates/loro/src/lib.rs @@ -13,6 +13,7 @@ use loro_internal::handler::HandlerTrait; use loro_internal::handler::ValueOrHandler; use loro_internal::loro::CommitOptions; use loro_internal::undo::{OnPop, OnPush}; +use loro_internal::JsonSchema; use loro_internal::LoroDoc as InnerLoroDoc; use loro_internal::OpLog; @@ -289,6 +290,18 @@ impl LoroDoc { self.doc.import_with(bytes, origin.into()) } + /// Import the json schema updates. + /// + /// only supports backward compatibility but not forward compatibility. + pub fn import_json_updates>(&self, json: T) -> Result<(), LoroError> { + self.doc.import_json_updates(json) + } + + /// Export the current state with json-string format of the document. + pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema { + self.doc.export_json_updates(vv) + } + /// Export all the ops not included in the given `VersionVector` pub fn export_from(&self, vv: &VersionVector) -> Vec { self.doc.export_from(vv) diff --git a/crates/loro/tests/loro_rust_test.rs b/crates/loro/tests/loro_rust_test.rs index 0aa22c56..3ac890e7 100644 --- a/crates/loro/tests/loro_rust_test.rs +++ b/crates/loro/tests/loro_rust_test.rs @@ -357,6 +357,24 @@ fn map() -> LoroResult<()> { Ok(()) } +#[test] +fn tree() { + use loro::{LoroDoc, ToJson}; + + let doc = LoroDoc::new(); + doc.set_peer_id(1).unwrap(); + let tree = doc.get_tree("tree"); + let root = tree.create(None).unwrap(); + let root2 = tree.create(None).unwrap(); + tree.mov(root2, root).unwrap(); + let root_meta = tree.get_meta(root).unwrap(); + root_meta.insert("color", "red").unwrap(); + assert_eq!( + tree.get_value_with_meta().to_json(), + r#"[{"parent":null,"meta":{"color":"red"},"id":"0@1","index":0,"position":"80"},{"parent":"0@1","meta":{},"id":"1@1","index":0,"position":"80"}]"# + ) +} + fn check_sync_send(_doc: impl Sync + Send) {} #[test] diff --git a/crates/rle/Cargo.toml b/crates/rle/Cargo.toml index 196e3b20..be6c2a80 100644 --- a/crates/rle/Cargo.toml +++ b/crates/rle/Cargo.toml @@ -14,7 +14,7 @@ keywords = ["crdt", "local-first"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -smallvec = "1.8.0" +smallvec = {workspace=true} num = "0.4.0" enum-as-inner = "0.6.0" arref = "0.1.0" diff --git a/docs/JsonSchema.md b/docs/JsonSchema.md new file mode 100644 index 00000000..788d38b3 --- /dev/null +++ b/docs/JsonSchema.md @@ -0,0 +1,391 @@ +# JSON Schema for Loro's OpLog + +## Introduction + +Loro supports multiple data structures and introduces many new concepts. Having only binary export formats would make it difficult for developers to understand the underlying processes. Better transparency leads to better developer experience. A human-readable JSON representation enables users to better understand and operate the document and to develop related tools. + +To better understand this document, you may first need to understand how Loro stores historical editing data: + +- [OpLog](https://www.loro.dev/docs/advanced/doc_state_and_oplog) +- [`Change`, `Operation`](https://www.loro.dev/docs/advanced/op_and_change) +- [`Replayable Event Graph (REG)`](https://www.loro.dev/docs/advanced/replayable_event_graph) + +It should be noted that considering the usage scenario, JSON Schema only supports backward compatibility but not forward compatibility. + +# Specification + +## Root object + +The root object contains all `Change`s, `Op`s, and critical metadata like start/end versions and schema version. + +We will also extract the 64-bit integer PeerID to the beginning of the document and replace it internally with incrementing numbers starting from zero: 0, 1, 2, 3... This significantly reduces the document size and enhances readability. + +```ts +{ + "schema_version": number, + "start_version": Map, + "peers": string[], + "changes": Change[], +} +``` + +- `schema_version`: the version of the schema that the document is encoded with. It's 1 for the current specification. +- `start_version`: the start `Frontiers` version of the document. They are represented as a map from the decimal string representation of `PeerID` to `Counter`. +- `peers`: the list of peers in the document. We represent all PeerIDs as decimal strings to avoid exceeding JavaScript's number limit. +- `changes`: the list of changes in the document. + +## Changes + +`Change`s are crucial in the OpLog. A REG([Replay event graph](https://www.loro.dev/docs/advanced/replayable_event_graph)) is a directed acyclic graph where each node is a `Change`, and each edge is a causal dependency between `Change`s. The metadata of the `Change`s helps us reconstruct the graph. + +You can also attach a commit message to a `Change` like you usually do with Git's commit. + +```ts +{ + "id": string, + "timestamp": number, + "deps": OpID[], + "lamport": number, + "msg": string, + "ops": Op[] +} + +type OpID = `${number}@${PeerID}`; +``` + +- `id`: the string representation of the unique `ID` of each `Change`, in the form of `{Counter}@{PeerID}` which is the `@` character connecting `Counter` and `PeerID`. Of course, This `PeerID` is the index of peers in the global context. +- `timestamp`: the number of Unix timestamp when the change is committed. [Timestamp is not recorded by default](https://loro.dev/docs/advanced/timestamp) +- `deps`: a list of causal dependency of this `Change`, each item is the `ID` represented by a string. +- `lamport`: the lamport timestamp of the `Change`. +- `msg`: the commit message. +- `ops`: all of the `Op` in the `Change`. + +## Operations + +Operation (abbreviated as `Op`) is the most complex part of the document. Loro currently supports multiple containers `List`, `Map`, `RichText`, `Movable List` and `Movable Tree`. Each data structure has several different `Op`s. + +But in general, each `Op` is composed of the `ContainerID` of the container that created it, a counter, and the corresponding content of the `Op`. + +```ts +type Op = { + "container": ContainerID, + "counter": number, + "content": OpContent // Its detailed definition is elaborated below, with different types for different Containers. +}; + +type OpContent = ListOp | TextOp | MapOp | TreeOp | MovableListOp | UnknownOp; +type ContainerID = + | `cid:root-${string}:${ContainerType}` + | `cid:${number}@${PeerID}:${ContainerType}`; +``` + +- `container`: the `ContainerID` of the container that created this `Op`, represented by a string starts with `cid:`. +- `counter`: the counter part of the OpID +- `content`: the semantic content of the `Op`, it is different for each field depending on the `Container`. + +The following is the **content** of each container。 + +### List + +```ts +type ListOp = ListInsertOp | ListDeleteOp; +``` + +#### Insert + +```ts +type ListInsertOp = { + "type": "insert", + "pos": number, + "value": LoroValue +} +``` + +- `type`: `insert`. +- `pos`: the index of the insert operation. +- `value`: the insert content which is a list of `LoroValue` + +#### Delete + +```ts +type ListDeleteOp = { + "type": "delete", + "pos": number, + "len": number, + "start_id": OpID +} +``` + +- `type`: `delete`. +- `pos`: the start index of the deletion. +- `len`: the length of deleted content. +- `start_id`: the string id of start element deleted. + +### MovableList + +```ts +type MovableListOp = ListInsertOp | ListDeleteOp | MovableListMoveOp | MovableListSetOp; +``` + +#### Insert + +```ts +type ListInsertOp = { + "type": "insert", + "pos": number, + "value": LoroValue +} +``` + +- `type`: `insert`, +- `pos`: the index of the insert operation. +- `value`: the insert content which is a list of `LoroValue` + +#### Delete + +```ts +type ListDeleteOp = { + "type": "delete", + "pos": number, + "len": number, + "start_id": OpID +} +``` + +- `type`: `delete` +- `pos`: the start index of the deletion. +- `len`: the length of deleted content. +- `start_id`: the string id of start element deleted. + +#### Move + +```ts +type MovableListMoveOp = { + "type": "move", + "from": number, + "to": number, + "elem_id": ElemID +} + +type ElemID = `L${number}@${PeerID}` +``` + +- `type`:`insert`, `delete`, `move` or `set`. +- `from`: the index of the element before is moved. +- `to`: the index of the index moved to after moving out the element +- `elem_id`: the ID (described by lamport@peer) of the element moved. + +#### Set + +```ts +type MovableListSetOp = { + "type": "set", + "elem_id": ElemID, + "value": LoroValue +} + +type ElemID = `L${number}@${PeerID}` +``` + +- `type`:`insert`, `delete`, `move` or `set`. +- `elem_id`: the ID (described by lamport@peer) of the element replaced. +- `value`: the value set. + +### Map + +```ts +type MapOp = MapInsertOp | MapDeleteOp; +``` + +#### Insert + +```ts +type MapInsertOp = { + "type": "insert", + "key": string, + "value": LoroValue +} +``` + +- `type`: `insert`. +- `key`: the key of the insertion. +- `value`: the value of the insertion. + +#### Delete + +```ts +type MapDeleteOp = { + "type": "delete", + "key": string +} +``` + +- `type`: `delete`. +- `key`: the key of the deletion + +### Text + +```ts +type TextOp = TextInsertOp | TextDeleteOp | TextMarkOp | TextMarkEndOp; +``` + +#### Insert + +```ts +type TextInsertOp = { + "type": "insert", + "pos": number, + "text": string +} +``` + +`type`: `insert`. +`pos`: the index of the insert operation. The position is based on the Unicode code point length. +`text`: the string of the insertion. + +#### Delete + +```ts +type TextDeleteOp = { + "type": "delete", + "pos": number, + "len": number, + "start_id": OpID +} +``` + +`type`: `delete`. +`pos`: the index of the deletion. The position is based on the Unicode code point length. +`len`: the length of the text deleted. +`start_id`: the string id of the beginning element deleted. + + +#### Mark + +```ts +type TextMarkOp = { + "type": "mark", + "start": number, + "end": number, + "style_key": string, + "style_value": LoroValue, + "info": number +} +``` + +`type`: `mark` +`start`: the start index of text need to mark. The position is based on the Unicode code point length. +`end`: the end index of text need to mark. The position is based on the Unicode code point length. +`style_key`: the key of style, it is customizable. +`style_value`: the value of style, it is customizable. +`info`: the config of the style, whether to expand the style when inserting new text around it. + +#### MarkEnd + +```ts +type TextMarkEndOp = { + "type": "mark_end" +} +``` + +`type`: `mark_end`. + +### Tree + +```ts +type TreeOp = TreeCreateOp | TreeMoveOp | TreeDeleteOp; +``` + +#### Create + +```ts +type TreeCreateOp = { + "type": "create", + "target": TreeID, + "parent": TreeID | null, + "fractional_index": string +} + +type TreeID = `${number}@${PeerID}` +``` + +- `type`: `create`. +- `target`: the string format of target `TreeID` moved. +- `parent`: the string format of `TreeID` or `null`. If it is `null`, the target node will be a root node. +- `fractional_index`: the fractional index with hex string format of the target node. + +#### Move + +```ts +type TreeMoveOp = { + "type": "move", + "target": TreeID, + "parent": TreeID | null, + "fractional_index": string +} + +type TreeID = `${number}@${PeerID}` +``` + +- `type`: `move`. +- `target`: the string format of target `TreeID` moved. +- `parent`: the string format of `TreeID` or `null`. If it is `null`, the target node will be a root node. +- `fractional_index`: the fractional index with hex string format of the target node. + +#### Delete + +```ts +type TreeDeleteOp = { + "type": "delete", + "target": TreeID +} + +type TreeID = `${number}@${PeerID}` +``` + +- `type`: `delete`. +- `target`: the string format of target `TreeID` deleted. + +### Unknown + +To support forward compatibility, we have an unknown type. When an `Op` with a newly supported Container from a newer version is decoded into the older version, it will be treated as an unknown type in a more general form, such as binary and string. When the new version decodes an unknown `Op`, the newer version of Loro will know its true type and decode correctly. + +```ts +type UnknownOp = { + "type": "unknown", + "prop": number, + "value_type": string, + "value": `${EncodeValue}` +} +``` + +- `type`: just an unknown type. +- `prop`: a property of the encoded op, it's a number. +- `value_type`: the type of `EncodeValue`. +- `value`: common data types used in encoding with json string format. + +## Value + +In this section, we will introduction two *Value* in Loro. One is `LoroValue`, it's an enum of data types supported by Loro, such as the value inserted by `List` or `Map`. + +The another is `EncodedValue`, it's just used in encoding module for unknown type. + +### LoroValue + +These are data types supported by Loro and its json format: + +- `null`: `null` +- `Bool`: `true` or `false` +- `F64`: `number`(float) +- `I64`: `number` or `bigint` (signed) +- `Binary`: `UInt8Array` +- `String`: `string` +- `List`: `Array` +- `Map`: `Map` +- `Container`: the id of container. `🦜:cid:{Counter}@{PeerID}:{ContainerType}` or `🦜:cid:root-{Name}:{ContainerType}` + +Note: Compared with the string format, we add a prefix `🦜:` when encoding the json format of `ContainerID` to prevent users from saving the string format of `ContainerID` and misinterpreting it as `ContainerID` when decoding. + +### EncodedValue + +The `EncodedValue` is the specific type used by Loro when encoding, it's an internal value, users do not need to get it clear. It is specially designed to handle the schema mismatch due to forward and backward compatibility. In JSON encoding schema, the `EncodedValue` will be encoded as an object. diff --git a/loro-js/src/index.ts b/loro-js/src/index.ts index cf1d4c6c..70b912c8 100644 --- a/loro-js/src/index.ts +++ b/loro-js/src/index.ts @@ -564,11 +564,34 @@ declare module "loro-wasm" { T extends Record = Record, > { new (): LoroTree; + /** + * Create a new tree node as the child of parent and return a `LoroTreeNode` instance. + * If the parent is undefined, the tree node will be a root node. + * + * If the index is not provided, the new node will be appended to the end. + * + * @example + * ```ts + * import { Loro } from "loro-crdt"; + * + * const doc = new Loro(); + * const tree = doc.getTree("tree"); + * const root = tree.createNode(); + * const node = tree.createNode(undefined, 0); + * + * // undefined + * // / \ + * // node root + * ``` + */ createNode(parent?: TreeID, index?: number): LoroTreeNode; move(target: TreeID, parent?: TreeID, index?: number): void; delete(target: TreeID): void; has(target: TreeID): boolean; - getNodeByID(target: TreeID): LoroTreeNode; + /** + * Get LoroTreeNode by the TreeID. + */ + getNodeByID(target: TreeID): LoroTreeNode; subscribe(listener: Listener): number; } @@ -579,9 +602,35 @@ declare module "loro-wasm" { * Get the associated metadata map container of a tree node. */ readonly data: LoroMap; + /** + * Create a new node as the child of the current node and + * return an instance of `LoroTreeNode`. + * + * If the index is not provided, the new node will be appended to the end. + * + * @example + * ```typescript + * import { Loro } from "loro-crdt"; + * + * let doc = new Loro(); + * let tree = doc.getTree("tree"); + * let root = tree.createNode(); + * let node = root.createNode(); + * let node2 = root.createNode(0); + * // root + * // / \ + * // node2 node + * ``` + */ createNode(index?: number): LoroTreeNode; move(parent?: LoroTreeNode, index?: number): void; parent(): LoroTreeNode | undefined; + /** + * Get the children of this node. + * + * The objects returned are new js objects each time because they need to cross + * the WASM boundary. + */ children(): Array>; } diff --git a/loro-js/tests/json.test.ts b/loro-js/tests/json.test.ts new file mode 100644 index 00000000..217abcf0 --- /dev/null +++ b/loro-js/tests/json.test.ts @@ -0,0 +1,165 @@ +import { expect, it } from "vitest"; +import { + Loro, + LoroMap, + TextOp, +} from "../src"; + +it("json encoding", () => { + const doc = new Loro(); + const text = doc.getText("text"); + text.insert(0, "123"); + const map = doc.getMap("map"); + const list = doc.getList("list"); + const movableList = doc.getMovableList("movableList"); + const tree = doc.getTree("tree"); + const subMap = map.setContainer("subMap", new LoroMap()); + subMap.set("foo", "bar"); + list.push("foo"); + list.push("🦜"); + movableList.push("move list"); + movableList.push("🦜"); + movableList.move(1, 0); + const root = tree.createNode(undefined); + const child = tree.createNode(root.id); + child.data.set("tree", "abc"); + text.mark({ start: 0, end: 3 }, "bold", true); + const json = doc.exportJsonUpdates(); + // console.log(json.changes[0].ops); + const doc2 = new Loro(); + doc2.importJsonUpdates(json); +}); + +it("json decoding", () => { + const v15Json = `{ + "schema_version": 1, + "start_version": {}, + "peers": [ + "14944917281143706156" + ], + "changes": [ + { + "id": "0@0", + "timestamp": 0, + "deps": [], + "lamport": 0, + "msg": null, + "ops": [ + { + "container": "cid:root-text:Text", + "content": { + "type": "insert", + "pos": 0, + "text": "123" + }, + "counter": 0 + }, + { + "container": "cid:root-map:Map", + "content": { + "type": "insert", + "key": "subMap", + "value": "🦜:cid:3@0:Map" + }, + "counter": 3 + }, + { + "container": "cid:3@0:Map", + "content": { + "type": "insert", + "key": "foo", + "value": "bar" + }, + "counter": 4 + }, + { + "container": "cid:root-list:List", + "content": { + "type": "insert", + "pos": 0, + "value": [ + "foo", + "🦜" + ] + }, + "counter": 5 + }, + { + "container": "cid:root-tree:Tree", + "content": { + "type": "move", + "target": "7@0", + "parent": null + }, + "counter": 7 + }, + { + "container": "cid:root-tree:Tree", + "content": { + "type": "move", + "target": "8@0", + "parent": "7@0" + }, + "counter": 8 + }, + { + "container": "cid:8@0:Map", + "content": { + "type": "insert", + "key": "tree", + "value": "abc" + }, + "counter": 9 + }, + { + "container": "cid:root-text:Text", + "content": { + "type": "mark", + "start": 0, + "end": 3, + "style_key": "bold", + "style_value": true, + "info": 132 + }, + "counter": 10 + }, + { + "container": "cid:root-text:Text", + "content": { + "type": "mark_end" + }, + "counter": 11 + } + ] + } + ] + }`; + const doc = new Loro(); + doc.importJsonUpdates(v15Json); + // console.log(doc.exportJsonUpdates()); +}); + +it("test some type correctness", () => { + const doc = new Loro(); + doc.setPeerId(0); + doc.getText("text").insert(0, "123"); + doc.commit(); + doc.getText("text").delete(2, 1); + doc.getText("text").delete(1, 1); + doc.getText("text").delete(0, 1); + doc.commit(); + const updates = doc.exportJsonUpdates(); + expect(updates.start_version).toBeDefined(); + expect(updates.changes.length).toBe(1); + expect(updates.changes[0].ops[0].content).toStrictEqual({ + type: "insert", + pos: 0, + text: "123", + } as TextOp); + expect(updates.changes[0].ops[1].content).toStrictEqual({ + type: "delete", + pos: 2, + len: -3, + start_id: "0@0", + } as TextOp); +});