Merge pull request #1288 from zed-industries/line-ending-fixes

Line ending fixes

Cargo.lock

@@ -4910,6 +4910,7 @@ dependencies = [
  "parking_lot 0.11.2",
  "postage",
  "rand 0.8.5",
+ "regex",
  "smallvec",
  "sum_tree",
  "util",

crates/text/Cargo.toml

@@ -23,6 +23,7 @@ log = { version = "0.4.16", features = ["kv_unstable_serde"] }
 parking_lot = "0.11"
 postage = { version = "0.4.1", features = ["futures-traits"] }
 rand = { version = "0.8.3", optional = true }
+regex = "1.5"
 smallvec = { version = "1.6", features = ["union"] }
 
 [dev-dependencies]

crates/text/src/tests.rs

@@ -43,7 +43,7 @@ fn test_random_edits(mut rng: StdRng) {
         .take(reference_string_len)
         .collect::<String>();
     let mut buffer = Buffer::new(0, 0, reference_string.clone().into());
-    reference_string = reference_string.replace("\r", "");
+    LineEnding::strip_carriage_returns(&mut reference_string);
 
     buffer.history.group_interval = Duration::from_millis(rng.gen_range(0..=200));
     let mut buffer_versions = Vec::new();
@@ -58,7 +58,6 @@ fn test_random_edits(mut rng: StdRng) {
         for (old_range, new_text) in edits.iter().rev() {
             reference_string.replace_range(old_range.clone(), &new_text);
         }
-        reference_string = reference_string.replace("\r", "");
 
         assert_eq!(buffer.text(), reference_string);
         log::info!(
@@ -154,14 +153,25 @@ fn test_random_edits(mut rng: StdRng) {
 
 #[test]
 fn test_line_endings() {
-    let mut buffer = Buffer::new(0, 0, "one\r\ntwo".into());
-    assert_eq!(buffer.text(), "one\ntwo");
+    assert_eq!(LineEnding::detect(&"🍐✅\n".repeat(1000)), LineEnding::Unix);
+    assert_eq!(LineEnding::detect(&"abcd\n".repeat(1000)), LineEnding::Unix);
+    assert_eq!(
+        LineEnding::detect(&"🍐✅\r\n".repeat(1000)),
+        LineEnding::Windows
+    );
+    assert_eq!(
+        LineEnding::detect(&"abcd\r\n".repeat(1000)),
+        LineEnding::Windows
+    );
+
+    let mut buffer = Buffer::new(0, 0, "one\r\ntwo\rthree".into());
+    assert_eq!(buffer.text(), "one\ntwo\nthree");
     assert_eq!(buffer.line_ending(), LineEnding::Windows);
     buffer.check_invariants();
 
-    buffer.edit([(buffer.len()..buffer.len(), "\r\nthree")]);
+    buffer.edit([(buffer.len()..buffer.len(), "\r\nfour")]);
     buffer.edit([(0..0, "zero\r\n")]);
-    assert_eq!(buffer.text(), "zero\none\ntwo\nthree");
+    assert_eq!(buffer.text(), "zero\none\ntwo\nthree\nfour");
     assert_eq!(buffer.line_ending(), LineEnding::Windows);
     buffer.check_invariants();
 }

crates/text/src/text.rs

@@ -18,6 +18,7 @@ pub use anchor::*;
 use anyhow::Result;
 use clock::ReplicaId;
 use collections::{HashMap, HashSet};
+use lazy_static::lazy_static;
 use locator::Locator;
 use operation_queue::OperationQueue;
 pub use patch::Patch;
@@ -26,10 +27,12 @@ pub use point_utf16::*;
 use postage::{barrier, oneshot, prelude::*};
 #[cfg(any(test, feature = "test-support"))]
 pub use random_char_iter::*;
+use regex::Regex;
 use rope::TextDimension;
 pub use rope::{Chunks, Rope, TextSummary};
 pub use selection::*;
 use std::{
+    borrow::Cow,
     cmp::{self, Ordering},
     future::Future,
     iter::Iterator,
@@ -42,6 +45,10 @@ pub use subscription::*;
 pub use sum_tree::Bias;
 use sum_tree::{FilterCursor, SumTree};
 
+lazy_static! {
+    static ref CARRIAGE_RETURNS_REGEX: Regex = Regex::new("\r\n|\r").unwrap();
+}
+
 pub type TransactionId = clock::Local;
 
 pub struct Buffer {
@@ -1472,6 +1479,15 @@ impl Buffer {
 
         log::info!("mutating buffer {} with {:?}", self.replica_id, edits);
         let op = self.edit(edits.iter().cloned());
+        if let Operation::Edit(edit) = &op {
+            assert_eq!(edits.len(), edit.new_text.len());
+            for (edit, new_text) in edits.iter_mut().zip(&edit.new_text) {
+                edit.1 = new_text.clone();
+            }
+        } else {
+            unreachable!()
+        }
+
         (edits, op)
     }
 
@@ -2353,9 +2369,13 @@ impl LineEnding {
     }
 
     pub fn detect(text: &str) -> Self {
-        if let Some(ix) = text[..cmp::min(text.len(), 1000)].find(&['\n']) {
-            let text = text.as_bytes();
-            if ix > 0 && text[ix - 1] == b'\r' {
+        let mut max_ix = cmp::min(text.len(), 1000);
+        while !text.is_char_boundary(max_ix) {
+            max_ix -= 1;
+        }
+
+        if let Some(ix) = text[..max_ix].find(&['\n']) {
+            if ix > 0 && text.as_bytes()[ix - 1] == b'\r' {
                 Self::Windows
             } else {
                 Self::Unix
@@ -2366,12 +2386,14 @@ impl LineEnding {
     }
 
     pub fn strip_carriage_returns(text: &mut String) {
-        text.retain(|c| c != '\r')
+        if let Cow::Owned(replaced) = CARRIAGE_RETURNS_REGEX.replace_all(text, "\n") {
+            *text = replaced;
+        }
     }
 
     fn strip_carriage_returns_from_arc(text: Arc<str>) -> Arc<str> {
-        if text.contains('\r') {
-            text.replace('\r', "").into()
+        if let Cow::Owned(replaced) = CARRIAGE_RETURNS_REGEX.replace_all(&text, "\n") {
+            replaced.into()
         } else {
             text
         }