From 08e9b2e8481bb87f3ae76843d1f24f4d5a29f0bc Mon Sep 17 00:00:00 2001 From: KCaverly Date: Mon, 30 Oct 2023 13:32:47 -0400 Subject: [PATCH 1/2] added parsing support for <|S| |E|> spans --- crates/assistant/src/codegen.rs | 110 +++++++++++++++++++++++++------- crates/assistant/src/prompts.rs | 18 +++--- 2 files changed, 96 insertions(+), 32 deletions(-) diff --git a/crates/assistant/src/codegen.rs b/crates/assistant/src/codegen.rs index b6ef6b5cfa..b26b1713b2 100644 --- a/crates/assistant/src/codegen.rs +++ b/crates/assistant/src/codegen.rs @@ -117,7 +117,7 @@ impl Codegen { let (mut hunks_tx, mut hunks_rx) = mpsc::channel(1); let diff = cx.background().spawn(async move { - let chunks = strip_markdown_codeblock(response.await?); + let chunks = strip_invalid_spans_from_codeblock(response.await?); futures::pin_mut!(chunks); let mut diff = StreamingDiff::new(selected_text.to_string()); @@ -278,12 +278,13 @@ impl Codegen { } } -fn strip_markdown_codeblock( +fn strip_invalid_spans_from_codeblock( stream: impl Stream>, ) -> impl Stream> { let mut first_line = true; let mut buffer = String::new(); - let mut starts_with_fenced_code_block = false; + let mut starts_with_markdown_codeblock = false; + let mut includes_start_or_end_span = false; stream.filter_map(move |chunk| { let chunk = match chunk { Ok(chunk) => chunk, @@ -291,11 +292,31 @@ fn strip_markdown_codeblock( }; buffer.push_str(&chunk); + if buffer.len() > "<|S|".len() && buffer.starts_with("<|S|") { + includes_start_or_end_span = true; + + buffer = buffer + .strip_prefix("<|S|>") + .or_else(|| buffer.strip_prefix("<|S|")) + .unwrap_or(&buffer) + .to_string(); + } else if buffer.ends_with("|E|>") { + includes_start_or_end_span = true; + } else if buffer.starts_with("<|") + || buffer.starts_with("<|S") + || buffer.starts_with("<|S|") + || buffer.ends_with("|") + || buffer.ends_with("|E") + || buffer.ends_with("|E|") + { + return future::ready(None); + } + if first_line { if buffer == "" || buffer == "`" || buffer == "``" { return future::ready(None); } else if buffer.starts_with("```") { - starts_with_fenced_code_block = true; + starts_with_markdown_codeblock = true; if let Some(newline_ix) = buffer.find('\n') { buffer.replace_range(..newline_ix + 1, ""); first_line = false; @@ -305,16 +326,26 @@ fn strip_markdown_codeblock( } } - let text = if starts_with_fenced_code_block { - buffer + let mut text = buffer.to_string(); + if starts_with_markdown_codeblock { + text = text .strip_suffix("\n```\n") - .or_else(|| buffer.strip_suffix("\n```")) - .or_else(|| buffer.strip_suffix("\n``")) - .or_else(|| buffer.strip_suffix("\n`")) - .or_else(|| buffer.strip_suffix('\n')) - .unwrap_or(&buffer) - } else { - &buffer + .or_else(|| text.strip_suffix("\n```")) + .or_else(|| text.strip_suffix("\n``")) + .or_else(|| text.strip_suffix("\n`")) + .or_else(|| text.strip_suffix('\n')) + .unwrap_or(&text) + .to_string(); + } + + if includes_start_or_end_span { + text = text + .strip_suffix("|E|>") + .or_else(|| text.strip_suffix("E|>")) + .or_else(|| text.strip_prefix("|>")) + .or_else(|| text.strip_prefix(">")) + .unwrap_or(&text) + .to_string(); }; if text.contains('\n') { @@ -327,6 +358,7 @@ fn strip_markdown_codeblock( } else { Some(Ok(buffer.clone())) }; + buffer = remainder; future::ready(result) }) @@ -537,50 +569,82 @@ mod tests { } #[gpui::test] - async fn test_strip_markdown_codeblock() { + async fn test_strip_invalid_spans_from_codeblock() { assert_eq!( - strip_markdown_codeblock(chunks("Lorem ipsum dolor", 2)) + strip_invalid_spans_from_codeblock(chunks("Lorem ipsum dolor", 2)) .map(|chunk| chunk.unwrap()) .collect::() .await, "Lorem ipsum dolor" ); assert_eq!( - strip_markdown_codeblock(chunks("```\nLorem ipsum dolor", 2)) + strip_invalid_spans_from_codeblock(chunks("```\nLorem ipsum dolor", 2)) .map(|chunk| chunk.unwrap()) .collect::() .await, "Lorem ipsum dolor" ); assert_eq!( - strip_markdown_codeblock(chunks("```\nLorem ipsum dolor\n```", 2)) + strip_invalid_spans_from_codeblock(chunks("```\nLorem ipsum dolor\n```", 2)) .map(|chunk| chunk.unwrap()) .collect::() .await, "Lorem ipsum dolor" ); assert_eq!( - strip_markdown_codeblock(chunks("```\nLorem ipsum dolor\n```\n", 2)) + strip_invalid_spans_from_codeblock(chunks("```\nLorem ipsum dolor\n```\n", 2)) .map(|chunk| chunk.unwrap()) .collect::() .await, "Lorem ipsum dolor" ); assert_eq!( - strip_markdown_codeblock(chunks("```html\n```js\nLorem ipsum dolor\n```\n```", 2)) - .map(|chunk| chunk.unwrap()) - .collect::() - .await, + strip_invalid_spans_from_codeblock(chunks( + "```html\n```js\nLorem ipsum dolor\n```\n```", + 2 + )) + .map(|chunk| chunk.unwrap()) + .collect::() + .await, "```js\nLorem ipsum dolor\n```" ); assert_eq!( - strip_markdown_codeblock(chunks("``\nLorem ipsum dolor\n```", 2)) + strip_invalid_spans_from_codeblock(chunks("``\nLorem ipsum dolor\n```", 2)) .map(|chunk| chunk.unwrap()) .collect::() .await, "``\nLorem ipsum dolor\n```" ); + assert_eq!( + strip_invalid_spans_from_codeblock(chunks("<|S|Lorem ipsum|E|>", 2)) + .map(|chunk| chunk.unwrap()) + .collect::() + .await, + "Lorem ipsum" + ); + assert_eq!( + strip_invalid_spans_from_codeblock(chunks("<|S|>Lorem ipsum", 2)) + .map(|chunk| chunk.unwrap()) + .collect::() + .await, + "Lorem ipsum" + ); + + assert_eq!( + strip_invalid_spans_from_codeblock(chunks("```\n<|S|>Lorem ipsum\n```", 2)) + .map(|chunk| chunk.unwrap()) + .collect::() + .await, + "Lorem ipsum" + ); + assert_eq!( + strip_invalid_spans_from_codeblock(chunks("```\n<|S|Lorem ipsum|E|>\n```", 2)) + .map(|chunk| chunk.unwrap()) + .collect::() + .await, + "Lorem ipsum" + ); fn chunks(text: &str, size: usize) -> impl Stream> { stream::iter( text.chars() diff --git a/crates/assistant/src/prompts.rs b/crates/assistant/src/prompts.rs index dffcbc2923..66425d31f7 100644 --- a/crates/assistant/src/prompts.rs +++ b/crates/assistant/src/prompts.rs @@ -79,12 +79,12 @@ fn summarize(buffer: &BufferSnapshot, selected_range: Range) -> S if !flushed_selection { // The collapsed node ends after the selection starts, so we'll flush the selection first. summary.extend(buffer.text_for_range(offset..selected_range.start)); - summary.push_str("<|START|"); + summary.push_str("<|S|"); if selected_range.end == selected_range.start { summary.push_str(">"); } else { summary.extend(buffer.text_for_range(selected_range.clone())); - summary.push_str("|END|>"); + summary.push_str("|E|>"); } offset = selected_range.end; flushed_selection = true; @@ -106,12 +106,12 @@ fn summarize(buffer: &BufferSnapshot, selected_range: Range) -> S // Flush selection if we haven't already done so. if !flushed_selection && offset <= selected_range.start { summary.extend(buffer.text_for_range(offset..selected_range.start)); - summary.push_str("<|START|"); + summary.push_str("<|S|"); if selected_range.end == selected_range.start { summary.push_str(">"); } else { summary.extend(buffer.text_for_range(selected_range.clone())); - summary.push_str("|END|>"); + summary.push_str("|E|>"); } offset = selected_range.end; } @@ -259,7 +259,7 @@ pub(crate) mod tests { summarize(&snapshot, Point::new(1, 4)..Point::new(1, 4)), indoc! {" struct X { - <|START|>a: usize, + <|S|>a: usize, b: usize, } @@ -285,7 +285,7 @@ pub(crate) mod tests { impl X { fn new() -> Self { - let <|START|a |END|>= 1; + let <|S|a |E|>= 1; let b = 2; Self { a, b } } @@ -306,7 +306,7 @@ pub(crate) mod tests { } impl X { - <|START|> + <|S|> fn new() -> Self {} pub fn a(&self, param: bool) -> usize {} @@ -332,7 +332,7 @@ pub(crate) mod tests { pub fn b(&self) -> usize {} } - <|START|>"} + <|S|>"} ); // Ensure nested functions get collapsed properly. @@ -368,7 +368,7 @@ pub(crate) mod tests { assert_eq!( summarize(&snapshot, Point::new(0, 0)..Point::new(0, 0)), indoc! {" - <|START|>struct X { + <|S|>struct X { a: usize, b: usize, } From bbe53895efb25233a4ef42ea701054d1efdc0f59 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Wed, 1 Nov 2023 15:45:42 +0100 Subject: [PATCH 2/2] Return `ColorScaleSet`s from individual color scale functions (#3197) This PR adjusts the individual color scale functions to return `ColorScaleSet`s instead of `DefaultColorScaleSet`s. We only use the `DefaultColorScaleSet`s to simplify the construction of the scales, so it isn't necessary to surface them outside of the function. Release Notes: - N/A --- crates/theme2/src/default_colors.rs | 165 +++++++++++++++++----------- 1 file changed, 99 insertions(+), 66 deletions(-) diff --git a/crates/theme2/src/default_colors.rs b/crates/theme2/src/default_colors.rs index 5ef93d036f..8fb38e9661 100644 --- a/crates/theme2/src/default_colors.rs +++ b/crates/theme2/src/default_colors.rs @@ -299,43 +299,43 @@ impl From for ColorScaleSet { pub fn default_color_scales() -> ColorScales { ColorScales { - gray: gray().into(), - mauve: mauve().into(), - slate: slate().into(), - sage: sage().into(), - olive: olive().into(), - sand: sand().into(), - gold: gold().into(), - bronze: bronze().into(), - brown: brown().into(), - yellow: yellow().into(), - amber: amber().into(), - orange: orange().into(), - tomato: tomato().into(), - red: red().into(), - ruby: ruby().into(), - crimson: crimson().into(), - pink: pink().into(), - plum: plum().into(), - purple: purple().into(), - violet: violet().into(), - iris: iris().into(), - indigo: indigo().into(), - blue: blue().into(), - cyan: cyan().into(), - teal: teal().into(), - jade: jade().into(), - green: green().into(), - grass: grass().into(), - lime: lime().into(), - mint: mint().into(), - sky: sky().into(), - black: black().into(), - white: white().into(), + gray: gray(), + mauve: mauve(), + slate: slate(), + sage: sage(), + olive: olive(), + sand: sand(), + gold: gold(), + bronze: bronze(), + brown: brown(), + yellow: yellow(), + amber: amber(), + orange: orange(), + tomato: tomato(), + red: red(), + ruby: ruby(), + crimson: crimson(), + pink: pink(), + plum: plum(), + purple: purple(), + violet: violet(), + iris: iris(), + indigo: indigo(), + blue: blue(), + cyan: cyan(), + teal: teal(), + jade: jade(), + green: green(), + grass: grass(), + lime: lime(), + mint: mint(), + sky: sky(), + black: black(), + white: white(), } } -fn gray() -> DefaultColorScaleSet { +fn gray() -> ColorScaleSet { DefaultColorScaleSet { scale: "Gray", light: [ @@ -395,9 +395,10 @@ fn gray() -> DefaultColorScaleSet { "#ffffffed", ], } + .into() } -fn mauve() -> DefaultColorScaleSet { +fn mauve() -> ColorScaleSet { DefaultColorScaleSet { scale: "Mauve", light: [ @@ -457,9 +458,10 @@ fn mauve() -> DefaultColorScaleSet { "#fdfdffef", ], } + .into() } -fn slate() -> DefaultColorScaleSet { +fn slate() -> ColorScaleSet { DefaultColorScaleSet { scale: "Slate", light: [ @@ -519,9 +521,10 @@ fn slate() -> DefaultColorScaleSet { "#fcfdffef", ], } + .into() } -fn sage() -> DefaultColorScaleSet { +fn sage() -> ColorScaleSet { DefaultColorScaleSet { scale: "Sage", light: [ @@ -581,9 +584,10 @@ fn sage() -> DefaultColorScaleSet { "#fdfffeed", ], } + .into() } -fn olive() -> DefaultColorScaleSet { +fn olive() -> ColorScaleSet { DefaultColorScaleSet { scale: "Olive", light: [ @@ -643,9 +647,10 @@ fn olive() -> DefaultColorScaleSet { "#fdfffded", ], } + .into() } -fn sand() -> DefaultColorScaleSet { +fn sand() -> ColorScaleSet { DefaultColorScaleSet { scale: "Sand", light: [ @@ -705,9 +710,10 @@ fn sand() -> DefaultColorScaleSet { "#fffffded", ], } + .into() } -fn gold() -> DefaultColorScaleSet { +fn gold() -> ColorScaleSet { DefaultColorScaleSet { scale: "Gold", light: [ @@ -767,9 +773,10 @@ fn gold() -> DefaultColorScaleSet { "#fef7ede7", ], } + .into() } -fn bronze() -> DefaultColorScaleSet { +fn bronze() -> ColorScaleSet { DefaultColorScaleSet { scale: "Bronze", light: [ @@ -829,9 +836,10 @@ fn bronze() -> DefaultColorScaleSet { "#fff1e9ec", ], } + .into() } -fn brown() -> DefaultColorScaleSet { +fn brown() -> ColorScaleSet { DefaultColorScaleSet { scale: "Brown", light: [ @@ -891,9 +899,10 @@ fn brown() -> DefaultColorScaleSet { "#feecd4f2", ], } + .into() } -fn yellow() -> DefaultColorScaleSet { +fn yellow() -> ColorScaleSet { DefaultColorScaleSet { scale: "Yellow", light: [ @@ -953,9 +962,10 @@ fn yellow() -> DefaultColorScaleSet { "#fef6baf6", ], } + .into() } -fn amber() -> DefaultColorScaleSet { +fn amber() -> ColorScaleSet { DefaultColorScaleSet { scale: "Amber", light: [ @@ -1015,9 +1025,10 @@ fn amber() -> DefaultColorScaleSet { "#ffe7b3ff", ], } + .into() } -fn orange() -> DefaultColorScaleSet { +fn orange() -> ColorScaleSet { DefaultColorScaleSet { scale: "Orange", light: [ @@ -1077,9 +1088,10 @@ fn orange() -> DefaultColorScaleSet { "#ffe0c2ff", ], } + .into() } -fn tomato() -> DefaultColorScaleSet { +fn tomato() -> ColorScaleSet { DefaultColorScaleSet { scale: "Tomato", light: [ @@ -1139,9 +1151,10 @@ fn tomato() -> DefaultColorScaleSet { "#ffd6cefb", ], } + .into() } -fn red() -> DefaultColorScaleSet { +fn red() -> ColorScaleSet { DefaultColorScaleSet { scale: "Red", light: [ @@ -1201,9 +1214,10 @@ fn red() -> DefaultColorScaleSet { "#ffd1d9ff", ], } + .into() } -fn ruby() -> DefaultColorScaleSet { +fn ruby() -> ColorScaleSet { DefaultColorScaleSet { scale: "Ruby", light: [ @@ -1263,9 +1277,10 @@ fn ruby() -> DefaultColorScaleSet { "#ffd3e2fe", ], } + .into() } -fn crimson() -> DefaultColorScaleSet { +fn crimson() -> ColorScaleSet { DefaultColorScaleSet { scale: "Crimson", light: [ @@ -1325,9 +1340,10 @@ fn crimson() -> DefaultColorScaleSet { "#ffd5eafd", ], } + .into() } -fn pink() -> DefaultColorScaleSet { +fn pink() -> ColorScaleSet { DefaultColorScaleSet { scale: "Pink", light: [ @@ -1387,9 +1403,10 @@ fn pink() -> DefaultColorScaleSet { "#ffd3ecfd", ], } + .into() } -fn plum() -> DefaultColorScaleSet { +fn plum() -> ColorScaleSet { DefaultColorScaleSet { scale: "Plum", light: [ @@ -1449,9 +1466,10 @@ fn plum() -> DefaultColorScaleSet { "#feddfef4", ], } + .into() } -fn purple() -> DefaultColorScaleSet { +fn purple() -> ColorScaleSet { DefaultColorScaleSet { scale: "Purple", light: [ @@ -1511,9 +1529,10 @@ fn purple() -> DefaultColorScaleSet { "#f1ddfffa", ], } + .into() } -fn violet() -> DefaultColorScaleSet { +fn violet() -> ColorScaleSet { DefaultColorScaleSet { scale: "Violet", light: [ @@ -1573,9 +1592,10 @@ fn violet() -> DefaultColorScaleSet { "#e3defffe", ], } + .into() } -fn iris() -> DefaultColorScaleSet { +fn iris() -> ColorScaleSet { DefaultColorScaleSet { scale: "Iris", light: [ @@ -1635,9 +1655,10 @@ fn iris() -> DefaultColorScaleSet { "#e1e0fffe", ], } + .into() } -fn indigo() -> DefaultColorScaleSet { +fn indigo() -> ColorScaleSet { DefaultColorScaleSet { scale: "Indigo", light: [ @@ -1697,9 +1718,10 @@ fn indigo() -> DefaultColorScaleSet { "#d6e1ffff", ], } + .into() } -fn blue() -> DefaultColorScaleSet { +fn blue() -> ColorScaleSet { DefaultColorScaleSet { scale: "Blue", light: [ @@ -1759,9 +1781,10 @@ fn blue() -> DefaultColorScaleSet { "#c2e6ffff", ], } + .into() } -fn cyan() -> DefaultColorScaleSet { +fn cyan() -> ColorScaleSet { DefaultColorScaleSet { scale: "Cyan", light: [ @@ -1821,9 +1844,10 @@ fn cyan() -> DefaultColorScaleSet { "#bbf3fef7", ], } + .into() } -fn teal() -> DefaultColorScaleSet { +fn teal() -> ColorScaleSet { DefaultColorScaleSet { scale: "Teal", light: [ @@ -1883,9 +1907,10 @@ fn teal() -> DefaultColorScaleSet { "#b8ffebef", ], } + .into() } -fn jade() -> DefaultColorScaleSet { +fn jade() -> ColorScaleSet { DefaultColorScaleSet { scale: "Jade", light: [ @@ -1945,9 +1970,10 @@ fn jade() -> DefaultColorScaleSet { "#b8ffe1ef", ], } + .into() } -fn green() -> DefaultColorScaleSet { +fn green() -> ColorScaleSet { DefaultColorScaleSet { scale: "Green", light: [ @@ -2007,9 +2033,10 @@ fn green() -> DefaultColorScaleSet { "#bbffd7f0", ], } + .into() } -fn grass() -> DefaultColorScaleSet { +fn grass() -> ColorScaleSet { DefaultColorScaleSet { scale: "Grass", light: [ @@ -2069,9 +2096,10 @@ fn grass() -> DefaultColorScaleSet { "#ceffceef", ], } + .into() } -fn lime() -> DefaultColorScaleSet { +fn lime() -> ColorScaleSet { DefaultColorScaleSet { scale: "Lime", light: [ @@ -2131,9 +2159,10 @@ fn lime() -> DefaultColorScaleSet { "#e9febff7", ], } + .into() } -fn mint() -> DefaultColorScaleSet { +fn mint() -> ColorScaleSet { DefaultColorScaleSet { scale: "Mint", light: [ @@ -2193,9 +2222,10 @@ fn mint() -> DefaultColorScaleSet { "#cbfee9f5", ], } + .into() } -fn sky() -> DefaultColorScaleSet { +fn sky() -> ColorScaleSet { DefaultColorScaleSet { scale: "Sky", light: [ @@ -2255,9 +2285,10 @@ fn sky() -> DefaultColorScaleSet { "#c2f3ffff", ], } + .into() } -fn black() -> DefaultColorScaleSet { +fn black() -> ColorScaleSet { DefaultColorScaleSet { scale: "Black", light: [ @@ -2317,9 +2348,10 @@ fn black() -> DefaultColorScaleSet { "#000000f2", ], } + .into() } -fn white() -> DefaultColorScaleSet { +fn white() -> ColorScaleSet { DefaultColorScaleSet { scale: "White", light: [ @@ -2379,4 +2411,5 @@ fn white() -> DefaultColorScaleSet { "#fffffff2", ], } + .into() }