diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index dd299c0381f..496f71efe35 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -1470,7 +1470,7 @@ pub(crate) fn parse_text(grammar: &Grammar, text: &Rope, old_tree: Option) .parse_with_options( &mut move |offset, _| { chunks.seek(offset); - chunks.next().unwrap_or("").as_bytes() + chunks.peek_bytes().unwrap_or_default() }, old_tree.as_ref(), None, diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs index 26a53bfd5ba..0e151f74f3e 100644 --- a/crates/language/src/syntax_map.rs +++ b/crates/language/src/syntax_map.rs @@ -349,8 +349,6 @@ struct ChangeRegionSet(Vec); struct TextProvider<'a>(&'a Rope); -struct ByteChunks<'a>(text::Chunks<'a>); - pub(crate) struct QueryCursorHandle(Option); impl SyntaxMap { @@ -1642,7 +1640,8 @@ fn parse_text( .parse_with_options( &mut move |offset, _| { chunks.seek(start_byte + offset); - chunks.next().unwrap_or("").as_bytes() + // Tree-sitter can request bytes inside a UTF-8 character. + chunks.peek_bytes().unwrap_or_default() }, old_tree, progress_callback @@ -2248,18 +2247,10 @@ impl std::fmt::Debug for SyntaxLayerEntry { } impl<'a> tree_sitter::TextProvider<&'a [u8]> for TextProvider<'a> { - type I = ByteChunks<'a>; + type I = text::Bytes<'a>; fn text(&mut self, node: tree_sitter::Node) -> Self::I { - ByteChunks(self.0.chunks_in_range(node.byte_range())) - } -} - -impl<'a> Iterator for ByteChunks<'a> { - type Item = &'a [u8]; - - fn next(&mut self) -> Option { - self.0.next().map(str::as_bytes) + self.0.bytes_in_range(node.byte_range()) } } diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs index c86338e7bf0..2dd6723ec69 100644 --- a/crates/language/src/syntax_map/syntax_map_tests.rs +++ b/crates/language/src/syntax_map/syntax_map_tests.rs @@ -14,6 +14,87 @@ use tree_sitter::Node; use unindent::Unindent as _; use util::test::marked_text_ranges; +#[test] +fn test_parse_text_inside_multibyte_character() -> anyhow::Result<()> { + let language = rust_lang(); + let grammar = language.grammar().expect("Rust grammar"); + for prefix in ["", "prefix "] { + for character in ["и", "中", "🧘"] { + let source = format!("// {character}\nfn main() {{}}"); + let text = Rope::from(format!("{prefix}{source}").as_str()); + for character_offset in 1..character.len() { + let range = tree_sitter::Range { + start_byte: 3 + character_offset, + start_point: tree_sitter::Point { + row: 0, + column: 3 + character_offset, + }, + end_byte: source.len(), + end_point: text.max_point().to_ts_point(), + }; + let tree = parse_text(grammar, &text, prefix.len(), &[range], None, &mut None)?; + + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&grammar.ts_language)?; + parser.set_included_ranges(&[range])?; + let expected_tree = parser + .parse(source.as_bytes(), None) + .expect("parse contiguous bytes"); + assert_eq!( + tree.root_node().to_sexp(), + expected_tree.root_node().to_sexp() + ); + assert_eq!( + tree.root_node().byte_range(), + expected_tree.root_node().byte_range() + ); + assert_eq!(tree.root_node().end_byte(), source.len()); + } + } + } + Ok(()) +} + +#[test] +fn test_query_text_inside_multibyte_character() -> anyhow::Result<()> { + let language = rust_lang(); + let grammar = language.grammar().expect("Rust grammar"); + let query = Query::new( + &grammar.ts_language, + "((ERROR) @error (#match? @error \".*\"))", + )?; + for character in ["и", "中", "🧘"] { + let source = format!("// {character}\nfn main() {{}}"); + let text = Rope::from(source.as_str()); + for character_offset in 1..character.len() { + let range = tree_sitter::Range { + start_byte: 3 + character_offset, + start_point: tree_sitter::Point { + row: 0, + column: 3 + character_offset, + }, + end_byte: text.len(), + end_point: text.max_point().to_ts_point(), + }; + let tree = parse_text(grammar, &text, 0, &[range], None, &mut None)?; + let mut cursor = QueryCursor::new(); + let mut matches = cursor.matches(&query, tree.root_node(), TextProvider(&text)); + let query_match = matches.next().expect("predicate matches the error node"); + let node = query_match.captures.first().expect("error capture").node; + assert_eq!(node.start_byte(), range.start_byte); + assert!(!source.is_char_boundary(node.start_byte())); + assert_eq!( + tree_sitter::TextProvider::text(&mut TextProvider(&text), node) + .flatten() + .copied() + .collect::>(), + &source.as_bytes()[node.byte_range()], + ); + } + } + Ok(()) +} + #[test] fn test_splice_included_ranges() { let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)]; diff --git a/crates/rope/src/rope.rs b/crates/rope/src/rope.rs index 4d5aa410ae7..d31a5dbbb37 100644 --- a/crates/rope/src/rope.rs +++ b/crates/rope/src/rope.rs @@ -962,6 +962,18 @@ impl<'a> Chunks<'a> { } pub fn peek(&self) -> Option<&'a str> { + let (chunk, slice_range) = self.current_chunk()?; + Some(&chunk.text[slice_range]) + } + + /// Unlike `peek`, this can return bytes starting or ending inside a UTF-8 character. + /// The initial offset passed to `new` must still be a character boundary. + pub fn peek_bytes(&self) -> Option<&'a [u8]> { + let (chunk, slice_range) = self.current_chunk()?; + Some(&chunk.text.as_bytes()[slice_range]) + } + + fn current_chunk(&self) -> Option<(&'a Chunk, Range)> { if !self.offset_is_valid() { return None; } @@ -978,26 +990,12 @@ impl<'a> Chunks<'a> { slice_start..slice_end }; - Some(&chunk.text[slice_range]) + Some((chunk, slice_range)) } /// Returns bitmaps that represent character positions and tab positions pub fn peek_with_bitmaps(&self) -> Option> { - if !self.offset_is_valid() { - return None; - } - - let chunk = self.chunks.item()?; - let chunk_start = *self.chunks.start(); - let slice_range = if self.reversed { - let slice_start = cmp::max(chunk_start, self.range.start) - chunk_start; - let slice_end = self.offset - chunk_start; - slice_start..slice_end - } else { - let slice_start = self.offset - chunk_start; - let slice_end = cmp::min(self.chunks.end(), self.range.end) - chunk_start; - slice_start..slice_end - }; + let (chunk, slice_range) = self.current_chunk()?; let chunk_start_offset = slice_range.start; let slice_text = &chunk.text[slice_range]; @@ -1745,6 +1743,42 @@ mod tests { assert_eq!(rope.text(), text); } + #[test] + fn test_chunks_peek_bytes() { + let text = "aи中🧘z".repeat(128); + let rope = Rope::from(text.as_str()); + for range in [0..text.len(), 1..text.len() - 1, 0..2] { + let mut chunks = rope.chunks_in_range(range.clone()); + for offset in (range.clone()).chain(range.clone().rev()) { + chunks.seek(offset); + let bytes = chunks.peek_bytes().expect("chunk bytes"); + assert!(!bytes.is_empty()); + assert_eq!(bytes, &text.as_bytes()[offset..offset + bytes.len()]); + assert!(offset + bytes.len() <= range.end); + if text.is_char_boundary(offset) && text.is_char_boundary(offset + bytes.len()) { + assert_eq!(chunks.peek().map(str::as_bytes), Some(bytes)); + } + } + chunks.seek(range.end); + assert_eq!(chunks.peek_bytes(), None); + } + + for range in [0..text.len(), 2..text.len() - 1, 2..3] { + let mut chunks = rope.reversed_chunks_in_range(range.clone()); + let offsets = range.start + 1..=range.end; + for offset in offsets.clone().rev().chain(offsets) { + chunks.seek(offset); + let bytes = chunks.peek_bytes().expect("reversed chunk bytes"); + assert!(!bytes.is_empty()); + assert_eq!(bytes, &text.as_bytes()[offset - bytes.len()..offset]); + assert!(offset - bytes.len() >= range.start); + } + chunks.seek(range.start); + assert_eq!(chunks.peek_bytes(), None); + } + assert_eq!(Rope::new().chunks().peek_bytes(), None); + } + #[test] fn test_clip() { let rope = Rope::from("🧘");