mirror of
https://github.com/zed-industries/zed.git
synced 2026-11-19 06:59:22 +01:00
Fix UTF-8 boundary crashes in syntax parsing (#65273)
Tree-sitter input callbacks receive byte offsets, which can fall inside a UTF-8 character. Both language parser entry points previously sought to that offset, sliced the rope chunk as `str`, and only then converted it to bytes. This could panic while reparsing after an agent edit. - Add `Chunks::peek_bytes` and use it in both parser callbacks, preserving the exact requested offset without allocating or rounding to a character boundary. - Share the chunk slice-range computation with string and bitmap peeks, preserving their existing UTF-8 requirements. Document that constructing a `Chunks` cursor still requires an aligned initial offset. - Use `Rope::bytes_in_range` for query text providers and add a query-predicate regression for error nodes starting inside multibyte characters. - Add parser regressions for two-, three-, and four-byte characters and nonzero syntax-layer origins; compare results with parsing contiguous bytes. - Exercise forward/reverse seeks across rope chunks, bounded ranges starting or ending inside characters, and EOF/empty input. Release Notes: - Fixed a crash when syntax parsing requested bytes inside a multibyte character.
This commit is contained in:
@@ -1470,7 +1470,7 @@ pub(crate) fn parse_text(grammar: &Grammar, text: &Rope, old_tree: Option<Tree>)
|
||||
.parse_with_options(
|
||||
&mut move |offset, _| {
|
||||
chunks.seek(offset);
|
||||
chunks.next().unwrap_or("").as_bytes()
|
||||
chunks.peek_bytes().unwrap_or_default()
|
||||
},
|
||||
old_tree.as_ref(),
|
||||
None,
|
||||
|
||||
@@ -349,8 +349,6 @@ struct ChangeRegionSet(Vec<ChangedRegion>);
|
||||
|
||||
struct TextProvider<'a>(&'a Rope);
|
||||
|
||||
struct ByteChunks<'a>(text::Chunks<'a>);
|
||||
|
||||
pub(crate) struct QueryCursorHandle(Option<QueryCursor>);
|
||||
|
||||
impl SyntaxMap {
|
||||
@@ -1642,7 +1640,8 @@ fn parse_text(
|
||||
.parse_with_options(
|
||||
&mut move |offset, _| {
|
||||
chunks.seek(start_byte + offset);
|
||||
chunks.next().unwrap_or("").as_bytes()
|
||||
// Tree-sitter can request bytes inside a UTF-8 character.
|
||||
chunks.peek_bytes().unwrap_or_default()
|
||||
},
|
||||
old_tree,
|
||||
progress_callback
|
||||
@@ -2248,18 +2247,10 @@ impl std::fmt::Debug for SyntaxLayerEntry {
|
||||
}
|
||||
|
||||
impl<'a> tree_sitter::TextProvider<&'a [u8]> for TextProvider<'a> {
|
||||
type I = ByteChunks<'a>;
|
||||
type I = text::Bytes<'a>;
|
||||
|
||||
fn text(&mut self, node: tree_sitter::Node) -> Self::I {
|
||||
ByteChunks(self.0.chunks_in_range(node.byte_range()))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for ByteChunks<'a> {
|
||||
type Item = &'a [u8];
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
self.0.next().map(str::as_bytes)
|
||||
self.0.bytes_in_range(node.byte_range())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,87 @@ use tree_sitter::Node;
|
||||
use unindent::Unindent as _;
|
||||
use util::test::marked_text_ranges;
|
||||
|
||||
#[test]
|
||||
fn test_parse_text_inside_multibyte_character() -> anyhow::Result<()> {
|
||||
let language = rust_lang();
|
||||
let grammar = language.grammar().expect("Rust grammar");
|
||||
for prefix in ["", "prefix "] {
|
||||
for character in ["и", "中", "🧘"] {
|
||||
let source = format!("// {character}\nfn main() {{}}");
|
||||
let text = Rope::from(format!("{prefix}{source}").as_str());
|
||||
for character_offset in 1..character.len() {
|
||||
let range = tree_sitter::Range {
|
||||
start_byte: 3 + character_offset,
|
||||
start_point: tree_sitter::Point {
|
||||
row: 0,
|
||||
column: 3 + character_offset,
|
||||
},
|
||||
end_byte: source.len(),
|
||||
end_point: text.max_point().to_ts_point(),
|
||||
};
|
||||
let tree = parse_text(grammar, &text, prefix.len(), &[range], None, &mut None)?;
|
||||
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser.set_language(&grammar.ts_language)?;
|
||||
parser.set_included_ranges(&[range])?;
|
||||
let expected_tree = parser
|
||||
.parse(source.as_bytes(), None)
|
||||
.expect("parse contiguous bytes");
|
||||
assert_eq!(
|
||||
tree.root_node().to_sexp(),
|
||||
expected_tree.root_node().to_sexp()
|
||||
);
|
||||
assert_eq!(
|
||||
tree.root_node().byte_range(),
|
||||
expected_tree.root_node().byte_range()
|
||||
);
|
||||
assert_eq!(tree.root_node().end_byte(), source.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_text_inside_multibyte_character() -> anyhow::Result<()> {
|
||||
let language = rust_lang();
|
||||
let grammar = language.grammar().expect("Rust grammar");
|
||||
let query = Query::new(
|
||||
&grammar.ts_language,
|
||||
"((ERROR) @error (#match? @error \".*\"))",
|
||||
)?;
|
||||
for character in ["и", "中", "🧘"] {
|
||||
let source = format!("// {character}\nfn main() {{}}");
|
||||
let text = Rope::from(source.as_str());
|
||||
for character_offset in 1..character.len() {
|
||||
let range = tree_sitter::Range {
|
||||
start_byte: 3 + character_offset,
|
||||
start_point: tree_sitter::Point {
|
||||
row: 0,
|
||||
column: 3 + character_offset,
|
||||
},
|
||||
end_byte: text.len(),
|
||||
end_point: text.max_point().to_ts_point(),
|
||||
};
|
||||
let tree = parse_text(grammar, &text, 0, &[range], None, &mut None)?;
|
||||
let mut cursor = QueryCursor::new();
|
||||
let mut matches = cursor.matches(&query, tree.root_node(), TextProvider(&text));
|
||||
let query_match = matches.next().expect("predicate matches the error node");
|
||||
let node = query_match.captures.first().expect("error capture").node;
|
||||
assert_eq!(node.start_byte(), range.start_byte);
|
||||
assert!(!source.is_char_boundary(node.start_byte()));
|
||||
assert_eq!(
|
||||
tree_sitter::TextProvider::text(&mut TextProvider(&text), node)
|
||||
.flatten()
|
||||
.copied()
|
||||
.collect::<Vec<_>>(),
|
||||
&source.as_bytes()[node.byte_range()],
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_splice_included_ranges() {
|
||||
let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
|
||||
|
||||
+50
-16
@@ -962,6 +962,18 @@ impl<'a> Chunks<'a> {
|
||||
}
|
||||
|
||||
pub fn peek(&self) -> Option<&'a str> {
|
||||
let (chunk, slice_range) = self.current_chunk()?;
|
||||
Some(&chunk.text[slice_range])
|
||||
}
|
||||
|
||||
/// Unlike `peek`, this can return bytes starting or ending inside a UTF-8 character.
|
||||
/// The initial offset passed to `new` must still be a character boundary.
|
||||
pub fn peek_bytes(&self) -> Option<&'a [u8]> {
|
||||
let (chunk, slice_range) = self.current_chunk()?;
|
||||
Some(&chunk.text.as_bytes()[slice_range])
|
||||
}
|
||||
|
||||
fn current_chunk(&self) -> Option<(&'a Chunk, Range<usize>)> {
|
||||
if !self.offset_is_valid() {
|
||||
return None;
|
||||
}
|
||||
@@ -978,26 +990,12 @@ impl<'a> Chunks<'a> {
|
||||
slice_start..slice_end
|
||||
};
|
||||
|
||||
Some(&chunk.text[slice_range])
|
||||
Some((chunk, slice_range))
|
||||
}
|
||||
|
||||
/// Returns bitmaps that represent character positions and tab positions
|
||||
pub fn peek_with_bitmaps(&self) -> Option<ChunkBitmaps<'a>> {
|
||||
if !self.offset_is_valid() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let chunk = self.chunks.item()?;
|
||||
let chunk_start = *self.chunks.start();
|
||||
let slice_range = if self.reversed {
|
||||
let slice_start = cmp::max(chunk_start, self.range.start) - chunk_start;
|
||||
let slice_end = self.offset - chunk_start;
|
||||
slice_start..slice_end
|
||||
} else {
|
||||
let slice_start = self.offset - chunk_start;
|
||||
let slice_end = cmp::min(self.chunks.end(), self.range.end) - chunk_start;
|
||||
slice_start..slice_end
|
||||
};
|
||||
let (chunk, slice_range) = self.current_chunk()?;
|
||||
let chunk_start_offset = slice_range.start;
|
||||
let slice_text = &chunk.text[slice_range];
|
||||
|
||||
@@ -1745,6 +1743,42 @@ mod tests {
|
||||
assert_eq!(rope.text(), text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunks_peek_bytes() {
|
||||
let text = "aи中🧘z".repeat(128);
|
||||
let rope = Rope::from(text.as_str());
|
||||
for range in [0..text.len(), 1..text.len() - 1, 0..2] {
|
||||
let mut chunks = rope.chunks_in_range(range.clone());
|
||||
for offset in (range.clone()).chain(range.clone().rev()) {
|
||||
chunks.seek(offset);
|
||||
let bytes = chunks.peek_bytes().expect("chunk bytes");
|
||||
assert!(!bytes.is_empty());
|
||||
assert_eq!(bytes, &text.as_bytes()[offset..offset + bytes.len()]);
|
||||
assert!(offset + bytes.len() <= range.end);
|
||||
if text.is_char_boundary(offset) && text.is_char_boundary(offset + bytes.len()) {
|
||||
assert_eq!(chunks.peek().map(str::as_bytes), Some(bytes));
|
||||
}
|
||||
}
|
||||
chunks.seek(range.end);
|
||||
assert_eq!(chunks.peek_bytes(), None);
|
||||
}
|
||||
|
||||
for range in [0..text.len(), 2..text.len() - 1, 2..3] {
|
||||
let mut chunks = rope.reversed_chunks_in_range(range.clone());
|
||||
let offsets = range.start + 1..=range.end;
|
||||
for offset in offsets.clone().rev().chain(offsets) {
|
||||
chunks.seek(offset);
|
||||
let bytes = chunks.peek_bytes().expect("reversed chunk bytes");
|
||||
assert!(!bytes.is_empty());
|
||||
assert_eq!(bytes, &text.as_bytes()[offset - bytes.len()..offset]);
|
||||
assert!(offset - bytes.len() >= range.start);
|
||||
}
|
||||
chunks.seek(range.start);
|
||||
assert_eq!(chunks.peek_bytes(), None);
|
||||
}
|
||||
assert_eq!(Rope::new().chunks().peek_bytes(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clip() {
|
||||
let rope = Rope::from("🧘");
|
||||
|
||||
Reference in New Issue
Block a user