Fix UTF-8 boundary crashes in syntax parsing (#65273)

Tree-sitter input callbacks receive byte offsets, which can fall inside
a UTF-8 character. Both language parser entry points previously sought
to that offset, sliced the rope chunk as `str`, and only then converted
it to bytes. This could panic while reparsing after an agent edit.

- Add `Chunks::peek_bytes` and use it in both parser callbacks,
preserving the exact requested offset without allocating or rounding to
a character boundary.
- Share the chunk slice-range computation with string and bitmap peeks,
preserving their existing UTF-8 requirements. Document that constructing
a `Chunks` cursor still requires an aligned initial offset.
- Use `Rope::bytes_in_range` for query text providers and add a
query-predicate regression for error nodes starting inside multibyte
characters.
- Add parser regressions for two-, three-, and four-byte characters and
nonzero syntax-layer origins; compare results with parsing contiguous
bytes.
- Exercise forward/reverse seeks across rope chunks, bounded ranges
starting or ending inside characters, and EOF/empty input.

Release Notes:

- Fixed a crash when syntax parsing requested bytes inside a multibyte
character.
This commit is contained in:
Ben Brandt
2026-10-07 11:03:59 +00:00
committed by GitHub
parent 98e168265b
commit e5ff5550e4
4 changed files with 136 additions and 30 deletions
+1 -1
View File
@@ -1470,7 +1470,7 @@ pub(crate) fn parse_text(grammar: &Grammar, text: &Rope, old_tree: Option<Tree>)
.parse_with_options(
&mut move |offset, _| {
chunks.seek(offset);
chunks.next().unwrap_or("").as_bytes()
chunks.peek_bytes().unwrap_or_default()
},
old_tree.as_ref(),
None,
+4 -13
View File
@@ -349,8 +349,6 @@ struct ChangeRegionSet(Vec<ChangedRegion>);
struct TextProvider<'a>(&'a Rope);
struct ByteChunks<'a>(text::Chunks<'a>);
pub(crate) struct QueryCursorHandle(Option<QueryCursor>);
impl SyntaxMap {
@@ -1642,7 +1640,8 @@ fn parse_text(
.parse_with_options(
&mut move |offset, _| {
chunks.seek(start_byte + offset);
chunks.next().unwrap_or("").as_bytes()
// Tree-sitter can request bytes inside a UTF-8 character.
chunks.peek_bytes().unwrap_or_default()
},
old_tree,
progress_callback
@@ -2248,18 +2247,10 @@ impl std::fmt::Debug for SyntaxLayerEntry {
}
impl<'a> tree_sitter::TextProvider<&'a [u8]> for TextProvider<'a> {
type I = ByteChunks<'a>;
type I = text::Bytes<'a>;
fn text(&mut self, node: tree_sitter::Node) -> Self::I {
ByteChunks(self.0.chunks_in_range(node.byte_range()))
}
}
impl<'a> Iterator for ByteChunks<'a> {
type Item = &'a [u8];
fn next(&mut self) -> Option<Self::Item> {
self.0.next().map(str::as_bytes)
self.0.bytes_in_range(node.byte_range())
}
}
@@ -14,6 +14,87 @@ use tree_sitter::Node;
use unindent::Unindent as _;
use util::test::marked_text_ranges;
#[test]
fn test_parse_text_inside_multibyte_character() -> anyhow::Result<()> {
let language = rust_lang();
let grammar = language.grammar().expect("Rust grammar");
for prefix in ["", "prefix "] {
for character in ["и", "中", "🧘"] {
let source = format!("// {character}\nfn main() {{}}");
let text = Rope::from(format!("{prefix}{source}").as_str());
for character_offset in 1..character.len() {
let range = tree_sitter::Range {
start_byte: 3 + character_offset,
start_point: tree_sitter::Point {
row: 0,
column: 3 + character_offset,
},
end_byte: source.len(),
end_point: text.max_point().to_ts_point(),
};
let tree = parse_text(grammar, &text, prefix.len(), &[range], None, &mut None)?;
let mut parser = tree_sitter::Parser::new();
parser.set_language(&grammar.ts_language)?;
parser.set_included_ranges(&[range])?;
let expected_tree = parser
.parse(source.as_bytes(), None)
.expect("parse contiguous bytes");
assert_eq!(
tree.root_node().to_sexp(),
expected_tree.root_node().to_sexp()
);
assert_eq!(
tree.root_node().byte_range(),
expected_tree.root_node().byte_range()
);
assert_eq!(tree.root_node().end_byte(), source.len());
}
}
}
Ok(())
}
#[test]
fn test_query_text_inside_multibyte_character() -> anyhow::Result<()> {
let language = rust_lang();
let grammar = language.grammar().expect("Rust grammar");
let query = Query::new(
&grammar.ts_language,
"((ERROR) @error (#match? @error \".*\"))",
)?;
for character in ["и", "中", "🧘"] {
let source = format!("// {character}\nfn main() {{}}");
let text = Rope::from(source.as_str());
for character_offset in 1..character.len() {
let range = tree_sitter::Range {
start_byte: 3 + character_offset,
start_point: tree_sitter::Point {
row: 0,
column: 3 + character_offset,
},
end_byte: text.len(),
end_point: text.max_point().to_ts_point(),
};
let tree = parse_text(grammar, &text, 0, &[range], None, &mut None)?;
let mut cursor = QueryCursor::new();
let mut matches = cursor.matches(&query, tree.root_node(), TextProvider(&text));
let query_match = matches.next().expect("predicate matches the error node");
let node = query_match.captures.first().expect("error capture").node;
assert_eq!(node.start_byte(), range.start_byte);
assert!(!source.is_char_boundary(node.start_byte()));
assert_eq!(
tree_sitter::TextProvider::text(&mut TextProvider(&text), node)
.flatten()
.copied()
.collect::<Vec<_>>(),
&source.as_bytes()[node.byte_range()],
);
}
}
Ok(())
}
#[test]
fn test_splice_included_ranges() {
let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
+50 -16
View File
@@ -962,6 +962,18 @@ impl<'a> Chunks<'a> {
}
pub fn peek(&self) -> Option<&'a str> {
let (chunk, slice_range) = self.current_chunk()?;
Some(&chunk.text[slice_range])
}
/// Unlike `peek`, this can return bytes starting or ending inside a UTF-8 character.
/// The initial offset passed to `new` must still be a character boundary.
pub fn peek_bytes(&self) -> Option<&'a [u8]> {
let (chunk, slice_range) = self.current_chunk()?;
Some(&chunk.text.as_bytes()[slice_range])
}
fn current_chunk(&self) -> Option<(&'a Chunk, Range<usize>)> {
if !self.offset_is_valid() {
return None;
}
@@ -978,26 +990,12 @@ impl<'a> Chunks<'a> {
slice_start..slice_end
};
Some(&chunk.text[slice_range])
Some((chunk, slice_range))
}
/// Returns bitmaps that represent character positions and tab positions
pub fn peek_with_bitmaps(&self) -> Option<ChunkBitmaps<'a>> {
if !self.offset_is_valid() {
return None;
}
let chunk = self.chunks.item()?;
let chunk_start = *self.chunks.start();
let slice_range = if self.reversed {
let slice_start = cmp::max(chunk_start, self.range.start) - chunk_start;
let slice_end = self.offset - chunk_start;
slice_start..slice_end
} else {
let slice_start = self.offset - chunk_start;
let slice_end = cmp::min(self.chunks.end(), self.range.end) - chunk_start;
slice_start..slice_end
};
let (chunk, slice_range) = self.current_chunk()?;
let chunk_start_offset = slice_range.start;
let slice_text = &chunk.text[slice_range];
@@ -1745,6 +1743,42 @@ mod tests {
assert_eq!(rope.text(), text);
}
#[test]
fn test_chunks_peek_bytes() {
let text = "aи中🧘z".repeat(128);
let rope = Rope::from(text.as_str());
for range in [0..text.len(), 1..text.len() - 1, 0..2] {
let mut chunks = rope.chunks_in_range(range.clone());
for offset in (range.clone()).chain(range.clone().rev()) {
chunks.seek(offset);
let bytes = chunks.peek_bytes().expect("chunk bytes");
assert!(!bytes.is_empty());
assert_eq!(bytes, &text.as_bytes()[offset..offset + bytes.len()]);
assert!(offset + bytes.len() <= range.end);
if text.is_char_boundary(offset) && text.is_char_boundary(offset + bytes.len()) {
assert_eq!(chunks.peek().map(str::as_bytes), Some(bytes));
}
}
chunks.seek(range.end);
assert_eq!(chunks.peek_bytes(), None);
}
for range in [0..text.len(), 2..text.len() - 1, 2..3] {
let mut chunks = rope.reversed_chunks_in_range(range.clone());
let offsets = range.start + 1..=range.end;
for offset in offsets.clone().rev().chain(offsets) {
chunks.seek(offset);
let bytes = chunks.peek_bytes().expect("reversed chunk bytes");
assert!(!bytes.is_empty());
assert_eq!(bytes, &text.as_bytes()[offset - bytes.len()..offset]);
assert!(offset - bytes.len() >= range.start);
}
chunks.seek(range.start);
assert_eq!(chunks.peek_bytes(), None);
}
assert_eq!(Rope::new().chunks().peek_bytes(), None);
}
#[test]
fn test_clip() {
let rope = Rope::from("🧘");