Files
zed/crates/worktree/Cargo.toml
T
Kirill Bulatov 8bbcd83968 Speed up project search over misclassified binary files (#64549)
Based on
https://github.com/zed-industries/zed/issues/38799#issuecomment-5758593194
finding.

Project search treated any NUL-free file as text, then read it whole,
ran chardetng over every byte (~34 MiB/s) and decoded it into a String
before searching. For raw assets (.yuv/.raw/.onnx/.glb/.tif in #38799)
that cost ~8 s and ~2.6× the file size in RAM per worker.

- Bound chardetng to a 1 MiB window starting at the first non-ASCII
byte.
- Stream-decode legacy/UTF-16 files in project search (`DecodingReader`)
instead of materializing them.
- Scan the single-line text pre-filter in 64 KiB blocks instead of
`read_line`.
- Run the control-byte check even without NULs; treat all high bytes as
text so legacy encodings (windows-1251, GBK, ...) stay text.
- Add magic numbers for TIFF, glTF, ELF, Mach-O, gzip/xz/zstd, SQLite,
HDF5, NumPy, safetensors, GGUF, fonts, media containers and others.
- Move the classifier into a `file_content` crate and use it for git
commit blobs too, replacing the separate NUL-only `is_binary_content`.

Measured on a 200 MiB NUL-free raw file: 8.5 s → ~1.5 s per file, no
whole-file allocation.

Release Notes:

- Improved project search speed and memory usage over misclassified
binary files
2026-09-30 17:39:08 +00:00

76 lines
1.9 KiB
TOML

[package]
name = "worktree"
version = "0.1.0"
edition.workspace = true
publish.workspace = true
license = "GPL-3.0-or-later"
[lib]
path = "src/worktree.rs"
doctest = false
[[test]]
name = "integration"
required-features = ["test-support"]
path = "tests/integration/worktree_tests.rs"
[lints]
workspace = true
[features]
test-support = [
"gpui/test-support",
"language/test-support",
"pretty_assertions",
"settings/test-support",
"text/test-support",
"util/test-support",
]
[dependencies]
anyhow.workspace = true
async-channel.workspace = true
async-lock.workspace = true
clock.workspace = true
collections.workspace = true
encoding_rs.workspace = true
file_content.workspace = true
fs.workspace = true
futures.workspace = true
futures-lite.workspace = true
fuzzy.workspace = true
git.workspace = true
gpui.workspace = true
ignore.workspace = true
language.workspace = true
log.workspace = true
parking_lot.workspace = true
paths.workspace = true
postage.workspace = true
pretty_assertions = { workspace = true, optional = true }
rpc = { workspace = true, features = ["gpui"] }
settings.workspace = true
smallvec.workspace = true
sum_tree.workspace = true
text.workspace = true
tracing.workspace = true
util.workspace = true
ztracing.workspace = true
[dev-dependencies]
clock = { workspace = true, features = ["test-support"] }
collections = { workspace = true, features = ["test-support"] }
gpui = { workspace = true, features = ["test-support"] }
paths = { workspace = true, features = ["test-support"] }
rand.workspace = true
rpc = { workspace = true, features = ["test-support"] }
serde_json.workspace = true
settings = { workspace = true, features = ["test-support"] }
util = { workspace = true, features = ["test-support"] }
worktree = { workspace = true, features = ["test-support"] }
zlog.workspace = true
[package.metadata.cargo-shear]
ignored = ["tracing"]