diff --git a/Cargo.lock b/Cargo.lock index c5964c2b42d..42d570f032e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1839,6 +1839,7 @@ name = "gix-dir" version = "0.28.0" dependencies = [ "bstr", + "criterion", "gix-discover", "gix-fs", "gix-ignore", @@ -1851,6 +1852,7 @@ dependencies = [ "gix-utils", "gix-worktree", "pretty_assertions", + "rustix", "thiserror 2.0.18", ] diff --git a/gix-dir/Cargo.toml b/gix-dir/Cargo.toml index 7b69120cff1..ee45b9c2d00 100644 --- a/gix-dir/Cargo.toml +++ b/gix-dir/Cargo.toml @@ -15,6 +15,11 @@ include = ["/src/**/*", "/LICENSE-*"] doctest = false test = false +[[bench]] +name = "dirwalk" +harness = false +path = "./benches/dirwalk.rs" + [features] ## Enable support for the SHA-1 hash by forwarding the feature to dependencies. sha1 = ["gix-index/sha1"] @@ -36,7 +41,11 @@ gix-utils = { version = "^0.3.5", path = "../gix-utils", features = ["bstr"] } bstr = { version = "1.12.0", default-features = false } thiserror = "2.0.18" +[target.'cfg(unix)'.dependencies] +rustix = { version = "1.1.2", default-features = false, features = ["system"] } + [dev-dependencies] +criterion = "0.8.2" gix-testtools = { path = "../tests/tools" } gix-fs = { path = "../gix-fs" } pretty_assertions = "1.4.0" diff --git a/gix-dir/benches/dirwalk.rs b/gix-dir/benches/dirwalk.rs new file mode 100644 index 00000000000..4b9a6ca8144 --- /dev/null +++ b/gix-dir/benches/dirwalk.rs @@ -0,0 +1,172 @@ +use std::{hint::black_box, path::PathBuf}; + +use bstr::ByteSlice; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use gix_dir::walk; +use gix_testtools::FixtureState; + +fn dirwalk(c: &mut Criterion) { + let fixture = gix_testtools::rust_fixture_read_only("dirwalk-benchmark", 2, |state| { + if let FixtureState::Uninitialized(root) = state { + create_clean_flat(&root.join("clean-flat"))?; + create_clean_wide(&root.join("clean-wide"))?; + create_untracked_wide(&root.join("untracked-wide"))?; + } + Ok(()) + }) + .expect("benchmark fixture can be created") + .0; + + let mut group = c.benchmark_group("dirwalk"); + for name in ["clean-flat", "clean-wide", "untracked-wide"] { + let scenario = Scenario::new(fixture.join(name)); + group.bench_with_input(BenchmarkId::from_parameter(name), &scenario, |b, scenario| { + b.iter(|| black_box(scenario.walk())); + }); + } +} + +criterion_group!(benches, dirwalk); +criterion_main!(benches); + +struct Scenario { + root: PathBuf, + git_dir_realpath: PathBuf, + index: gix_index::State, +} + +impl Scenario { + fn new(root: PathBuf) -> Self { + let git_dir = root.join(".git"); + let index = std::fs::read(git_dir.join("index")) + .map_err(|err| format!("cannot read benchmark index: {err}")) + .and_then(|bytes| { + gix_index::State::from_bytes( + &bytes, + std::time::UNIX_EPOCH.into(), + gix_index::hash::Kind::Sha1, + Default::default(), + ) + .map(|(index, _)| index) + .map_err(|err| format!("cannot decode benchmark index: {err}")) + }) + .expect("Git creates a valid benchmark index"); + assert!(index.untracked().is_some(), "Git must populate the UNTR cache"); + Scenario { + git_dir_realpath: gix_path::realpath(&git_dir).expect("git directory can be resolved"), + root, + index, + } + } + + fn walk(&self) -> walk::Outcome { + let mut pathspec = gix_pathspec::Search::from_specs( + std::iter::empty::(), + None, + "benchmark has no absolute pathspecs".as_ref(), + ) + .expect("empty pathspec is valid"); + let mut excludes = gix_worktree::Stack::from_state_and_ignore_case( + &self.root, + false, + gix_worktree::stack::State::IgnoreStack(gix_worktree::stack::state::Ignore::new( + Default::default(), + Default::default(), + None, + gix_worktree::stack::state::ignore::Source::WorktreeThenIdMappingIfNotSkipped, + Default::default(), + )), + &self.index, + self.index.path_backing(), + ); + let mut delegate = Ignore; + walk( + &self.root, + walk::Context { + should_interrupt: None, + git_dir_realpath: &self.git_dir_realpath, + current_dir: &self.root, + index: &self.index, + ignore_case_index_lookup: None, + pathspec: &mut pathspec, + pathspec_attributes: &mut |_, _, _, _| unreachable!("benchmark pathspecs have no attributes"), + excludes: Some(&mut excludes), + objects: &gix_object::find::Never, + explicit_traversal_root: None, + }, + walk::Options { + use_untracked_cache: true, + emit_untracked: walk::EmissionMode::CollapseDirectory, + ..Default::default() + }, + &mut delegate, + ) + .expect("benchmark dirwalk succeeds") + .0 + } +} + +struct Ignore; + +impl walk::Delegate for Ignore { + fn emit( + &mut self, + _entry: gix_dir::EntryRef<'_>, + _collapsed_directory_status: Option, + ) -> walk::Action { + std::ops::ControlFlow::Continue(()) + } +} + +fn create_clean_flat(root: &std::path::Path) -> gix_testtools::Result { + init(root)?; + for file_idx in 0..10_000 { + std::fs::write(root.join(format!("file-{file_idx:05}")), [])?; + } + commit_and_prime(root) +} + +fn create_clean_wide(root: &std::path::Path) -> gix_testtools::Result { + init(root)?; + create_wide_tree(root)?; + commit_and_prime(root) +} + +fn create_untracked_wide(root: &std::path::Path) -> gix_testtools::Result { + init(root)?; + std::fs::write(root.join("tracked"), [])?; + commit_and_prime(root)?; + create_wide_tree(root)?; + prime(root) +} + +fn create_wide_tree(root: &std::path::Path) -> gix_testtools::Result { + for dir_idx in 0..100 { + let dir = root.join(format!("dir-{dir_idx:03}")); + std::fs::create_dir(&dir)?; + for file_idx in 0..100 { + std::fs::write(dir.join(format!("file-{file_idx:03}")), [])?; + } + } + Ok(()) +} + +fn init(root: &std::path::Path) -> gix_testtools::Result { + std::fs::create_dir(root)?; + gix_testtools::git(root, "init --quiet")?; + gix_testtools::git(root, "config core.untrackedCache true")?; + gix_testtools::git(root, "config core.excludesFile .git/no-global-excludes")?; + Ok(()) +} + +fn commit_and_prime(root: &std::path::Path) -> gix_testtools::Result { + gix_testtools::git(root, "add .")?; + gix_testtools::git(root, "commit --quiet -m baseline")?; + prime(root) +} + +fn prime(root: &std::path::Path) -> gix_testtools::Result { + let status = gix_testtools::git(root, "status --porcelain")?; + black_box(status.as_bytes().as_bstr()); + Ok(()) +} diff --git a/gix-dir/src/walk/function.rs b/gix-dir/src/walk/function.rs index 674737ff20b..49ed7d133fd 100644 --- a/gix-dir/src/walk/function.rs +++ b/gix-dir/src/walk/function.rs @@ -104,6 +104,13 @@ pub fn walk( return Ok((out, root.to_owned())); } + let untracked_cache = crate::walk::untracked_cache::State::new( + worktree_root, + ctx.index, + ctx.pathspec, + ctx.explicit_traversal_root, + options, + ); let mut state = readdir::State::new(worktree_root, ctx.current_dir, options.for_deletion.is_some()); let may_collapse = root != worktree_root && state.may_collapse(¤t); let (action, _) = readdir::recursive( @@ -116,6 +123,7 @@ pub fn walk( delegate, &mut out, &mut state, + untracked_cache.as_ref().map(|cache| (cache, 0)), )?; if action.is_continue() { state.emit_remaining(may_collapse, options, &mut out, delegate); diff --git a/gix-dir/src/walk/mod.rs b/gix-dir/src/walk/mod.rs index 0529f097cca..d8dc2769826 100644 --- a/gix-dir/src/walk/mod.rs +++ b/gix-dir/src/walk/mod.rs @@ -142,6 +142,13 @@ pub enum ForDeletionMode { /// Options for use in [`walk()`](function::walk()) function. #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] pub struct Options<'a> { + /// If `true`, use a compatible `UNTR` index extension to avoid reading unchanged directories. + /// + /// Callers must only enable this when Git configuration permits use of the cache. + /// The cache is otherwise validated against the worktree and configured exclude files before use. + pub use_untracked_cache: bool, + /// The effective `core.excludesFile`, including Git's user-level default, to validate against the `UNTR` extension. + pub untracked_cache_excludes_file: Option<&'a std::path::Path>, /// If `true`, the filesystem will store paths as decomposed unicode, i.e. `รค` becomes `"a\u{308}"`, which means that /// we have to turn these forms back from decomposed to precomposed unicode before storing it in the index or generally /// using it. This also applies to input received from the command-line, so callers may have to be aware of this and @@ -308,3 +315,4 @@ pub enum Error { mod classify; pub(crate) mod function; mod readdir; +mod untracked_cache; diff --git a/gix-dir/src/walk/readdir.rs b/gix-dir/src/walk/readdir.rs index 56374cb2f86..422ce73d00b 100644 --- a/gix-dir/src/walk/readdir.rs +++ b/gix-dir/src/walk/readdir.rs @@ -15,6 +15,7 @@ use crate::{ EmissionMode::CollapseDirectory, Error, ForDeletionMode, Options, Outcome, classify, function::{can_recurse, emit_entry}, + untracked_cache, }, }; @@ -32,96 +33,120 @@ pub(super) fn recursive( delegate: &mut dyn Delegate, out: &mut Outcome, state: &mut State, + cache_dir: Option<(&untracked_cache::State<'_>, usize)>, ) -> Result<(Action, bool), Error> { if ctx.should_interrupt.is_some_and(|flag| flag.load(Ordering::Relaxed)) { return Err(Error::Interrupted); } - out.read_dir_calls += 1; - let entries = gix_fs::read_dir(current, opts.precompose_unicode).map_err(|err| Error::ReadDir { - path: current.to_owned(), - source: err, - })?; let mut num_entries = 0; let mark = state.mark(may_collapse); let mut prevent_collapse = false; - for entry in entries { - let entry = entry.map_err(|err| Error::DirEntry { - parent_directory: current.to_owned(), - source: err, - })?; - // Important to count right away, otherwise the directory could be seen as empty even though it's not. - // That is, this should be independent of the kind. - num_entries += 1; - - let prev_len = current_bstr.len(); - if prev_len != 0 { - current_bstr.push(b'/'); - } - let file_name = entry.file_name(); - current_bstr.extend_from_slice( - gix_path::try_os_str_into_bstr(Cow::Borrowed(file_name.as_ref())) - .expect("no illformed UTF-8") - .as_ref(), - ); - current.push(file_name); - let mut info = classify::path( - current, - current_bstr, - if prev_len == 0 { 0 } else { prev_len + 1 }, - None, - || entry.file_type().ok().map(Into::into), - opts, - ctx, - )?; - - if can_recurse( - current_bstr.as_bstr(), - info, - opts.for_deletion, - false, /* is root */ - delegate, - ) { - let subdir_may_collapse = state.may_collapse(current); - let (action, subdir_prevent_collapse) = recursive( - subdir_may_collapse, + let cached = cache_dir.and_then(|(cache, index)| { + cache + .directory(index, current, current_bstr.as_bstr(), current_info, ctx) + .map(|directory| (cache, index, directory)) + }); + if let Some((cache, directory_index, directory)) = cached { + let has_tracked_descendant = + untracked_cache::has_tracked_descendant(current_bstr.as_bstr(), opts.ignore_case, ctx); + prevent_collapse = has_tracked_descendant; + num_entries += usize::from(has_tracked_descendant); + for child_index in directory.sub_directories() { + let name = cache + .child_name(*child_index) + .expect("UNTR child indices were validated during decoding"); + num_entries += 1; + let (action, child_prevent_collapse) = visit( + name, + Some(entry::Kind::Directory), + || None, + false, + Some((cache, *child_index)), current, current_bstr, - info, ctx, opts, delegate, out, state, )?; - prevent_collapse |= subdir_prevent_collapse; + prevent_collapse |= child_prevent_collapse; if action.is_break() { return Ok((action, prevent_collapse)); } - } else { - if opts.for_deletion == Some(ForDeletionMode::IgnoredDirectoriesCanHideNestedRepositories) - && info.disk_kind == Some(entry::Kind::Directory) - && matches!(info.status, Status::Ignored(_)) - { - info.disk_kind = classify::maybe_upgrade_to_repository( - info.disk_kind, - true, - false, - current, - ctx.current_dir, - ctx.git_dir_realpath, - ); + } + for name in directory.untracked_entries() { + let is_directory = name.ends_with_str("/"); + let name = name.strip_suffix(b"/").unwrap_or(name.as_bstr()).as_bstr(); + if is_directory && cache.child_index(directory_index, name).is_some() { + continue; } - if !state.held_for_directory_collapse(current_bstr.as_bstr(), info, &opts) { - let action = emit_entry(Cow::Borrowed(current_bstr.as_bstr()), info, None, opts, out, delegate); - if action.is_break() { - return Ok((action, prevent_collapse)); - } + num_entries += 1; + let on_demand_disk_kind = (!is_directory) + .then(|| { + current + .join(gix_path::from_bstr(name)) + .symlink_metadata() + .ok() + .map(|metadata| metadata.file_type().into()) + }) + .flatten(); + let (action, child_prevent_collapse) = visit( + name, + is_directory.then_some(entry::Kind::Directory), + || on_demand_disk_kind, + is_directory, + None, + current, + current_bstr, + ctx, + opts, + delegate, + out, + state, + )?; + prevent_collapse |= child_prevent_collapse; + if action.is_break() { + return Ok((action, prevent_collapse)); + } + } + } else { + out.read_dir_calls += 1; + let entries = gix_fs::read_dir(current, opts.precompose_unicode).map_err(|err| Error::ReadDir { + path: current.to_owned(), + source: err, + })?; + for entry in entries { + let entry = entry.map_err(|err| Error::DirEntry { + parent_directory: current.to_owned(), + source: err, + })?; + // Count before classification so unreadable entries still keep a directory from appearing empty. + num_entries += 1; + let file_name = gix_path::try_os_str_into_bstr(Cow::Borrowed(entry.file_name().as_ref())) + .expect("no illformed UTF-8") + .into_owned(); + let (action, child_prevent_collapse) = visit( + file_name.as_bstr(), + None, + || entry.file_type().ok().map(Into::into), + false, + None, + current, + current_bstr, + ctx, + opts, + delegate, + out, + state, + )?; + prevent_collapse |= child_prevent_collapse; + if action.is_break() { + return Ok((action, prevent_collapse)); } } - current_bstr.truncate(prev_len); - current.pop(); } let res = mark.reduce_held_entries( @@ -139,6 +164,87 @@ pub(super) fn recursive( Ok((res, prevent_collapse)) } +#[expect(clippy::too_many_arguments)] +fn visit( + name: &BStr, + disk_kind: Option, + on_demand_disk_kind: impl FnOnce() -> Option, + cached_leaf_directory: bool, + cache_dir: Option<(&untracked_cache::State<'_>, usize)>, + current: &mut PathBuf, + current_bstr: &mut BString, + ctx: &mut Context<'_>, + opts: Options<'_>, + delegate: &mut dyn Delegate, + out: &mut Outcome, + state: &mut State, +) -> Result<(Action, bool), Error> { + let prev_len = current_bstr.len(); + if prev_len != 0 { + current_bstr.push(b'/'); + } + current_bstr.extend_from_slice(name); + current.push(gix_path::from_bstr(name)); + + let mut info = classify::path( + current, + current_bstr, + if prev_len == 0 { 0 } else { prev_len + 1 }, + disk_kind, + on_demand_disk_kind, + opts, + ctx, + )?; + + let mut prevent_collapse = false; + let action = if !cached_leaf_directory + && can_recurse( + current_bstr.as_bstr(), + info, + opts.for_deletion, + false, /* is root */ + delegate, + ) { + let subdir_may_collapse = state.may_collapse(current); + let (action, subdir_prevent_collapse) = recursive( + subdir_may_collapse, + current, + current_bstr, + info, + ctx, + opts, + delegate, + out, + state, + cache_dir, + )?; + prevent_collapse = subdir_prevent_collapse; + action + } else { + if opts.for_deletion == Some(ForDeletionMode::IgnoredDirectoriesCanHideNestedRepositories) + && info.disk_kind == Some(entry::Kind::Directory) + && matches!(info.status, Status::Ignored(_)) + { + info.disk_kind = classify::maybe_upgrade_to_repository( + info.disk_kind, + true, + false, + current, + ctx.current_dir, + ctx.git_dir_realpath, + ); + } + if state.held_for_directory_collapse(current_bstr.as_bstr(), info, &opts) { + std::ops::ControlFlow::Continue(()) + } else { + emit_entry(Cow::Borrowed(current_bstr.as_bstr()), info, None, opts, out, delegate) + } + }; + current_bstr.truncate(prev_len); + current.pop(); + Ok((action, prevent_collapse)) +} + pub(super) struct State { /// The entries to hold back until it's clear what to do with them. pub on_hold: Vec, diff --git a/gix-dir/src/walk/untracked_cache.rs b/gix-dir/src/walk/untracked_cache.rs new file mode 100644 index 00000000000..84dae9c11be --- /dev/null +++ b/gix-dir/src/walk/untracked_cache.rs @@ -0,0 +1,194 @@ +use std::path::Path; + +use bstr::{BStr, BString, ByteSlice}; + +use crate::walk::{Context, EmissionMode, Options, classify}; + +const DIR_SHOW_OTHER_DIRECTORIES: u32 = 1 << 1; +const DIR_HIDE_EMPTY_DIRECTORIES: u32 = 1 << 2; + +pub(super) struct State<'a> { + cache: &'a gix_index::extension::UntrackedCache, +} + +impl<'a> State<'a> { + pub(super) fn new( + worktree_root: &Path, + index: &'a gix_index::State, + pathspec: &gix_pathspec::Search, + explicit_traversal_root: Option<&Path>, + opts: Options<'_>, + ) -> Option { + let cache = index.untracked()?; + (opts.use_untracked_cache + && opts.emit_untracked == EmissionMode::CollapseDirectory + && opts.emit_ignored.is_none() + && opts.for_deletion.is_none() + && !opts.recurse_repositories + && !opts.classify_untracked_bare_repositories + && !opts.emit_tracked + && !opts.emit_empty_directories + && opts.emit_collapsed.is_none() + && pathspec.patterns().len() == 0 + && explicit_traversal_root.is_none_or(|root| root == worktree_root) + && cache.dir_flags() == DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES + && cache.exclude_filename_per_dir() == ".gitignore" + && opts.untracked_cache_excludes_file.map_or_else( + || cache.excludes_file().is_none().then_some(true), + |path| { + ignore_oid_at_path_matches( + path, + index.object_hash(), + cache + .excludes_file() + .map(gix_index::extension::untracked_cache::OidStat::id), + ) + }, + )? + && identifier_matches(cache.identifier().as_bstr(), worktree_root)? + && ignore_oid_at_path_matches( + &worktree_root.join(".git/info/exclude"), + index.object_hash(), + cache + .info_exclude() + .map(gix_index::extension::untracked_cache::OidStat::id), + )?) + .then_some(State { cache }) + } + + pub(super) fn directory( + &self, + index: usize, + current: &Path, + current_rela_path: &BStr, + current_info: classify::Outcome, + ctx: &Context<'_>, + ) -> Option<&'a gix_index::extension::untracked_cache::Directory> { + let directory = self.cache.directories().get(index)?; + let metadata = gix_index::fs::Metadata::from_path_no_follow(current).ok()?; + let stat = gix_index::entry::Stat::from_fs(&metadata).ok()?; + let expected_check_only = + !current_rela_path.is_empty() && current_info.status == crate::entry::Status::Untracked; + let cached_stat = directory.stat()?; + let stat_options = gix_index::entry::stat::Options { + use_nsec: cached_stat.mtime.nsecs != 0, + ..Default::default() + }; + (cached_stat.matches(&stat, stat_options) + && directory.check_only() == expected_check_only + && ignore_oid_matches(current, current_rela_path, ctx, directory.exclude_file_oid())?) + .then_some(directory) + } + + pub(super) fn child_index(&self, directory_index: usize, name: &BStr) -> Option { + self.cache + .directories() + .get(directory_index)? + .sub_directories() + .iter() + .copied() + .find(|index| { + self.cache + .directories() + .get(*index) + .is_some_and(|directory| directory.name() == name) + }) + } + + pub(super) fn child_name(&self, index: usize) -> Option<&'a BStr> { + self.cache + .directories() + .get(index) + .map(gix_index::extension::untracked_cache::Directory::name) + } +} + +#[cfg(windows)] +fn identifier_matches(identifier: &BStr, worktree_root: &Path) -> Option { + let location = identifier + .strip_prefix(b"Location ")? + .strip_suffix(b", system Windows\0")?; + Some( + std::fs::canonicalize(gix_path::from_bstr(location.as_bstr())).ok()? + == std::fs::canonicalize(worktree_root).ok()?, + ) +} + +#[cfg(not(windows))] +fn identifier_matches(identifier: &BStr, worktree_root: &Path) -> Option { + let worktree_location = gix_path::into_bstr(gix_path::realpath(worktree_root).ok()?); + Some(identifier == format!("Location {}, system {}\0", worktree_location, system_name()?)) +} + +#[cfg(unix)] +fn system_name() -> Option { + rustix::system::uname().sysname().to_str().ok().map(ToOwned::to_owned) +} + +#[cfg(not(any(unix, windows)))] +fn system_name() -> Option { + None +} + +pub(super) fn has_tracked_descendant(directory: &BStr, ignore_case: bool, ctx: &Context<'_>) -> bool { + ctx.ignore_case_index_lookup + .map_or_else( + || ctx.index.entry_closest_to_directory_or_directory(directory), + |lookup| { + ctx.index + .entry_closest_to_directory_or_directory_icase(directory, ignore_case, lookup) + }, + ) + .is_some() +} + +fn ignore_oid_matches( + current: &Path, + current_rela_path: &BStr, + ctx: &Context<'_>, + expected: Option, +) -> Option { + let ignore_path = current.join(".gitignore"); + match std::fs::read(&ignore_path) { + Ok(mut data) => { + let raw = gix_object::compute_hash(ctx.index.object_hash(), gix_object::Kind::Blob, &data).ok()?; + if Some(raw) == expected { + return Some(true); + } + data.push(b'\n'); + gix_object::compute_hash(ctx.index.object_hash(), gix_object::Kind::Blob, &data) + .ok() + .map(|id| Some(id) == expected) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + let mut rela_path = BString::from(current_rela_path); + if !rela_path.is_empty() { + rela_path.push(b'/'); + } + rela_path.extend_from_slice(b".gitignore"); + Some(ctx.index.entry_by_path(rela_path.as_bstr()).map(|entry| entry.id) == expected) + } + Err(_) => None, + } +} + +fn ignore_oid_at_path_matches( + path: &Path, + object_hash: gix_index::hash::Kind, + expected: Option, +) -> Option { + match std::fs::read(path) { + Ok(mut data) => { + let raw = gix_object::compute_hash(object_hash, gix_object::Kind::Blob, &data).ok()?; + if Some(raw) == expected { + return Some(true); + } + data.push(b'\n'); + gix_object::compute_hash(object_hash, gix_object::Kind::Blob, &data) + .ok() + .map(|id| Some(id) == expected) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Some(expected.is_none()), + Err(_) => None, + } +} diff --git a/gix-dir/tests/dir/walk.rs b/gix-dir/tests/dir/walk.rs index d4c2a7f9830..c700384608a 100644 --- a/gix-dir/tests/dir/walk.rs +++ b/gix-dir/tests/dir/walk.rs @@ -19,6 +19,47 @@ use crate::walk_utils::{ try_collect_filtered_opts, try_collect_filtered_opts_collect, try_collect_filtered_opts_collect_with_root, }; +#[test] +fn untracked_cache_avoids_reading_unchanged_directories() { + let root = fixture("only-untracked"); + let mut without_cache_options = options(); + without_cache_options.use_untracked_cache = false; + without_cache_options.emit_untracked = CollapseDirectory; + let ((without_cache, _), expected) = + collect(&root, None, |keep, ctx| walk(&root, ctx, without_cache_options, keep)); + + let mut with_cache_options = options(); + with_cache_options.emit_untracked = CollapseDirectory; + let ((with_cache, _), actual) = collect(&root, None, |keep, ctx| walk(&root, ctx, with_cache_options, keep)); + + assert_eq!(actual, expected, "UNTR replay must produce the same entries"); + assert!( + with_cache.read_dir_calls < without_cache.read_dir_calls, + "UNTR replay should reduce directory reads, got {} with cache and {} without", + with_cache.read_dir_calls, + without_cache.read_dir_calls + ); +} + +#[test] +fn untracked_cache_does_not_replay_descendants_after_inherited_ignore_changes() { + let root = fixture("untracked-cache-changed-parent-ignore"); + let mut without_cache_options = options(); + without_cache_options.use_untracked_cache = false; + without_cache_options.emit_untracked = CollapseDirectory; + let (_, expected) = collect(&root, None, |keep, ctx| walk(&root, ctx, without_cache_options, keep)); + + let mut with_cache_options = options(); + with_cache_options.emit_untracked = CollapseDirectory; + let (_, actual) = collect(&root, None, |keep, ctx| walk(&root, ctx, with_cache_options, keep)); + + assert_eq!( + actual, expected, + "rejecting a parent cache must invalidate descendants that inherited its excludes" + ); + assert_eq!(actual, [entry("child", Untracked, Directory)]); +} + #[test] #[cfg(unix)] fn root_is_fifo() { @@ -1056,9 +1097,9 @@ fn only_untracked_with_prefix_deletion() -> crate::Result { #[test] fn only_untracked() -> crate::Result { let root = fixture("only-untracked"); - let ((out, _root), entries) = collect(&root, None, |keep, ctx| walk(&root, ctx, options(), keep)); + let ((without_cache, _root), entries) = collect(&root, None, |keep, ctx| walk(&root, ctx, options(), keep)); assert_eq!( - out, + without_cache, walk::Outcome { read_dir_calls: 3, returned_entries: entries.len(), @@ -1108,13 +1149,12 @@ fn only_untracked() -> crate::Result { ) }); assert_eq!( - out, - walk::Outcome { - read_dir_calls: 3, - returned_entries: entries.len(), - seen_entries: 7 + 2, - }, - "There are 2 extra directories that we fold into, but ultimately discard" + out.read_dir_calls, 0, + "UNTR supplies the cached entries without opening directories" + ); + assert!( + out.read_dir_calls < without_cache.read_dir_calls, + "UNTR should reduce directory reads" ); assert_eq!( entries, @@ -1331,9 +1371,9 @@ fn expendable_and_precious() { assert_eq!( out, walk::Outcome { - read_dir_calls: 6, + read_dir_calls: 0, returned_entries: entries.len(), - seen_entries: 16 + 2, + seen_entries: 8, } ); @@ -1395,11 +1435,11 @@ fn subdir_untracked() -> crate::Result { assert_eq!( out, walk::Outcome { - read_dir_calls: 3, + read_dir_calls: 0, returned_entries: entries.len(), - seen_entries: 7 + 1, + seen_entries: 2, }, - "there is a folded directory we added" + "UNTR supplies only the entries needed to produce the folded directory" ); assert_eq!(entries, [entry("d/d", Untracked, Directory)]); Ok(()) @@ -1966,11 +2006,11 @@ fn untracked_and_ignored() -> crate::Result { assert_eq!( out, walk::Outcome { - read_dir_calls: 5, + read_dir_calls: 0, returned_entries: entries.len(), - seen_entries: 21 + 1, + seen_entries: 5, }, - "we still encounter the same amount of entries, and 1 folded directory" + "UNTR supplies only the entries needed for aggregation" ); assert_eq!( entries, @@ -4407,6 +4447,47 @@ fn type_mismatch_ignore_case() { ); } +#[test] +fn untracked_cache_honors_case_insensitive_tracked_directories() { + let root = fixture("untracked-cache-icase"); + let ((out, _root), entries) = try_collect_filtered_opts_collect( + &root, + None, + |keep, ctx| { + walk( + &root, + ctx, + walk::Options { + emit_untracked: CollapseDirectory, + ignore_case: true, + ..options() + }, + keep, + ) + }, + None::<&str>, + Options { + fresh_index: false, + ..Default::default() + }, + ) + .expect("success"); + assert_eq!( + out, + walk::Outcome { + read_dir_calls: 0, + returned_entries: 1, + seen_entries: 1, + }, + "the cached directory and its tracked contents are accounted for without filesystem reads" + ); + assert_eq!( + entries, + [entry("dir/untracked", Untracked, File)], + "case-folded tracked contents prevent collapsing their untracked sibling" + ); +} + #[test] fn type_mismatch_ignore_case_clash_dir_is_file() { let root = fixture("type-mismatch-icase-clash-dir-is-file"); diff --git a/gix-dir/tests/fixtures/generated-archives/.gitignore b/gix-dir/tests/fixtures/generated-archives/.gitignore index 5dede136750..3f483fe26a4 100644 --- a/gix-dir/tests/fixtures/generated-archives/.gitignore +++ b/gix-dir/tests/fixtures/generated-archives/.gitignore @@ -12,3 +12,5 @@ many.tar many-symlinks.tar # Uses `mkfifo`; FIFOs cannot be represented in archives extracted on Windows. fifo.tar +# Synthetic benchmark repositories are large and cheap to regenerate. +rust-dirwalk-benchmark.tar diff --git a/gix-dir/tests/fixtures/many-symlinks.sh b/gix-dir/tests/fixtures/many-symlinks.sh index a9fe584da2f..d8c939763f9 100755 --- a/gix-dir/tests/fixtures/many-symlinks.sh +++ b/gix-dir/tests/fixtures/many-symlinks.sh @@ -66,3 +66,13 @@ git init -q submodule-symlink rm -Rf sub ln -s ../module sub ) + +# Populate UNTR caches so every directory-walk test exercises cache eligibility. +find . -name .git -prune | while read -r git_dir +do + repo=${git_dir%/.git} + git -C "$repo" config core.untrackedCache true + git -C "$repo" config core.excludesFile .git/no-global-excludes + git -C "$repo" update-index --untracked-cache --index-version 2 + git -C "$repo" status --porcelain --ignore-submodules=all >/dev/null +done diff --git a/gix-dir/tests/fixtures/many.sh b/gix-dir/tests/fixtures/many.sh index 4d8850b616a..0f522126d21 100755 --- a/gix-dir/tests/fixtures/many.sh +++ b/gix-dir/tests/fixtures/many.sh @@ -183,6 +183,15 @@ git init only-untracked >c ) +git init untracked-cache-changed-parent-ignore +(cd untracked-cache-changed-parent-ignore + mkdir child + echo ignored >.gitignore + >child/ignored + git add .gitignore + git commit -m "track inherited ignore" +) + git init ignored-with-empty (cd ignored-with-empty echo "/target/" >> .gitignore @@ -328,6 +337,15 @@ git init type-mismatch-icase rm file-is-dir && mkdir File-is-Dir && >File-is-Dir/b ) +git init untracked-cache-icase +(cd untracked-cache-icase + mkdir Dir && >Dir/tracked + git add . && git commit -m "tracked directory" + mv Dir dir + >dir/untracked + git config core.ignoreCase true +) + git init type-mismatch-icase-clash-dir-is-file (cd type-mismatch-icase-clash-dir-is-file empty_oid=$(git hash-object -w --stdin /dev/null +done + +# Make the root cache stale while leaving its child's stat and local excludes unchanged. +>untracked-cache-changed-parent-ignore/.gitignore diff --git a/gix-dir/tests/walk_utils/mod.rs b/gix-dir/tests/walk_utils/mod.rs index 93196477e65..689b9f78d1b 100644 --- a/gix-dir/tests/walk_utils/mod.rs +++ b/gix-dir/tests/walk_utils/mod.rs @@ -18,12 +18,17 @@ pub fn fixture(name: &str) -> PathBuf { /// Default options pub fn options() -> walk::Options<'static> { - walk::Options::default() + walk::Options { + use_untracked_cache: true, + ..Default::default() + } } /// Default options pub fn options_emit_all() -> walk::Options<'static> { walk::Options { + use_untracked_cache: true, + untracked_cache_excludes_file: None, precompose_unicode: false, ignore_case: false, recurse_repositories: false, diff --git a/gix/src/config/cache/access.rs b/gix/src/config/cache/access.rs index 4c9357d9c2a..16da2845067 100644 --- a/gix/src/config/cache/access.rs +++ b/gix/src/config/cache/access.rs @@ -247,6 +247,19 @@ impl Cache { self.trusted_file_path(Core::EXCLUDES_FILE) } + #[cfg(feature = "excludes")] + pub(crate) fn effective_excludes_file(&self) -> Result, config::exclude_stack::Error> { + Ok(match self.excludes_file()? { + Some(user_path) => Some(user_path), + None => self.xdg_config_path("ignore")?, + }) + } + + #[cfg(feature = "dirwalk")] + pub(crate) fn use_untracked_cache(&self) -> bool { + self.resolved.boolean(Core::UNTRACKED_CACHE).ok().flatten() != Some(false) + } + /// A helper to obtain a file from trusted configuration at `section_name`, `subsection_name`, and `key`, which is interpolated /// if present. pub(crate) fn trusted_file_path( @@ -393,10 +406,7 @@ impl Cache { source: gix_worktree::stack::state::ignore::Source, buf: &mut Vec, ) -> Result { - let excludes_file = match self.excludes_file()? { - Some(user_path) => Some(user_path), - None => self.xdg_config_path("ignore")?, - }; + let excludes_file = self.effective_excludes_file()?; let parse_ignore = self.ignore_pattern_parser()?; Ok(gix_worktree::stack::state::Ignore::new( overrides.unwrap_or_default(), diff --git a/gix/src/config/tree/sections/core.rs b/gix/src/config/tree/sections/core.rs index 241f7eb04bd..08fe3cb7db3 100644 --- a/gix/src/config/tree/sections/core.rs +++ b/gix/src/config/tree/sections/core.rs @@ -62,6 +62,8 @@ impl Core { pub const SYMLINKS: keys::Boolean = keys::Boolean::new_boolean("symlinks", &config::Tree::CORE); /// The `core.trustCTime` key. pub const TRUST_C_TIME: keys::Boolean = keys::Boolean::new_boolean("trustCTime", &config::Tree::CORE); + /// The `core.untrackedCache` key. + pub const UNTRACKED_CACHE: keys::Any = keys::Any::new("untrackedCache", &config::Tree::CORE); /// The `core.worktree` key. pub const WORKTREE: keys::Any = keys::Any::new("worktree", &config::Tree::CORE) .with_environment_override("GIT_WORK_TREE") @@ -129,6 +131,7 @@ impl Section for Core { &Self::REPOSITORY_FORMAT_VERSION, &Self::SYMLINKS, &Self::TRUST_C_TIME, + &Self::UNTRACKED_CACHE, &Self::WORKTREE, &Self::PROTECT_HFS, &Self::PROTECT_NTFS, diff --git a/gix/src/dirwalk/options.rs b/gix/src/dirwalk/options.rs index 8c292821719..4628368c04b 100644 --- a/gix/src/dirwalk/options.rs +++ b/gix/src/dirwalk/options.rs @@ -26,6 +26,8 @@ impl Options { impl From for gix_dir::walk::Options<'static> { fn from(v: Options) -> Self { gix_dir::walk::Options { + use_untracked_cache: false, + untracked_cache_excludes_file: None, precompose_unicode: v.precompose_unicode, ignore_case: v.ignore_case, recurse_repositories: v.recurse_repositories, diff --git a/gix/src/repository/dirwalk.rs b/gix/src/repository/dirwalk.rs index eb8e4b6e5e7..b111b0831da 100644 --- a/gix/src/repository/dirwalk.rs +++ b/gix/src/repository/dirwalk.rs @@ -59,6 +59,9 @@ impl Repository { let fs_caps = self.filesystem_options()?; let accelerate_lookup = fs_caps.ignore_case.then(|| index.prepare_icase_backing()); let mut opts = gix_dir::walk::Options::from(options); + let excludes_file = self.config.effective_excludes_file()?; + opts.use_untracked_cache = self.config.use_untracked_cache(); + opts.untracked_cache_excludes_file = excludes_file.as_deref(); let worktree_relative_worktree_dirs_storage; if let Some(workdir) = self.workdir().filter(|_| opts.for_deletion.is_some()) { let linked_worktrees = self.worktrees()?; diff --git a/gix/src/status/index_worktree.rs b/gix/src/status/index_worktree.rs index 2b72ddd2bd6..006bab8eeec 100644 --- a/gix/src/status/index_worktree.rs +++ b/gix/src/status/index_worktree.rs @@ -136,6 +136,15 @@ impl Repository { new_root: Some(workdir.to_owned()), }, )?; + let excludes_file = self + .config + .effective_excludes_file() + .map_err(crate::repository::attributes::Error::from)?; + let mut dirwalk_options: Option> = options.dirwalk_options.map(Into::into); + if let Some(options) = dirwalk_options.as_mut() { + options.use_untracked_cache = self.config.use_untracked_cache(); + options.untracked_cache_excludes_file = excludes_file.as_deref(); + } let out = gix_status::index_as_worktree_with_renames( index, @@ -165,7 +174,7 @@ impl Repository { fscache, }, fscache, - dirwalk: options.dirwalk_options.map(Into::into), + dirwalk: dirwalk_options, rewrites: options.rewrites, }, )?; diff --git a/gix/tests/gix/status.rs b/gix/tests/gix/status.rs index e2da4de21d8..7e80fd7bee2 100644 --- a/gix/tests/gix/status.rs +++ b/gix/tests/gix/status.rs @@ -16,6 +16,60 @@ pub fn repo(name: &str) -> crate::Result { )?) } +#[test] +fn untracked_cache_is_used_unless_disabled_by_config() -> crate::Result { + let tmp = gix_testtools::tempfile::tempdir()?; + let root = tmp.path(); + gix_testtools::git(root, "init")?; + for directory in 0..16 { + let directory = root.join(format!("d{directory}")); + std::fs::create_dir(&directory)?; + std::fs::write(directory.join("tracked"), b"content")?; + } + gix_testtools::git(root, "add .")?; + gix_testtools::git(root, "config core.untrackedCache true")?; + gix_testtools::git(root, "update-index --untracked-cache --index-version 2")?; + gix_testtools::git(root, "status --porcelain")?; + + let enabled = gix::ThreadSafeRepository::open_opts(root, crate::util::restricted())?.to_thread_local(); + let disabled = gix::ThreadSafeRepository::open_opts( + root, + crate::util::restricted().cli_overrides(["core.untrackedCache=false"]), + )? + .to_thread_local(); + let kept = gix::ThreadSafeRepository::open_opts( + root, + crate::util::restricted().cli_overrides(["core.untrackedCache=keep"]), + )? + .to_thread_local(); + + let read_dir_calls = |repo: &gix::Repository| -> crate::Result { + let mut status = repo.status(gix::progress::Discard)?.into_iter(None)?; + for item in status.by_ref() { + item?; + } + Ok(status + .into_outcome() + .expect("iteration was exhausted") + .index_worktree + .dirwalk + .expect("untracked files are enabled") + .read_dir_calls) + }; + let with_cache = read_dir_calls(&enabled)?; + let with_keep = read_dir_calls(&kept)?; + let without_cache = read_dir_calls(&disabled)?; + assert!( + with_cache < without_cache, + "core.untrackedCache=true should reduce directory reads, got {with_cache} with it and {without_cache} without" + ); + assert_eq!( + with_keep, with_cache, + "Git's `keep` value should retain an existing cache" + ); + Ok(()) +} + mod into_iter { use gix::status::{Item, Submodule, tree_index::TrackRenames}; use gix_diff::Rewrites;