feat: ghost snapshot v2 (#8055)

This PR updates ghost snapshotting to avoid capturing oversized
untracked artifacts while keeping undo safe. Snapshot creation now
builds a temporary index from `git status --porcelain=2 -z`, writes a
tree and detached commit without touching refs, and records any ignored
large files/dirs in the snapshot report. Undo uses that metadata to
preserve large local artifacts while still cleaning up new transient
files.

codex-rs/Cargo.lock

@@ -44,7 +44,7 @@ dependencies = [
  "bytestring",
  "derive_more 2.0.1",
  "encoding_rs",
- "foldhash",
+ "foldhash 0.1.5",
  "futures-core",
  "http 0.2.12",
  "httparse",
@@ -139,7 +139,7 @@ dependencies = [
  "cfg-if",
  "derive_more 2.0.1",
  "encoding_rs",
- "foldhash",
+ "foldhash 0.1.5",
  "futures-core",
  "futures-util",
  "impl-more",
@@ -154,7 +154,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded",
  "smallvec",
- "socket2 0.6.0",
+ "socket2 0.6.1",
  "time",
  "tracing",
  "url",

codex-rs/core/src/tasks/ghost_snapshot.rs

@@ -8,8 +8,7 @@ use async_trait::async_trait;
 use codex_git::CreateGhostCommitOptions;
 use codex_git::GhostSnapshotReport;
 use codex_git::GitToolingError;
-use codex_git::capture_ghost_snapshot_report;
+use codex_git::create_ghost_commit_with_report;
-use codex_git::create_ghost_commit;
 use codex_protocol::models::ResponseItem;
 use codex_protocol::user_input::UserInput;
 use codex_utils_readiness::Readiness;
@@ -74,48 +73,30 @@ impl SessionTask for GhostSnapshotTask {
                 _ = async {
                     let repo_path = ctx_for_task.cwd.clone();
                     let ghost_snapshot = ctx_for_task.ghost_snapshot.clone();
-                    let ignore_large_untracked_dirs = ghost_snapshot.ignore_large_untracked_dirs;
+                    let ghost_snapshot_for_commit = ghost_snapshot.clone();
-                    // First, compute a snapshot report so we can warn about
-                    // large untracked directories before running the heavier
-                    // snapshot logic.
-                    if let Ok(Ok(report)) = tokio::task::spawn_blocking({
-                        let repo_path = repo_path.clone();
-                        let ghost_snapshot = ghost_snapshot.clone();
-                        move || {
-                            let options =
-                                CreateGhostCommitOptions::new(&repo_path).ghost_snapshot(ghost_snapshot);
-                            capture_ghost_snapshot_report(&options)
-                        }
-                    })
-                    .await
-                    {
-                        for message in
-                            format_snapshot_warnings(
-                                ghost_snapshot.ignore_large_untracked_files,
-                                ignore_large_untracked_dirs,
-                                &report,
-                            )
-                        {
-                            session
-                                .session
-                                .send_event(
-                                    &ctx_for_task,
-                                    EventMsg::Warning(WarningEvent { message }),
-                                )
-                                .await;
-                        }
-                    }
-
                     // Required to run in a dedicated blocking pool.
                     match tokio::task::spawn_blocking(move || {
                         let options =
-                            CreateGhostCommitOptions::new(&repo_path).ghost_snapshot(ghost_snapshot);
+                            CreateGhostCommitOptions::new(&repo_path).ghost_snapshot(ghost_snapshot_for_commit);
-                        create_ghost_commit(&options)
+                        create_ghost_commit_with_report(&options)
                     })
                     .await
                     {
-                        Ok(Ok(ghost_commit)) => {
+                        Ok(Ok((ghost_commit, report))) => {
                             info!("ghost snapshot blocking task finished");
+                            for message in format_snapshot_warnings(
+                                ghost_snapshot.ignore_large_untracked_files,
+                                ghost_snapshot.ignore_large_untracked_dirs,
+                                &report,
+                            ) {
+                                session
+                                    .session
+                                    .send_event(
+                                        &ctx_for_task,
+                                        EventMsg::Warning(WarningEvent { message }),
+                                    )
+                                    .await;
+                            }
                             session
                                 .session
                                 .record_conversation_items(&ctx, &[ResponseItem::GhostSnapshot {

codex-rs/utils/git/src/ghost_commits.rs

@@ -306,13 +306,14 @@ pub fn create_ghost_commit_with_report(
     let repo_prefix = repo_subdir(repo_root.as_path(), options.repo_path);
     let parent = resolve_head(repo_root.as_path())?;
     let force_include = prepare_force_include(repo_prefix.as_deref(), &options.force_include)?;
-    let existing_untracked = capture_existing_untracked(
+    let status_snapshot = capture_status_snapshot(
         repo_root.as_path(),
         repo_prefix.as_deref(),
         options.ghost_snapshot.ignore_large_untracked_files,
         options.ghost_snapshot.ignore_large_untracked_dirs,
         &force_include,
     )?;
+    let existing_untracked = status_snapshot.untracked;
 
     let warning_ignored_files = existing_untracked
         .ignored_untracked_files
@@ -336,6 +337,12 @@ pub fn create_ghost_commit_with_report(
         OsString::from("GIT_INDEX_FILE"),
         OsString::from(index_path.as_os_str()),
     )];
+    // Use a temporary index so snapshotting does not disturb the user's index state.
+    // Example plumbing sequence:
+    //   GIT_INDEX_FILE=/tmp/index git read-tree HEAD
+    //   GIT_INDEX_FILE=/tmp/index git add --all -- <paths>
+    //   GIT_INDEX_FILE=/tmp/index git write-tree
+    //   GIT_INDEX_FILE=/tmp/index git commit-tree <tree> -p <parent> -m "codex snapshot"
 
     // Pre-populate the temporary index with HEAD so unchanged tracked files
     // are included in the snapshot tree.
@@ -347,22 +354,12 @@ pub fn create_ghost_commit_with_report(
         )?;
     }
 
-    let mut add_args = vec![OsString::from("add"), OsString::from("--all")];
+    let mut index_paths = status_snapshot.tracked_paths;
-    if let Some(prefix) = repo_prefix.as_deref() {
+    index_paths.extend(existing_untracked.untracked_files_for_index.iter().cloned());
-        add_args.extend([OsString::from("--"), prefix.as_os_str().to_os_string()]);
+    let index_paths = dedupe_paths(index_paths);
-    }
+    // Stage tracked + new files into the temp index so write-tree reflects the working tree.
-
+    // We use `git add --all` to make deletions show up in the snapshot tree too.
-    run_git_for_status(repo_root.as_path(), add_args, Some(base_env.as_slice()))?;
+    add_paths_to_index(repo_root.as_path(), base_env.as_slice(), &index_paths)?;
-    remove_large_untracked_dirs_from_index(
-        repo_root.as_path(),
-        base_env.as_slice(),
-        &existing_untracked.ignored_large_untracked_dir_files,
-    )?;
-    remove_large_untracked_files_from_index(
-        repo_root.as_path(),
-        base_env.as_slice(),
-        &existing_untracked.ignored_untracked_files,
-    )?;
     if !force_include.is_empty() {
         let mut args = Vec::with_capacity(force_include.len() + 2);
         args.push(OsString::from("add"));
@@ -393,6 +390,8 @@ pub fn create_ghost_commit_with_report(
         result
     };
 
+    // `git commit-tree` writes a detached commit object without updating refs,
+    // which keeps snapshots out of the user's branch history.
     // Retrieve commit ID.
     let commit_id = run_git_for_stdout(
         repo_root.as_path(),
@@ -468,6 +467,9 @@ fn restore_to_commit_inner(
     repo_prefix: Option<&Path>,
     commit_id: &str,
 ) -> Result<(), GitToolingError> {
+    // `git restore` resets both the index and working tree to the snapshot commit.
+    // Example:
+    //   git restore --source <commit> --worktree --staged -- <prefix>
     let mut restore_args = vec![
         OsString::from("restore"),
         OsString::from("--source"),
@@ -490,22 +492,30 @@ fn restore_to_commit_inner(
 struct UntrackedSnapshot {
     files: Vec<PathBuf>,
     dirs: Vec<PathBuf>,
+    untracked_files_for_index: Vec<PathBuf>,
     ignored_untracked_files: Vec<IgnoredUntrackedFile>,
     ignored_large_untracked_dirs: Vec<LargeUntrackedDir>,
     ignored_large_untracked_dir_files: Vec<PathBuf>,
 }
 
-/// Captures the untracked and ignored entries under `repo_root`, optionally limited by `repo_prefix`.
+#[derive(Default)]
-/// Returns the result as an `UntrackedSnapshot`.
+struct StatusSnapshot {
-fn capture_existing_untracked(
+    tracked_paths: Vec<PathBuf>,
+    untracked: UntrackedSnapshot,
+}
+
+/// Captures the working tree status under `repo_root`, optionally limited by `repo_prefix`.
+/// Returns the result as a `StatusSnapshot`.
+fn capture_status_snapshot(
     repo_root: &Path,
     repo_prefix: Option<&Path>,
     ignore_large_untracked_files: Option<i64>,
     ignore_large_untracked_dirs: Option<i64>,
     force_include: &[PathBuf],
-) -> Result<UntrackedSnapshot, GitToolingError> {
+) -> Result<StatusSnapshot, GitToolingError> {
     // Ask git for the zero-delimited porcelain status so we can enumerate
-    // every untracked path (including ones filtered by prefix).
+    // tracked, untracked, and ignored entries (including ones filtered by prefix).
+    // This keeps the snapshot consistent without multiple git invocations.
     let mut args = vec![
         OsString::from("status"),
         OsString::from("--porcelain=2"),
@@ -519,55 +529,90 @@ fn capture_existing_untracked(
 
     let output = run_git_for_stdout_all(repo_root, args, None)?;
     if output.is_empty() {
-        return Ok(UntrackedSnapshot::default());
+        return Ok(StatusSnapshot::default());
     }
 
-    let mut snapshot = UntrackedSnapshot::default();
+    let mut snapshot = StatusSnapshot::default();
     let mut untracked_files_for_dir_scan: Vec<PathBuf> = Vec::new();
-    // Each entry is of the form "<code> <path>" where code is '?' (untracked)
+    let mut expect_rename_source = false;
-    // or '!' (ignored); everything else is irrelevant to this snapshot.
     for entry in output.split('\0') {
         if entry.is_empty() {
             continue;
         }
-        let mut parts = entry.splitn(2, ' ');
+        if expect_rename_source {
-        let code = parts.next();
+            let normalized = normalize_relative_path(Path::new(entry))?;
-        let path_part = parts.next();
+            snapshot.tracked_paths.push(normalized);
-        let (Some(code), Some(path_part)) = (code, path_part) else {
+            expect_rename_source = false;
-            continue;
-        };
-        if code != "?" && code != "!" {
-            continue;
-        }
-        if path_part.is_empty() {
             continue;
         }
 
-        let normalized = normalize_relative_path(Path::new(path_part))?;
+        let record_type = entry.as_bytes().first().copied().unwrap_or(b' ');
-        if should_ignore_for_snapshot(&normalized) {
+        match record_type {
-            continue;
+            b'?' | b'!' => {
-        }
+                let mut parts = entry.splitn(2, ' ');
-        let absolute = repo_root.join(&normalized);
+                let code = parts.next();
-        let is_dir = absolute.is_dir();
+                let path_part = parts.next();
-        if is_dir {
+                let (Some(code), Some(path_part)) = (code, path_part) else {
-            snapshot.dirs.push(normalized);
+                    continue;
-        } else if code == "?" {
+                };
-            untracked_files_for_dir_scan.push(normalized.clone());
+                if path_part.is_empty() {
-            if let Some(threshold) = ignore_large_untracked_files
+                    continue;
-                && threshold > 0
+                }
-                && !is_force_included(&normalized, force_include)
+
-                && let Ok(Some(byte_size)) = untracked_file_size(&absolute)
+                let normalized = normalize_relative_path(Path::new(path_part))?;
-                && byte_size > threshold
+                if should_ignore_for_snapshot(&normalized) {
-            {
+                    continue;
-                snapshot.ignored_untracked_files.push(IgnoredUntrackedFile {
+                }
-                    path: normalized,
+                let absolute = repo_root.join(&normalized);
-                    byte_size,
+                let is_dir = absolute.is_dir();
-                });
+                if is_dir {
-            } else {
+                    snapshot.untracked.dirs.push(normalized);
-                snapshot.files.push(normalized);
+                } else if code == "?" {
+                    untracked_files_for_dir_scan.push(normalized.clone());
+                    if let Some(threshold) = ignore_large_untracked_files
+                        && threshold > 0
+                        && !is_force_included(&normalized, force_include)
+                        && let Ok(Some(byte_size)) = untracked_file_size(&absolute)
+                        && byte_size > threshold
+                    {
+                        snapshot
+                            .untracked
+                            .ignored_untracked_files
+                            .push(IgnoredUntrackedFile {
+                                path: normalized,
+                                byte_size,
+                            });
+                    } else {
+                        snapshot.untracked.files.push(normalized.clone());
+                        snapshot
+                            .untracked
+                            .untracked_files_for_index
+                            .push(normalized);
+                    }
+                } else {
+                    snapshot.untracked.files.push(normalized);
+                }
             }
-        } else {
+            b'1' => {
-            snapshot.files.push(normalized);
+                if let Some(path) = extract_status_path_after_fields(entry, 8) {
+                    let normalized = normalize_relative_path(Path::new(path))?;
+                    snapshot.tracked_paths.push(normalized);
+                }
+            }
+            b'2' => {
+                if let Some(path) = extract_status_path_after_fields(entry, 9) {
+                    let normalized = normalize_relative_path(Path::new(path))?;
+                    snapshot.tracked_paths.push(normalized);
+                }
+                expect_rename_source = true;
+            }
+            b'u' => {
+                if let Some(path) = extract_status_path_after_fields(entry, 10) {
+                    let normalized = normalize_relative_path(Path::new(path))?;
+                    snapshot.tracked_paths.push(normalized);
+                }
+            }
+            _ => {}
         }
     }
 
@@ -576,7 +621,7 @@ fn capture_existing_untracked(
     {
         let ignored_large_untracked_dirs = detect_large_untracked_dirs(
             &untracked_files_for_dir_scan,
-            &snapshot.dirs,
+            &snapshot.untracked.dirs,
             Some(threshold),
         )
         .into_iter()
@@ -590,28 +635,69 @@ fn capture_existing_untracked(
                 .collect::<Vec<_>>();
 
             snapshot
+                .untracked
                 .files
                 .retain(|path| !ignored_dir_paths.iter().any(|dir| path.starts_with(dir)));
             snapshot
+                .untracked
                 .dirs
                 .retain(|path| !ignored_dir_paths.iter().any(|dir| path.starts_with(dir)));
-            snapshot.ignored_untracked_files.retain(|file| {
+            snapshot
+                .untracked
+                .untracked_files_for_index
+                .retain(|path| !ignored_dir_paths.iter().any(|dir| path.starts_with(dir)));
+            snapshot.untracked.ignored_untracked_files.retain(|file| {
                 !ignored_dir_paths
                     .iter()
                     .any(|dir| file.path.starts_with(dir))
             });
 
-            snapshot.ignored_large_untracked_dir_files = untracked_files_for_dir_scan
+            snapshot.untracked.ignored_large_untracked_dir_files = untracked_files_for_dir_scan
                 .into_iter()
                 .filter(|path| ignored_dir_paths.iter().any(|dir| path.starts_with(dir)))
                 .collect();
-            snapshot.ignored_large_untracked_dirs = ignored_large_untracked_dirs;
+            snapshot.untracked.ignored_large_untracked_dirs = ignored_large_untracked_dirs;
         }
     }
 
     Ok(snapshot)
 }
 
+/// Captures the untracked and ignored entries under `repo_root`, optionally limited by `repo_prefix`.
+/// Returns the result as an `UntrackedSnapshot`.
+fn capture_existing_untracked(
+    repo_root: &Path,
+    repo_prefix: Option<&Path>,
+    ignore_large_untracked_files: Option<i64>,
+    ignore_large_untracked_dirs: Option<i64>,
+    force_include: &[PathBuf],
+) -> Result<UntrackedSnapshot, GitToolingError> {
+    Ok(capture_status_snapshot(
+        repo_root,
+        repo_prefix,
+        ignore_large_untracked_files,
+        ignore_large_untracked_dirs,
+        force_include,
+    )?
+    .untracked)
+}
+
+fn extract_status_path_after_fields(record: &str, fields_before_path: i64) -> Option<&str> {
+    if fields_before_path <= 0 {
+        return None;
+    }
+    let mut spaces = 0_i64;
+    for (idx, byte) in record.as_bytes().iter().enumerate() {
+        if *byte == b' ' {
+            spaces += 1;
+            if spaces == fields_before_path {
+                return record.get((idx + 1)..).filter(|path| !path.is_empty());
+            }
+        }
+    }
+    None
+}
+
 fn should_ignore_for_snapshot(path: &Path) -> bool {
     path.components().any(|component| {
         if let Component::Normal(name) = component
@@ -657,27 +743,7 @@ fn untracked_file_size(path: &Path) -> io::Result<Option<i64>> {
     Ok(Some(len_i64))
 }
 
-fn remove_large_untracked_files_from_index(
+fn add_paths_to_index(
-    repo_root: &Path,
-    env: &[(OsString, OsString)],
-    ignored: &[IgnoredUntrackedFile],
-) -> Result<(), GitToolingError> {
-    let paths = ignored
-        .iter()
-        .map(|entry| entry.path.clone())
-        .collect::<Vec<_>>();
-    remove_paths_from_index(repo_root, env, &paths)
-}
-
-fn remove_large_untracked_dirs_from_index(
-    repo_root: &Path,
-    env: &[(OsString, OsString)],
-    paths: &[PathBuf],
-) -> Result<(), GitToolingError> {
-    remove_paths_from_index(repo_root, env, paths)
-}
-
-fn remove_paths_from_index(
     repo_root: &Path,
     env: &[(OsString, OsString)],
     paths: &[PathBuf],
@@ -686,20 +752,32 @@ fn remove_paths_from_index(
         return Ok(());
     }
 
-    const CHUNK_SIZE: usize = 64;
+    let chunk_size = usize::try_from(64_i64).unwrap_or(1);
-    for chunk in paths.chunks(CHUNK_SIZE) {
+    for chunk in paths.chunks(chunk_size) {
         let mut args = vec![
-            OsString::from("update-index"),
+            OsString::from("add"),
-            OsString::from("--force-remove"),
+            OsString::from("--all"),
             OsString::from("--"),
         ];
         args.extend(chunk.iter().map(|path| path.as_os_str().to_os_string()));
+        // Chunk the argv to avoid oversized command lines on large repos.
         run_git_for_status(repo_root, args, Some(env))?;
     }
 
     Ok(())
 }
 
+fn dedupe_paths(paths: Vec<PathBuf>) -> Vec<PathBuf> {
+    let mut seen = HashSet::new();
+    let mut result = Vec::new();
+    for path in paths {
+        if seen.insert(path.clone()) {
+            result.push(path);
+        }
+    }
+    result
+}
+
 fn merge_preserved_untracked_files(
     mut files: Vec<PathBuf>,
     ignored: &[IgnoredUntrackedFile],