mirror of
https://github.com/openai/codex.git
synced 2026-11-13 22:47:20 +00:00
Coordinate rollout compression with active thread writers (#44138)
## Why Cold rollout files can still have active writers. Compression must not replace their files while writes are pending, including background work that outlives the local thread store. ## What changed - Share cross-process writer locks between the rollout compressor and local thread store. Skip busy threads and recheck file state under coordination before publishing compressed files. - Retain writer ownership through recorder background I/O and compressed rollout materialization. Wait for writers to exit on shutdown or discard, and stop queued file work before deletion. - Route rollout metadata patches through owned recorders, reject competing writers before updating SQLite, and serialize metadata appends with shutdown. ## Testing Add regression coverage for compression with live and detached writers, parallel cold-file compression, publication locking, metadata ownership conflicts, updates to compressed rollouts, concurrent shutdown, and discarding deferred items. GitOrigin-RevId: c52f4037eb2019e8095b554e29bf531af95aa4bd
This commit is contained in:
1 parent
20f109eadb
commit
73a1148c9c
20 files changed
+671
-177
No files matched your search
@@ -166,7 +166,7 @@ fn referenced_thread_error(thread_id: codex_protocol::ThreadId) -> ThreadStoreEr
|
||||
async fn delete_thread_after_reference_check(
|
||||
store: &LocalThreadStore,
|
||||
mut thread_rollouts: ThreadRollouts,
|
||||
writer_guards: &mut Vec<super::writer_lock::WriterLockGuard>,
|
||||
writer_guards: &mut Vec<super::WriterLockGuard>,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let thread_id = thread_rollouts.thread_id;
|
||||
let thread_id_str = thread_id.to_string();
|
||||
@@ -206,9 +206,17 @@ async fn delete_thread_after_reference_check(
|
||||
super::thread_history::delete_thread(store, rollout_id).await?;
|
||||
}
|
||||
|
||||
// Drop the recorder before removing files, but retain its writer lock until cleanup finishes.
|
||||
if let Some(entry) = store.live_recorders.lock().await.remove(&thread_id) {
|
||||
// Stop queued file work before removing files, retaining ownership until cleanup finishes.
|
||||
let live_entry = store.live_recorders.lock().await.remove(&thread_id);
|
||||
if let Some(entry) = live_entry {
|
||||
writer_guards.push(entry.writer_lock);
|
||||
entry
|
||||
.recorder
|
||||
.discard()
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to stop thread writer before deletion: {err}"),
|
||||
})?;
|
||||
}
|
||||
let found_rollout_path = !thread_rollouts.paths.is_empty();
|
||||
for rollout_path in thread_rollouts.paths {
|
||||
@@ -546,8 +554,7 @@ mod tests {
|
||||
)
|
||||
.expect("child session file");
|
||||
let _owner_guard = owner
|
||||
.writer_lock_coordinator
|
||||
.acquire(child_thread_id)
|
||||
.acquire_writer_lock(child_thread_id)
|
||||
.expect("acquire child writer lock");
|
||||
|
||||
let error = store
|
||||
@@ -570,8 +577,7 @@ mod tests {
|
||||
let owner = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
let _owner_guard = owner
|
||||
.writer_lock_coordinator
|
||||
.acquire(thread_id)
|
||||
.acquire_writer_lock(thread_id)
|
||||
.expect("acquire writer lock before rollout exists");
|
||||
|
||||
let error = store
|
||||
|
||||
Reference in new issue
Block a user