Enforce writer ownership for thread archive and deletion (#35031)

## Why

Paginated threads allow only one app-server process to write at a time. Archive and delete operations must not mutate a thread or its spawned descendants while another process owns any of them.

## What changed

- Acquire paginated writer locks for the full archive or deletion set before changing rollout files, including descendants whose rollout has not materialized yet.
- Add a batch archive store operation so ownership conflicts are detected before any thread in the subtree is archived.
- Return ownership conflicts as JSON-RPC invalid-request errors and document the behavior.

## Testing

- Cover archive and deletion conflicts for owned threads and descendants, including unmaterialized rollouts.
- Verify deletion still removes rollouts with unreadable metadata.

GitOrigin-RevId: f62a3e0c1ee2631cea48dd9145b588a1486e2425
This commit is contained in:
Owen Lin authored and copyberry committed 2026-07-23 23:18:06 +00:00
1 parent 9fc4e5a7aa
commit 963316583b
12 files changed
+437 -49

No files matched your search

@@ -2,10 +2,58 @@ use super::LocalThreadStore;
use super::helpers::matching_rollout_file_name;
use super::helpers::scoped_rollout_path;
use crate::ArchiveThreadParams;
use crate::ArchiveThreadsParams;
use crate::ThreadStoreError;
use crate::ThreadStoreResult;
use chrono::Utc;
use codex_rollout::find_thread_path_by_id_str;
use tracing::warn;
pub(super) async fn archive_threads(
store: &LocalThreadStore,
params: ArchiveThreadsParams,
) -> ThreadStoreResult<Vec<codex_protocol::ThreadId>> {
let thread_ids = params.thread_ids;
if thread_ids.is_empty() {
return Ok(Vec::new());
}
let mut lock_thread_ids = params.writer_lock_thread_ids;
lock_thread_ids.extend(thread_ids.iter().copied());
lock_thread_ids.sort_unstable_by_key(ToString::to_string);
lock_thread_ids.dedup();
let mut _live_writer_guards = Vec::with_capacity(lock_thread_ids.len());
for thread_id in &lock_thread_ids {
_live_writer_guards.push(store.live_writer_locks.lock(*thread_id).await);
if store
.live_recorders
.lock()
.await
.get(thread_id)
.is_some_and(|entry| entry.writer_lock.is_some())
{
return Err(ThreadStoreError::Conflict {
message: format!("thread {thread_id} already has an active writer"),
});
}
}
let _writer_guards = store
.acquire_paginated_writer_locks(&lock_thread_ids)
.await?;
let parent_thread_id = thread_ids[0];
let mut archived_thread_ids = Vec::new();
for thread_id in thread_ids {
match archive_thread(store, ArchiveThreadParams { thread_id }).await {
Ok(()) => archived_thread_ids.push(thread_id),
Err(err) if archived_thread_ids.is_empty() => return Err(err),
Err(err) => warn!(
"failed to archive spawned descendant thread {thread_id} while archiving {parent_thread_id}: {err}"
),
}
}
Ok(archived_thread_ids)
}
pub(super) async fn archive_thread(
store: &LocalThreadStore,
@@ -64,6 +112,7 @@ mod tests {
use chrono::Utc;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::ThreadHistoryMode;
use codex_rollout::ARCHIVED_SESSIONS_SUBDIR;
use codex_utils_absolute_path::test_support::PathExt;
use pretty_assertions::assert_eq;
@@ -77,6 +126,50 @@ mod tests {
use crate::local::LocalThreadStore;
use crate::local::test_support::test_config;
use crate::local::test_support::write_session_file;
use crate::local::test_support::write_session_file_with_history_mode;
#[tokio::test]
async fn archive_threads_rejects_owned_descendants_before_archiving_anything() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let owner = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let parent_uuid = Uuid::from_u128(203);
let parent_thread_id =
ThreadId::from_string(&parent_uuid.to_string()).expect("valid parent thread id");
let parent_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-00",
parent_uuid,
ThreadHistoryMode::Paginated,
)
.expect("parent session file");
let child_uuid = Uuid::from_u128(204);
let child_thread_id =
ThreadId::from_string(&child_uuid.to_string()).expect("valid child thread id");
let child_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-01",
child_uuid,
ThreadHistoryMode::Paginated,
)
.expect("child session file");
let _owner_guard = owner
.writer_lock_coordinator
.acquire(child_thread_id)
.expect("acquire child writer lock");
let error = store
.archive_threads(ArchiveThreadsParams {
thread_ids: vec![parent_thread_id, child_thread_id],
writer_lock_thread_ids: Vec::new(),
})
.await
.expect_err("owned descendant should block archive");
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
assert!(parent_path.exists());
assert!(child_path.exists());
}
#[tokio::test]
async fn archive_thread_moves_rollout_to_archived_collection() {
@@ -34,7 +34,8 @@ pub(super) async fn delete_thread(
if reference_index.reference_count(thread_id) > 0 {
return Err(referenced_thread_error(thread_id));
}
delete_thread_after_reference_check(store, thread_id).await
let mut writer_guards = store.acquire_paginated_writer_locks(&[thread_id]).await?;
delete_thread_after_reference_check(store, thread_id, &mut writer_guards).await
}
pub(super) async fn delete_threads(
@@ -50,7 +51,7 @@ pub(super) async fn delete_threads(
let mut lock_thread_ids: Vec<_> = deletion_set.iter().copied().collect();
lock_thread_ids.sort_unstable_by_key(ToString::to_string);
let mut _live_writer_guards = Vec::with_capacity(lock_thread_ids.len());
for thread_id in lock_thread_ids {
for &thread_id in &lock_thread_ids {
_live_writer_guards.push(store.live_writer_locks.lock(thread_id).await);
}
@@ -78,8 +79,11 @@ pub(super) async fn delete_threads(
}
}
let mut writer_guards = store
.acquire_paginated_writer_locks(&lock_thread_ids)
.await?;
for thread_id in thread_ids {
match delete_thread_after_reference_check(store, thread_id).await {
match delete_thread_after_reference_check(store, thread_id, &mut writer_guards).await {
Ok(()) | Err(ThreadStoreError::ThreadNotFound { .. }) => {}
Err(err) => return Err(err),
}
@@ -106,6 +110,7 @@ fn referenced_thread_error(thread_id: codex_protocol::ThreadId) -> ThreadStoreEr
async fn delete_thread_after_reference_check(
store: &LocalThreadStore,
thread_id: codex_protocol::ThreadId,
writer_guards: &mut Vec<super::writer_lock::WriterLockGuard>,
) -> ThreadStoreResult<()> {
let thread_id_str = thread_id.to_string();
let state_db_ctx = store.state_db().await;
@@ -145,12 +150,15 @@ async fn delete_thread_after_reference_check(
}
}
// Drop the recorder before removing files, but retain its writer lock until cleanup finishes.
let _writer_lock = store
if let Some(writer_lock) = store
.live_recorders
.lock()
.await
.remove(&thread_id)
.and_then(|entry| entry._writer_lock);
.and_then(|entry| entry.writer_lock)
{
writer_guards.push(writer_lock);
}
let found_rollout_path = !rollout_paths.is_empty();
for rollout_path in rollout_paths {
delete_rollout_file(store, rollout_path.as_path(), thread_id)?;
@@ -395,6 +403,89 @@ mod tests {
assert!(!child_path.exists());
}
#[tokio::test]
async fn delete_threads_rejects_owned_descendants_before_deleting_anything() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let owner = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let parent_uuid = Uuid::from_u128(309);
let parent_thread_id =
ThreadId::from_string(&parent_uuid.to_string()).expect("valid parent thread id");
let parent_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-00",
parent_uuid,
ThreadHistoryMode::Paginated,
)
.expect("parent session file");
let child_uuid = Uuid::from_u128(310);
let child_thread_id =
ThreadId::from_string(&child_uuid.to_string()).expect("valid child thread id");
let child_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-01",
child_uuid,
ThreadHistoryMode::Paginated,
)
.expect("child session file");
let _owner_guard = owner
.writer_lock_coordinator
.acquire(child_thread_id)
.expect("acquire child writer lock");
let error = store
.delete_threads(DeleteThreadsParams {
thread_ids: vec![parent_thread_id, child_thread_id],
})
.await
.expect_err("owned descendant should block deletion");
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
assert!(parent_path.exists());
assert!(child_path.exists());
}
#[tokio::test]
async fn delete_threads_rejects_owned_thread_before_rollout_materializes() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let owner = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let thread_id = ThreadId::default();
let _owner_guard = owner
.writer_lock_coordinator
.acquire(thread_id)
.expect("acquire writer lock before rollout exists");
let error = store
.delete_threads(DeleteThreadsParams {
thread_ids: vec![thread_id],
})
.await
.expect_err("owned thread should block deletion before rollout exists");
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
}
#[tokio::test]
async fn delete_threads_removes_rollout_with_unreadable_metadata() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let uuid = Uuid::from_u128(311);
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
let rollout_path =
write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file");
std::fs::write(&rollout_path, "{not json}\n").expect("damage rollout metadata");
store
.delete_threads(DeleteThreadsParams {
thread_ids: vec![thread_id],
})
.await
.expect("delete rollout with unreadable metadata");
assert!(!rollout_path.exists());
}
#[tokio::test]
async fn delete_rollout_file_treats_vanished_path_as_already_deleted() {
let home = TempDir::new().expect("temp dir");
+57 -3
View File
@@ -34,6 +34,7 @@ use tokio::sync::OwnedMutexGuard;
use crate::AppendThreadItemsParams;
use crate::ArchiveThreadParams;
use crate::ArchiveThreadsParams;
use crate::CreateThreadParams;
use crate::DeleteThreadParams;
use crate::DeleteThreadsParams;
@@ -91,7 +92,7 @@ struct LiveRecorderEntry {
// canonical SessionMeta is durable. Retain the mode captured when live persistence was opened
// so missing SQLite rows can still be seeded.
history_mode: ThreadHistoryMode,
_writer_lock: Option<WriterLockGuard>,
writer_lock: Option<WriterLockGuard>,
}
#[derive(Default)]
@@ -207,6 +208,42 @@ impl LocalThreadStore {
Ok(())
}
async fn acquire_paginated_writer_locks(
&self,
thread_ids: &[ThreadId],
) -> ThreadStoreResult<Vec<WriterLockGuard>> {
let mut writer_locks = Vec::new();
for &thread_id in thread_ids {
if self
.live_recorders
.lock()
.await
.get(&thread_id)
.is_some_and(|entry| entry.writer_lock.is_some())
{
continue;
}
// Only a readable legacy header proves no paginated writer can own this id. Missing
// lazy rollouts and damaged headers must conservatively try the lock.
let history_mode = match read_thread::resolve_rollout_path(
self, thread_id, /*include_archived*/ true,
)
.await?
{
Some(rollout_path) => codex_rollout::read_session_meta_line(rollout_path.as_path())
.await
.ok()
.map(|meta_line| meta_line.meta.history_mode),
None => None,
};
if !matches!(history_mode, Some(ThreadHistoryMode::Legacy)) {
writer_locks.push(self.writer_lock_coordinator.acquire(thread_id)?);
}
}
Ok(writer_locks)
}
async fn insert_live_recorder(
&self,
thread_id: ThreadId,
@@ -222,7 +259,7 @@ impl LocalThreadStore {
entry.insert(LiveRecorderEntry {
recorder,
history_mode,
_writer_lock: writer_lock,
writer_lock,
});
Ok(())
}
@@ -402,7 +439,24 @@ impl ThreadStore for LocalThreadStore {
}
fn archive_thread(&self, params: ArchiveThreadParams) -> ThreadStoreFuture<'_, ()> {
Box::pin(async move { archive_thread::archive_thread(self, params).await })
Box::pin(async move {
archive_thread::archive_threads(
self,
ArchiveThreadsParams {
thread_ids: vec![params.thread_id],
writer_lock_thread_ids: Vec::new(),
},
)
.await
.map(|_| ())
})
}
fn archive_threads(
&self,
params: ArchiveThreadsParams,
) -> ThreadStoreFuture<'_, Vec<ThreadId>> {
Box::pin(async move { archive_thread::archive_threads(self, params).await })
}
fn unarchive_thread(&self, params: ArchiveThreadParams) -> ThreadStoreFuture<'_, StoredThread> {