Enforce single-writer ownership for all thread histories (#36389)

## Why

Legacy thread histories did not use the cross-process writer ownership guard already applied to paginated histories.

## What changed

- Acquire and retain a writer lock whenever a legacy or paginated thread is created or resumed.
- Apply the same ownership checks when archiving or deleting either history mode.
- Release ownership when initialization fails or the active thread shuts down, allowing another process to resume it.

## Testing

- Cover competing create and resume operations for both history modes.
- Cover archive and deletion conflicts for legacy and paginated descendants.

GitOrigin-RevId: 1be79b01966ad2634ed79eb96c0aa0db8eafe2fa
This commit is contained in:
Owen Lin authored and copyberry committed 2026-07-31 20:49:08 +00:00
1 parent 1bef168976
commit dc60dadce6
16 files changed
+240 -366

No files matched your search

@@ -29,21 +29,13 @@ pub(super) async fn archive_threads(
let mut _live_writer_guards = Vec::with_capacity(lock_thread_ids.len());
for thread_id in &lock_thread_ids {
_live_writer_guards.push(store.live_writer_locks.lock(*thread_id).await);
if store
.live_recorders
.lock()
.await
.get(thread_id)
.is_some_and(|entry| entry.writer_lock.is_some())
{
if store.live_recorders.lock().await.contains_key(thread_id) {
return Err(ThreadStoreError::Conflict {
message: format!("thread {thread_id} already has an active writer"),
});
}
}
let _writer_guards = store
.acquire_paginated_writer_locks(&lock_thread_ids)
.await?;
let _writer_guards = store.acquire_writer_locks(&lock_thread_ids).await?;
let parent_thread_id = thread_ids[0];
let mut archived_thread_ids = Vec::new();
@@ -173,42 +165,53 @@ mod tests {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let owner = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let parent_uuid = Uuid::from_u128(203);
let parent_thread_id =
ThreadId::from_string(&parent_uuid.to_string()).expect("valid parent thread id");
let parent_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-00",
parent_uuid,
ThreadHistoryMode::Paginated,
)
.expect("parent session file");
let child_uuid = Uuid::from_u128(204);
let child_thread_id =
ThreadId::from_string(&child_uuid.to_string()).expect("valid child thread id");
let child_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-01",
child_uuid,
ThreadHistoryMode::Paginated,
)
.expect("child session file");
let _owner_guard = owner
.writer_lock_coordinator
.acquire(child_thread_id)
.expect("acquire child writer lock");
for (parent_uuid, child_uuid, history_mode) in [
(
Uuid::from_u128(203),
Uuid::from_u128(204),
ThreadHistoryMode::Legacy,
),
(
Uuid::from_u128(206),
Uuid::from_u128(207),
ThreadHistoryMode::Paginated,
),
] {
let parent_thread_id =
ThreadId::from_string(&parent_uuid.to_string()).expect("valid parent thread id");
let parent_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-00",
parent_uuid,
history_mode,
)
.expect("parent session file");
let child_thread_id =
ThreadId::from_string(&child_uuid.to_string()).expect("valid child thread id");
let child_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-01",
child_uuid,
history_mode,
)
.expect("child session file");
let _owner_guard = owner
.writer_lock_coordinator
.acquire(child_thread_id)
.expect("acquire child writer lock");
let error = store
.archive_threads(ArchiveThreadsParams {
thread_ids: vec![parent_thread_id, child_thread_id],
writer_lock_thread_ids: Vec::new(),
})
.await
.expect_err("owned descendant should block archive");
let error = store
.archive_threads(ArchiveThreadsParams {
thread_ids: vec![parent_thread_id, child_thread_id],
writer_lock_thread_ids: Vec::new(),
})
.await
.expect_err("owned descendant should block archive");
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
assert!(parent_path.exists());
assert!(child_path.exists());
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
assert!(parent_path.exists());
assert!(child_path.exists());
}
}
#[tokio::test]
@@ -35,7 +35,7 @@ pub(super) async fn delete_thread(
if reference_index.reference_count(thread_id) > 0 {
return Err(referenced_thread_error(thread_id));
}
let mut writer_guards = store.acquire_paginated_writer_locks(&[thread_id]).await?;
let mut writer_guards = store.acquire_writer_locks(&[thread_id]).await?;
delete_thread_after_reference_check(store, thread_id, &mut writer_guards).await
}
@@ -84,9 +84,7 @@ pub(super) async fn delete_threads(
}
}
let mut writer_guards = store
.acquire_paginated_writer_locks(&lock_thread_ids)
.await?;
let mut writer_guards = store.acquire_writer_locks(&lock_thread_ids).await?;
for thread_id in thread_ids {
match delete_thread_after_reference_check(store, thread_id, &mut writer_guards).await {
Ok(()) | Err(ThreadStoreError::ThreadNotFound { .. }) => {}
@@ -157,14 +155,8 @@ async fn delete_thread_after_reference_check(
super::thread_history::delete_thread(store, thread_id).await?;
// Drop the recorder before removing files, but retain its writer lock until cleanup finishes.
if let Some(writer_lock) = store
.live_recorders
.lock()
.await
.remove(&thread_id)
.and_then(|entry| entry.writer_lock)
{
writer_guards.push(writer_lock);
if let Some(entry) = store.live_recorders.lock().await.remove(&thread_id) {
writer_guards.push(entry.writer_lock);
}
let found_rollout_path = !rollout_paths.is_empty();
for rollout_path in rollout_paths {
@@ -412,41 +404,52 @@ mod tests {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let owner = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let parent_uuid = Uuid::from_u128(309);
let parent_thread_id =
ThreadId::from_string(&parent_uuid.to_string()).expect("valid parent thread id");
let parent_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-00",
parent_uuid,
ThreadHistoryMode::Paginated,
)
.expect("parent session file");
let child_uuid = Uuid::from_u128(310);
let child_thread_id =
ThreadId::from_string(&child_uuid.to_string()).expect("valid child thread id");
let child_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-01",
child_uuid,
ThreadHistoryMode::Paginated,
)
.expect("child session file");
let _owner_guard = owner
.writer_lock_coordinator
.acquire(child_thread_id)
.expect("acquire child writer lock");
for (parent_uuid, child_uuid, history_mode) in [
(
Uuid::from_u128(309),
Uuid::from_u128(310),
ThreadHistoryMode::Legacy,
),
(
Uuid::from_u128(312),
Uuid::from_u128(313),
ThreadHistoryMode::Paginated,
),
] {
let parent_thread_id =
ThreadId::from_string(&parent_uuid.to_string()).expect("valid parent thread id");
let parent_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-00",
parent_uuid,
history_mode,
)
.expect("parent session file");
let child_thread_id =
ThreadId::from_string(&child_uuid.to_string()).expect("valid child thread id");
let child_path = write_session_file_with_history_mode(
home.path(),
"2025-01-03T12-00-01",
child_uuid,
history_mode,
)
.expect("child session file");
let _owner_guard = owner
.writer_lock_coordinator
.acquire(child_thread_id)
.expect("acquire child writer lock");
let error = store
.delete_threads(DeleteThreadsParams {
thread_ids: vec![parent_thread_id, child_thread_id],
})
.await
.expect_err("owned descendant should block deletion");
let error = store
.delete_threads(DeleteThreadsParams {
thread_ids: vec![parent_thread_id, child_thread_id],
})
.await
.expect_err("owned descendant should block deletion");
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
assert!(parent_path.exists());
assert!(child_path.exists());
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
assert!(parent_path.exists());
assert!(child_path.exists());
}
}
#[tokio::test]
+2 -10
View File
@@ -30,11 +30,7 @@ pub(super) async fn create_thread(
let _live_writer_guard = store.live_writer_locks.lock(thread_id).await;
let history_mode = params.history_mode;
store.ensure_live_recorder_absent(thread_id).await?;
let writer_lock = if matches!(history_mode, ThreadHistoryMode::Paginated) {
Some(store.writer_lock_coordinator.acquire(thread_id)?)
} else {
None
};
let writer_lock = store.writer_lock_coordinator.acquire(thread_id)?;
let recorder = create_thread::create_thread(store, params).await?;
store
.insert_live_recorder(thread_id, recorder, history_mode, writer_lock)
@@ -47,6 +43,7 @@ pub(super) async fn resume_thread(
) -> ThreadStoreResult<()> {
let _live_writer_guard = store.live_writer_locks.lock(params.thread_id).await;
store.ensure_live_recorder_absent(params.thread_id).await?;
let writer_lock = store.writer_lock_coordinator.acquire(params.thread_id)?;
let history_mode = if let Some(history) = params.history.as_deref() {
canonical_history_mode_from_rollout_items(history)
} else if let Some(rollout_path) = params.rollout_path.as_ref() {
@@ -103,11 +100,6 @@ pub(super) async fn resume_thread(
model_provider_id: params.metadata.model_provider.clone(),
generate_memories: matches!(params.metadata.memory_mode, ThreadMemoryMode::Enabled),
};
let writer_lock = if matches!(history_mode, ThreadHistoryMode::Paginated) {
Some(store.writer_lock_coordinator.acquire(params.thread_id)?)
} else {
None
};
let recorder = RolloutRecorder::new(&config, RolloutRecorderParams::resume(rollout_path))
.await
.map_err(|err| ThreadStoreError::Internal {
+88 -38
View File
@@ -107,7 +107,7 @@ struct LiveRecorderEntry {
// canonical SessionMeta is durable. Retain the mode captured when live persistence was opened
// so missing SQLite rows can still be seeded.
history_mode: ThreadHistoryMode,
writer_lock: Option<WriterLockGuard>,
writer_lock: WriterLockGuard,
}
#[derive(Default)]
@@ -264,38 +264,16 @@ impl LocalThreadStore {
Ok(())
}
async fn acquire_paginated_writer_locks(
async fn acquire_writer_locks(
&self,
thread_ids: &[ThreadId],
) -> ThreadStoreResult<Vec<WriterLockGuard>> {
let mut writer_locks = Vec::new();
let mut writer_locks = Vec::with_capacity(thread_ids.len());
for &thread_id in thread_ids {
if self
.live_recorders
.lock()
.await
.get(&thread_id)
.is_some_and(|entry| entry.writer_lock.is_some())
{
if self.live_recorders.lock().await.contains_key(&thread_id) {
continue;
}
// Only a readable legacy header proves no paginated writer can own this id. Missing
// lazy rollouts and damaged headers must conservatively try the lock.
let history_mode = match read_thread::resolve_rollout_path(
self, thread_id, /*include_archived*/ true,
)
.await?
{
Some(rollout_path) => codex_rollout::read_session_meta_line(rollout_path.as_path())
.await
.ok()
.map(|meta_line| meta_line.meta.history_mode),
None => None,
};
if !matches!(history_mode, Some(ThreadHistoryMode::Legacy)) {
writer_locks.push(self.writer_lock_coordinator.acquire(thread_id)?);
}
writer_locks.push(self.writer_lock_coordinator.acquire(thread_id)?);
}
Ok(writer_locks)
}
@@ -305,7 +283,7 @@ impl LocalThreadStore {
thread_id: ThreadId,
recorder: RolloutRecorder,
history_mode: ThreadHistoryMode,
writer_lock: Option<WriterLockGuard>,
writer_lock: WriterLockGuard,
) -> ThreadStoreResult<()> {
match self.live_recorders.lock().await.entry(thread_id) {
Entry::Occupied(entry) => Err(ThreadStoreError::InvalidRequest {
@@ -1200,6 +1178,8 @@ mod tests {
async fn create_thread_rejects_missing_cwd() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let competing_store =
LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
for history_mode in [ThreadHistoryMode::Legacy, ThreadHistoryMode::Paginated] {
let thread_id = ThreadId::default();
let mut params = create_thread_params(thread_id);
@@ -1219,10 +1199,10 @@ mod tests {
let mut valid_params = create_thread_params(thread_id);
valid_params.history_mode = history_mode;
store
competing_store
.create_thread(valid_params)
.await
.expect("failed initialization should release writer ownership");
.expect("failed initialization should release cross-process writer ownership");
}
}
@@ -1249,10 +1229,7 @@ mod tests {
.path()
.join("thread-writer-locks")
.join(format!("{thread_id}.lock"));
assert_eq!(
lock_path.exists(),
matches!(history_mode, ThreadHistoryMode::Paginated)
);
assert!(lock_path.exists());
store
.discard_thread(thread_id)
.await
@@ -1335,6 +1312,61 @@ mod tests {
assert_rollout_contains_message(rollout_path.as_path(), "after resume").await;
}
#[tokio::test]
async fn live_writers_reject_cross_process_create_and_resume() {
let home = TempDir::new().expect("temp dir");
let config = test_config(home.path());
let primary = LocalThreadStore::new(config.clone(), /*state_db*/ None);
let secondary = LocalThreadStore::new(config, /*state_db*/ None);
for history_mode in [ThreadHistoryMode::Legacy, ThreadHistoryMode::Paginated] {
let thread_id = ThreadId::default();
let mut create_params = create_thread_params(thread_id);
create_params.history_mode = history_mode;
primary
.create_thread(create_params.clone())
.await
.expect("create live thread");
primary
.persist_thread(thread_id)
.await
.expect("persist thread for resume");
let rollout_path = primary
.live_rollout_path(thread_id)
.await
.expect("load rollout path");
let resume_params = ResumeThreadParams {
thread_id,
rollout_path: Some(rollout_path),
history: None,
include_archived: true,
metadata: thread_metadata(),
};
let error = secondary
.create_thread(create_params)
.await
.expect_err("competing create should fail");
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
let error = secondary
.resume_thread(resume_params.clone())
.await
.expect_err("competing resume should fail");
assert!(matches!(error, ThreadStoreError::Conflict { .. }));
primary
.shutdown_thread(thread_id)
.await
.expect("shutdown should release writer ownership");
secondary
.resume_thread(resume_params)
.await
.expect("resume after shutdown should acquire writer ownership");
}
}
#[tokio::test]
async fn create_thread_rejects_duplicate_live_writer() {
let home = TempDir::new().expect("temp dir");
@@ -1387,14 +1419,21 @@ mod tests {
async fn resume_thread_rejects_missing_cwd() {
let home = TempDir::new().expect("temp dir");
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let uuid = uuid::Uuid::from_u128(407);
let competing_store =
LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
let uuid = uuid::Uuid::from_u128(408);
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
let rollout_path =
write_session_file(home.path(), "2025-01-04T11-30-00", uuid).expect("session file");
let rollout_path = write_session_file_with_history_mode(
home.path(),
"2025-01-04T11-30-00",
uuid,
ThreadHistoryMode::Paginated,
)
.expect("session file");
let err = store
.resume_thread(ResumeThreadParams {
thread_id,
rollout_path: Some(rollout_path),
rollout_path: Some(rollout_path.clone()),
history: None,
include_archived: true,
metadata: ThreadPersistenceMetadata {
@@ -1408,6 +1447,17 @@ mod tests {
assert!(matches!(err, ThreadStoreError::InvalidRequest { .. }));
assert!(err.to_string().contains("requires a cwd"));
competing_store
.resume_thread(ResumeThreadParams {
thread_id,
rollout_path: Some(rollout_path),
history: None,
include_archived: true,
metadata: thread_metadata(),
})
.await
.expect("failed initialization should release cross-process writer ownership");
}
#[tokio::test]