mirror of
https://github.com/openai/codex.git
synced 2026-11-17 14:58:36 +00:00
Unify thread metadata updates above store (#22236)
- make ThreadStore::update_thread_metadata accept a broad range of metadata patches - keep ThreadStore::append_items as raw canonical history append (no metadata side effects) - in the local store, write these metadata updates to a combination of sqlite and rollout jsonl files for backwards-compat. It special cases which fields need to go into jsonl vs sqlite vs whatever, confining the awkwardness to just this implementation - in remote stores we can simply persist the metadata directly to a database, no special casing required. - move the "implicit metadata updates triggered by appending rollout items" from the RolloutRecorder (which is local-threadstore-specific) to the LiveThread layer above the ThreadStore, inside of a private helper utility called ThreadMetadataSync. LiveThread calls ThreadStore append_items and update_metadata separately. - Add a generic update metadata method to ThreadManager that works on both live threads and "cold" threads - Call that ThreadManager method from app server code, so app server doesn't need to worry about whether the thread is live or not
This commit is contained in:
31 files changed
+2382
-762
No files matched your search
@@ -372,10 +372,7 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result<
|
||||
/*thread_source*/ None,
|
||||
BaseInstructions::default(),
|
||||
Vec::new(),
|
||||
EventPersistenceMode::Limited,
|
||||
),
|
||||
/*state_db_ctx*/ None,
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -386,7 +383,7 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result<
|
||||
);
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
.record_canonical_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "buffered-event".to_string(),
|
||||
phase: None,
|
||||
@@ -401,7 +398,7 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result<
|
||||
);
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
.record_canonical_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
message: "first-user-message".to_string(),
|
||||
images: None,
|
||||
@@ -453,16 +450,13 @@ async fn persist_reports_filesystem_error_and_retries_buffered_items() -> std::i
|
||||
/*thread_source*/ None,
|
||||
BaseInstructions::default(),
|
||||
Vec::new(),
|
||||
EventPersistenceMode::Limited,
|
||||
),
|
||||
/*state_db_ctx*/ None,
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await?;
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
.record_canonical_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "buffered-before-persist".to_string(),
|
||||
phase: None,
|
||||
@@ -498,7 +492,6 @@ async fn persist_reports_filesystem_error_and_retries_buffered_items() -> std::i
|
||||
#[tokio::test]
|
||||
async fn writer_state_retries_write_error_before_reporting_flush_success() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let rollout_path = home.path().join("rollout.jsonl");
|
||||
File::create(&rollout_path)?;
|
||||
let read_only_file = std::fs::OpenOptions::new().read(true).open(&rollout_path)?;
|
||||
@@ -508,10 +501,6 @@ async fn writer_state_retries_write_error_before_reporting_flush_success() -> st
|
||||
/*meta*/ None,
|
||||
home.path().to_path_buf(),
|
||||
rollout_path.clone(),
|
||||
/*state_db_ctx*/ None,
|
||||
/*state_builder*/ None,
|
||||
config.model_provider_id.clone(),
|
||||
config.generate_memories,
|
||||
);
|
||||
state.add_items(vec![RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
@@ -530,216 +519,6 @@ async fn writer_state_retries_write_error_before_reporting_flush_success() -> st
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_irrelevant_events_coalesce_state_db_updated_at() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
|
||||
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
state_db
|
||||
.mark_backfill_complete(/*last_watermark*/ None)
|
||||
.await
|
||||
.expect("backfill should be complete");
|
||||
|
||||
let thread_id = ThreadId::new();
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::new(
|
||||
thread_id,
|
||||
/*forked_from_id*/ None,
|
||||
SessionSource::Cli,
|
||||
/*thread_source*/ None,
|
||||
BaseInstructions::default(),
|
||||
Vec::new(),
|
||||
EventPersistenceMode::Limited,
|
||||
),
|
||||
Some(state_db.clone()),
|
||||
/*state_builder*/ None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
message: "first-user-message".to_string(),
|
||||
images: None,
|
||||
local_images: Vec::new(),
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
))])
|
||||
.await?;
|
||||
recorder.persist().await?;
|
||||
recorder.flush().await?;
|
||||
let initial_thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load")
|
||||
.expect("thread should exist");
|
||||
let initial_updated_at = initial_thread.updated_at;
|
||||
let initial_title = initial_thread.title.clone();
|
||||
let initial_first_user_message = initial_thread.first_user_message.clone();
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "assistant text".to_string(),
|
||||
phase: None,
|
||||
memory_citation: None,
|
||||
},
|
||||
))])
|
||||
.await?;
|
||||
recorder.flush().await?;
|
||||
|
||||
let updated_thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after agent message")
|
||||
.expect("thread should still exist");
|
||||
|
||||
assert_eq!(updated_thread.updated_at, initial_updated_at);
|
||||
assert_eq!(updated_thread.title, initial_title);
|
||||
assert_eq!(
|
||||
updated_thread.first_user_message,
|
||||
initial_first_user_message
|
||||
);
|
||||
|
||||
tokio::time::sleep(THREAD_UPDATED_AT_TOUCH_INTERVAL + Duration::from_millis(10)).await;
|
||||
|
||||
recorder
|
||||
.record_items(&[RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "more assistant text".to_string(),
|
||||
phase: None,
|
||||
memory_citation: None,
|
||||
},
|
||||
))])
|
||||
.await?;
|
||||
recorder.flush().await?;
|
||||
|
||||
let refreshed_thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after refresh")
|
||||
.expect("thread should still exist");
|
||||
assert!(refreshed_thread.updated_at > initial_updated_at);
|
||||
assert_eq!(refreshed_thread.title, initial_title);
|
||||
assert_eq!(
|
||||
refreshed_thread.first_user_message,
|
||||
initial_first_user_message
|
||||
);
|
||||
|
||||
recorder.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_flushes_pending_metadata_irrelevant_updated_at() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
|
||||
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
state_db
|
||||
.mark_backfill_complete(/*last_watermark*/ None)
|
||||
.await
|
||||
.expect("backfill should be complete");
|
||||
|
||||
let thread_id = ThreadId::new();
|
||||
let rollout_path = home.path().join("rollout.jsonl");
|
||||
let initial_updated_at = Utc.with_ymd_and_hms(2026, 5, 7, 7, 37, 8).unwrap();
|
||||
let builder = ThreadMetadataBuilder::new(
|
||||
thread_id,
|
||||
rollout_path.clone(),
|
||||
initial_updated_at,
|
||||
SessionSource::Cli,
|
||||
);
|
||||
state_db
|
||||
.upsert_thread(&builder.build(config.model_provider_id.as_str()))
|
||||
.await
|
||||
.expect("thread should be inserted");
|
||||
|
||||
File::create(&rollout_path)?;
|
||||
let rollout_file = std::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&rollout_path)?;
|
||||
let mut state = RolloutWriterState::new(
|
||||
Some(tokio::fs::File::from_std(rollout_file)),
|
||||
/*deferred_log_file_info*/ None,
|
||||
/*meta*/ None,
|
||||
home.path().to_path_buf(),
|
||||
rollout_path,
|
||||
Some(state_db.clone()),
|
||||
Some(builder),
|
||||
config.model_provider_id.clone(),
|
||||
config.generate_memories,
|
||||
);
|
||||
let pending_updated_at = initial_updated_at + chrono::Duration::seconds(1);
|
||||
state.thread_updated_at_touch.pending_touch = Some((thread_id, pending_updated_at));
|
||||
|
||||
state.shutdown().await?;
|
||||
|
||||
assert_eq!(
|
||||
state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after shutdown")
|
||||
.expect("thread should still exist")
|
||||
.updated_at,
|
||||
pending_updated_at
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_irrelevant_events_fall_back_to_upsert_when_thread_missing() -> std::io::Result<()>
|
||||
{
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
|
||||
let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone())
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
let thread_id = ThreadId::new();
|
||||
let rollout_path = home.path().join("rollout.jsonl");
|
||||
let builder = ThreadMetadataBuilder::new(
|
||||
thread_id,
|
||||
rollout_path.clone(),
|
||||
Utc::now(),
|
||||
SessionSource::Cli,
|
||||
);
|
||||
let items = vec![RolloutItem::EventMsg(EventMsg::AgentMessage(
|
||||
AgentMessageEvent {
|
||||
message: "assistant text".to_string(),
|
||||
phase: None,
|
||||
memory_citation: None,
|
||||
},
|
||||
))];
|
||||
|
||||
let mut thread_updated_at_touch = ThreadUpdatedAtTouch::default();
|
||||
sync_thread_state_after_write(
|
||||
Some(state_db.as_ref()),
|
||||
rollout_path.as_path(),
|
||||
Some(&builder),
|
||||
items.as_slice(),
|
||||
config.model_provider_id.as_str(),
|
||||
/*new_thread_memory_mode*/ None,
|
||||
&mut thread_updated_at_touch,
|
||||
)
|
||||
.await;
|
||||
|
||||
let thread = state_db
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load after fallback")
|
||||
.expect("thread should be inserted after fallback");
|
||||
assert_eq!(thread.id, thread_id);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
|
||||
Reference in new issue
Block a user