Unify thread metadata updates above store (#22236)

- make ThreadStore::update_thread_metadata accept a broad range of
metadata patches
- keep ThreadStore::append_items as raw canonical history append (no
metadata side effects)
- in the local store, write these metadata updates to a combination of
sqlite and rollout jsonl files for backwards-compat. It special cases
which fields need to go into jsonl vs sqlite vs whatever, confining the
awkwardness to just this implementation
- in remote stores we can simply persist the metadata directly to a
database, no special casing required.
- move the "implicit metadata updates triggered by appending rollout
items" from the RolloutRecorder (which is local-threadstore-specific) to
the LiveThread layer above the ThreadStore, inside of a private helper
utility called ThreadMetadataSync. LiveThread calls ThreadStore
append_items and update_metadata separately.
- Add a generic update metadata method to ThreadManager that works on
both live threads and "cold" threads
- Call that ThreadManager method from app server code, so app server
doesn't need to worry about whether the thread is live or not

codex-rs/thread-store/src/local/mod.rs

@@ -37,6 +37,18 @@ use crate::ThreadStoreResult;
 use crate::UpdateThreadMetadataParams;
 
 /// Local filesystem/SQLite-backed implementation of [`ThreadStore`].
+///
+/// Local storage has two compatibility surfaces. Rollout JSONL files are the
+/// durable replay format and remain readable without SQLite, including older
+/// files that encode metadata in `SessionMeta` items and name-index entries.
+/// The SQLite state DB, when available, is the queryable metadata index used by
+/// list/read paths for fast lookup.
+///
+/// Live appends still write canonical JSONL history, but append-derived
+/// metadata is observed above the store and applied through
+/// [`ThreadStore::update_thread_metadata`]. This implementation applies that
+/// patch literally to SQLite while keeping the JSONL/name-index compatibility
+/// behavior needed for SQLite-less reads, repair, and old local rollout files.
 #[derive(Clone)]
 pub struct LocalThreadStore {
     pub(super) config: LocalThreadStoreConfig,
@@ -270,6 +282,8 @@ impl ThreadStore for LocalThreadStore {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+
     use codex_protocol::ThreadId;
     use codex_protocol::models::BaseInstructions;
     use codex_protocol::protocol::EventMsg;
@@ -280,6 +294,7 @@ mod tests {
     use tempfile::TempDir;
 
     use super::*;
+    use crate::LiveThread;
     use crate::ThreadEventPersistenceMode;
     use crate::ThreadPersistenceMetadata;
     use crate::local::test_support::test_config;
@@ -335,6 +350,267 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn raw_append_items_does_not_update_sqlite_metadata() {
+        // This pins the ThreadStore contract: raw appends are history-only. Callers that need
+        // metadata updates must use LiveThread or call update_thread_metadata explicitly.
+        let home = TempDir::new().expect("temp dir");
+        let config = test_config(home.path());
+        let runtime = codex_state::StateRuntime::init(
+            config.sqlite_home.clone(),
+            config.default_model_provider_id.clone(),
+        )
+        .await
+        .expect("state db should initialize");
+        let store = LocalThreadStore::new(config, Some(runtime.clone()));
+        let thread_id = ThreadId::default();
+
+        store
+            .create_thread(create_thread_params(thread_id))
+            .await
+            .expect("create live thread");
+        store
+            .append_items(AppendThreadItemsParams {
+                thread_id,
+                items: vec![user_message_item("raw append")],
+            })
+            .await
+            .expect("append raw item");
+        store.flush_thread(thread_id).await.expect("flush thread");
+
+        assert_eq!(
+            runtime
+                .get_thread(thread_id)
+                .await
+                .expect("sqlite metadata read"),
+            None
+        );
+    }
+
+    #[tokio::test]
+    async fn live_thread_observes_appended_items_into_sqlite_metadata() {
+        let home = TempDir::new().expect("temp dir");
+        let config = test_config(home.path());
+        let runtime = codex_state::StateRuntime::init(
+            config.sqlite_home.clone(),
+            config.default_model_provider_id.clone(),
+        )
+        .await
+        .expect("state db should initialize");
+        let store = Arc::new(LocalThreadStore::new(config, Some(runtime.clone())));
+        let thread_id = ThreadId::default();
+        let live_thread = LiveThread::create(store.clone(), create_thread_params(thread_id))
+            .await
+            .expect("create live thread");
+
+        live_thread
+            .append_items(&[user_message_item("observed append")])
+            .await
+            .expect("append observed item");
+        live_thread.flush().await.expect("flush thread");
+
+        let metadata = runtime
+            .get_thread(thread_id)
+            .await
+            .expect("sqlite metadata read")
+            .expect("sqlite metadata");
+        assert_eq!(
+            metadata.first_user_message.as_deref(),
+            Some("observed append")
+        );
+        assert_eq!(metadata.preview.as_deref(), Some("observed append"));
+        assert_eq!(metadata.title, "observed append");
+    }
+
+    #[tokio::test]
+    async fn live_thread_shutdown_does_not_materialize_empty_thread_metadata() {
+        let home = TempDir::new().expect("temp dir");
+        let config = test_config(home.path());
+        let runtime = codex_state::StateRuntime::init(
+            config.sqlite_home.clone(),
+            config.default_model_provider_id.clone(),
+        )
+        .await
+        .expect("state db should initialize");
+        let store = Arc::new(LocalThreadStore::new(config, Some(runtime.clone())));
+        let thread_id = ThreadId::default();
+        let live_thread = LiveThread::create(store.clone(), create_thread_params(thread_id))
+            .await
+            .expect("create live thread");
+        let rollout_path = store
+            .live_rollout_path(thread_id)
+            .await
+            .expect("live rollout path");
+
+        live_thread.shutdown().await.expect("shutdown thread");
+
+        assert!(
+            !tokio::fs::try_exists(rollout_path.as_path())
+                .await
+                .expect("rollout path should be checkable")
+        );
+        assert_eq!(
+            runtime
+                .get_thread(thread_id)
+                .await
+                .expect("sqlite metadata read"),
+            None
+        );
+    }
+
+    #[tokio::test]
+    async fn live_thread_shutdown_with_buffered_items_materializes_before_metadata_read() {
+        let home = TempDir::new().expect("temp dir");
+        let config = test_config(home.path());
+        let runtime = codex_state::StateRuntime::init(
+            config.sqlite_home.clone(),
+            config.default_model_provider_id.clone(),
+        )
+        .await
+        .expect("state db should initialize");
+        let store = Arc::new(LocalThreadStore::new(config, Some(runtime.clone())));
+        let thread_id = ThreadId::default();
+        let live_thread = LiveThread::create(store.clone(), create_thread_params(thread_id))
+            .await
+            .expect("create live thread");
+        let rollout_path = store
+            .live_rollout_path(thread_id)
+            .await
+            .expect("live rollout path");
+
+        live_thread
+            .append_items(&[RolloutItem::EventMsg(EventMsg::TokenCount(
+                codex_protocol::protocol::TokenCountEvent {
+                    info: None,
+                    rate_limits: None,
+                },
+            ))])
+            .await
+            .expect("append metadata-only item");
+        live_thread.shutdown().await.expect("shutdown thread");
+
+        assert!(
+            tokio::fs::try_exists(rollout_path.as_path())
+                .await
+                .expect("rollout path should be checkable")
+        );
+        let metadata = runtime
+            .get_thread(thread_id)
+            .await
+            .expect("sqlite metadata read")
+            .expect("sqlite metadata");
+        assert_eq!(metadata.rollout_path, rollout_path);
+    }
+
+    #[tokio::test]
+    async fn live_thread_resume_loads_history_before_observing_metadata() {
+        let home = TempDir::new().expect("temp dir");
+        let config = test_config(home.path());
+        let runtime = codex_state::StateRuntime::init(
+            config.sqlite_home.clone(),
+            config.default_model_provider_id.clone(),
+        )
+        .await
+        .expect("state db should initialize");
+        let store = Arc::new(LocalThreadStore::new(config, Some(runtime.clone())));
+        let uuid = uuid::Uuid::from_u128(401);
+        let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
+        let rollout_path =
+            write_session_file(home.path(), "2025-01-03T17-00-00", uuid).expect("session file");
+        let live_thread = LiveThread::resume(
+            store,
+            ResumeThreadParams {
+                thread_id,
+                rollout_path: Some(rollout_path),
+                history: None,
+                include_archived: false,
+                metadata: ThreadPersistenceMetadata {
+                    cwd: Some(home.path().to_path_buf()),
+                    model_provider: "different-provider".to_string(),
+                    memory_mode: ThreadMemoryMode::Enabled,
+                },
+                event_persistence_mode: ThreadEventPersistenceMode::Limited,
+            },
+        )
+        .await
+        .expect("resume live thread");
+
+        live_thread
+            .append_items(&[user_message_item("new live append")])
+            .await
+            .expect("append after resume");
+
+        let metadata = runtime
+            .get_thread(thread_id)
+            .await
+            .expect("sqlite metadata read")
+            .expect("sqlite metadata");
+        assert_eq!(
+            metadata.created_at.to_rfc3339(),
+            "2025-01-03T17:00:00+00:00"
+        );
+        assert_eq!(metadata.model_provider, "test-provider");
+        assert_eq!(
+            metadata.first_user_message.as_deref(),
+            Some("Hello from user")
+        );
+    }
+
+    #[tokio::test]
+    async fn live_thread_resume_loads_history_from_explicit_external_rollout_path() {
+        let home = TempDir::new().expect("temp dir");
+        let external_home = TempDir::new().expect("external temp dir");
+        let config = test_config(home.path());
+        let runtime = codex_state::StateRuntime::init(
+            config.sqlite_home.clone(),
+            config.default_model_provider_id.clone(),
+        )
+        .await
+        .expect("state db should initialize");
+        let store = Arc::new(LocalThreadStore::new(config, Some(runtime.clone())));
+        let uuid = uuid::Uuid::from_u128(402);
+        let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
+        let rollout_path = write_session_file(external_home.path(), "2025-01-03T17-30-00", uuid)
+            .expect("external session file");
+        let live_thread = LiveThread::resume(
+            store,
+            ResumeThreadParams {
+                thread_id,
+                rollout_path: Some(rollout_path),
+                history: None,
+                include_archived: false,
+                metadata: ThreadPersistenceMetadata {
+                    cwd: Some(home.path().to_path_buf()),
+                    model_provider: "different-provider".to_string(),
+                    memory_mode: ThreadMemoryMode::Enabled,
+                },
+                event_persistence_mode: ThreadEventPersistenceMode::Limited,
+            },
+        )
+        .await
+        .expect("resume external live thread");
+
+        live_thread
+            .append_items(&[user_message_item("new external append")])
+            .await
+            .expect("append after external resume");
+
+        let metadata = runtime
+            .get_thread(thread_id)
+            .await
+            .expect("sqlite metadata read")
+            .expect("sqlite metadata");
+        assert_eq!(
+            metadata.created_at.to_rfc3339(),
+            "2025-01-03T17:30:00+00:00"
+        );
+        assert_eq!(metadata.model_provider, "test-provider");
+        assert_eq!(
+            metadata.first_user_message.as_deref(),
+            Some("Hello from user")
+        );
+    }
+
     #[tokio::test]
     async fn create_thread_rejects_missing_cwd() {
         let home = TempDir::new().expect("temp dir");