From 553df1c691fe8bf7747e50da22f1342984495ae0 Mon Sep 17 00:00:00 2001 From: jif Date: Tue, 8 Sep 2026 12:48:18 +0000 Subject: [PATCH] Add dedicated memory v2 consolidation and read prompts (#43813) ## What changed - Consolidate v2 rollout summaries into `memory_summary.md` without generating `raw_memories.md` or requiring `MEMORY.md`. Validate the summary's required sections and size below 10,000 UTF-8 bytes. - Add v2 read instructions for selective history retrieval, evidence-grounded preferences, citations, and explicit memory edits. Split injected instructions into fragments to preserve the complete summary within fragment byte limits. - Record memory citation usage in the store selected by `memories.version`. ## Testing Add coverage for v2 consolidation without a handbook, summary validation, version-isolated memory reads, and resetting both memory versions while preserving threads. GitOrigin-RevId: 1d895fb4a23a973f1a45ba03be07a1f480c10227 --- .../app-server/tests/suite/v2/memory_read.rs | 89 +++++++++++++++++++ .../app-server/tests/suite/v2/memory_reset.rs | 84 +++++++++-------- codex-rs/app-server/tests/suite/v2/mod.rs | 1 + codex-rs/core/src/stream_events_utils.rs | 23 ++--- codex-rs/ext/memories/src/extension.rs | 29 +++++- codex-rs/ext/memories/src/prompts.rs | 13 ++- codex-rs/ext/memories/src/prompts_tests.rs | 31 +++++++ .../templates/memories/read_path_v2.md | 44 +++++++++ codex-rs/memories/write/src/phase2.rs | 31 +++++-- codex-rs/memories/write/src/prompts.rs | 21 ++++- codex-rs/memories/write/src/startup_tests.rs | 78 ++++++++++++---- codex-rs/memories/write/src/workspace.rs | 49 +++++++--- .../memories/write/src/workspace_tests.rs | 28 ++++++ .../templates/memories/consolidation_v2.md | 52 +++++++++++ 14 files changed, 487 insertions(+), 86 deletions(-) create mode 100644 codex-rs/app-server/tests/suite/v2/memory_read.rs create mode 100644 codex-rs/ext/memories/templates/memories/read_path_v2.md create mode 100644 codex-rs/memories/write/templates/memories/consolidation_v2.md diff --git a/codex-rs/app-server/tests/suite/v2/memory_read.rs b/codex-rs/app-server/tests/suite/v2/memory_read.rs new file mode 100644 index 0000000000..cb675f95f5 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/memory_read.rs @@ -0,0 +1,89 @@ +//! Exercises version selection through app-server and the model request boundary. + +use anyhow::Result; +use app_test_support::MockResponsesConfig; +use app_test_support::TestAppServer; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStatus; +use codex_app_server_protocol::UserInput; +use codex_features::Feature; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_once; +use core_test_support::responses::sse; +use core_test_support::responses::start_mock_server; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; + +#[tokio::test] +async fn memory_read_version_selects_only_its_own_context() -> Result<()> { + for (version, expected, excluded) in [ + ("v1", "legacy-only-marker", "v2-only-marker"), + ("v2", "v2-only-marker", "legacy-only-marker"), + ] { + let server = start_mock_server().await; + let home = TempDir::new()?; + MockResponsesConfig::new(&server.uri()) + .enable_feature(Feature::MemoryTool) + .write(home.path())?; + for (directory, marker) in [ + ("memories", "legacy-only-marker"), + ("memories_v2", "v2-only-marker"), + ] { + tokio::fs::create_dir_all(home.path().join(directory)).await?; + tokio::fs::write( + home.path().join(directory).join("memory_summary.md"), + format!("v1\n{marker}\n"), + ) + .await?; + } + let mock = mount_sse_once( + &server, + sse(vec![ + ev_response_created("response"), + ev_assistant_message("message", "Done"), + ev_completed("response"), + ]), + ) + .await; + let mut app = TestAppServer::builder() + .with_codex_home(home.path()) + .build_initialized() + .await?; + let start = app + .send_thread_start_request_with_auto_env(ThreadStartParams { + // Skip background consolidation so it cannot consume the read test's response. + ephemeral: Some(true), + config: Some( + [ + ("memories.version".to_string(), json!(version)), + ("memories.generate_memories".to_string(), json!(false)), + ] + .into(), + ), + ..Default::default() + }) + .await?; + let response: ThreadStartResponse = app.read_response(start).await?; + app.send_turn_start_request(TurnStartParams { + thread_id: response.thread.id, + input: vec![UserInput::Text { + text: "Use my memory preferences".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let completed: TurnCompletedNotification = app.read_notification("turn/completed").await?; + assert_eq!(completed.turn.status, TurnStatus::Completed); + let request = mock.single_request(); + assert!(request.body_contains_text(expected)); + assert!(!request.body_contains_text(excluded)); + } + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/memory_reset.rs b/codex-rs/app-server/tests/suite/v2/memory_reset.rs index 8e89fa9c20..d0274bb88d 100644 --- a/codex-rs/app-server/tests/suite/v2/memory_reset.rs +++ b/codex-rs/app-server/tests/suite/v2/memory_reset.rs @@ -4,6 +4,7 @@ use app_test_support::TestAppServer; use chrono::Utc; use codex_app_server_protocol::MemoryResetResponse; use codex_features::Feature; +use codex_protocol::MemoryVersion; use codex_protocol::ThreadId; use codex_protocol::protocol::SessionSource; use codex_state::Stage1JobClaimOutcome; @@ -28,16 +29,18 @@ async fn memory_reset_clears_memory_files_and_rows_preserves_threads() -> Result .write(codex_home.path())?; let state_db = init_state_db(codex_home.path()).await?; - let memory_root = codex_home.path().join("memories"); - tokio::fs::create_dir_all(memory_root.join("rollout_summaries")).await?; - tokio::fs::write(memory_root.join("MEMORY.md"), "stale memory\n").await?; - tokio::fs::write( - memory_root.join("rollout_summaries").join("stale.md"), - "stale rollout summary\n", - ) - .await?; - - let thread_id = seed_stage1_output(&state_db, codex_home.path()).await?; + let mut thread_ids = Vec::new(); + for version in [MemoryVersion::V1, MemoryVersion::V2] { + let root = codex_home.path().join(version.directory_name()); + tokio::fs::create_dir_all(root.join("rollout_summaries")).await?; + tokio::fs::write(root.join("memory_summary.md"), "v1\nstale memory\n").await?; + tokio::fs::write( + root.join("rollout_summaries/stale.md"), + "stale rollout summary\n", + ) + .await?; + thread_ids.push(seed_stage1_output(&state_db, codex_home.path(), version).await?); + } let mut mcp = TestAppServer::builder() .with_codex_home(codex_home.path()) @@ -51,26 +54,37 @@ async fn memory_reset_clears_memory_files_and_rows_preserves_threads() -> Result let _: MemoryResetResponse = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(request_id)).await??; - let stage1_outputs = state_db - .memories() - .list_stage1_outputs_for_global(/*n*/ 10) - .await?; - assert_eq!(stage1_outputs, Vec::new()); - assert_eq!( - state_db.get_thread_memory_mode(thread_id).await?.as_deref(), - Some("enabled") - ); - - let mut remaining_entries = tokio::fs::read_dir(&memory_root).await?; - assert!( - remaining_entries.next_entry().await?.is_none(), - "memory root should be empty after reset" - ); + for version in [MemoryVersion::V1, MemoryVersion::V2] { + let outputs = state_db + .memories_for_version(version) + .await? + .list_stage1_outputs_for_global(/*n*/ 10) + .await?; + assert_eq!(outputs, Vec::new()); + let root = codex_home.path().join(version.directory_name()); + assert!( + tokio::fs::read_dir(root) + .await? + .next_entry() + .await? + .is_none() + ); + } + for thread_id in thread_ids { + assert_eq!( + state_db.get_thread_memory_mode(thread_id).await?.as_deref(), + Some("enabled") + ); + } Ok(()) } -async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> Result { +async fn seed_stage1_output( + state_db: &Arc, + codex_home: &Path, + version: MemoryVersion, +) -> Result { let now = Utc::now(); let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; let worker_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; @@ -85,8 +99,8 @@ async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> let metadata = builder.build("mock_provider"); state_db.upsert_thread(&metadata).await?; - let claim = state_db - .memories() + let store = state_db.memories_for_version(version).await?; + let claim = store .try_claim_stage1_job( thread_id, worker_id, @@ -99,23 +113,23 @@ async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> anyhow::bail!("unexpected stage1 claim outcome: {claim:?}"); }; assert!( - state_db - .memories() + store .mark_stage1_job_succeeded( thread_id, ownership_token.as_str(), now.timestamp(), - "raw memory", + if version == MemoryVersion::V1 { + "raw memory" + } else { + "" + }, "rollout summary", /*rollout_slug*/ None, ) .await?, "stage1 success should be recorded" ); - state_db - .memories() - .enqueue_global_consolidation(now.timestamp()) - .await?; + store.enqueue_global_consolidation(now.timestamp()).await?; Ok(thread_id) } diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 673dadf834..c7bc32d01f 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -57,6 +57,7 @@ mod mcp_resource_origin; mod mcp_server_elicitation; mod mcp_server_status; mod mcp_tool; +mod memory_read; mod memory_reset; mod misalignment_policy; mod model_auto_review; diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index 8c87eaf717..9de40b03e5 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -112,17 +112,20 @@ pub(crate) async fn record_completed_response_item_with_finalized_facts( .await; } mark_thread_memory_mode_polluted_if_external_context(sess, turn_context, item).await; + let memory_usage_db = match sess.services.state_db.as_ref() { + Some(db) => db + .memories_for_version(turn_context.config.memories.version) + .await + .ok(), + None => None, + }; let has_memory_citation = if let Some(memory_citation) = finalized_facts.and_then(|facts| facts.memory_citation.as_ref()) { - record_stage1_output_usage_for_memory_citation( - sess.services.state_db.as_ref(), - memory_citation, - ) - .await - } else { - record_stage1_output_usage_and_detect_memory_citation(sess.services.state_db.as_ref(), item) + record_stage1_output_usage_for_memory_citation(memory_usage_db.as_ref(), memory_citation) .await + } else { + record_stage1_output_usage_and_detect_memory_citation(memory_usage_db.as_ref(), item).await }; if has_memory_citation { sess.record_memory_citation_for_turn(&turn_context.sub_id) @@ -159,7 +162,7 @@ pub(crate) async fn mark_thread_memory_mode_polluted_if_external_context( } async fn record_stage1_output_usage_and_detect_memory_citation( - state_db_ctx: Option<&state_db::StateDbHandle>, + state_db_ctx: Option<&codex_state::MemoryStore>, item: &ResponseItem, ) -> bool { let Some(raw_text) = raw_assistant_output_text_from_item(item) else { @@ -174,7 +177,7 @@ async fn record_stage1_output_usage_and_detect_memory_citation( } async fn record_stage1_output_usage_for_memory_citation( - state_db_ctx: Option<&state_db::StateDbHandle>, + state_db_ctx: Option<&codex_state::MemoryStore>, memory_citation: &MemoryCitation, ) -> bool { let thread_ids = thread_ids_from_memory_citation(memory_citation); @@ -183,7 +186,7 @@ async fn record_stage1_output_usage_for_memory_citation( } if let Some(db) = state_db_ctx { - let _ = db.memories().record_stage1_output_usage(&thread_ids).await; + let _ = db.record_stage1_output_usage(&thread_ids).await; } true } diff --git a/codex-rs/ext/memories/src/extension.rs b/codex-rs/ext/memories/src/extension.rs index c58533112e..3db27e4984 100644 --- a/codex-rs/ext/memories/src/extension.rs +++ b/codex-rs/ext/memories/src/extension.rs @@ -1,6 +1,8 @@ use std::sync::Arc; use codex_core::config::Config; +use codex_core::context::ContextualUserFragment; +use codex_core::context::MemoryContextFragment; use codex_extension_api::ConfigContributor; use codex_extension_api::ContentItemKind; use codex_extension_api::ContextContributor; @@ -65,15 +67,36 @@ impl ContextContributor for MemoriesExtension { return Vec::new(); } - build_memory_tool_developer_instructions(&config.codex_home, config.version) - .await + let Some(instructions) = + build_memory_tool_developer_instructions(&config.codex_home, config.version).await + else { + return Vec::new(); + }; + let instructions = match config.version { + MemoryVersion::V1 => vec![instructions], + MemoryVersion::V2 => { + // Keep the complete summary while respecting each fragment's byte cap. + let mut remaining = instructions.as_str(); + let mut fragments = Vec::new(); + while !remaining.is_empty() { + let end = remaining.floor_char_boundary(remaining.len().min(8_900)); + fragments.push( + MemoryContextFragment::ReadInstructions(remaining[..end].to_string()) + .render(), + ); + remaining = &remaining[end..]; + } + fragments + } + }; + instructions + .into_iter() .map(|instructions| { PromptFragment::developer_policy( instructions, ContentItemKind("memories.instructions".to_string()), ) }) - .into_iter() .collect() }) } diff --git a/codex-rs/ext/memories/src/prompts.rs b/codex-rs/ext/memories/src/prompts.rs index e307934d6a..333a3c6918 100644 --- a/codex-rs/ext/memories/src/prompts.rs +++ b/codex-rs/ext/memories/src/prompts.rs @@ -14,6 +14,13 @@ static MEMORY_TOOL_DEVELOPER_INSTRUCTIONS_TEMPLATE: LazyLock