mirror of
https://github.com/openai/codex.git
synced 2026-10-20 00:18:49 +00:00
Support paginated thread forks (#35220)
## Why `thread/fork` rejected threads using paginated history, so they could not use the same fork workflow as legacy threads. ## What changed - Create paginated forks by referencing a frozen source-history prefix while persisting only child-owned records in the new rollout. - Support latest, `lastTurnId`, and `beforeTurnId` boundaries, including inherited lineage, model-context reconstruction, interruption markers, and approval-reviewer inheritance. - Make paginated reads and occurrence search traverse referenced history while excluding source records added after the fork. - Coordinate fork preparation with archive and delete operations, and materialize compressed rollouts before they become fork references. ## Testing Added app-server and thread-store coverage for boundary selection, active turns, inherited reads and search, compressed lineage, and concurrent source lifecycle operations. GitOrigin-RevId: 66052d69517d7fb97a3b7817ab77a39c5ad995fd
This commit is contained in:
Generated
+1
@@ -4133,6 +4133,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -324,6 +324,7 @@ impl ExternalAgentSessionImporter {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: Some(MultiAgentVersion::V1),
|
||||
history_mode: ThreadHistoryMode::Legacy,
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: uuid::Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
|
||||
@@ -3999,9 +3999,7 @@ impl ThreadRequestProcessor {
|
||||
/*include_history*/ false,
|
||||
)
|
||||
.await?;
|
||||
if matches!(source_thread.history_mode, ThreadHistoryMode::Paginated) {
|
||||
return Err(method_not_found("paginated_threads is not supported yet"));
|
||||
}
|
||||
let paginated_source = matches!(source_thread.history_mode, ThreadHistoryMode::Paginated);
|
||||
if last_turn_id.is_some() && before_turn_id.is_some() {
|
||||
return Err(invalid_request(
|
||||
"`beforeTurnId` cannot be combined with `lastTurnId`",
|
||||
@@ -4012,35 +4010,69 @@ impl ThreadRequestProcessor {
|
||||
"`deferGoalContinuation` cannot be combined with `ephemeral`",
|
||||
));
|
||||
}
|
||||
let mut source_thread = self
|
||||
.read_stored_thread_for_resume(&thread_id, path.as_ref(), /*include_history*/ true)
|
||||
.await?;
|
||||
let source_thread_id = source_thread.thread_id;
|
||||
let source_thread_name = source_thread
|
||||
.name
|
||||
.as_deref()
|
||||
.and_then(codex_core::util::normalize_thread_name);
|
||||
let source_history_items = source_thread
|
||||
.history
|
||||
.take()
|
||||
.map(|history| history.items)
|
||||
.ok_or_else(|| {
|
||||
internal_error(format!(
|
||||
"thread {source_thread_id} did not include persisted history"
|
||||
))
|
||||
})?;
|
||||
let source_history_items = Arc::new(source_history_items);
|
||||
let history_items = match (last_turn_id.as_deref(), before_turn_id.as_deref()) {
|
||||
(Some(last_turn_id), None) => Arc::new(
|
||||
truncate_rollout_after_turn_id(&source_history_items, last_turn_id)
|
||||
.map_err(|err| core_thread_write_error("truncate thread for fork", err))?,
|
||||
),
|
||||
(None, Some(before_turn_id)) => Arc::new(
|
||||
truncate_rollout_before_turn_id(&source_history_items, before_turn_id)
|
||||
.map_err(|err| core_thread_write_error("truncate thread for fork", err))?,
|
||||
),
|
||||
(None, None) => Arc::clone(&source_history_items),
|
||||
(Some(_), Some(_)) => unreachable!("fork boundaries are mutually exclusive"),
|
||||
let prepared_fork = if paginated_source {
|
||||
if ephemeral {
|
||||
return Err(invalid_request(
|
||||
"ephemeral paginated thread/fork is not supported",
|
||||
));
|
||||
}
|
||||
let boundary = match (last_turn_id.as_deref(), before_turn_id.as_deref()) {
|
||||
(Some(turn_id), None) => {
|
||||
codex_thread_store::ForkBoundary::ThroughTurn(turn_id.to_string())
|
||||
}
|
||||
(None, Some(turn_id)) => {
|
||||
codex_thread_store::ForkBoundary::BeforeTurn(turn_id.to_string())
|
||||
}
|
||||
(None, None) => codex_thread_store::ForkBoundary::Latest,
|
||||
(Some(_), Some(_)) => unreachable!("fork boundaries are mutually exclusive"),
|
||||
};
|
||||
Some(
|
||||
self.thread_store
|
||||
.prepare_fork(codex_thread_store::PrepareForkParams {
|
||||
thread_id: source_thread_id,
|
||||
boundary,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
ThreadStoreError::InvalidRequest { message } => invalid_request(message),
|
||||
ThreadStoreError::ThreadNotFound { thread_id } => {
|
||||
invalid_request(format!("no rollout found for thread id {thread_id}"))
|
||||
}
|
||||
ThreadStoreError::Unsupported { .. } => {
|
||||
method_not_found("paginated_threads is not supported yet")
|
||||
}
|
||||
err => internal_error(format!("failed to prepare paginated fork: {err}")),
|
||||
})?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let source_history_items = if let Some(prepared_fork) = prepared_fork.as_ref() {
|
||||
Arc::clone(&prepared_fork.model_context)
|
||||
} else {
|
||||
let mut source_thread = self
|
||||
.read_stored_thread_for_resume(
|
||||
&thread_id,
|
||||
path.as_ref(),
|
||||
/*include_history*/ true,
|
||||
)
|
||||
.await?;
|
||||
Arc::new(
|
||||
source_thread
|
||||
.history
|
||||
.take()
|
||||
.map(|history| history.items)
|
||||
.ok_or_else(|| {
|
||||
internal_error(format!(
|
||||
"thread {source_thread_id} did not include persisted history"
|
||||
))
|
||||
})?,
|
||||
)
|
||||
};
|
||||
let history_cwd = Some(source_thread.cwd.clone());
|
||||
|
||||
@@ -4066,11 +4098,6 @@ impl ThreadRequestProcessor {
|
||||
} else {
|
||||
Some(cli_overrides)
|
||||
};
|
||||
let thread_history = InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id: source_thread_id,
|
||||
history: Arc::clone(&history_items),
|
||||
rollout_path: source_thread.rollout_path.clone(),
|
||||
});
|
||||
let runtime_workspace_roots = runtime_workspace_roots.map(resolve_runtime_workspace_roots);
|
||||
let mut typesafe_overrides = self.build_thread_config_overrides(
|
||||
model,
|
||||
@@ -4087,8 +4114,37 @@ impl ThreadRequestProcessor {
|
||||
/*personality*/ None,
|
||||
);
|
||||
typesafe_overrides.ephemeral = ephemeral.then_some(true);
|
||||
let latest_context = if paginated_source
|
||||
&& typesafe_overrides.approvals_reviewer.is_none()
|
||||
&& !request_overrides
|
||||
.as_ref()
|
||||
.is_some_and(|overrides| overrides.contains_key("approvals_reviewer"))
|
||||
{
|
||||
if let Ok(parent) = self.thread_manager.get_thread(source_thread_id).await {
|
||||
typesafe_overrides.approvals_reviewer =
|
||||
Some(parent.config_snapshot().await.approvals_reviewer);
|
||||
None
|
||||
} else if last_turn_id.is_some() || before_turn_id.is_some() {
|
||||
Some(
|
||||
self.thread_store
|
||||
.load_latest_model_context(StoreLoadThreadHistoryParams {
|
||||
thread_id: source_thread_id,
|
||||
include_archived: true,
|
||||
})
|
||||
.await
|
||||
.map_err(thread_store_resume_read_error)?
|
||||
.items,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
merge_persisted_approvals_reviewer(
|
||||
&source_history_items,
|
||||
latest_context
|
||||
.as_deref()
|
||||
.unwrap_or_else(|| source_history_items.as_ref()),
|
||||
request_overrides.as_ref(),
|
||||
&mut typesafe_overrides,
|
||||
);
|
||||
@@ -4101,30 +4157,64 @@ impl ThreadRequestProcessor {
|
||||
let goals_enabled = config.features.enabled(Feature::Goals);
|
||||
|
||||
let fallback_model_provider = config.model_provider_id.clone();
|
||||
let parent_trace = self.request_trace_context(&request_id).await;
|
||||
let thread_source = thread_source.map(Into::into);
|
||||
|
||||
let (history_items, new_thread) = if let Some(prepared_fork) = prepared_fork {
|
||||
let history_items = Arc::clone(&prepared_fork.model_context);
|
||||
let new_thread = self
|
||||
.thread_manager
|
||||
.fork_prepared_thread(
|
||||
config,
|
||||
prepared_fork,
|
||||
thread_source,
|
||||
parent_trace,
|
||||
supports_openai_form_elicitation,
|
||||
)
|
||||
.await;
|
||||
(history_items, new_thread)
|
||||
} else {
|
||||
let history_items = match (last_turn_id.as_deref(), before_turn_id.as_deref()) {
|
||||
(Some(last_turn_id), None) => Arc::new(
|
||||
truncate_rollout_after_turn_id(&source_history_items, last_turn_id)
|
||||
.map_err(|err| core_thread_write_error("truncate thread for fork", err))?,
|
||||
),
|
||||
(None, Some(before_turn_id)) => Arc::new(
|
||||
truncate_rollout_before_turn_id(&source_history_items, before_turn_id)
|
||||
.map_err(|err| core_thread_write_error("truncate thread for fork", err))?,
|
||||
),
|
||||
(None, None) => Arc::clone(&source_history_items),
|
||||
(Some(_), Some(_)) => unreachable!("fork boundaries are mutually exclusive"),
|
||||
};
|
||||
let new_thread = self
|
||||
.thread_manager
|
||||
.fork_thread_from_history(
|
||||
ForkSnapshot::Interrupted,
|
||||
config,
|
||||
InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id: source_thread_id,
|
||||
history: Arc::clone(&history_items),
|
||||
rollout_path: source_thread.rollout_path.clone(),
|
||||
}),
|
||||
thread_source,
|
||||
parent_trace,
|
||||
supports_openai_form_elicitation,
|
||||
)
|
||||
.await;
|
||||
(history_items, new_thread)
|
||||
};
|
||||
let NewThread {
|
||||
thread_id,
|
||||
thread: forked_thread,
|
||||
session_configured,
|
||||
..
|
||||
} = self
|
||||
.thread_manager
|
||||
.fork_thread_from_history(
|
||||
ForkSnapshot::Interrupted,
|
||||
config,
|
||||
thread_history,
|
||||
thread_source.map(Into::into),
|
||||
self.request_trace_context(&request_id).await,
|
||||
supports_openai_form_elicitation,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| match err.details() {
|
||||
CodexErrorDetails::Io(_) | CodexErrorDetails::Json(_) => {
|
||||
invalid_request(format!("failed to load thread {source_thread_id}: {err}"))
|
||||
}
|
||||
CodexErrorDetails::InvalidRequest(message) => invalid_request(message.clone()),
|
||||
_ => internal_error(format!("error forking thread: {err}")),
|
||||
})?;
|
||||
} = new_thread.map_err(|err| match err.details() {
|
||||
CodexErrorDetails::Io(_) | CodexErrorDetails::Json(_) => {
|
||||
invalid_request(format!("failed to load thread {source_thread_id}: {err}"))
|
||||
}
|
||||
CodexErrorDetails::InvalidRequest(message) => invalid_request(message.clone()),
|
||||
_ => internal_error(format!("error forking thread: {err}")),
|
||||
})?;
|
||||
|
||||
Self::set_app_server_client_info(
|
||||
forked_thread.as_ref(),
|
||||
@@ -4196,12 +4286,12 @@ impl ThreadRequestProcessor {
|
||||
// pathless, so they rebuild their visible history from the copied source history instead.
|
||||
let mut thread = if session_configured.rollout_path.is_some() {
|
||||
let stored_thread = self
|
||||
.read_stored_thread_for_new_fork(thread_id, include_turns)
|
||||
.read_stored_thread_for_new_fork(thread_id, include_turns && !paginated_source)
|
||||
.await?;
|
||||
self.stored_thread_to_api_thread(
|
||||
stored_thread,
|
||||
fallback_model_provider.as_str(),
|
||||
include_turns,
|
||||
include_turns && !paginated_source,
|
||||
)
|
||||
} else {
|
||||
let mut thread = build_thread_from_snapshot(
|
||||
@@ -4222,6 +4312,9 @@ impl ThreadRequestProcessor {
|
||||
}
|
||||
thread
|
||||
};
|
||||
if paginated_source && include_turns {
|
||||
thread.turns = self.paginated_thread_full_turns(thread_id).await?;
|
||||
}
|
||||
if let Some(name) = source_thread_name {
|
||||
set_thread_name_from_title(&mut thread, name);
|
||||
}
|
||||
|
||||
@@ -132,6 +132,7 @@ async fn get_conversation_summary_by_thread_id_reads_pathless_store_thread() ->
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
|
||||
@@ -210,6 +210,7 @@ async fn thread_delete_with_non_local_thread_store_does_not_create_local_persist
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
|
||||
@@ -2,6 +2,7 @@ use anyhow::Result;
|
||||
use app_test_support::ChatGptAuthFixture;
|
||||
use app_test_support::MockResponsesConfig;
|
||||
use app_test_support::TestAppServer;
|
||||
use app_test_support::create_fake_paginated_rollout;
|
||||
use app_test_support::create_fake_rollout;
|
||||
use app_test_support::create_fake_rollout_with_token_usage;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
@@ -10,6 +11,7 @@ use app_test_support::rollout_path;
|
||||
use app_test_support::to_response;
|
||||
use app_test_support::write_chatgpt_auth;
|
||||
use codex_app_server_protocol::ApprovalsReviewer;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::JSONRPCError;
|
||||
use codex_app_server_protocol::JSONRPCMessage;
|
||||
use codex_app_server_protocol::JSONRPCResponse;
|
||||
@@ -18,28 +20,43 @@ use codex_app_server_protocol::ServerNotification;
|
||||
use codex_app_server_protocol::SessionSource;
|
||||
use codex_app_server_protocol::ThreadForkParams;
|
||||
use codex_app_server_protocol::ThreadForkResponse;
|
||||
use codex_app_server_protocol::ThreadHistoryMode;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::ThreadListParams;
|
||||
use codex_app_server_protocol::ThreadListResponse;
|
||||
use codex_app_server_protocol::ThreadReadParams;
|
||||
use codex_app_server_protocol::ThreadReadResponse;
|
||||
use codex_app_server_protocol::ThreadResumeParams;
|
||||
use codex_app_server_protocol::ThreadResumeResponse;
|
||||
use codex_app_server_protocol::ThreadSearchOccurrencesParams;
|
||||
use codex_app_server_protocol::ThreadSearchOccurrencesResponse;
|
||||
use codex_app_server_protocol::ThreadSource;
|
||||
use codex_app_server_protocol::ThreadStartParams;
|
||||
use codex_app_server_protocol::ThreadStartResponse;
|
||||
use codex_app_server_protocol::ThreadStartedNotification;
|
||||
use codex_app_server_protocol::ThreadStatus;
|
||||
use codex_app_server_protocol::ThreadStatusChangedNotification;
|
||||
use codex_app_server_protocol::ThreadTurnsListParams;
|
||||
use codex_app_server_protocol::ThreadTurnsListResponse;
|
||||
use codex_app_server_protocol::TurnItemsView;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
use codex_app_server_protocol::TurnStartResponse;
|
||||
use codex_app_server_protocol::TurnStatus;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
use codex_config::types::AuthCredentialsStoreMode;
|
||||
use codex_features::Feature;
|
||||
use codex_login::REFRESH_TOKEN_URL_OVERRIDE_ENV_VAR;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::items::TurnItem as CoreTurnItem;
|
||||
use codex_protocol::items::UserMessageItem;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::protocol::TurnStartedEvent;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_rollout::append_rollout_item_to_path;
|
||||
@@ -267,6 +284,17 @@ async fn thread_fork_creates_new_thread_and_emits_started() -> Result<()> {
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_fork_preserves_persisted_approvals_reviewer() -> Result<()> {
|
||||
assert_thread_fork_preserves_persisted_approvals_reviewer(ThreadHistoryMode::Legacy).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paginated_thread_fork_preserves_persisted_approvals_reviewer() -> Result<()> {
|
||||
assert_thread_fork_preserves_persisted_approvals_reviewer(ThreadHistoryMode::Paginated).await
|
||||
}
|
||||
|
||||
async fn assert_thread_fork_preserves_persisted_approvals_reviewer(
|
||||
history_mode: ThreadHistoryMode,
|
||||
) -> Result<()> {
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let codex_home = TempDir::new()?;
|
||||
MockResponsesConfig::new(&server.uri()).write(codex_home.path())?;
|
||||
@@ -278,7 +306,10 @@ async fn thread_fork_preserves_persisted_approvals_reviewer() -> Result<()> {
|
||||
.await?;
|
||||
|
||||
let start_id = mcp
|
||||
.send_thread_start_request_with_auto_env(ThreadStartParams::default())
|
||||
.send_thread_start_request_with_auto_env(ThreadStartParams {
|
||||
history_mode: Some(history_mode),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let start_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
@@ -331,6 +362,20 @@ async fn thread_fork_preserves_persisted_approvals_reviewer() -> Result<()> {
|
||||
)
|
||||
.await??;
|
||||
|
||||
if matches!(history_mode, ThreadHistoryMode::Paginated) {
|
||||
let fork_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: thread.id.clone(),
|
||||
last_turn_id: Some(turn.id.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let ThreadForkResponse {
|
||||
approvals_reviewer, ..
|
||||
} = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(fork_id)).await??;
|
||||
assert_eq!(approvals_reviewer, ApprovalsReviewer::AutoReview);
|
||||
}
|
||||
|
||||
(thread.id, turn.id)
|
||||
};
|
||||
|
||||
@@ -340,8 +385,8 @@ async fn thread_fork_preserves_persisted_approvals_reviewer() -> Result<()> {
|
||||
.await?;
|
||||
let fork_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: source_thread_id,
|
||||
last_turn_id: Some(source_turn_id),
|
||||
thread_id: source_thread_id.clone(),
|
||||
last_turn_id: Some(source_turn_id.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
@@ -356,11 +401,38 @@ async fn thread_fork_preserves_persisted_approvals_reviewer() -> Result<()> {
|
||||
|
||||
assert_eq!(approvals_reviewer, ApprovalsReviewer::AutoReview);
|
||||
|
||||
if matches!(history_mode, ThreadHistoryMode::Paginated) {
|
||||
let fork_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: source_thread_id,
|
||||
last_turn_id: Some(source_turn_id),
|
||||
approvals_reviewer: Some(ApprovalsReviewer::User),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let ThreadForkResponse {
|
||||
approvals_reviewer, ..
|
||||
} = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(fork_id)).await??;
|
||||
assert_eq!(approvals_reviewer, ApprovalsReviewer::User);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_fork_at_last_turn_id_keeps_only_terminal_prefix() -> Result<()> {
|
||||
assert_thread_fork_at_named_boundary_keeps_only_terminal_prefix(ThreadHistoryMode::Legacy).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paginated_thread_fork_at_named_boundaries_keeps_only_terminal_prefix() -> Result<()> {
|
||||
assert_thread_fork_at_named_boundary_keeps_only_terminal_prefix(ThreadHistoryMode::Paginated)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn assert_thread_fork_at_named_boundary_keeps_only_terminal_prefix(
|
||||
history_mode: ThreadHistoryMode,
|
||||
) -> Result<()> {
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let codex_home = TempDir::new()?;
|
||||
MockResponsesConfig::new(&server.uri()).write(codex_home.path())?;
|
||||
@@ -371,7 +443,10 @@ async fn thread_fork_at_last_turn_id_keeps_only_terminal_prefix() -> Result<()>
|
||||
.await?;
|
||||
|
||||
let start_id = mcp
|
||||
.send_thread_start_request_with_auto_env(ThreadStartParams::default())
|
||||
.send_thread_start_request_with_auto_env(ThreadStartParams {
|
||||
history_mode: Some(history_mode),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let ThreadStartResponse {
|
||||
thread: source_thread,
|
||||
@@ -430,8 +505,10 @@ async fn thread_fork_at_last_turn_id_keeps_only_terminal_prefix() -> Result<()>
|
||||
.iter()
|
||||
.all(|turn| turn.status == TurnStatus::Completed)
|
||||
);
|
||||
assert_eq!(forked_thread.forked_from_id, Some(source_thread_id));
|
||||
assert_eq!(forked_thread.preview, "first");
|
||||
assert_eq!(forked_thread.forked_from_id, Some(source_thread_id.clone()));
|
||||
if history_mode == ThreadHistoryMode::Legacy {
|
||||
assert_eq!(forked_thread.preview, "first");
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(source_path.as_path())?,
|
||||
original_contents,
|
||||
@@ -440,7 +517,18 @@ async fn thread_fork_at_last_turn_id_keeps_only_terminal_prefix() -> Result<()>
|
||||
|
||||
let forked_path = forked_thread.path.clone().expect("forked thread path");
|
||||
let forked_contents = std::fs::read_to_string(forked_path.as_path())?;
|
||||
assert!(forked_contents.contains(turn_ids[1].as_str()));
|
||||
if history_mode == ThreadHistoryMode::Paginated {
|
||||
assert!(
|
||||
read_session_meta_line(forked_path.as_path())
|
||||
.await?
|
||||
.meta
|
||||
.history_base
|
||||
.is_some()
|
||||
);
|
||||
assert!(!forked_contents.contains(turn_ids[1].as_str()));
|
||||
} else {
|
||||
assert!(forked_contents.contains(turn_ids[1].as_str()));
|
||||
}
|
||||
assert!(!forked_contents.contains(turn_ids[2].as_str()));
|
||||
|
||||
let started = loop {
|
||||
@@ -457,6 +545,28 @@ async fn thread_fork_at_last_turn_id_keeps_only_terminal_prefix() -> Result<()>
|
||||
};
|
||||
assert!(started.thread.turns.is_empty());
|
||||
|
||||
if history_mode == ThreadHistoryMode::Paginated {
|
||||
let before_fork_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: source_thread_id,
|
||||
before_turn_id: Some(turn_ids[2].clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let ThreadForkResponse {
|
||||
thread: before_fork,
|
||||
..
|
||||
} = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(before_fork_id)).await??;
|
||||
assert_eq!(
|
||||
before_fork
|
||||
.turns
|
||||
.iter()
|
||||
.map(|turn| turn.id.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
turn_ids[..2]
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1083,12 +1193,12 @@ async fn thread_fork_rejects_unmaterialized_thread() -> Result<()> {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_fork_rejects_paginated_thread() -> Result<()> {
|
||||
async fn thread_fork_creates_reference_backed_paginated_thread() -> Result<()> {
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let codex_home = TempDir::new()?;
|
||||
MockResponsesConfig::new(&server.uri()).write(codex_home.path())?;
|
||||
|
||||
let conversation_id = create_fake_rollout(
|
||||
let conversation_id = create_fake_paginated_rollout(
|
||||
codex_home.path(),
|
||||
"2025-01-05T12-00-00",
|
||||
"2025-01-05T12:00:00Z",
|
||||
@@ -1096,27 +1206,31 @@ async fn thread_fork_rejects_paginated_thread() -> Result<()> {
|
||||
Some("mock_provider"),
|
||||
/*git_info*/ None,
|
||||
)?;
|
||||
let rollout_path = codex_home
|
||||
.path()
|
||||
.join("sessions")
|
||||
.join("2025")
|
||||
.join("01")
|
||||
.join("05")
|
||||
.join(format!(
|
||||
"rollout-2025-01-05T12-00-00-{conversation_id}.jsonl"
|
||||
));
|
||||
let mut lines = std::fs::read_to_string(&rollout_path)?
|
||||
.lines()
|
||||
.map(serde_json::from_str::<Value>)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
lines[0]["payload"]["history_mode"] = json!("paginated");
|
||||
let contents = lines
|
||||
.into_iter()
|
||||
.map(|line| line.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
std::fs::write(&rollout_path, format!("{contents}\n"))?;
|
||||
|
||||
let source_path = rollout_path(
|
||||
codex_home.path(),
|
||||
"2025-01-05T12-00-00",
|
||||
conversation_id.as_str(),
|
||||
);
|
||||
for item in [
|
||||
RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
|
||||
turn_id: "turn-1".to_string(),
|
||||
trace_id: None,
|
||||
started_at: Some(10),
|
||||
model_context_window: None,
|
||||
collaboration_mode_kind: Default::default(),
|
||||
})),
|
||||
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: "turn-1".to_string(),
|
||||
last_agent_message: None,
|
||||
error: None,
|
||||
started_at: Some(10),
|
||||
completed_at: Some(20),
|
||||
duration_ms: Some(10_000),
|
||||
time_to_first_token_ms: None,
|
||||
})),
|
||||
] {
|
||||
append_rollout_item_to_path(source_path.as_path(), &item).await?;
|
||||
}
|
||||
let mut mcp = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.without_auto_env()
|
||||
@@ -1125,21 +1239,396 @@ async fn thread_fork_rejects_paginated_thread() -> Result<()> {
|
||||
|
||||
let fork_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: conversation_id,
|
||||
thread_id: conversation_id.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let fork_err: JSONRPCError = timeout(
|
||||
let ThreadForkResponse {
|
||||
thread: forked_thread,
|
||||
..
|
||||
} = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(fork_id)).await??;
|
||||
assert_eq!(forked_thread.forked_from_id, Some(conversation_id.clone()));
|
||||
assert_eq!(forked_thread.turns.len(), 1);
|
||||
let forked_thread_id = forked_thread.id.clone();
|
||||
let forked_path = forked_thread.path.expect("forked rollout path");
|
||||
assert!(!std::fs::read_to_string(forked_path.as_path())?.contains("Saved user message"));
|
||||
let meta = read_session_meta_line(forked_path.as_path()).await?;
|
||||
let history_base = meta.meta.history_base.expect("history base");
|
||||
assert_eq!(
|
||||
history_base.thread_id,
|
||||
ThreadId::from_string(conversation_id.as_str())?
|
||||
);
|
||||
|
||||
let turn_id = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: forked_thread_id,
|
||||
input: vec![UserInput::Text {
|
||||
text: "Continue from the fork".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let _: TurnStartResponse = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_id)).await??;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_error_message(RequestId::Integer(fork_id)),
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
let requests = server.received_requests().await.expect("wiremock requests");
|
||||
let response_request = requests
|
||||
.iter()
|
||||
.find(|request| request.url.path().ends_with("/responses"))
|
||||
.expect("forked turn response request");
|
||||
let request_body = response_request.body_json::<Value>()?;
|
||||
let model_input = request_body["input"]
|
||||
.as_array()
|
||||
.expect("response input array");
|
||||
let model_input = serde_json::to_string(model_input)?;
|
||||
assert!(model_input.contains("Saved user message"));
|
||||
assert!(model_input.contains("Continue from the fork"));
|
||||
|
||||
assert_eq!(fork_err.error.code, -32601);
|
||||
// excludeTurns only controls response hydration; it must not change the inherited prefix.
|
||||
let exclude_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: conversation_id,
|
||||
exclude_turns: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let ThreadForkResponse {
|
||||
thread: excluded_turns_thread,
|
||||
..
|
||||
} = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(exclude_id)).await??;
|
||||
assert!(excluded_turns_thread.turns.is_empty());
|
||||
let excluded_turns_path = excluded_turns_thread.path.expect("forked rollout path");
|
||||
let excluded_turns_meta = read_session_meta_line(excluded_turns_path.as_path()).await?;
|
||||
assert_eq!(excluded_turns_meta.meta.history_base, Some(history_base));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_fork_freezes_active_paginated_turn_as_interrupted() -> Result<()> {
|
||||
assert_thread_fork_freezes_active_paginated_turn_as_interrupted(MultiAgentVersion::V1).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_fork_persists_developer_interruption_marker_for_multi_agent_v2() -> Result<()> {
|
||||
assert_thread_fork_freezes_active_paginated_turn_as_interrupted(MultiAgentVersion::V2).await
|
||||
}
|
||||
|
||||
async fn assert_thread_fork_freezes_active_paginated_turn_as_interrupted(
|
||||
multi_agent_version: MultiAgentVersion,
|
||||
) -> Result<()> {
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
let codex_home = TempDir::new()?;
|
||||
let config = MockResponsesConfig::new(&server.uri());
|
||||
let (config, expected_marker_role, thread_source) = match multi_agent_version {
|
||||
MultiAgentVersion::V2 => (
|
||||
config.enable_feature(Feature::MultiAgentV2),
|
||||
"developer",
|
||||
Some(ThreadSource::Subagent),
|
||||
),
|
||||
MultiAgentVersion::V1 => (config, "user", None),
|
||||
MultiAgentVersion::Disabled => unreachable!("interruption markers require agent support"),
|
||||
};
|
||||
config.write(codex_home.path())?;
|
||||
let source_thread_id = create_fake_paginated_rollout(
|
||||
codex_home.path(),
|
||||
"2025-01-05T12-00-00",
|
||||
"2025-01-05T12:00:00Z",
|
||||
"Saved user message",
|
||||
Some("mock_provider"),
|
||||
/*git_info*/ None,
|
||||
)?;
|
||||
let source_path = rollout_path(codex_home.path(), "2025-01-05T12-00-00", &source_thread_id);
|
||||
let source_id = ThreadId::from_string(source_thread_id.as_str())?;
|
||||
let user_response_item = |id: &str| {
|
||||
RolloutItem::ResponseItem(ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputText {
|
||||
text: format!("{id} model input"),
|
||||
}],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
})
|
||||
};
|
||||
let completed_user_item = |id: &str, completed_at_ms| {
|
||||
RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
|
||||
thread_id: source_id,
|
||||
turn_id: "active-turn".to_string(),
|
||||
item: CoreTurnItem::UserMessage(UserMessageItem {
|
||||
id: id.to_string(),
|
||||
client_id: None,
|
||||
content: vec![codex_protocol::user_input::UserInput::Text {
|
||||
text: format!("{id} needle"),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
}),
|
||||
completed_at_ms,
|
||||
}))
|
||||
};
|
||||
append_rollout_item_to_path(
|
||||
source_path.as_path(),
|
||||
&RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent {
|
||||
turn_id: "active-turn".to_string(),
|
||||
trace_id: None,
|
||||
started_at: Some(10),
|
||||
model_context_window: None,
|
||||
collaboration_mode_kind: Default::default(),
|
||||
})),
|
||||
)
|
||||
.await?;
|
||||
append_rollout_item_to_path(source_path.as_path(), &user_response_item("before-fork")).await?;
|
||||
append_rollout_item_to_path(
|
||||
source_path.as_path(),
|
||||
&completed_user_item("before-fork", /*completed_at_ms*/ 1),
|
||||
)
|
||||
.await?;
|
||||
let mut mcp = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.without_auto_env()
|
||||
.build_initialized()
|
||||
.await?;
|
||||
|
||||
let invalid_fork_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: source_thread_id.clone(),
|
||||
last_turn_id: Some("active-turn".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let invalid_fork = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_error_message(RequestId::Integer(invalid_fork_id)),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(
|
||||
fork_err.error.message,
|
||||
"paginated_threads is not supported yet"
|
||||
invalid_fork.error.message,
|
||||
"lastTurnId 'active-turn' identifies an in-progress turn"
|
||||
);
|
||||
|
||||
let ThreadForkResponse {
|
||||
thread: forked_thread,
|
||||
..
|
||||
} = mcp
|
||||
.request(|request_id| ClientRequest::ThreadFork {
|
||||
request_id,
|
||||
params: ThreadForkParams {
|
||||
thread_id: source_thread_id.clone(),
|
||||
thread_source,
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
let forked_thread_id = forked_thread.id.clone();
|
||||
let forked_path = forked_thread.path.expect("forked rollout path");
|
||||
let child_rollout = std::fs::read_to_string(forked_path.as_path())?
|
||||
.lines()
|
||||
.map(serde_json::from_str::<RolloutLine>)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
assert!(matches!(
|
||||
child_rollout.as_slice(),
|
||||
[
|
||||
RolloutLine { item: RolloutItem::SessionMeta(_), .. },
|
||||
RolloutLine {
|
||||
item: RolloutItem::EventMsg(EventMsg::ThreadSettingsApplied(_)),
|
||||
..
|
||||
},
|
||||
RolloutLine {
|
||||
item: RolloutItem::ResponseItem(codex_protocol::models::ResponseItem::Message {
|
||||
role,
|
||||
..
|
||||
}),
|
||||
..
|
||||
},
|
||||
RolloutLine {
|
||||
item: RolloutItem::EventMsg(EventMsg::TurnAborted(aborted)),
|
||||
..
|
||||
},
|
||||
] if role == expected_marker_role && aborted.turn_id.as_deref() == Some("active-turn")
|
||||
));
|
||||
|
||||
append_rollout_item_to_path(source_path.as_path(), &user_response_item("after-fork")).await?;
|
||||
append_rollout_item_to_path(
|
||||
source_path.as_path(),
|
||||
&completed_user_item("after-fork", /*completed_at_ms*/ 2),
|
||||
)
|
||||
.await?;
|
||||
append_rollout_item_to_path(
|
||||
source_path.as_path(),
|
||||
&RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: "active-turn".to_string(),
|
||||
last_agent_message: None,
|
||||
error: None,
|
||||
started_at: Some(10),
|
||||
completed_at: Some(20),
|
||||
duration_ms: Some(10_000),
|
||||
time_to_first_token_ms: None,
|
||||
})),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let ThreadTurnsListResponse { data: turns, .. } = mcp
|
||||
.request(|request_id| ClientRequest::ThreadTurnsList {
|
||||
request_id,
|
||||
params: ThreadTurnsListParams {
|
||||
thread_id: forked_thread_id.clone(),
|
||||
cursor: None,
|
||||
limit: None,
|
||||
sort_direction: None,
|
||||
items_view: None,
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(turns.len(), 1);
|
||||
assert_eq!(turns[0].id, "active-turn");
|
||||
assert_eq!(turns[0].status, TurnStatus::Interrupted);
|
||||
assert_eq!(turns[0].items.len(), 1);
|
||||
assert!(matches!(
|
||||
&turns[0].items[0],
|
||||
ThreadItem::UserMessage { id, .. } if id == "before-fork"
|
||||
));
|
||||
|
||||
let search: ThreadSearchOccurrencesResponse = mcp
|
||||
.request(|request_id| ClientRequest::ThreadSearchOccurrences {
|
||||
request_id,
|
||||
params: ThreadSearchOccurrencesParams {
|
||||
thread_id: forked_thread_id.clone(),
|
||||
search_term: "needle".to_string(),
|
||||
cursor: None,
|
||||
limit: Some(1),
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(search.data.len(), 1);
|
||||
assert_eq!(search.data[0].item_id, "before-fork");
|
||||
assert!(search.next_cursor.is_none());
|
||||
let searched_turns: ThreadTurnsListResponse = mcp
|
||||
.request(|request_id| ClientRequest::ThreadTurnsList {
|
||||
request_id,
|
||||
params: ThreadTurnsListParams {
|
||||
thread_id: forked_thread_id.clone(),
|
||||
cursor: Some(search.data[0].turn_cursor.clone()),
|
||||
limit: Some(1),
|
||||
sort_direction: None,
|
||||
items_view: None,
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(searched_turns.data, turns);
|
||||
|
||||
let ThreadForkResponse {
|
||||
thread: nested_fork,
|
||||
..
|
||||
} = mcp
|
||||
.request(|request_id| ClientRequest::ThreadFork {
|
||||
request_id,
|
||||
params: ThreadForkParams {
|
||||
thread_id: forked_thread_id.clone(),
|
||||
last_turn_id: Some("active-turn".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
let ThreadTurnsListResponse {
|
||||
data: nested_turns, ..
|
||||
} = mcp
|
||||
.request(|request_id| ClientRequest::ThreadTurnsList {
|
||||
request_id,
|
||||
params: ThreadTurnsListParams {
|
||||
thread_id: nested_fork.id,
|
||||
cursor: None,
|
||||
limit: None,
|
||||
sort_direction: None,
|
||||
items_view: None,
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(nested_turns, turns);
|
||||
|
||||
let ThreadForkResponse {
|
||||
thread: nested_before,
|
||||
..
|
||||
} = mcp
|
||||
.request(|request_id| ClientRequest::ThreadFork {
|
||||
request_id,
|
||||
params: ThreadForkParams {
|
||||
thread_id: forked_thread_id.clone(),
|
||||
before_turn_id: Some("active-turn".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert!(nested_before.turns.is_empty());
|
||||
|
||||
drop(mcp);
|
||||
let mut resumed_app_server = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.without_auto_env()
|
||||
.build_initialized()
|
||||
.await?;
|
||||
let ThreadResumeResponse {
|
||||
thread: resumed_thread,
|
||||
..
|
||||
} = resumed_app_server
|
||||
.request(|request_id| ClientRequest::ThreadResume {
|
||||
request_id,
|
||||
params: ThreadResumeParams {
|
||||
thread_id: forked_thread_id,
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
let mut expected_resumed_turns = turns;
|
||||
for turn in &mut expected_resumed_turns {
|
||||
turn.items_view = TurnItemsView::Full;
|
||||
}
|
||||
assert_eq!(resumed_thread.turns, expected_resumed_turns);
|
||||
|
||||
let _: TurnStartResponse = resumed_app_server
|
||||
.request(|request_id| ClientRequest::TurnStart {
|
||||
request_id,
|
||||
params: TurnStartParams {
|
||||
thread_id: resumed_thread.id,
|
||||
input: vec![UserInput::Text {
|
||||
text: "Continue after cold resume".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
resumed_app_server.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
let requests = server.received_requests().await.expect("response requests");
|
||||
let request_body = requests
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|request| request.url.path().ends_with("/responses"))
|
||||
.expect("cold-resumed model request")
|
||||
.body_json::<Value>()?;
|
||||
let model_input = request_body["input"].as_array().expect("model input");
|
||||
assert!(model_input.iter().any(|item| {
|
||||
item["role"] == expected_marker_role
|
||||
&& item["content"].as_array().is_some_and(|content| {
|
||||
content.iter().any(|fragment| {
|
||||
fragment["text"]
|
||||
.as_str()
|
||||
.is_some_and(|text| text.to_ascii_lowercase().contains("interrupt"))
|
||||
})
|
||||
})
|
||||
}));
|
||||
let serialized_input = serde_json::to_string(model_input)?;
|
||||
assert!(serialized_input.contains("Saved user message"));
|
||||
assert!(serialized_input.contains("before-fork model input"));
|
||||
assert!(!serialized_input.contains("after-fork model input"));
|
||||
assert!(serialized_input.contains("Continue after cold resume"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -495,6 +495,7 @@ async fn thread_search_occurrences_reads_paginated_projection() -> Result<()> {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: codex_protocol::protocol::ThreadHistoryMode::Paginated,
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
@@ -642,6 +643,75 @@ async fn thread_search_occurrences_reads_paginated_projection() -> Result<()> {
|
||||
assert_eq!(data[2].snippet_match_range.end, 15);
|
||||
assert_eq!(next_cursor, None);
|
||||
|
||||
let fork_request_id = mcp
|
||||
.send_thread_fork_request(ThreadForkParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let ThreadForkResponse { thread, .. } =
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(fork_request_id)).await??;
|
||||
let forked_thread_id = thread.id;
|
||||
let source_resume_id = mcp
|
||||
.send_thread_resume_request(ThreadResumeParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let _: ThreadResumeResponse =
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(source_resume_id)).await??;
|
||||
for (target_thread_id, text) in [
|
||||
(thread_id.to_string(), "excluded parent needle"),
|
||||
(forked_thread_id.clone(), "child needle"),
|
||||
] {
|
||||
let turn_id = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: target_thread_id,
|
||||
input: vec![UserInput::Text {
|
||||
text: text.to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let _: TurnStartResponse =
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(turn_id)).await??;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
}
|
||||
let request_id = mcp
|
||||
.send_thread_search_occurrences_request(ThreadSearchOccurrencesParams {
|
||||
thread_id: forked_thread_id.clone(),
|
||||
search_term: "needle".to_string(),
|
||||
cursor: None,
|
||||
limit: Some(6),
|
||||
})
|
||||
.await?;
|
||||
let ThreadSearchOccurrencesResponse { data, next_cursor } =
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(request_id)).await??;
|
||||
assert_eq!(data.len(), 6);
|
||||
assert!(
|
||||
data.iter()
|
||||
.all(|occurrence| !occurrence.snippet.contains("excluded parent needle"))
|
||||
);
|
||||
let next_cursor = next_cursor.expect("search should continue into child history");
|
||||
let request_id = mcp
|
||||
.send_thread_search_occurrences_request(ThreadSearchOccurrencesParams {
|
||||
thread_id: forked_thread_id,
|
||||
search_term: "needle".to_string(),
|
||||
cursor: Some(next_cursor),
|
||||
limit: Some(6),
|
||||
})
|
||||
.await?;
|
||||
let ThreadSearchOccurrencesResponse { data, next_cursor } =
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(request_id)).await??;
|
||||
assert_eq!(data.len(), 1);
|
||||
assert!(data[0].snippet.contains("child needle"));
|
||||
assert_eq!(next_cursor, None);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1473,6 +1543,7 @@ async fn paginated_history_lists_use_projected_turns_and_items() -> Result<()> {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: codex_protocol::protocol::ThreadHistoryMode::Paginated,
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
@@ -2161,6 +2232,7 @@ async fn seed_pathless_store_thread(
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
|
||||
@@ -227,6 +227,7 @@ async fn thread_unarchive_preserves_pathless_store_metadata() -> Result<()> {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
|
||||
@@ -49,6 +49,7 @@ use crate::mcp_tool_call::McpToolApprovalMetadata;
|
||||
use crate::mcp_tool_call::build_guardian_mcp_tool_review_request;
|
||||
use crate::mcp_tool_call::is_mcp_tool_approval_question_id;
|
||||
use crate::mcp_tool_call::mcp_approvals_reviewer;
|
||||
use crate::session::ForkPersistence;
|
||||
use crate::session::GitEnrichmentPolicy;
|
||||
use crate::session::SUBMISSION_CHANNEL_CAPACITY;
|
||||
use crate::session::SessionIo;
|
||||
@@ -115,6 +116,7 @@ pub(crate) async fn run_codex_thread_interactive(
|
||||
extensions: Arc::clone(&parent_session.services.extensions),
|
||||
conversation_history,
|
||||
requested_history_mode: None,
|
||||
fork_persistence: ForkPersistence::Copied,
|
||||
session_source: SessionSource::SubAgent(subagent_source.clone()),
|
||||
forked_from_thread_id,
|
||||
parent_thread_id: Some(parent_session.thread_id),
|
||||
|
||||
@@ -115,6 +115,7 @@ use codex_protocol::permissions::NetworkSandboxPolicy;
|
||||
use codex_protocol::protocol::AdditionalContextEntry;
|
||||
use codex_protocol::protocol::FileChange;
|
||||
use codex_protocol::protocol::HasLegacyEvent;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::InterAgentCommunication;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::ItemStartedEvent;
|
||||
@@ -413,6 +414,15 @@ pub(crate) enum GitEnrichmentPolicy {
|
||||
Skip,
|
||||
}
|
||||
|
||||
/// Controls which fork history belongs in the newly created thread's own rollout.
|
||||
pub(crate) enum ForkPersistence {
|
||||
Copied,
|
||||
Referenced {
|
||||
history_base: Option<HistoryPosition>,
|
||||
inherited_item_count: usize,
|
||||
},
|
||||
}
|
||||
|
||||
pub(crate) struct SessionSpawnArgs {
|
||||
pub(crate) config: Config,
|
||||
pub(crate) allow_provider_model_fallback: bool,
|
||||
@@ -428,6 +438,7 @@ pub(crate) struct SessionSpawnArgs {
|
||||
pub(crate) extensions: Arc<codex_extension_api::ExtensionRegistry<crate::config::Config>>,
|
||||
pub(crate) conversation_history: InitialHistory,
|
||||
pub(crate) requested_history_mode: Option<ThreadHistoryMode>,
|
||||
pub(crate) fork_persistence: ForkPersistence,
|
||||
pub(crate) session_source: SessionSource,
|
||||
pub(crate) forked_from_thread_id: Option<ThreadId>,
|
||||
pub(crate) parent_thread_id: Option<ThreadId>,
|
||||
@@ -523,6 +534,7 @@ impl Session {
|
||||
extensions,
|
||||
conversation_history,
|
||||
requested_history_mode,
|
||||
fork_persistence,
|
||||
session_source,
|
||||
forked_from_thread_id,
|
||||
parent_thread_id,
|
||||
@@ -716,6 +728,7 @@ impl Session {
|
||||
tx_event.clone(),
|
||||
agent_status_tx.clone(),
|
||||
conversation_history,
|
||||
fork_persistence,
|
||||
session_source_clone,
|
||||
skills_service,
|
||||
plugins_manager,
|
||||
@@ -1352,16 +1365,29 @@ impl Session {
|
||||
|
||||
let thread_settings_applied =
|
||||
RolloutItem::EventMsg(handlers::thread_settings_applied_event(self).await);
|
||||
if is_paginated_subagent {
|
||||
// Paginated subagents persist inherited model context while creating the live
|
||||
// thread so the copied prefix is not observed as child-owned metadata.
|
||||
self.persist_rollout_items(&[thread_settings_applied]).await;
|
||||
} else {
|
||||
// Keep the copied prefix and the child's effective settings in one append so a
|
||||
// cold resume cannot observe inherited settings as the child's latest value.
|
||||
rollout_items.push(thread_settings_applied);
|
||||
self.persist_rollout_items(&rollout_items).await;
|
||||
match &self.fork_persistence {
|
||||
ForkPersistence::Referenced {
|
||||
inherited_item_count,
|
||||
..
|
||||
} => {
|
||||
// Ancestor records remain behind history_base; only effective child
|
||||
// settings and boundaries synthesized by snapshot processing are local.
|
||||
rollout_items.drain(..*inherited_item_count);
|
||||
rollout_items.insert(0, thread_settings_applied);
|
||||
}
|
||||
ForkPersistence::Copied if is_paginated_subagent => {
|
||||
// Paginated subagents already persist inherited context when their live
|
||||
// thread is created.
|
||||
rollout_items.clear();
|
||||
rollout_items.push(thread_settings_applied);
|
||||
}
|
||||
ForkPersistence::Copied => {
|
||||
// Keep the copied prefix and effective child settings in one append so a
|
||||
// cold resume cannot observe inherited settings as the latest value.
|
||||
rollout_items.push(thread_settings_applied);
|
||||
}
|
||||
}
|
||||
self.persist_rollout_items(&rollout_items).await;
|
||||
|
||||
// Forked threads should remain file-backed immediately after startup.
|
||||
self.ensure_rollout_materialized().await;
|
||||
|
||||
@@ -54,6 +54,7 @@ pub(crate) struct Session {
|
||||
pub(crate) guardian_review_session: GuardianReviewSessionManager,
|
||||
pub(crate) services: SessionServices,
|
||||
pub(super) git_enrichment_policy: GitEnrichmentPolicy,
|
||||
pub(super) fork_persistence: ForkPersistence,
|
||||
pub(super) next_internal_sub_id: AtomicU64,
|
||||
}
|
||||
|
||||
@@ -501,6 +502,7 @@ impl Session {
|
||||
tx_event: Sender<Event>,
|
||||
agent_status: watch::Sender<AgentStatus>,
|
||||
mut initial_history: InitialHistory,
|
||||
fork_persistence: ForkPersistence,
|
||||
session_source: SessionSource,
|
||||
skills_service: Arc<SkillsService>,
|
||||
plugins_manager: Arc<PluginsManager>,
|
||||
@@ -631,6 +633,10 @@ impl Session {
|
||||
selected_capability_roots: selected_capability_roots.clone(),
|
||||
multi_agent_version: initial_multi_agent_version,
|
||||
history_mode: session_configuration.history_mode,
|
||||
history_base: match &fork_persistence {
|
||||
ForkPersistence::Copied => None,
|
||||
ForkPersistence::Referenced { history_base, .. } => *history_base,
|
||||
},
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: initial_auto_compact_window_ids
|
||||
.window_id
|
||||
@@ -646,6 +652,7 @@ impl Session {
|
||||
},
|
||||
};
|
||||
if is_paginated_subagent
|
||||
&& matches!(&fork_persistence, ForkPersistence::Copied)
|
||||
&& let InitialHistory::Forked(items) = &initial_history
|
||||
{
|
||||
LiveThread::create_with_inherited_model_context(
|
||||
@@ -1172,6 +1179,7 @@ impl Session {
|
||||
guardian_review_session: GuardianReviewSessionManager::default(),
|
||||
services,
|
||||
git_enrichment_policy,
|
||||
fork_persistence,
|
||||
next_internal_sub_id: AtomicU64::new(0),
|
||||
});
|
||||
if let Some(network_policy_decider_session) = network_policy_decider_session {
|
||||
@@ -1262,6 +1270,10 @@ impl Session {
|
||||
|
||||
// record_initial_history can emit events. We record only after the SessionConfiguredEvent is emitted.
|
||||
Box::pin(sess.record_initial_history(initial_history)).await;
|
||||
if matches!(&sess.fork_persistence, ForkPersistence::Referenced { .. }) {
|
||||
// Keep the source reserved until the child's history reference is durable.
|
||||
sess.try_ensure_rollout_materialized().await?;
|
||||
}
|
||||
{
|
||||
let mut state = sess.state.lock().await;
|
||||
state.queue_pending_session_start_source(session_start_source);
|
||||
|
||||
@@ -4309,6 +4309,7 @@ async fn attach_thread_persistence(session: &mut Session) -> PathBuf {
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
subagent_history_start_ordinal: None,
|
||||
history_base: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
@@ -5352,6 +5353,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_packaged_zsh() {
|
||||
tx_event,
|
||||
agent_status_tx,
|
||||
InitialHistory::New,
|
||||
ForkPersistence::Copied,
|
||||
SessionSource::Exec,
|
||||
skills_service,
|
||||
plugins_manager,
|
||||
@@ -5640,6 +5642,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
|
||||
guardian_review_session: crate::guardian::GuardianReviewSessionManager::default(),
|
||||
services,
|
||||
git_enrichment_policy: GitEnrichmentPolicy::Fresh,
|
||||
fork_persistence: ForkPersistence::Copied,
|
||||
next_internal_sub_id: AtomicU64::new(0),
|
||||
};
|
||||
|
||||
@@ -5743,6 +5746,7 @@ async fn make_session_with_config_and_rx(
|
||||
tx_event,
|
||||
agent_status_tx,
|
||||
InitialHistory::New,
|
||||
ForkPersistence::Copied,
|
||||
SessionSource::Exec,
|
||||
skills_service,
|
||||
plugins_manager,
|
||||
@@ -5852,6 +5856,7 @@ async fn make_session_with_history_source_and_agent_control_and_rx(
|
||||
tx_event,
|
||||
agent_status_tx,
|
||||
initial_history,
|
||||
ForkPersistence::Copied,
|
||||
session_source,
|
||||
skills_service,
|
||||
plugins_manager,
|
||||
@@ -7120,6 +7125,7 @@ async fn shutdown_complete_does_not_append_to_thread_store_after_shutdown() {
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
subagent_history_start_ordinal: None,
|
||||
history_base: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
@@ -7198,6 +7204,7 @@ async fn submission_loop_channel_close_runs_full_thread_teardown() {
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
subagent_history_start_ordinal: None,
|
||||
history_base: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
@@ -7808,6 +7815,7 @@ where
|
||||
guardian_review_session: crate::guardian::GuardianReviewSessionManager::default(),
|
||||
services,
|
||||
git_enrichment_policy: GitEnrichmentPolicy::Fresh,
|
||||
fork_persistence: ForkPersistence::Copied,
|
||||
next_internal_sub_id: AtomicU64::new(0),
|
||||
});
|
||||
|
||||
@@ -9593,6 +9601,7 @@ async fn attach_in_memory_thread_store(
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
subagent_history_start_ordinal: None,
|
||||
history_base: None,
|
||||
initial_window_id: Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
cwd: Some(config.cwd.to_path_buf()),
|
||||
|
||||
@@ -742,6 +742,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() {
|
||||
extensions: codex_extension_api::empty_extension_registry(),
|
||||
conversation_history: InitialHistory::New,
|
||||
requested_history_mode: None,
|
||||
fork_persistence: ForkPersistence::Copied,
|
||||
session_source: SessionSource::SubAgent(SubAgentSource::Other(
|
||||
GUARDIAN_REVIEWER_NAME.to_string(),
|
||||
)),
|
||||
|
||||
@@ -10,6 +10,7 @@ use crate::environment_selection::TurnEnvironmentSnapshot;
|
||||
use crate::environment_selection::default_thread_environment_selections;
|
||||
use crate::mcp::McpManager;
|
||||
use crate::rollout::truncation;
|
||||
use crate::session::ForkPersistence;
|
||||
use crate::session::GitEnrichmentPolicy;
|
||||
use crate::session::INITIAL_SUBMIT_ID;
|
||||
use crate::session::SessionIo;
|
||||
@@ -69,6 +70,7 @@ use codex_thread_store::InMemoryThreadStore;
|
||||
use codex_thread_store::LoadThreadHistoryParams;
|
||||
use codex_thread_store::LocalThreadStore;
|
||||
use codex_thread_store::LocalThreadStoreConfig;
|
||||
use codex_thread_store::PreparedFork;
|
||||
use codex_thread_store::ReadThreadByRolloutPathParams;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
use codex_thread_store::StoredModelContext;
|
||||
@@ -160,6 +162,12 @@ pub enum ForkSnapshot {
|
||||
Interrupted,
|
||||
}
|
||||
|
||||
struct ForkHistory {
|
||||
snapshot: ForkSnapshot,
|
||||
initial_history: InitialHistory,
|
||||
persistence: ForkPersistence,
|
||||
}
|
||||
|
||||
/// Preserve legacy `fork_thread(usize, ...)` callsites by mapping them to the
|
||||
/// existing truncate-before-nth-user-message snapshot mode.
|
||||
impl From<usize> for ForkSnapshot {
|
||||
@@ -760,6 +768,7 @@ impl ThreadManager {
|
||||
session_source,
|
||||
/*parent_thread_id*/ None,
|
||||
forked_from_thread_id,
|
||||
ForkPersistence::Copied,
|
||||
thread_source,
|
||||
options.dynamic_tools,
|
||||
options.metrics_service_name,
|
||||
@@ -866,6 +875,7 @@ impl ThreadManager {
|
||||
session_source,
|
||||
/*parent_thread_id*/ None,
|
||||
/*forked_from_thread_id*/ None,
|
||||
ForkPersistence::Copied,
|
||||
thread_source,
|
||||
Vec::new(),
|
||||
/*metrics_service_name*/ None,
|
||||
@@ -899,6 +909,7 @@ impl ThreadManager {
|
||||
agent_control,
|
||||
/*parent_thread_id*/ None,
|
||||
/*forked_from_thread_id*/ None,
|
||||
ForkPersistence::Copied,
|
||||
/*thread_source*/ None,
|
||||
Vec::new(),
|
||||
/*metrics_service_name*/ None,
|
||||
@@ -939,6 +950,7 @@ impl ThreadManager {
|
||||
session_source,
|
||||
/*parent_thread_id*/ None,
|
||||
/*forked_from_thread_id*/ None,
|
||||
ForkPersistence::Copied,
|
||||
thread_source,
|
||||
Vec::new(),
|
||||
/*metrics_service_name*/ None,
|
||||
@@ -1071,9 +1083,12 @@ impl ThreadManager {
|
||||
S: Into<ForkSnapshot>,
|
||||
{
|
||||
self.fork_thread_with_initial_history(
|
||||
snapshot.into(),
|
||||
config,
|
||||
history,
|
||||
ForkHistory {
|
||||
snapshot: snapshot.into(),
|
||||
initial_history: history,
|
||||
persistence: ForkPersistence::Copied,
|
||||
},
|
||||
thread_source,
|
||||
parent_trace,
|
||||
supports_openai_form_elicitation,
|
||||
@@ -1081,15 +1096,54 @@ impl ThreadManager {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn fork_thread_with_initial_history(
|
||||
/// Fork prepared reference-backed history using the same snapshot semantics as copied forks.
|
||||
pub async fn fork_prepared_thread(
|
||||
&self,
|
||||
snapshot: ForkSnapshot,
|
||||
config: Config,
|
||||
history: InitialHistory,
|
||||
prepared: PreparedFork,
|
||||
thread_source: Option<ThreadSource>,
|
||||
parent_trace: Option<W3cTraceContext>,
|
||||
supports_openai_form_elicitation: bool,
|
||||
) -> CodexResult<NewThread> {
|
||||
let history = InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id: prepared.source_thread_id,
|
||||
history: Arc::clone(&prepared.model_context),
|
||||
rollout_path: None,
|
||||
});
|
||||
let fork_persistence = ForkPersistence::Referenced {
|
||||
history_base: prepared.history_base,
|
||||
inherited_item_count: prepared.model_context.len(),
|
||||
};
|
||||
let result = self
|
||||
.fork_thread_with_initial_history(
|
||||
config,
|
||||
ForkHistory {
|
||||
snapshot: ForkSnapshot::Interrupted,
|
||||
initial_history: history,
|
||||
persistence: fork_persistence,
|
||||
},
|
||||
thread_source,
|
||||
parent_trace,
|
||||
supports_openai_form_elicitation,
|
||||
)
|
||||
.await;
|
||||
drop(prepared);
|
||||
result
|
||||
}
|
||||
|
||||
async fn fork_thread_with_initial_history(
|
||||
&self,
|
||||
config: Config,
|
||||
fork_history: ForkHistory,
|
||||
thread_source: Option<ThreadSource>,
|
||||
parent_trace: Option<W3cTraceContext>,
|
||||
supports_openai_form_elicitation: bool,
|
||||
) -> CodexResult<NewThread> {
|
||||
let ForkHistory {
|
||||
snapshot,
|
||||
initial_history: history,
|
||||
persistence: fork_persistence,
|
||||
} = fork_history;
|
||||
// `forked_from_id()` describes this history's existing lineage. When
|
||||
// forking a resumed thread, the child copies the resumed thread itself.
|
||||
let source_thread_id = match &history {
|
||||
@@ -1123,6 +1177,7 @@ impl ThreadManager {
|
||||
agent_control,
|
||||
/*parent_thread_id*/ None,
|
||||
source_thread_id,
|
||||
fork_persistence,
|
||||
thread_source,
|
||||
Vec::new(),
|
||||
/*metrics_service_name*/ None,
|
||||
@@ -1469,6 +1524,7 @@ impl ThreadManagerState {
|
||||
session_source,
|
||||
parent_thread_id,
|
||||
forked_from_thread_id,
|
||||
ForkPersistence::Copied,
|
||||
thread_source,
|
||||
Vec::new(),
|
||||
metrics_service_name,
|
||||
@@ -1512,6 +1568,7 @@ impl ThreadManagerState {
|
||||
session_source,
|
||||
parent_thread_id,
|
||||
/*forked_from_thread_id*/ None,
|
||||
ForkPersistence::Copied,
|
||||
thread_source,
|
||||
Vec::new(),
|
||||
/*metrics_service_name*/ None,
|
||||
@@ -1559,6 +1616,7 @@ impl ThreadManagerState {
|
||||
session_source,
|
||||
parent_thread_id,
|
||||
forked_from_thread_id,
|
||||
ForkPersistence::Copied,
|
||||
thread_source,
|
||||
Vec::new(),
|
||||
/*metrics_service_name*/ None,
|
||||
@@ -1583,6 +1641,7 @@ impl ThreadManagerState {
|
||||
agent_control: AgentControl,
|
||||
parent_thread_id: Option<ThreadId>,
|
||||
forked_from_thread_id: Option<ThreadId>,
|
||||
fork_persistence: ForkPersistence,
|
||||
thread_source: Option<ThreadSource>,
|
||||
dynamic_tools: Vec<codex_protocol::dynamic_tools::DynamicToolSpec>,
|
||||
metrics_service_name: Option<String>,
|
||||
@@ -1602,6 +1661,7 @@ impl ThreadManagerState {
|
||||
self.session_source.clone(),
|
||||
parent_thread_id,
|
||||
forked_from_thread_id,
|
||||
fork_persistence,
|
||||
thread_source,
|
||||
dynamic_tools,
|
||||
metrics_service_name,
|
||||
@@ -1628,6 +1688,7 @@ impl ThreadManagerState {
|
||||
session_source: SessionSource,
|
||||
parent_thread_id: Option<ThreadId>,
|
||||
forked_from_thread_id: Option<ThreadId>,
|
||||
fork_persistence: ForkPersistence,
|
||||
thread_source: Option<ThreadSource>,
|
||||
dynamic_tools: Vec<codex_protocol::dynamic_tools::DynamicToolSpec>,
|
||||
metrics_service_name: Option<String>,
|
||||
@@ -1718,6 +1779,7 @@ impl ThreadManagerState {
|
||||
extensions: Arc::clone(&self.extensions),
|
||||
conversation_history: initial_history,
|
||||
requested_history_mode: history_mode,
|
||||
fork_persistence,
|
||||
session_source,
|
||||
forked_from_thread_id,
|
||||
parent_thread_id,
|
||||
|
||||
@@ -11,6 +11,7 @@ use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::ResumedHistory;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_protocol::protocol::ThreadSettingsAppliedEvent;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::responses::ev_completed;
|
||||
@@ -168,6 +169,15 @@ fn thread_settings_applied_item(snapshot: ThreadConfigSnapshot) -> RolloutItem {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn fork_thread_from_history_does_not_require_source_rollout_path() {
|
||||
assert_copied_fork_persists_inherited_history(ThreadHistoryMode::Legacy).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn copied_paginated_fork_persists_inherited_history() {
|
||||
assert_copied_fork_persists_inherited_history(ThreadHistoryMode::Paginated).await;
|
||||
}
|
||||
|
||||
async fn assert_copied_fork_persists_inherited_history(history_mode: ThreadHistoryMode) {
|
||||
skip_if_no_network!();
|
||||
|
||||
let server = MockServer::start().await;
|
||||
@@ -179,11 +189,15 @@ async fn fork_thread_from_history_does_not_require_source_rollout_path() {
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(sse, "text/event-stream"),
|
||||
)
|
||||
.expect(1)
|
||||
.expect(if matches!(history_mode, ThreadHistoryMode::Paginated) {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
})
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let mut builder = test_codex();
|
||||
let mut builder = test_codex().with_history_mode(history_mode);
|
||||
let test = builder.build(&server).await.expect("create conversation");
|
||||
let codex = test.codex.clone();
|
||||
let thread_manager = test.thread_manager.clone();
|
||||
@@ -200,11 +214,16 @@ async fn fork_thread_from_history_does_not_require_source_rollout_path() {
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.expect("submit initial user turn");
|
||||
let _ = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let source_path = codex.rollout_path().expect("source rollout path");
|
||||
let source_items = read_rollout_items(&source_path);
|
||||
let source_meta = codex_rollout::read_session_meta_line(source_path.as_path())
|
||||
.await
|
||||
.expect("read source session metadata");
|
||||
let mut supplied_history = vec![RolloutItem::SessionMeta(source_meta)];
|
||||
supplied_history.extend(source_items.iter().cloned());
|
||||
let NewThread {
|
||||
thread: forked_thread,
|
||||
..
|
||||
@@ -214,7 +233,7 @@ async fn fork_thread_from_history_does_not_require_source_rollout_path() {
|
||||
test.config.clone(),
|
||||
InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id: test.session_configured.thread_id,
|
||||
history: Arc::new(source_items.clone()),
|
||||
history: Arc::new(supplied_history),
|
||||
rollout_path: None,
|
||||
}),
|
||||
/*thread_source*/ None,
|
||||
@@ -228,16 +247,64 @@ async fn fork_thread_from_history_does_not_require_source_rollout_path() {
|
||||
let forked_items = read_rollout_items(&forked_path);
|
||||
let forked_items = forked_items
|
||||
.iter()
|
||||
.map(|item| serde_json::to_value(item).unwrap())
|
||||
.map(|item| serde_json::to_value(item).expect("serialize forked rollout item"))
|
||||
.collect::<Vec<_>>();
|
||||
let source_items = source_items
|
||||
.iter()
|
||||
.map(|item| serde_json::to_value(item).unwrap())
|
||||
.map(|item| serde_json::to_value(item).expect("serialize source rollout item"))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
forked_items.starts_with(&source_items),
|
||||
"forked history should start with the supplied source history"
|
||||
);
|
||||
|
||||
if matches!(history_mode, ThreadHistoryMode::Paginated) {
|
||||
forked_thread
|
||||
.shutdown_and_wait()
|
||||
.await
|
||||
.expect("shutdown copied paginated fork");
|
||||
let resumed_history = codex_rollout::RolloutRecorder::get_rollout_history(&forked_path)
|
||||
.await
|
||||
.expect("load copied paginated fork history");
|
||||
let resumed = thread_manager
|
||||
.resume_thread_with_history(
|
||||
test.config.clone(),
|
||||
resumed_history,
|
||||
codex_core::test_support::auth_manager_from_auth(
|
||||
codex_login::CodexAuth::from_api_key("dummy"),
|
||||
),
|
||||
/*parent_trace*/ None,
|
||||
/*supports_openai_form_elicitation*/ false,
|
||||
)
|
||||
.await
|
||||
.expect("resume copied paginated fork")
|
||||
.thread;
|
||||
resumed
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "continue after cold resume".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
responsesapi_client_metadata: None,
|
||||
additional_context: Default::default(),
|
||||
thread_settings: Default::default(),
|
||||
})
|
||||
.await
|
||||
.expect("start resumed turn");
|
||||
wait_for_event(&resumed, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
||||
let requests = server.received_requests().await.expect("response requests");
|
||||
let input = serde_json::to_string(
|
||||
&requests
|
||||
.last()
|
||||
.expect("resumed model request")
|
||||
.body_json::<serde_json::Value>()
|
||||
.expect("response request body")["input"],
|
||||
)
|
||||
.expect("serialize model input");
|
||||
assert!(input.contains("fork me from stored history"));
|
||||
assert!(input.contains("continue after cold resume"));
|
||||
}
|
||||
}
|
||||
|
||||
fn read_rollout_items(path: &std::path::Path) -> Vec<RolloutItem> {
|
||||
|
||||
@@ -89,20 +89,24 @@ pub(crate) fn materialize_rollout_for_append_blocking(path: &Path) -> io::Result
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let result: io::Result<()> = (|| {
|
||||
let permissions = std::fs::metadata(compressed_path.as_path())?.permissions();
|
||||
let metadata = std::fs::metadata(compressed_path.as_path())?;
|
||||
let permissions = metadata.permissions();
|
||||
let mut output = create_file_with_permissions(temp_path.as_path(), &permissions)?;
|
||||
{
|
||||
let input = File::open(compressed_path.as_path())?;
|
||||
let mut decoder = zstd::stream::read::Decoder::new(input)?;
|
||||
let mut output = create_file_with_permissions(temp_path.as_path(), &permissions)?;
|
||||
io::copy(&mut decoder, &mut output)?;
|
||||
output.flush()?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
output.flush()?;
|
||||
output.sync_all()?;
|
||||
match std::fs::hard_link(temp_path.as_path(), plain_path.as_path()) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(_) => persist_temp_file_noclobber(temp_path.as_path(), plain_path.as_path())?,
|
||||
}
|
||||
output.set_times(std::fs::FileTimes::new().set_modified(metadata.modified()?))?;
|
||||
output.sync_all()?;
|
||||
drop(output);
|
||||
let _ = std::fs::remove_file(temp_path.as_path());
|
||||
match std::fs::remove_file(compressed_path.as_path()) {
|
||||
Ok(()) => {}
|
||||
|
||||
@@ -404,6 +404,8 @@ async fn resume_materializes_compressed_rollout_path() -> anyhow::Result<()> {
|
||||
write_rollout(&rollout_path, thread_id, "hello before resume")?;
|
||||
compress_now(&rollout_path)?;
|
||||
let compressed_path = compressed_rollout_path(&rollout_path);
|
||||
set_old_mtime(compressed_path.as_path())?;
|
||||
let compressed_modified = fs::metadata(compressed_path.as_path())?.modified()?;
|
||||
|
||||
let InitialHistory::Resumed(history) =
|
||||
RolloutRecorder::get_rollout_history(compressed_path.as_path()).await?
|
||||
@@ -421,6 +423,7 @@ async fn resume_materializes_compressed_rollout_path() -> anyhow::Result<()> {
|
||||
assert_eq!(recorder.rollout_path(), rollout_path.as_path());
|
||||
assert!(rollout_path.exists());
|
||||
assert!(!compressed_path.exists());
|
||||
assert!(fs::metadata(rollout_path.as_path())?.modified()? > compressed_modified);
|
||||
recorder
|
||||
.record_canonical_items(&[RolloutItem::EventMsg(EventMsg::UserMessage(
|
||||
UserMessageEvent {
|
||||
|
||||
@@ -39,6 +39,13 @@ pub use compression::existing_rollout_path;
|
||||
pub use compression::open_rollout_line_reader;
|
||||
pub use compression::plain_rollout_path;
|
||||
pub use compression::spawn_rollout_compression_worker;
|
||||
|
||||
/// Materializes a compressed rollout as plain JSONL before another rollout references it.
|
||||
pub async fn materialize_rollout_for_reference(
|
||||
path: &std::path::Path,
|
||||
) -> std::io::Result<std::path::PathBuf> {
|
||||
compression::materialize_rollout_for_append(path).await
|
||||
}
|
||||
pub use config::Config;
|
||||
pub use config::RolloutConfig;
|
||||
pub use config::RolloutConfigView;
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::io::Seek;
|
||||
use std::io::SeekFrom;
|
||||
use std::path::Path;
|
||||
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
@@ -20,10 +21,15 @@ pub(crate) enum RolloutOrdinalState {
|
||||
}
|
||||
|
||||
impl RolloutOrdinalState {
|
||||
pub(crate) fn for_new_rollout(history_mode: ThreadHistoryMode) -> Self {
|
||||
pub(crate) fn for_new_rollout(
|
||||
history_mode: ThreadHistoryMode,
|
||||
history_base: Option<HistoryPosition>,
|
||||
) -> Self {
|
||||
match history_mode {
|
||||
ThreadHistoryMode::Legacy => Self::Legacy,
|
||||
ThreadHistoryMode::Paginated => Self::Paginated { next: Some(0) },
|
||||
ThreadHistoryMode::Paginated => Self::Paginated {
|
||||
next: Some(history_base.map_or(0, |base| base.end_ordinal_exclusive)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,7 @@ use crate::state_db::StateDbHandle;
|
||||
use codex_git_utils::collect_git_info;
|
||||
use codex_git_utils::get_git_repo_root;
|
||||
use codex_protocol::protocol::GitInfo as ProtocolGitInfo;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::InitialHistory;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::ResumedHistory;
|
||||
@@ -103,6 +104,7 @@ pub enum RolloutRecorderParams {
|
||||
selected_capability_roots: Vec<SelectedCapabilityRoot>,
|
||||
multi_agent_version: Option<MultiAgentVersion>,
|
||||
history_mode: ThreadHistoryMode,
|
||||
history_base: Option<HistoryPosition>,
|
||||
subagent_history_start_ordinal: Option<u64>,
|
||||
initial_window_id: Option<String>,
|
||||
},
|
||||
@@ -197,6 +199,7 @@ impl RolloutRecorderParams {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: Default::default(),
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: None,
|
||||
}
|
||||
@@ -247,6 +250,16 @@ impl RolloutRecorderParams {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_history_base(mut self, history_base: Option<HistoryPosition>) -> Self {
|
||||
if let Self::Create {
|
||||
history_base: base, ..
|
||||
} = &mut self
|
||||
{
|
||||
*base = history_base;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_subagent_history_start_ordinal(
|
||||
mut self,
|
||||
subagent_history_start_ordinal: Option<u64>,
|
||||
@@ -797,10 +810,12 @@ impl RolloutRecorder {
|
||||
selected_capability_roots,
|
||||
multi_agent_version,
|
||||
history_mode,
|
||||
history_base,
|
||||
subagent_history_start_ordinal,
|
||||
initial_window_id,
|
||||
} => {
|
||||
let ordinal_state = RolloutOrdinalState::for_new_rollout(history_mode);
|
||||
let ordinal_state =
|
||||
RolloutOrdinalState::for_new_rollout(history_mode, history_base);
|
||||
let log_file_info = precompute_log_file_info(config, conversation_id)?;
|
||||
let path = log_file_info.path.clone();
|
||||
let thread_id = log_file_info.conversation_id;
|
||||
@@ -838,7 +853,7 @@ impl RolloutRecorder {
|
||||
selected_capability_roots,
|
||||
memory_mode: (!config.generate_memories()).then_some("disabled".to_string()),
|
||||
history_mode,
|
||||
history_base: None,
|
||||
history_base,
|
||||
subagent_history_start_ordinal,
|
||||
multi_agent_version,
|
||||
context_window: initial_window_id.map(SessionContextWindow::new),
|
||||
@@ -1564,6 +1579,7 @@ fn precompute_log_file_info(
|
||||
}
|
||||
|
||||
fn open_log_file(path: &Path) -> std::io::Result<File> {
|
||||
let refresh_modified_time = !compression::plain_rollout_path(path).try_exists()?;
|
||||
let path = compression::materialize_rollout_for_append_blocking(path)?;
|
||||
let Some(parent) = path.parent() else {
|
||||
return Err(IoError::other(format!(
|
||||
@@ -1577,6 +1593,9 @@ fn open_log_file(path: &Path) -> std::io::Result<File> {
|
||||
.append(true)
|
||||
.create(true)
|
||||
.open(path)?;
|
||||
if refresh_modified_time {
|
||||
file.set_times(std::fs::FileTimes::new().set_modified(std::time::SystemTime::now()))?;
|
||||
}
|
||||
ensure_rollout_is_newline_terminated(&mut file)?;
|
||||
Ok(file)
|
||||
}
|
||||
@@ -1831,6 +1850,8 @@ pub async fn append_rollout_item_to_path(
|
||||
async fn open_rollout_for_append(
|
||||
path: &Path,
|
||||
) -> std::io::Result<(PathBuf, tokio::fs::File, RolloutOrdinalState)> {
|
||||
let refresh_modified_time =
|
||||
!tokio::fs::try_exists(compression::plain_rollout_path(path)).await?;
|
||||
let path = compression::materialize_rollout_for_append(path).await?;
|
||||
let path_for_open = path.clone();
|
||||
let (file, ordinal_state) = tokio::task::spawn_blocking(move || {
|
||||
@@ -1838,6 +1859,9 @@ async fn open_rollout_for_append(
|
||||
.read(true)
|
||||
.append(true)
|
||||
.open(path_for_open.as_path())?;
|
||||
if refresh_modified_time {
|
||||
file.set_times(std::fs::FileTimes::new().set_modified(std::time::SystemTime::now()))?;
|
||||
}
|
||||
ensure_rollout_is_newline_terminated(&mut file)?;
|
||||
let ordinal_state = ordinal_state_for_rollout(&mut file, path_for_open.as_path())?;
|
||||
Ok::<_, std::io::Error>((file, ordinal_state))
|
||||
|
||||
@@ -9,6 +9,7 @@ use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::AgentMessageEvent;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
@@ -145,6 +146,25 @@ fn append_repair_terminates_nonempty_rollout_tail() -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opening_existing_rollout_preserves_modified_time() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let rollout_path = home.path().join("rollout.jsonl");
|
||||
write_paginated_rollout(&rollout_path, ThreadId::default(), &[])?;
|
||||
let modified = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000);
|
||||
File::options()
|
||||
.write(true)
|
||||
.open(&rollout_path)?
|
||||
.set_times(std::fs::FileTimes::new().set_modified(modified))?;
|
||||
|
||||
drop(open_log_file(&rollout_path)?);
|
||||
assert_eq!(fs::metadata(&rollout_path)?.modified()?, modified);
|
||||
|
||||
drop(open_rollout_for_append(&rollout_path).await?);
|
||||
assert_eq!(fs::metadata(&rollout_path)?.modified()?, modified);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn state_db_init_backfills_before_returning() -> anyhow::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
@@ -590,6 +610,60 @@ async fn recorder_materializes_on_flush_with_pending_items() -> std::io::Result<
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn referenced_paginated_rollout_starts_at_history_cutoff_and_resumes() -> std::io::Result<()>
|
||||
{
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let config = test_config(home.path());
|
||||
let history_base = HistoryPosition {
|
||||
thread_id: ThreadId::new(),
|
||||
end_ordinal_exclusive: 41,
|
||||
end_byte_offset: 1,
|
||||
};
|
||||
let recorder = RolloutRecorder::new(
|
||||
&config,
|
||||
RolloutRecorderParams::new(
|
||||
ThreadId::new(),
|
||||
/*forked_from_id*/ None,
|
||||
/*parent_thread_id*/ None,
|
||||
SessionSource::Exec,
|
||||
/*thread_source*/ None,
|
||||
"test_originator".to_string(),
|
||||
BaseInstructions::default(),
|
||||
Vec::new(),
|
||||
)
|
||||
.with_history_mode(ThreadHistoryMode::Paginated)
|
||||
.with_history_base(Some(history_base)),
|
||||
)
|
||||
.await?;
|
||||
let rollout_path = recorder.rollout_path().to_path_buf();
|
||||
recorder.persist().await?;
|
||||
recorder.shutdown().await?;
|
||||
|
||||
let resumed =
|
||||
RolloutRecorder::new(&config, RolloutRecorderParams::resume(rollout_path.clone())).await?;
|
||||
resumed
|
||||
.record_canonical_items(&[agent_message_item("first child record")])
|
||||
.await?;
|
||||
resumed.flush().await?;
|
||||
resumed.shutdown().await?;
|
||||
|
||||
let resumed =
|
||||
RolloutRecorder::new(&config, RolloutRecorderParams::resume(rollout_path.clone())).await?;
|
||||
resumed
|
||||
.record_canonical_items(&[agent_message_item("second child record")])
|
||||
.await?;
|
||||
resumed.flush().await?;
|
||||
assert_eq!(
|
||||
read_rollout_lines(&rollout_path)?
|
||||
.into_iter()
|
||||
.map(|line| line.ordinal)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![Some(41), Some(42), Some(43)]
|
||||
);
|
||||
resumed.shutdown().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recorder_omits_ordinals_from_legacy_rollouts() -> std::io::Result<()> {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
|
||||
@@ -38,3 +38,4 @@ pretty_assertions = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
uuid = { workspace = true }
|
||||
zstd = { workspace = true }
|
||||
|
||||
@@ -135,6 +135,7 @@ mod tests {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: ThreadHistoryMode::Legacy,
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: uuid::Uuid::now_v7().to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
@@ -371,6 +372,7 @@ mod tests {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode,
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: uuid::Uuid::now_v7().to_string(),
|
||||
metadata: thread_metadata(),
|
||||
@@ -488,7 +490,7 @@ impl InMemoryThreadStore {
|
||||
memory_mode: matches!(params.metadata.memory_mode, ThreadMemoryMode::Disabled)
|
||||
.then_some("disabled".to_string()),
|
||||
history_mode: params.history_mode,
|
||||
history_base: None,
|
||||
history_base: params.history_base,
|
||||
subagent_history_start_ordinal: params.subagent_history_start_ordinal,
|
||||
multi_agent_version: params.multi_agent_version,
|
||||
context_window: Some(SessionContextWindow::new(params.initial_window_id.clone())),
|
||||
|
||||
@@ -30,6 +30,7 @@ pub use types::CreateThreadParams;
|
||||
pub use types::DeleteThreadParams;
|
||||
pub use types::DeleteThreadsParams;
|
||||
pub use types::ExtraConfig;
|
||||
pub use types::ForkBoundary;
|
||||
pub use types::GitInfoPatch;
|
||||
pub use types::ItemPage;
|
||||
pub use types::ItemSortKey;
|
||||
@@ -37,6 +38,8 @@ pub use types::ListItemsParams;
|
||||
pub use types::ListThreadsParams;
|
||||
pub use types::ListTurnsParams;
|
||||
pub use types::LoadThreadHistoryParams;
|
||||
pub use types::PrepareForkParams;
|
||||
pub use types::PreparedFork;
|
||||
pub use types::ReadThreadByRolloutPathParams;
|
||||
pub use types::ReadThreadParams;
|
||||
pub use types::ResumeThreadParams;
|
||||
|
||||
@@ -22,6 +22,10 @@ pub(super) async fn archive_threads(
|
||||
lock_thread_ids.extend(thread_ids.iter().copied());
|
||||
lock_thread_ids.sort_unstable_by_key(ToString::to_string);
|
||||
lock_thread_ids.dedup();
|
||||
let mut _lifecycle_guards = Vec::with_capacity(lock_thread_ids.len());
|
||||
for thread_id in &lock_thread_ids {
|
||||
_lifecycle_guards.push(store.live_writer_locks.lock_lifecycle(*thread_id).await);
|
||||
}
|
||||
let mut _live_writer_guards = Vec::with_capacity(lock_thread_ids.len());
|
||||
for thread_id in &lock_thread_ids {
|
||||
_live_writer_guards.push(store.live_writer_locks.lock(*thread_id).await);
|
||||
@@ -109,6 +113,8 @@ pub(super) async fn archive_thread(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
@@ -128,6 +134,40 @@ mod tests {
|
||||
use crate::local::test_support::write_session_file;
|
||||
use crate::local::test_support::write_session_file_with_history_mode;
|
||||
|
||||
#[tokio::test]
|
||||
async fn archive_waits_for_fork_reservation_without_holding_writer_lock() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let uuid = Uuid::from_u128(205);
|
||||
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id");
|
||||
let active_path = write_session_file_with_history_mode(
|
||||
home.path(),
|
||||
"2025-01-03T12-00-00",
|
||||
uuid,
|
||||
ThreadHistoryMode::Paginated,
|
||||
)
|
||||
.expect("session file");
|
||||
let reservation = store.live_writer_locks.reserve_lifecycle(thread_id).await;
|
||||
let mut archive = Box::pin(store.archive_thread(ArchiveThreadParams { thread_id }));
|
||||
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut archive => panic!("archive completed while the source was reserved: {result:?}"),
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
let writer_guard = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
store.live_writer_locks.lock(thread_id),
|
||||
)
|
||||
.await
|
||||
.expect("pending archive should not hold the writer lock");
|
||||
drop(writer_guard);
|
||||
drop(reservation);
|
||||
|
||||
archive.await.expect("archive reserved thread");
|
||||
assert!(!active_path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn archive_threads_rejects_owned_descendants_before_archiving_anything() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
|
||||
@@ -41,6 +41,7 @@ pub(super) async fn create_thread(
|
||||
.with_selected_capability_roots(params.selected_capability_roots)
|
||||
.with_multi_agent_version(params.multi_agent_version)
|
||||
.with_history_mode(params.history_mode)
|
||||
.with_history_base(params.history_base)
|
||||
.with_subagent_history_start_ordinal(params.subagent_history_start_ordinal)
|
||||
.with_initial_window_id(params.initial_window_id),
|
||||
)
|
||||
|
||||
@@ -29,6 +29,7 @@ pub(super) async fn delete_thread(
|
||||
params: DeleteThreadParams,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let thread_id = params.thread_id;
|
||||
let _lifecycle_guard = store.live_writer_locks.lock_lifecycle(thread_id).await;
|
||||
let _live_writer_guard = store.live_writer_locks.lock(thread_id).await;
|
||||
let reference_index = scan_reference_index(store).await?;
|
||||
if reference_index.reference_count(thread_id) > 0 {
|
||||
@@ -50,6 +51,10 @@ pub(super) async fn delete_threads(
|
||||
let deletion_set: HashSet<_> = thread_ids.iter().copied().collect();
|
||||
let mut lock_thread_ids: Vec<_> = deletion_set.iter().copied().collect();
|
||||
lock_thread_ids.sort_unstable_by_key(ToString::to_string);
|
||||
let mut _lifecycle_guards = Vec::with_capacity(lock_thread_ids.len());
|
||||
for thread_id in &lock_thread_ids {
|
||||
_lifecycle_guards.push(store.live_writer_locks.lock_lifecycle(*thread_id).await);
|
||||
}
|
||||
let mut _live_writer_guards = Vec::with_capacity(lock_thread_ids.len());
|
||||
for &thread_id in &lock_thread_ids {
|
||||
_live_writer_guards.push(store.live_writer_locks.lock(thread_id).await);
|
||||
|
||||
@@ -5,6 +5,7 @@ mod helpers;
|
||||
mod list_threads;
|
||||
mod live_writer;
|
||||
mod model_context;
|
||||
mod paginated_fork;
|
||||
mod read_thread;
|
||||
// This lands before the reader PRs that consume the shared lineage resolver.
|
||||
#[allow(dead_code)]
|
||||
@@ -31,6 +32,9 @@ use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio::sync::OwnedMutexGuard;
|
||||
use tokio::sync::OwnedRwLockReadGuard;
|
||||
use tokio::sync::OwnedRwLockWriteGuard;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::AppendThreadItemsParams;
|
||||
use crate::ArchiveThreadParams;
|
||||
@@ -43,6 +47,8 @@ use crate::ListItemsParams;
|
||||
use crate::ListThreadsParams;
|
||||
use crate::ListTurnsParams;
|
||||
use crate::LoadThreadHistoryParams;
|
||||
use crate::PrepareForkParams;
|
||||
use crate::PreparedFork;
|
||||
use crate::ReadThreadByRolloutPathParams;
|
||||
use crate::ReadThreadParams;
|
||||
use crate::ResumeThreadParams;
|
||||
@@ -99,19 +105,55 @@ struct LiveRecorderEntry {
|
||||
struct LiveWriterLocks {
|
||||
// Keep per-thread locks after a writer goes idle. Removing one while another caller is about
|
||||
// to acquire it could let two operations for the same thread run at once.
|
||||
by_thread: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
|
||||
by_thread: Mutex<HashMap<ThreadId, Arc<ThreadCoordination>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ThreadCoordination {
|
||||
// Serialize writes and capture consistent fork snapshots.
|
||||
writer: Arc<Mutex<()>>,
|
||||
// Forks hold a shared lease until their child reference is durable; deletion, archive, and
|
||||
// unarchive require exclusive access. Keeping this separate from `writer` lets the source
|
||||
// accept writes during child initialization, including MCP startup that can take 30 seconds.
|
||||
// Operations that need both locks must acquire `lifecycle` before `writer`.
|
||||
lifecycle: Arc<RwLock<()>>,
|
||||
}
|
||||
|
||||
impl LiveWriterLocks {
|
||||
async fn lock(&self, thread_id: ThreadId) -> OwnedMutexGuard<()> {
|
||||
let lock = self
|
||||
.by_thread
|
||||
async fn coordination(&self, thread_id: ThreadId) -> Arc<ThreadCoordination> {
|
||||
self.by_thread
|
||||
.lock()
|
||||
.await
|
||||
.entry(thread_id)
|
||||
.or_insert_with(|| Arc::new(Mutex::new(())))
|
||||
.clone();
|
||||
lock.lock_owned().await
|
||||
.or_default()
|
||||
.clone()
|
||||
}
|
||||
|
||||
async fn lock(&self, thread_id: ThreadId) -> OwnedMutexGuard<()> {
|
||||
self.coordination(thread_id)
|
||||
.await
|
||||
.writer
|
||||
.clone()
|
||||
.lock_owned()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_lifecycle(&self, thread_id: ThreadId) -> OwnedRwLockReadGuard<()> {
|
||||
self.coordination(thread_id)
|
||||
.await
|
||||
.lifecycle
|
||||
.clone()
|
||||
.read_owned()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn lock_lifecycle(&self, thread_id: ThreadId) -> OwnedRwLockWriteGuard<()> {
|
||||
self.coordination(thread_id)
|
||||
.await
|
||||
.lifecycle
|
||||
.clone()
|
||||
.write_owned()
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -388,6 +430,10 @@ impl ThreadStore for LocalThreadStore {
|
||||
Box::pin(async move { model_context::load_latest_model_context(self, params).await })
|
||||
}
|
||||
|
||||
fn prepare_fork(&self, params: PrepareForkParams) -> ThreadStoreFuture<'_, PreparedFork> {
|
||||
Box::pin(async move { paginated_fork::prepare(self, params).await })
|
||||
}
|
||||
|
||||
fn read_thread(&self, params: ReadThreadParams) -> ThreadStoreFuture<'_, StoredThread> {
|
||||
Box::pin(async move { read_thread::read_thread(self, params).await })
|
||||
}
|
||||
@@ -1313,6 +1359,15 @@ mod tests {
|
||||
RolloutItem::EventMsg(EventMsg::UserMessage(event)) if event.message == "Hello from user"
|
||||
)
|
||||
}));
|
||||
|
||||
let error = store
|
||||
.prepare_fork(PrepareForkParams {
|
||||
thread_id,
|
||||
boundary: crate::ForkBoundary::Latest,
|
||||
})
|
||||
.await
|
||||
.expect_err("external rollouts cannot be referenced by thread id");
|
||||
assert!(error.to_string().contains("must be in Codex home"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1575,6 +1630,7 @@ mod tests {
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: ThreadHistoryMode::Legacy,
|
||||
history_base: None,
|
||||
subagent_history_start_ordinal: None,
|
||||
initial_window_id: uuid::Uuid::now_v7().to_string(),
|
||||
metadata: thread_metadata(),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
@@ -75,6 +76,35 @@ pub(super) async fn load_latest_model_context(
|
||||
})
|
||||
}
|
||||
|
||||
/// Loads startup context from a fork's frozen inherited prefix.
|
||||
pub(super) async fn load_for_fork(
|
||||
lineage: RolloutLineage,
|
||||
history_base: Option<HistoryPosition>,
|
||||
) -> ThreadStoreResult<Vec<RolloutItem>> {
|
||||
let source_path = lineage
|
||||
.segments()
|
||||
.last()
|
||||
.map(|segment| segment.rollout_path.as_path())
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: "fork lineage has no source segment".to_string(),
|
||||
})?;
|
||||
let session_meta = codex_rollout::read_session_meta_line(source_path)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"failed to read session metadata {}: {err}",
|
||||
source_path.display()
|
||||
),
|
||||
})?;
|
||||
match history_base {
|
||||
Some(history_base) => {
|
||||
let lineage = lineage.truncate_at(history_base).await?;
|
||||
scan_model_context_from_lineage(lineage, session_meta).await
|
||||
}
|
||||
None => Ok(vec![RolloutItem::SessionMeta(session_meta)]),
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_model_context_from_lineage(
|
||||
lineage: RolloutLineage,
|
||||
session_meta: SessionMetaLine,
|
||||
|
||||
@@ -81,6 +81,44 @@ async fn loads_latest_checkpoint_with_required_turn_metadata() {
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fork_context_excludes_items_after_frozen_cutoff() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let uuid = Uuid::from_u128(/*v*/ 1007);
|
||||
let thread_id = ThreadId::from_string(&uuid.to_string()).expect("thread id");
|
||||
let path = write_ordinaled_paginated_rollout(
|
||||
home.path(),
|
||||
"2025-01-03T13-00-06",
|
||||
uuid,
|
||||
[turn_started("frozen-turn"), user_message("frozen message")],
|
||||
);
|
||||
let history_base =
|
||||
history_position(path.as_path(), thread_id, /*end_ordinal_exclusive*/ 3);
|
||||
append_items(path.as_path(), [user_message("later message")]);
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let lineage = store
|
||||
.resolve_rollout_lineage(thread_id)
|
||||
.await
|
||||
.expect("resolve source lineage");
|
||||
let session_meta = codex_rollout::read_session_meta_line(path.as_path())
|
||||
.await
|
||||
.expect("read source metadata");
|
||||
|
||||
let context = load_for_fork(lineage, Some(history_base))
|
||||
.await
|
||||
.expect("load frozen fork context");
|
||||
|
||||
let expected = vec![
|
||||
RolloutItem::SessionMeta(session_meta),
|
||||
turn_started("frozen-turn"),
|
||||
user_message("frozen message"),
|
||||
];
|
||||
assert_eq!(
|
||||
serde_json::to_value(context).expect("serialize fork context"),
|
||||
serde_json::to_value(expected).expect("serialize expected fork context")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn loads_turn_metadata_across_an_older_checkpoint() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::LocalThreadStore;
|
||||
use super::live_writer;
|
||||
use super::model_context;
|
||||
use super::thread_history::find_source_turn;
|
||||
use super::thread_history::find_visible_turn;
|
||||
use crate::ForkBoundary;
|
||||
use crate::PrepareForkParams;
|
||||
use crate::PreparedFork;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
pub(super) async fn prepare(
|
||||
store: &LocalThreadStore,
|
||||
params: PrepareForkParams,
|
||||
) -> ThreadStoreResult<PreparedFork> {
|
||||
let PrepareForkParams {
|
||||
thread_id,
|
||||
boundary,
|
||||
} = params;
|
||||
let source_reservation = store.live_writer_locks.reserve_lifecycle(thread_id).await;
|
||||
// Keep the source reserved until persistence and lineage materialization finish, even if the
|
||||
// caller cancels fork preparation.
|
||||
let lineage_store = store.clone();
|
||||
let (lineage, source_reservation) = tokio::spawn(async move {
|
||||
match live_writer::persist_thread(&lineage_store, thread_id).await {
|
||||
Ok(()) | Err(ThreadStoreError::ThreadNotFound { .. }) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let lineage = lineage_store
|
||||
.resolve_rollout_lineage_for_reference(thread_id)
|
||||
.await?;
|
||||
Ok::<_, ThreadStoreError>((lineage, source_reservation))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to resolve fork lineage: {err}"),
|
||||
})??;
|
||||
let source_segment = lineage
|
||||
.segments()
|
||||
.last()
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: "fork lineage has no source segment".to_string(),
|
||||
})?;
|
||||
if !matches!(boundary, ForkBoundary::Latest) {
|
||||
for segment in lineage
|
||||
.segments()
|
||||
.iter()
|
||||
.take(lineage.segments().len().saturating_sub(1))
|
||||
{
|
||||
let _ancestor_writer_guard = store.live_writer_locks.lock(segment.thread_id()).await;
|
||||
super::thread_history_materialization::materialize_to_sqlite(
|
||||
store,
|
||||
segment.thread_id(),
|
||||
segment.rollout_path.as_path(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
let source_writer_guard = store.live_writer_locks.lock(thread_id).await;
|
||||
super::thread_history_materialization::materialize_to_sqlite(
|
||||
store,
|
||||
thread_id,
|
||||
source_segment.rollout_path.as_path(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let latest_projection_state = super::thread_history::projection_state(store, thread_id)
|
||||
.await?
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: format!("missing projection state for paginated thread {thread_id}"),
|
||||
})?;
|
||||
let latest_position = HistoryPosition {
|
||||
thread_id,
|
||||
end_ordinal_exclusive: latest_projection_state.next_ordinal,
|
||||
end_byte_offset: latest_projection_state.next_byte_offset,
|
||||
};
|
||||
let pool = store.thread_history_db().await?;
|
||||
let position = match boundary {
|
||||
ForkBoundary::Latest => latest_position,
|
||||
ForkBoundary::ThroughTurn(turn_id) => {
|
||||
let row = find_visible_turn(pool, &lineage, turn_id.as_str()).await?;
|
||||
if row.status == "inProgress" {
|
||||
return Err(ThreadStoreError::InvalidRequest {
|
||||
message: format!("lastTurnId '{turn_id}' identifies an in-progress turn"),
|
||||
});
|
||||
}
|
||||
let rollout_end_ordinal = row
|
||||
.rollout_end_ordinal
|
||||
.ok_or_else(|| missing_turn_position(turn_id.as_str()))?;
|
||||
let rollout_end_byte_offset = row
|
||||
.rollout_end_byte_offset
|
||||
.ok_or_else(|| missing_turn_position(turn_id.as_str()))?;
|
||||
HistoryPosition {
|
||||
thread_id: row.physical_thread_id,
|
||||
end_ordinal_exclusive: u64::try_from(rollout_end_ordinal)
|
||||
.map_err(|_| invalid_turn_position(turn_id.as_str()))?
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| invalid_turn_position(turn_id.as_str()))?,
|
||||
end_byte_offset: u64::try_from(rollout_end_byte_offset)
|
||||
.map_err(|_| invalid_turn_position(turn_id.as_str()))?,
|
||||
}
|
||||
}
|
||||
ForkBoundary::BeforeTurn(turn_id) => {
|
||||
let row = find_source_turn(pool, &lineage, turn_id.as_str()).await?;
|
||||
if row.rollout_end_ordinal == Some(row.rollout_ordinal) {
|
||||
return Err(ThreadStoreError::InvalidRequest {
|
||||
message: format!("turn {turn_id} does not have a persisted start boundary"),
|
||||
});
|
||||
}
|
||||
let rollout_byte_offset = row
|
||||
.rollout_byte_offset
|
||||
.ok_or_else(|| missing_turn_position(turn_id.as_str()))?;
|
||||
HistoryPosition {
|
||||
thread_id: row.physical_thread_id,
|
||||
end_ordinal_exclusive: u64::try_from(row.rollout_ordinal)
|
||||
.map_err(|_| invalid_turn_position(turn_id.as_str()))?,
|
||||
end_byte_offset: u64::try_from(rollout_byte_offset)
|
||||
.map_err(|_| invalid_turn_position(turn_id.as_str()))?,
|
||||
}
|
||||
}
|
||||
};
|
||||
let segment_index = lineage
|
||||
.segments()
|
||||
.iter()
|
||||
.position(|segment| segment.thread_id() == position.thread_id)
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: "fork position is outside the source lineage".to_string(),
|
||||
})?;
|
||||
if lineage.segments()[segment_index].end.is_some_and(|end| {
|
||||
position.end_ordinal_exclusive > end.end_ordinal_exclusive
|
||||
|| position.end_byte_offset > end.end_byte_offset
|
||||
}) {
|
||||
return Err(ThreadStoreError::InvalidRequest {
|
||||
message: "fork boundary exceeds inherited source history".to_string(),
|
||||
});
|
||||
}
|
||||
let history_base =
|
||||
if position.end_ordinal_exclusive == lineage.segments()[segment_index].start_ordinal() {
|
||||
segment_index
|
||||
.checked_sub(1)
|
||||
.and_then(|index| lineage.segments()[index].end)
|
||||
} else {
|
||||
Some(position)
|
||||
};
|
||||
drop(source_writer_guard);
|
||||
let model_context = Arc::new(model_context::load_for_fork(lineage, history_base).await?);
|
||||
|
||||
Ok(PreparedFork::new(
|
||||
thread_id,
|
||||
history_base,
|
||||
model_context,
|
||||
source_reservation,
|
||||
))
|
||||
}
|
||||
|
||||
fn missing_turn_position(turn_id: &str) -> ThreadStoreError {
|
||||
ThreadStoreError::InvalidRequest {
|
||||
message: format!("turn {turn_id} does not have persisted rollout positions"),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_turn_position(turn_id: &str) -> ThreadStoreError {
|
||||
ThreadStoreError::Internal {
|
||||
message: format!("invalid rollout position for turn {turn_id}"),
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ use crate::ThreadStoreResult;
|
||||
pub(super) struct RolloutLineageSegment {
|
||||
pub(super) thread_id: ThreadId,
|
||||
pub(super) rollout_path: PathBuf,
|
||||
pub(super) start_ordinal: u64,
|
||||
pub(super) end: Option<HistoryPosition>,
|
||||
}
|
||||
|
||||
@@ -32,6 +33,29 @@ impl LocalThreadStore {
|
||||
pub(super) async fn resolve_rollout_lineage(
|
||||
&self,
|
||||
requested_thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
self.resolve_rollout_lineage_with_representation(
|
||||
requested_thread_id,
|
||||
LineageRepresentation::Existing,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_rollout_lineage_for_reference(
|
||||
&self,
|
||||
requested_thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
self.resolve_rollout_lineage_with_representation(
|
||||
requested_thread_id,
|
||||
LineageRepresentation::PlainForReference,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn resolve_rollout_lineage_with_representation(
|
||||
&self,
|
||||
requested_thread_id: ThreadId,
|
||||
representation: LineageRepresentation,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
let mut segments = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
@@ -42,10 +66,34 @@ impl LocalThreadStore {
|
||||
if !seen.insert(thread_id) {
|
||||
return Err(malformed_lineage(requested_thread_id, "cycle detected"));
|
||||
}
|
||||
let _writer_guard = match representation {
|
||||
LineageRepresentation::Existing => None,
|
||||
LineageRepresentation::PlainForReference => {
|
||||
Some(self.live_writer_locks.lock(thread_id).await)
|
||||
}
|
||||
};
|
||||
let rollout_path =
|
||||
read_thread::resolve_rollout_path(self, thread_id, /*include_archived*/ true)
|
||||
.await?
|
||||
.ok_or_else(|| malformed_lineage(thread_id, "missing source rollout"))?;
|
||||
let rollout_path = match representation {
|
||||
LineageRepresentation::Existing => rollout_path,
|
||||
LineageRepresentation::PlainForReference => {
|
||||
let rollout_path = super::helpers::scoped_rollout_path(
|
||||
self.config.codex_home.clone(),
|
||||
rollout_path.as_path(),
|
||||
"Codex home",
|
||||
)?;
|
||||
codex_rollout::materialize_rollout_for_reference(rollout_path.as_path())
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"failed to materialize referenced rollout {}: {err}",
|
||||
rollout_path.display()
|
||||
),
|
||||
})?
|
||||
}
|
||||
};
|
||||
let meta = codex_rollout::read_session_meta_line(rollout_path.as_path())
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
@@ -69,9 +117,16 @@ impl LocalThreadStore {
|
||||
if let Some(end) = end {
|
||||
validate_cutoff_bounds(requested_thread_id, rollout_path.as_path(), &end).await?;
|
||||
}
|
||||
let start_ordinal = match meta.meta.history_base {
|
||||
Some(base) => base.end_ordinal_exclusive.checked_add(1).ok_or_else(|| {
|
||||
malformed_lineage(requested_thread_id, "source ordinal overflow")
|
||||
})?,
|
||||
None => 1,
|
||||
};
|
||||
segments.push(RolloutLineageSegment {
|
||||
thread_id,
|
||||
rollout_path,
|
||||
start_ordinal,
|
||||
end,
|
||||
});
|
||||
|
||||
@@ -85,27 +140,50 @@ impl LocalThreadStore {
|
||||
segments.reverse();
|
||||
Ok(RolloutLineage { segments })
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_rollout_lineage_at(
|
||||
&self,
|
||||
end: HistoryPosition,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
let mut lineage = self.resolve_rollout_lineage(end.thread_id).await?;
|
||||
let Some(segment) = lineage.segments.last_mut() else {
|
||||
return Err(ThreadStoreError::Internal {
|
||||
message: "rollout lineage has no segments".to_string(),
|
||||
});
|
||||
};
|
||||
validate_cutoff_bounds(end.thread_id, segment.rollout_path.as_path(), &end).await?;
|
||||
segment.end = Some(end);
|
||||
Ok(lineage)
|
||||
}
|
||||
#[derive(Clone, Copy)]
|
||||
enum LineageRepresentation {
|
||||
Existing,
|
||||
PlainForReference,
|
||||
}
|
||||
|
||||
impl RolloutLineage {
|
||||
pub(super) fn segments(&self) -> &[RolloutLineageSegment] {
|
||||
self.segments.as_slice()
|
||||
}
|
||||
|
||||
pub(super) fn segment_index_for_ordinal(&self, ordinal: u64) -> Option<usize> {
|
||||
self.segments.iter().position(|segment| {
|
||||
ordinal >= segment.start_ordinal()
|
||||
&& segment
|
||||
.end_ordinal()
|
||||
.is_none_or(|end_ordinal| ordinal < end_ordinal)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn truncate_at(
|
||||
mut self,
|
||||
end: HistoryPosition,
|
||||
) -> ThreadStoreResult<RolloutLineage> {
|
||||
let segment_index = self
|
||||
.segments
|
||||
.iter()
|
||||
.position(|segment| segment.thread_id == end.thread_id)
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: "fork position is outside the source lineage".to_string(),
|
||||
})?;
|
||||
self.segments.truncate(segment_index + 1);
|
||||
let segment = self
|
||||
.segments
|
||||
.last_mut()
|
||||
.ok_or_else(|| ThreadStoreError::Internal {
|
||||
message: "rollout lineage has no segments".to_string(),
|
||||
})?;
|
||||
validate_cutoff_bounds(end.thread_id, segment.rollout_path.as_path(), &end).await?;
|
||||
segment.end = Some(end);
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl RolloutLineageSegment {
|
||||
@@ -114,7 +192,7 @@ impl RolloutLineageSegment {
|
||||
}
|
||||
|
||||
pub(super) fn start_ordinal(&self) -> u64 {
|
||||
1
|
||||
self.start_ordinal
|
||||
}
|
||||
|
||||
pub(super) fn end_ordinal(&self) -> Option<u64> {
|
||||
|
||||
@@ -33,7 +33,7 @@ async fn resolves_nested_lineage_with_empty_intermediate_segments() {
|
||||
let middle_end = history_position(
|
||||
middle_path.as_path(),
|
||||
middle,
|
||||
/*end_ordinal_exclusive*/ 1,
|
||||
/*end_ordinal_exclusive*/ 5,
|
||||
);
|
||||
let child_path = write_rollout(
|
||||
home.path(),
|
||||
@@ -53,16 +53,19 @@ async fn resolves_nested_lineage_with_empty_intermediate_segments() {
|
||||
RolloutLineageSegment {
|
||||
thread_id: root,
|
||||
rollout_path: root_path.clone(),
|
||||
start_ordinal: 1,
|
||||
end: Some(root_end),
|
||||
},
|
||||
RolloutLineageSegment {
|
||||
thread_id: middle,
|
||||
rollout_path: middle_path.clone(),
|
||||
start_ordinal: 5,
|
||||
end: Some(middle_end),
|
||||
},
|
||||
RolloutLineageSegment {
|
||||
thread_id: child,
|
||||
rollout_path: child_path,
|
||||
start_ordinal: 6,
|
||||
end: None,
|
||||
},
|
||||
]
|
||||
@@ -117,11 +120,14 @@ async fn resolves_lineage_at_explicit_history_position() {
|
||||
let end = history_position(
|
||||
child_path.as_path(),
|
||||
child,
|
||||
/*end_ordinal_exclusive*/ 2,
|
||||
/*end_ordinal_exclusive*/ 6,
|
||||
);
|
||||
|
||||
let lineage = store
|
||||
.resolve_rollout_lineage_at(end)
|
||||
.resolve_rollout_lineage(child)
|
||||
.await
|
||||
.expect("resolve child lineage")
|
||||
.truncate_at(end)
|
||||
.await
|
||||
.expect("resolve explicit position");
|
||||
|
||||
@@ -131,11 +137,13 @@ async fn resolves_lineage_at_explicit_history_position() {
|
||||
RolloutLineageSegment {
|
||||
thread_id: root,
|
||||
rollout_path: root_path.clone(),
|
||||
start_ordinal: 1,
|
||||
end: Some(root_end),
|
||||
},
|
||||
RolloutLineageSegment {
|
||||
thread_id: child,
|
||||
rollout_path: child_path.clone(),
|
||||
start_ordinal: 5,
|
||||
end: Some(end),
|
||||
},
|
||||
]
|
||||
@@ -235,8 +243,9 @@ fn write_rollout_under(
|
||||
) -> std::path::PathBuf {
|
||||
fs::create_dir_all(directory.as_path()).expect("create rollout directory");
|
||||
let path = directory.join(format!("rollout-2026-07-16T00-00-00-{thread_id}.jsonl"));
|
||||
let initial_ordinal = history_base.map_or(0, |base| base.end_ordinal_exclusive);
|
||||
let mut lines = vec![rollout_line(
|
||||
/*ordinal*/ 0,
|
||||
initial_ordinal,
|
||||
RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
session_id: thread_id.into(),
|
||||
@@ -248,7 +257,10 @@ fn write_rollout_under(
|
||||
git: None,
|
||||
}),
|
||||
)];
|
||||
for ordinal in 1..next_ordinal {
|
||||
for offset in 1..next_ordinal {
|
||||
let ordinal = initial_ordinal
|
||||
.checked_add(offset)
|
||||
.expect("fixture ordinal");
|
||||
lines.push(rollout_line(
|
||||
ordinal,
|
||||
RolloutItem::EventMsg(codex_protocol::protocol::EventMsg::ShutdownComplete),
|
||||
@@ -280,11 +292,16 @@ fn history_position(
|
||||
}
|
||||
|
||||
fn rollout_end_byte_offset(path: &Path, end_ordinal_exclusive: u64) -> u64 {
|
||||
let line_count = usize::try_from(end_ordinal_exclusive).expect("ordinal fits usize");
|
||||
let bytes = fs::read(path).expect("read rollout");
|
||||
let end_byte_offset = bytes
|
||||
.split_inclusive(|byte| *byte == b'\n')
|
||||
.take(line_count)
|
||||
.take_while(|line| {
|
||||
serde_json::from_slice::<RolloutLine>(line)
|
||||
.expect("parse rollout fixture")
|
||||
.ordinal
|
||||
.expect("paginated rollout ordinal")
|
||||
< end_ordinal_exclusive
|
||||
})
|
||||
.map(<[u8]>::len)
|
||||
.sum::<usize>();
|
||||
u64::try_from(end_byte_offset).expect("rollout byte offset fits u64")
|
||||
|
||||
@@ -11,10 +11,13 @@ use crate::ThreadStoreResult;
|
||||
mod read;
|
||||
mod search;
|
||||
mod segment_paging;
|
||||
mod turn_lookup;
|
||||
|
||||
pub(super) use read::list_items;
|
||||
pub(super) use read::list_turns;
|
||||
pub(super) use search::search_thread_occurrences;
|
||||
pub(super) use turn_lookup::find_source_turn;
|
||||
pub(super) use turn_lookup::find_visible_turn;
|
||||
|
||||
/// A valid complete rollout line with its absolute byte span in durable JSONL.
|
||||
///
|
||||
@@ -28,30 +31,55 @@ pub(super) struct ProjectedRolloutLine {
|
||||
pub changes: ThreadHistoryChangeSet,
|
||||
}
|
||||
|
||||
pub(super) async fn next_rollout_byte_offset(
|
||||
pub(super) struct RolloutProjectionState {
|
||||
pub next_byte_offset: u64,
|
||||
pub next_ordinal: u64,
|
||||
}
|
||||
|
||||
pub(super) async fn projection_state(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
) -> ThreadStoreResult<u64> {
|
||||
) -> ThreadStoreResult<Option<RolloutProjectionState>> {
|
||||
let db_path = store.config.sqlite.thread_history_db_path();
|
||||
if !tokio::fs::try_exists(db_path.as_path())
|
||||
.await
|
||||
.map_err(thread_history_error)?
|
||||
{
|
||||
return Ok(0);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let pool = store.thread_history_db().await?;
|
||||
let offset = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT next_rollout_byte_offset FROM thread_history_projection_state WHERE thread_id = ?",
|
||||
let state = sqlx::query_as::<_, (i64, i64)>(
|
||||
r#"
|
||||
SELECT next_rollout_byte_offset, next_rollout_ordinal
|
||||
FROM thread_history_projection_state
|
||||
WHERE thread_id = ?
|
||||
"#,
|
||||
)
|
||||
.bind(thread_id.to_string())
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(thread_history_error)?
|
||||
.unwrap_or(0);
|
||||
u64::try_from(offset).map_err(|_| ThreadStoreError::Internal {
|
||||
message: format!("thread history projection for {thread_id} has a negative byte offset"),
|
||||
})
|
||||
.map_err(thread_history_error)?;
|
||||
state
|
||||
.map(|(next_byte_offset, next_ordinal)| {
|
||||
Ok(RolloutProjectionState {
|
||||
next_byte_offset: u64::try_from(next_byte_offset).map_err(|_| {
|
||||
ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"thread history projection for {thread_id} has a negative byte offset"
|
||||
),
|
||||
}
|
||||
})?,
|
||||
next_ordinal: u64::try_from(next_ordinal).map_err(|_| {
|
||||
ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"thread history projection for {thread_id} has a negative ordinal"
|
||||
),
|
||||
}
|
||||
})?,
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(super) async fn apply_projection(
|
||||
@@ -59,6 +87,7 @@ pub(super) async fn apply_projection(
|
||||
thread_id: ThreadId,
|
||||
start_offset: u64,
|
||||
next_offset: u64,
|
||||
initial_ordinal: u64,
|
||||
projections: Vec<ProjectedRolloutLine>,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let pool = store.thread_history_db().await?;
|
||||
@@ -81,7 +110,8 @@ WHERE thread_id = ?
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(thread_history_error)?;
|
||||
let (expected_offset, mut next_ordinal) = projection_state.unwrap_or((0, 0));
|
||||
let (expected_offset, mut next_ordinal) =
|
||||
projection_state.unwrap_or((0, sqlite_integer(initial_ordinal, "rollout ordinal")?));
|
||||
let start_offset = sqlite_integer(start_offset, "rollout byte offset")?;
|
||||
if expected_offset != start_offset {
|
||||
return Err(ThreadStoreError::Internal {
|
||||
@@ -311,9 +341,9 @@ WHERE thread_id = ?
|
||||
for item in changes.changed_items {
|
||||
let item_id = item.item.id().to_string();
|
||||
let item_json = serde_json::to_string(&item.item).map_err(thread_history_error)?;
|
||||
// The same item can appear again with a newer snapshot. Replace its JSON, but keep the
|
||||
// creation ordinal and timestamp from the first record so item identity and age stay
|
||||
// stable. Track the latest snapshot separately for incremental replay.
|
||||
// Completed items are immutable: local producers emit ItemCompleted exactly once per
|
||||
// item. Tolerate an unexpected duplicate defensively so it cannot poison materialization,
|
||||
// preserving the original creation ordinal and timestamp while updating its snapshot.
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO thread_items (
|
||||
|
||||
@@ -5,9 +5,12 @@ use serde::Serialize;
|
||||
use sqlx::Row;
|
||||
|
||||
use super::super::LocalThreadStore;
|
||||
use super::super::rollout_lineage::RolloutLineage;
|
||||
use super::segment_paging::page_item_rows;
|
||||
use super::segment_paging::page_turn_rows;
|
||||
use super::segment_paging::validate_page_size;
|
||||
use super::sqlite_integer;
|
||||
use super::turn_lookup::find_source_turn;
|
||||
use crate::ItemPage;
|
||||
use crate::ListItemsParams;
|
||||
use crate::ListTurnsParams;
|
||||
@@ -36,7 +39,6 @@ pub(super) enum CursorScope {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct HistoryCursor {
|
||||
pub requested_thread_id: ThreadId,
|
||||
pub physical_thread_id: ThreadId,
|
||||
pub rollout_ordinal: u64,
|
||||
pub include_anchor: bool,
|
||||
pub scope: CursorScope,
|
||||
@@ -92,7 +94,7 @@ pub(in crate::local) async fn list_turns(
|
||||
for turn in page.rows {
|
||||
let items = match params.items_view {
|
||||
StoredTurnItemsView::NotLoaded => Vec::new(),
|
||||
StoredTurnItemsView::Summary => load_summary_items(pool, &turn).await?,
|
||||
StoredTurnItemsView::Summary => load_summary_items(pool, &lineage, &turn).await?,
|
||||
};
|
||||
turns.push(StoredTurn {
|
||||
turn_id: turn.turn_id,
|
||||
@@ -168,22 +170,58 @@ pub(super) async fn validate_thread_for_paginated_reads(
|
||||
|
||||
async fn load_summary_items(
|
||||
pool: &sqlx::SqlitePool,
|
||||
lineage: &RolloutLineage,
|
||||
turn: &StoredTurnRow,
|
||||
) -> ThreadStoreResult<Vec<StoredThreadItem>> {
|
||||
let (item_owner, first_user_item_id, final_agent_item_id) =
|
||||
if matches!(turn.status, StoredTurnStatus::Interrupted)
|
||||
&& turn.first_user_item_id.is_none()
|
||||
&& turn.final_agent_item_id.is_none()
|
||||
{
|
||||
let source = find_source_turn(pool, lineage, turn.turn_id.as_str()).await?;
|
||||
(
|
||||
source.physical_thread_id,
|
||||
source.first_user_item_id,
|
||||
source.final_agent_item_id,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
turn.position.physical_thread_id,
|
||||
turn.first_user_item_id.clone(),
|
||||
turn.final_agent_item_id.clone(),
|
||||
)
|
||||
};
|
||||
let Some(segment) = lineage
|
||||
.segments()
|
||||
.iter()
|
||||
.find(|segment| segment.thread_id() == item_owner)
|
||||
else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let start_ordinal = sqlite_integer(segment.start_ordinal(), "rollout ordinal")?;
|
||||
let end_ordinal = segment
|
||||
.end_ordinal()
|
||||
.map(|ordinal| sqlite_integer(ordinal, "rollout ordinal"))
|
||||
.transpose()?;
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT turn_id, item_id, updated_at_ordinal, created_at_ms, item_json
|
||||
FROM thread_items
|
||||
WHERE thread_id = ?
|
||||
AND turn_id = ?
|
||||
AND rollout_ordinal >= ?
|
||||
AND (? IS NULL OR rollout_ordinal < ?)
|
||||
AND (item_id = ? OR item_id = ?)
|
||||
ORDER BY rollout_ordinal ASC
|
||||
"#,
|
||||
)
|
||||
.bind(turn.position.physical_thread_id.to_string())
|
||||
.bind(item_owner.to_string())
|
||||
.bind(turn.turn_id.as_str())
|
||||
.bind(turn.first_user_item_id.as_deref())
|
||||
.bind(turn.final_agent_item_id.as_deref())
|
||||
.bind(start_ordinal)
|
||||
.bind(end_ordinal)
|
||||
.bind(end_ordinal)
|
||||
.bind(first_user_item_id.as_deref())
|
||||
.bind(final_agent_item_id.as_deref())
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(super::thread_history_error)?;
|
||||
@@ -209,14 +247,13 @@ pub(super) fn parse_cursor(
|
||||
pub(super) fn serialize_cursor(
|
||||
requested_thread_id: ThreadId,
|
||||
scope: CursorScope,
|
||||
position: PhysicalHistoryPosition,
|
||||
rollout_ordinal: i64,
|
||||
include_anchor: bool,
|
||||
) -> ThreadStoreResult<String> {
|
||||
let rollout_ordinal = u64::try_from(position.rollout_ordinal)
|
||||
.map_err(|_| invalid_cursor("negative rollout ordinal"))?;
|
||||
let rollout_ordinal =
|
||||
u64::try_from(rollout_ordinal).map_err(|_| invalid_cursor("negative rollout ordinal"))?;
|
||||
serde_json::to_string(&HistoryCursor {
|
||||
requested_thread_id,
|
||||
physical_thread_id: position.physical_thread_id,
|
||||
rollout_ordinal,
|
||||
include_anchor,
|
||||
scope,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fs;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
@@ -16,6 +17,7 @@ use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::ItemSortKey;
|
||||
use crate::SearchThreadOccurrencesParams;
|
||||
use crate::SortDirection;
|
||||
use crate::StoredTurnError;
|
||||
use crate::StoredTurnStatus;
|
||||
@@ -441,7 +443,7 @@ async fn lineage_reads_page_across_parent_and_child_segments() {
|
||||
(root_id, "root-1", 1, Some("root-user"), Some("root-agent")),
|
||||
(root_id, "root-2", 4, None, None),
|
||||
(root_id, "excluded-root", 6, None, None),
|
||||
(child_id, "child-1", 1, None, None),
|
||||
(child_id, "child-1", 7, None, None),
|
||||
] {
|
||||
insert_turn(
|
||||
db,
|
||||
@@ -460,7 +462,7 @@ async fn lineage_reads_page_across_parent_and_child_segments() {
|
||||
(root_id, "root-1", "root-agent", 3),
|
||||
(root_id, "root-2", "root-2-item", 5),
|
||||
(root_id, "excluded-root", "excluded-item", 7),
|
||||
(child_id, "child-1", "child-item", 2),
|
||||
(child_id, "child-1", "child-item", 8),
|
||||
] {
|
||||
insert_item(db, thread_id, turn_id, item_id, ordinal).await;
|
||||
}
|
||||
@@ -568,6 +570,65 @@ async fn lineage_reads_page_across_parent_and_child_segments() {
|
||||
assert!(matches!(error, ThreadStoreError::InvalidRequest { .. }));
|
||||
}
|
||||
|
||||
let first_occurrences = store
|
||||
.search_thread_occurrences(SearchThreadOccurrencesParams {
|
||||
thread_id: child_id,
|
||||
search_term: "item".to_string(),
|
||||
cursor: None,
|
||||
page_size: 2,
|
||||
})
|
||||
.await
|
||||
.expect("first inherited occurrence page");
|
||||
assert_eq!(
|
||||
first_occurrences
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| item.item_id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["root-agent", "root-2-item"]
|
||||
);
|
||||
let inherited_turn = store
|
||||
.list_turns(turn_params(
|
||||
child_id,
|
||||
Some(first_occurrences.items[0].turn_cursor.clone()),
|
||||
/*page_size*/ 1,
|
||||
SortDirection::Asc,
|
||||
StoredTurnItemsView::NotLoaded,
|
||||
))
|
||||
.await
|
||||
.expect("navigate to inherited occurrence turn");
|
||||
assert_eq!(turn_ids(&inherited_turn), vec!["root-1"]);
|
||||
let child_occurrences = store
|
||||
.search_thread_occurrences(SearchThreadOccurrencesParams {
|
||||
thread_id: child_id,
|
||||
search_term: "item".to_string(),
|
||||
cursor: first_occurrences.next_cursor,
|
||||
page_size: 2,
|
||||
})
|
||||
.await
|
||||
.expect("continue inherited occurrence search");
|
||||
assert_eq!(child_occurrences.items[0].item_id, "child-item");
|
||||
assert_eq!(child_occurrences.next_cursor, None);
|
||||
|
||||
let gap_cursor = serde_json::to_string(&HistoryCursor {
|
||||
requested_thread_id: child_id,
|
||||
rollout_ordinal: 6,
|
||||
include_anchor: true,
|
||||
scope: CursorScope::Turns,
|
||||
})
|
||||
.expect("serialize cursor in metadata gap");
|
||||
let error = store
|
||||
.list_turns(turn_params(
|
||||
child_id,
|
||||
Some(gap_cursor),
|
||||
/*page_size*/ 1,
|
||||
SortDirection::Asc,
|
||||
StoredTurnItemsView::NotLoaded,
|
||||
))
|
||||
.await
|
||||
.expect_err("cursor cannot point to segment metadata");
|
||||
assert!(matches!(error, ThreadStoreError::InvalidRequest { .. }));
|
||||
|
||||
let (_other_home, other_store, other_thread_id) =
|
||||
store_with_mode(ThreadHistoryMode::Paginated).await;
|
||||
let error = other_store
|
||||
@@ -583,6 +644,62 @@ async fn lineage_reads_page_across_parent_and_child_segments() {
|
||||
assert!(matches!(error, ThreadStoreError::InvalidRequest { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inherited_search_excludes_turns_created_after_the_fork() {
|
||||
let (home, store, child_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
|
||||
let source_id = ThreadId::default();
|
||||
let source_path = rollout_path(home.path(), source_id);
|
||||
write_rollout_with_end(
|
||||
source_path.as_path(),
|
||||
source_id,
|
||||
/*history_base*/ None,
|
||||
/*next_ordinal*/ 5,
|
||||
);
|
||||
write_rollout_with_end(
|
||||
rollout_path(home.path(), child_id).as_path(),
|
||||
child_id,
|
||||
Some(history_position(
|
||||
source_path.as_path(),
|
||||
source_id,
|
||||
/*end_ordinal_exclusive*/ 3,
|
||||
)),
|
||||
/*next_ordinal*/ 2,
|
||||
);
|
||||
let db = history_db(&store).await;
|
||||
insert_turn(
|
||||
db,
|
||||
source_id,
|
||||
"hidden-turn",
|
||||
/*rollout_ordinal*/ 3,
|
||||
"completed",
|
||||
/*error_json*/ None,
|
||||
/*first_user_item_id*/ Some("hidden-item"),
|
||||
/*final_agent_item_id*/ None,
|
||||
)
|
||||
.await;
|
||||
insert_item(
|
||||
db,
|
||||
source_id,
|
||||
"hidden-turn",
|
||||
"hidden-item",
|
||||
/*rollout_ordinal*/ 2,
|
||||
)
|
||||
.await;
|
||||
|
||||
let occurrences = store
|
||||
.search_thread_occurrences(SearchThreadOccurrencesParams {
|
||||
thread_id: child_id,
|
||||
search_term: "hidden".to_string(),
|
||||
cursor: None,
|
||||
page_size: 1,
|
||||
})
|
||||
.await
|
||||
.expect("search inherited history");
|
||||
|
||||
assert!(occurrences.items.is_empty());
|
||||
assert_eq!(occurrences.next_cursor, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lineage_reads_nested_forks() {
|
||||
let (home, store, child_id) = store_with_mode(ThreadHistoryMode::Paginated).await;
|
||||
@@ -593,7 +710,7 @@ async fn lineage_reads_nested_forks() {
|
||||
root_path.as_path(),
|
||||
root_id,
|
||||
/*history_base*/ None,
|
||||
/*next_ordinal*/ 3,
|
||||
/*next_ordinal*/ 5,
|
||||
);
|
||||
let middle_path = rollout_path(home.path(), middle_id);
|
||||
write_rollout_with_end(
|
||||
@@ -602,9 +719,9 @@ async fn lineage_reads_nested_forks() {
|
||||
Some(history_position(
|
||||
root_path.as_path(),
|
||||
root_id,
|
||||
/*end_ordinal_exclusive*/ 3,
|
||||
/*end_ordinal_exclusive*/ 4,
|
||||
)),
|
||||
/*next_ordinal*/ 2,
|
||||
/*next_ordinal*/ 3,
|
||||
);
|
||||
write_rollout_with_end(
|
||||
rollout_path(home.path(), child_id).as_path(),
|
||||
@@ -612,28 +729,46 @@ async fn lineage_reads_nested_forks() {
|
||||
Some(history_position(
|
||||
middle_path.as_path(),
|
||||
middle_id,
|
||||
/*end_ordinal_exclusive*/ 2,
|
||||
/*end_ordinal_exclusive*/ 7,
|
||||
)),
|
||||
/*next_ordinal*/ 2,
|
||||
);
|
||||
let db = history_db(&store).await;
|
||||
for (thread_id, turn_id) in [
|
||||
(root_id, "root"),
|
||||
(middle_id, "middle"),
|
||||
(child_id, "child"),
|
||||
for (thread_id, turn_id, ordinal, status, first_user_item_id) in [
|
||||
(root_id, "root", 1, "completed", None),
|
||||
(root_id, "shared", 2, "completed", Some("before-fork")),
|
||||
(middle_id, "shared", 5, "interrupted", None),
|
||||
(middle_id, "middle", 6, "completed", None),
|
||||
(child_id, "child", 8, "completed", None),
|
||||
] {
|
||||
insert_turn(
|
||||
db,
|
||||
thread_id,
|
||||
turn_id,
|
||||
/*rollout_ordinal*/ 1,
|
||||
"completed",
|
||||
ordinal,
|
||||
status,
|
||||
/*error_json*/ None,
|
||||
/*first_user_item_id*/ None,
|
||||
first_user_item_id,
|
||||
/*final_agent_item_id*/ None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
insert_item(
|
||||
db,
|
||||
root_id,
|
||||
"shared",
|
||||
"before-fork",
|
||||
/*rollout_ordinal*/ 3,
|
||||
)
|
||||
.await;
|
||||
insert_item(
|
||||
db,
|
||||
root_id,
|
||||
"shared",
|
||||
"after-fork",
|
||||
/*rollout_ordinal*/ 4,
|
||||
)
|
||||
.await;
|
||||
|
||||
let first_descending_page = store
|
||||
.list_turns(turn_params(
|
||||
@@ -652,11 +787,66 @@ async fn lineage_reads_nested_forks() {
|
||||
first_descending_page.next_cursor,
|
||||
/*page_size*/ 2,
|
||||
SortDirection::Desc,
|
||||
StoredTurnItemsView::NotLoaded,
|
||||
StoredTurnItemsView::Summary,
|
||||
))
|
||||
.await
|
||||
.expect("second nested descending page");
|
||||
assert_eq!(turn_ids(&second_descending_page), vec!["root"]);
|
||||
assert_eq!(turn_ids(&second_descending_page), vec!["shared", "root"]);
|
||||
assert_eq!(
|
||||
second_descending_page.turns[0].status,
|
||||
StoredTurnStatus::Interrupted
|
||||
);
|
||||
assert_eq!(
|
||||
second_descending_page.turns[0].items,
|
||||
vec![expected_item(
|
||||
"shared",
|
||||
"before-fork",
|
||||
/*rollout_ordinal*/ 3
|
||||
)]
|
||||
);
|
||||
|
||||
let ascending_page = store
|
||||
.list_turns(turn_params(
|
||||
child_id,
|
||||
/*cursor*/ None,
|
||||
/*page_size*/ 2,
|
||||
SortDirection::Asc,
|
||||
StoredTurnItemsView::Summary,
|
||||
))
|
||||
.await
|
||||
.expect("first nested ascending page");
|
||||
assert_eq!(turn_ids(&ascending_page), vec!["root", "shared"]);
|
||||
|
||||
let mut held_connections = Vec::new();
|
||||
for _ in 0..4 {
|
||||
held_connections.push(db.acquire().await.expect("hold history connection"));
|
||||
}
|
||||
let occurrences = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
store.search_thread_occurrences(SearchThreadOccurrencesParams {
|
||||
thread_id: child_id,
|
||||
search_term: "o".to_string(),
|
||||
cursor: None,
|
||||
page_size: 1,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("inherited search releases its row connection")
|
||||
.expect("search inherited occurrence");
|
||||
assert_eq!(occurrences.items[0].item_id, "before-fork");
|
||||
assert!(occurrences.next_cursor.is_some());
|
||||
|
||||
let occurrence_turn = store
|
||||
.list_turns(turn_params(
|
||||
child_id,
|
||||
Some(occurrences.items[0].turn_cursor.clone()),
|
||||
/*page_size*/ 1,
|
||||
SortDirection::Asc,
|
||||
StoredTurnItemsView::NotLoaded,
|
||||
))
|
||||
.await
|
||||
.expect("navigate to effective occurrence turn");
|
||||
assert_eq!(turn_ids(&occurrence_turn), vec!["shared"]);
|
||||
}
|
||||
|
||||
async fn store_with_mode(history_mode: ThreadHistoryMode) -> (TempDir, LocalThreadStore, ThreadId) {
|
||||
@@ -707,9 +897,10 @@ fn write_rollout_with_end(
|
||||
next_ordinal: u64,
|
||||
) {
|
||||
fs::create_dir_all(path.parent().expect("rollout parent")).expect("create rollout parent");
|
||||
let initial_ordinal = history_base.map_or(0, |base| base.end_ordinal_exclusive);
|
||||
let mut lines = vec![RolloutLine {
|
||||
timestamp: "2026-07-16T00:00:00.000Z".to_string(),
|
||||
ordinal: Some(0),
|
||||
ordinal: Some(initial_ordinal),
|
||||
item: RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
session_id: thread_id.into(),
|
||||
@@ -721,7 +912,10 @@ fn write_rollout_with_end(
|
||||
git: None,
|
||||
}),
|
||||
}];
|
||||
for ordinal in 1..next_ordinal {
|
||||
for offset in 1..next_ordinal {
|
||||
let ordinal = initial_ordinal
|
||||
.checked_add(offset)
|
||||
.expect("fixture ordinal");
|
||||
lines.push(RolloutLine {
|
||||
timestamp: "2026-07-16T00:00:00.000Z".to_string(),
|
||||
ordinal: Some(ordinal),
|
||||
@@ -761,11 +955,16 @@ fn history_position(
|
||||
}
|
||||
|
||||
fn rollout_end_byte_offset(path: &std::path::Path, end_ordinal_exclusive: u64) -> u64 {
|
||||
let line_count = usize::try_from(end_ordinal_exclusive).expect("ordinal fits usize");
|
||||
let bytes = fs::read(path).expect("read rollout");
|
||||
let end_byte_offset = bytes
|
||||
.split_inclusive(|byte| *byte == b'\n')
|
||||
.take(line_count)
|
||||
.take_while(|line| {
|
||||
serde_json::from_slice::<RolloutLine>(line)
|
||||
.expect("parse rollout fixture")
|
||||
.ordinal
|
||||
.expect("paginated rollout ordinal")
|
||||
< end_ordinal_exclusive
|
||||
})
|
||||
.map(<[u8]>::len)
|
||||
.sum::<usize>();
|
||||
u64::try_from(end_byte_offset).expect("rollout byte offset fits u64")
|
||||
@@ -821,8 +1020,9 @@ async fn insert_item(
|
||||
item_id: &str,
|
||||
rollout_ordinal: i64,
|
||||
) {
|
||||
let (item_type, item_json) = fixture_item(item_id);
|
||||
sqlx::query(
|
||||
"INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, updated_at_ordinal, created_at_ms, item_json) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
"INSERT INTO thread_items (thread_id, turn_id, item_id, rollout_ordinal, updated_at_ordinal, created_at_ms, item_type, item_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind(thread_id.to_string())
|
||||
.bind(turn_id)
|
||||
@@ -830,7 +1030,8 @@ async fn insert_item(
|
||||
.bind(rollout_ordinal)
|
||||
.bind(rollout_ordinal)
|
||||
.bind(rollout_ordinal * 1_000)
|
||||
.bind(format!(r#"{{"type":"userMessage","id":"{item_id}","content":[]}}"#))
|
||||
.bind(item_type)
|
||||
.bind(item_json)
|
||||
.execute(db)
|
||||
.await
|
||||
.expect("insert item fixture");
|
||||
@@ -892,8 +1093,23 @@ fn expected_item(turn_id: &str, item_id: &str, rollout_ordinal: u64) -> StoredTh
|
||||
item_id: item_id.to_string(),
|
||||
updated_at_ordinal: rollout_ordinal,
|
||||
created_at_ms: i64::try_from(rollout_ordinal).expect("fixture ordinal fits i64") * 1_000,
|
||||
item_json: format!(r#"{{"type":"userMessage","id":"{item_id}","content":[]}}"#)
|
||||
.into_bytes(),
|
||||
item_json: fixture_item(item_id).1.into_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_item(item_id: &str) -> (&'static str, String) {
|
||||
if item_id.contains("agent") {
|
||||
(
|
||||
"agentMessage",
|
||||
format!(r#"{{"type":"agentMessage","id":"{item_id}","text":"{item_id} item"}}"#),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"userMessage",
|
||||
format!(
|
||||
r#"{{"type":"userMessage","id":"{item_id}","content":[{{"type":"text","text":"{item_id}"}}]}}"#
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
@@ -14,10 +15,11 @@ use sqlx::Row;
|
||||
|
||||
use super::super::LocalThreadStore;
|
||||
use super::read::CursorScope;
|
||||
use super::read::PhysicalHistoryPosition;
|
||||
use super::read::serialize_cursor;
|
||||
use super::read::validate_thread_for_paginated_reads;
|
||||
use super::sqlite_integer;
|
||||
use super::thread_history_error;
|
||||
use super::turn_lookup::find_visible_turn;
|
||||
use crate::SearchTextRange;
|
||||
use crate::SearchThreadOccurrencesParams;
|
||||
use crate::StoredThreadOccurrence;
|
||||
@@ -71,13 +73,43 @@ pub(in crate::local) async fn search_thread_occurrences(
|
||||
params.thread_id,
|
||||
¶ms.search_term,
|
||||
)?;
|
||||
let next_rollout_ordinal = cursor
|
||||
.as_ref()
|
||||
.map_or(0, |cursor| cursor.next_rollout_ordinal);
|
||||
let matcher = LiteralMatcher::new(params.search_term.as_str());
|
||||
let lineage = store.resolve_rollout_lineage(params.thread_id).await?;
|
||||
let cursor_segment = cursor
|
||||
.as_ref()
|
||||
.map(|cursor| {
|
||||
lineage
|
||||
.segment_index_for_ordinal(
|
||||
u64::try_from(cursor.next_rollout_ordinal)
|
||||
.map_err(|_| invalid_cursor("negative rollout ordinal"))?,
|
||||
)
|
||||
.ok_or_else(|| invalid_cursor("position outside thread lineage"))
|
||||
})
|
||||
.transpose()?;
|
||||
let pool = store.thread_history_db().await?;
|
||||
let mut rows = sqlx::query(
|
||||
r#"
|
||||
let mut items = Vec::with_capacity(params.page_size);
|
||||
let mut effective_turn_ordinals = HashMap::new();
|
||||
for (segment_index, segment) in lineage
|
||||
.segments()
|
||||
.iter()
|
||||
.enumerate()
|
||||
.skip(cursor_segment.unwrap_or(0))
|
||||
{
|
||||
let segment_start_ordinal = sqlite_integer(segment.start_ordinal(), "rollout ordinal")?;
|
||||
let next_rollout_ordinal = if Some(segment_index) == cursor_segment {
|
||||
cursor
|
||||
.as_ref()
|
||||
.map_or(0, |cursor| cursor.next_rollout_ordinal)
|
||||
} else {
|
||||
segment_start_ordinal
|
||||
};
|
||||
let end_rollout_ordinal = segment
|
||||
.end_ordinal()
|
||||
.map(|ordinal| sqlite_integer(ordinal, "rollout ordinal"))
|
||||
.transpose()?
|
||||
.unwrap_or(i64::MAX);
|
||||
let mut rows = sqlx::query(
|
||||
r#"
|
||||
SELECT turn_id, item_id, rollout_ordinal, item_json, turn_rollout_ordinal
|
||||
FROM (
|
||||
SELECT
|
||||
@@ -93,6 +125,9 @@ FROM (
|
||||
WHERE items.thread_id = ?
|
||||
AND items.item_type = 'userMessage'
|
||||
AND items.rollout_ordinal >= ?
|
||||
AND items.rollout_ordinal < ?
|
||||
AND turns.rollout_ordinal >= ?
|
||||
AND turns.rollout_ordinal < ?
|
||||
|
||||
UNION ALL
|
||||
|
||||
@@ -110,71 +145,107 @@ FROM (
|
||||
WHERE turns.thread_id = ?
|
||||
AND turns.final_agent_item_id IS NOT NULL
|
||||
AND items.rollout_ordinal >= ?
|
||||
AND items.rollout_ordinal < ?
|
||||
AND turns.rollout_ordinal >= ?
|
||||
AND turns.rollout_ordinal < ?
|
||||
)
|
||||
ORDER BY rollout_ordinal ASC
|
||||
"#,
|
||||
)
|
||||
.bind(params.thread_id.to_string())
|
||||
.bind(next_rollout_ordinal)
|
||||
.bind(params.thread_id.to_string())
|
||||
.bind(next_rollout_ordinal)
|
||||
.fetch(pool);
|
||||
)
|
||||
.bind(segment.thread_id().to_string())
|
||||
.bind(next_rollout_ordinal)
|
||||
.bind(end_rollout_ordinal)
|
||||
.bind(segment_start_ordinal)
|
||||
.bind(end_rollout_ordinal)
|
||||
.bind(segment.thread_id().to_string())
|
||||
.bind(next_rollout_ordinal)
|
||||
.bind(end_rollout_ordinal)
|
||||
.bind(segment_start_ordinal)
|
||||
.bind(end_rollout_ordinal)
|
||||
.fetch(pool);
|
||||
|
||||
let mut items = Vec::with_capacity(params.page_size);
|
||||
while let Some(row) = rows.try_next().await.map_err(thread_history_error)? {
|
||||
let row = candidate_row(row)?;
|
||||
let item = serde_json::from_str::<ThreadItem>(row.item_json.as_str()).map_err(|err| {
|
||||
ThreadStoreError::Internal {
|
||||
message: format!("failed to deserialize stored thread item: {err}"),
|
||||
}
|
||||
})?;
|
||||
let Some(text) = searchable_text(&item) else {
|
||||
continue;
|
||||
};
|
||||
let first_occurrence_index = cursor
|
||||
.as_ref()
|
||||
.filter(|cursor| cursor.next_rollout_ordinal == row.rollout_ordinal)
|
||||
.map_or(0, |cursor| cursor.next_occurrence_index);
|
||||
let remaining = params
|
||||
.page_size
|
||||
.saturating_add(1)
|
||||
.saturating_sub(items.len());
|
||||
let turn_cursor = serialize_cursor(
|
||||
params.thread_id,
|
||||
CursorScope::Turns,
|
||||
PhysicalHistoryPosition {
|
||||
physical_thread_id: params.thread_id,
|
||||
rollout_ordinal: row.turn_rollout_ordinal,
|
||||
},
|
||||
/*include_anchor*/ true,
|
||||
)?;
|
||||
for (occurrence_index, matched) in matcher
|
||||
.find_ranges(
|
||||
let mut matching_rows = Vec::new();
|
||||
let mut matching_occurrences = 0_usize;
|
||||
while let Some(row) = rows.try_next().await.map_err(thread_history_error)? {
|
||||
let row = candidate_row(row)?;
|
||||
let item =
|
||||
serde_json::from_str::<ThreadItem>(row.item_json.as_str()).map_err(|err| {
|
||||
ThreadStoreError::Internal {
|
||||
message: format!("failed to deserialize stored thread item: {err}"),
|
||||
}
|
||||
})?;
|
||||
let Some(text) = searchable_text(&item) else {
|
||||
continue;
|
||||
};
|
||||
let first_occurrence_index = cursor
|
||||
.as_ref()
|
||||
.filter(|cursor| cursor.next_rollout_ordinal == row.rollout_ordinal)
|
||||
.map_or(0, |cursor| cursor.next_occurrence_index);
|
||||
let remaining = params
|
||||
.page_size
|
||||
.saturating_add(1)
|
||||
.saturating_sub(items.len())
|
||||
.saturating_sub(matching_occurrences);
|
||||
let matches = matcher.find_ranges(
|
||||
text.as_ref(),
|
||||
first_occurrence_index.saturating_add(remaining),
|
||||
)
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.skip(first_occurrence_index)
|
||||
{
|
||||
if items.len() == params.page_size {
|
||||
return Ok(ThreadOccurrenceSearchPage {
|
||||
items,
|
||||
next_cursor: Some(serialize_cursor_for_search(SearchCursor {
|
||||
thread_id: params.thread_id,
|
||||
search_term: params.search_term,
|
||||
next_rollout_ordinal: row.rollout_ordinal,
|
||||
next_occurrence_index: occurrence_index,
|
||||
})?),
|
||||
});
|
||||
);
|
||||
if matches.len() <= first_occurrence_index {
|
||||
continue;
|
||||
}
|
||||
matching_occurrences += matches.len() - first_occurrence_index;
|
||||
matching_rows.push((row, text.into_owned(), matches, first_occurrence_index));
|
||||
if matching_occurrences
|
||||
== params
|
||||
.page_size
|
||||
.saturating_add(1)
|
||||
.saturating_sub(items.len())
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
drop(rows);
|
||||
|
||||
for (row, text, matches, first_occurrence_index) in matching_rows {
|
||||
let turn_rollout_ordinal = if segment_index + 1 == lineage.segments().len() {
|
||||
row.turn_rollout_ordinal
|
||||
} else if let Some(ordinal) = effective_turn_ordinals.get(&row.turn_id) {
|
||||
*ordinal
|
||||
} else {
|
||||
let ordinal = find_visible_turn(pool, &lineage, row.turn_id.as_str())
|
||||
.await?
|
||||
.rollout_ordinal;
|
||||
effective_turn_ordinals.insert(row.turn_id.clone(), ordinal);
|
||||
ordinal
|
||||
};
|
||||
let turn_cursor = serialize_cursor(
|
||||
params.thread_id,
|
||||
CursorScope::Turns,
|
||||
turn_rollout_ordinal,
|
||||
/*include_anchor*/ true,
|
||||
)?;
|
||||
for (occurrence_index, matched) in
|
||||
matches.into_iter().enumerate().skip(first_occurrence_index)
|
||||
{
|
||||
if items.len() == params.page_size {
|
||||
return Ok(ThreadOccurrenceSearchPage {
|
||||
items,
|
||||
next_cursor: Some(serialize_cursor_for_search(SearchCursor {
|
||||
thread_id: params.thread_id,
|
||||
search_term: params.search_term,
|
||||
next_rollout_ordinal: row.rollout_ordinal,
|
||||
next_occurrence_index: occurrence_index,
|
||||
})?),
|
||||
});
|
||||
}
|
||||
items.push(occurrence_in_item(
|
||||
row.turn_id.as_str(),
|
||||
row.item_id.as_str(),
|
||||
text.as_str(),
|
||||
matched,
|
||||
turn_cursor.as_str(),
|
||||
));
|
||||
}
|
||||
items.push(occurrence_in_item(
|
||||
row.turn_id.as_str(),
|
||||
row.item_id.as_str(),
|
||||
text.as_ref(),
|
||||
matched,
|
||||
turn_cursor.as_str(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,9 @@ pub(super) async fn page_turn_rows(
|
||||
) -> ThreadStoreResult<SegmentPage<StoredTurnRow>> {
|
||||
let cursor = parse_cursor(cursor, requested_thread_id, CursorScope::Turns)?;
|
||||
let mut rows = Vec::new();
|
||||
for (segment, segment_cursor) in segments_from_cursor(lineage, direction, cursor.as_ref())? {
|
||||
for (segment_index, segment, segment_cursor) in
|
||||
segments_from_cursor(lineage, direction, cursor.as_ref())?
|
||||
{
|
||||
let remaining = remaining_limit(page_size, rows.len())?;
|
||||
if remaining == 0 {
|
||||
break;
|
||||
@@ -71,6 +73,19 @@ WHERE thread_id =
|
||||
);
|
||||
query.push_bind(segment.thread_id().to_string());
|
||||
push_segment_range(&mut query, segment)?;
|
||||
for newer_segment in &lineage.segments()[segment_index + 1..] {
|
||||
query
|
||||
.push(" AND NOT EXISTS (SELECT 1 FROM thread_turns AS newer_turn WHERE newer_turn.thread_id = ")
|
||||
.push_bind(newer_segment.thread_id().to_string())
|
||||
.push(" AND newer_turn.turn_id = thread_turns.turn_id AND newer_turn.rollout_ordinal >= ")
|
||||
.push_bind(sqlite_integer(newer_segment.start_ordinal())?);
|
||||
if let Some(end_ordinal) = newer_segment.end_ordinal() {
|
||||
query
|
||||
.push(" AND newer_turn.rollout_ordinal < ")
|
||||
.push_bind(sqlite_integer(end_ordinal)?);
|
||||
}
|
||||
query.push(")");
|
||||
}
|
||||
push_cursor_clause(&mut query, direction, segment_cursor)?;
|
||||
push_order_and_limit(&mut query, direction, remaining);
|
||||
rows.extend(
|
||||
@@ -113,7 +128,7 @@ pub(super) async fn page_item_rows(
|
||||
CursorScope::ItemsByCreatedAtOrdinal,
|
||||
)?;
|
||||
let mut rows = Vec::new();
|
||||
for (segment, segment_cursor) in
|
||||
for (_, segment, segment_cursor) in
|
||||
segments_from_cursor(lineage, params.sort_direction, cursor.as_ref())?
|
||||
{
|
||||
let remaining = remaining_limit(params.page_size, rows.len())?;
|
||||
@@ -168,12 +183,6 @@ async fn page_updated_item_rows(
|
||||
params.thread_id,
|
||||
CursorScope::ItemsByUpdatedAtOrdinal,
|
||||
)?;
|
||||
if cursor
|
||||
.as_ref()
|
||||
.is_some_and(|cursor| cursor.physical_thread_id != params.thread_id)
|
||||
{
|
||||
return Err(invalid_cursor("unknown physical segment"));
|
||||
}
|
||||
let mut query = QueryBuilder::<Sqlite>::new(
|
||||
r#"
|
||||
SELECT turn_id, item_id, updated_at_ordinal AS rollout_ordinal, updated_at_ordinal, created_at_ms, item_json
|
||||
@@ -230,22 +239,15 @@ fn segments_from_cursor<'a>(
|
||||
lineage: &'a RolloutLineage,
|
||||
direction: SortDirection,
|
||||
cursor: Option<&'a HistoryCursor>,
|
||||
) -> ThreadStoreResult<Vec<(&'a RolloutLineageSegment, Option<&'a HistoryCursor>)>> {
|
||||
) -> ThreadStoreResult<Vec<(usize, &'a RolloutLineageSegment, Option<&'a HistoryCursor>)>> {
|
||||
let segments = lineage.segments();
|
||||
let cursor_index = cursor
|
||||
.map(|cursor| {
|
||||
segments
|
||||
.iter()
|
||||
.position(|segment| segment.thread_id() == cursor.physical_thread_id)
|
||||
.ok_or_else(|| invalid_cursor("unknown physical segment"))
|
||||
lineage
|
||||
.segment_index_for_ordinal(cursor.rollout_ordinal)
|
||||
.ok_or_else(|| invalid_cursor("position outside thread lineage"))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(cursor) = cursor
|
||||
&& let Some(index) = cursor_index
|
||||
&& !cursor_in_segment(cursor, &segments[index])
|
||||
{
|
||||
return Err(invalid_cursor("position outside physical segment"));
|
||||
}
|
||||
let indexes: Vec<usize> = match direction {
|
||||
SortDirection::Asc => (cursor_index.unwrap_or(0)..segments.len()).collect(),
|
||||
SortDirection::Desc => {
|
||||
@@ -261,19 +263,11 @@ fn segments_from_cursor<'a>(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
(&segments[index], segment_cursor)
|
||||
(index, &segments[index], segment_cursor)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn cursor_in_segment(cursor: &HistoryCursor, segment: &RolloutLineageSegment) -> bool {
|
||||
let ordinal = cursor.rollout_ordinal;
|
||||
ordinal >= segment.start_ordinal()
|
||||
&& segment
|
||||
.end_ordinal()
|
||||
.is_none_or(|end_ordinal| ordinal < end_ordinal)
|
||||
}
|
||||
|
||||
fn push_segment_range(
|
||||
query: &mut QueryBuilder<Sqlite>,
|
||||
segment: &RolloutLineageSegment,
|
||||
@@ -344,7 +338,7 @@ fn finish_page<T: HasPosition>(
|
||||
serialize_cursor(
|
||||
requested_thread_id,
|
||||
scope.clone(),
|
||||
row.position(),
|
||||
row.position().rollout_ordinal,
|
||||
/*include_anchor*/ true,
|
||||
)
|
||||
})
|
||||
@@ -355,7 +349,7 @@ fn finish_page<T: HasPosition>(
|
||||
serialize_cursor(
|
||||
requested_thread_id,
|
||||
scope,
|
||||
row.position(),
|
||||
row.position().rollout_ordinal,
|
||||
/*include_anchor*/ false,
|
||||
)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
use codex_protocol::ThreadId;
|
||||
use sqlx::Row;
|
||||
|
||||
use super::super::rollout_lineage::RolloutLineage;
|
||||
use super::super::rollout_lineage::RolloutLineageSegment;
|
||||
use super::sqlite_integer;
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
pub(in crate::local) struct TurnRow {
|
||||
pub physical_thread_id: ThreadId,
|
||||
pub rollout_ordinal: i64,
|
||||
pub rollout_byte_offset: Option<i64>,
|
||||
pub rollout_end_ordinal: Option<i64>,
|
||||
pub rollout_end_byte_offset: Option<i64>,
|
||||
pub status: String,
|
||||
pub first_user_item_id: Option<String>,
|
||||
pub final_agent_item_id: Option<String>,
|
||||
}
|
||||
|
||||
pub(in crate::local) async fn find_source_turn(
|
||||
pool: &sqlx::SqlitePool,
|
||||
lineage: &RolloutLineage,
|
||||
turn_id: &str,
|
||||
) -> ThreadStoreResult<TurnRow> {
|
||||
find_turn(pool, lineage.segments().iter(), turn_id).await
|
||||
}
|
||||
|
||||
pub(in crate::local) async fn find_visible_turn(
|
||||
pool: &sqlx::SqlitePool,
|
||||
lineage: &RolloutLineage,
|
||||
turn_id: &str,
|
||||
) -> ThreadStoreResult<TurnRow> {
|
||||
find_turn(pool, lineage.segments().iter().rev(), turn_id).await
|
||||
}
|
||||
|
||||
async fn find_turn<'a>(
|
||||
pool: &sqlx::SqlitePool,
|
||||
segments: impl Iterator<Item = &'a RolloutLineageSegment>,
|
||||
turn_id: &str,
|
||||
) -> ThreadStoreResult<TurnRow> {
|
||||
for segment in segments {
|
||||
if let Some(row) = query_turn_row(pool, segment, turn_id).await? {
|
||||
return Ok(row);
|
||||
}
|
||||
}
|
||||
Err(ThreadStoreError::InvalidRequest {
|
||||
message: format!("turn not found: {turn_id}"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn query_turn_row(
|
||||
pool: &sqlx::SqlitePool,
|
||||
segment: &RolloutLineageSegment,
|
||||
turn_id: &str,
|
||||
) -> ThreadStoreResult<Option<TurnRow>> {
|
||||
let end_ordinal = segment
|
||||
.end_ordinal()
|
||||
.map(|ordinal| sqlite_integer(ordinal, "rollout ordinal"))
|
||||
.transpose()?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
SELECT
|
||||
rollout_ordinal,
|
||||
rollout_byte_offset,
|
||||
rollout_end_ordinal,
|
||||
rollout_end_byte_offset,
|
||||
status,
|
||||
first_user_item_id,
|
||||
final_agent_item_id
|
||||
FROM thread_turns
|
||||
WHERE thread_id = ?
|
||||
AND turn_id = ?
|
||||
AND rollout_ordinal >= ?
|
||||
AND (? IS NULL OR rollout_ordinal < ?)
|
||||
"#,
|
||||
)
|
||||
.bind(segment.thread_id().to_string())
|
||||
.bind(turn_id)
|
||||
.bind(sqlite_integer(segment.start_ordinal(), "rollout ordinal")?)
|
||||
.bind(end_ordinal)
|
||||
.bind(end_ordinal)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|err| ThreadStoreError::Internal {
|
||||
message: format!("failed to resolve logical turn: {err}"),
|
||||
})
|
||||
.map(|row| {
|
||||
row.map(|row| TurnRow {
|
||||
physical_thread_id: segment.thread_id(),
|
||||
rollout_ordinal: row.get("rollout_ordinal"),
|
||||
rollout_byte_offset: row.get("rollout_byte_offset"),
|
||||
rollout_end_ordinal: row.get("rollout_end_ordinal"),
|
||||
rollout_end_byte_offset: row.get("rollout_end_byte_offset"),
|
||||
status: row.get("status"),
|
||||
first_user_item_id: row.get("first_user_item_id"),
|
||||
final_agent_item_id: row.get("final_agent_item_id"),
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -26,17 +26,22 @@ pub(super) async fn materialize_to_sqlite(
|
||||
thread_id: ThreadId,
|
||||
rollout_path: &Path,
|
||||
) -> ThreadStoreResult<()> {
|
||||
let start_offset = super::thread_history::next_rollout_byte_offset(store, thread_id).await?;
|
||||
let start_offset = super::thread_history::projection_state(store, thread_id)
|
||||
.await?
|
||||
.map_or(0, |state| state.next_byte_offset);
|
||||
let (lines, next_offset) = read_complete_rollout_lines(rollout_path, start_offset).await?;
|
||||
// Empty valid records can still consume bytes through blank or rejected complete lines.
|
||||
if lines.is_empty() && start_offset == next_offset {
|
||||
return Ok(());
|
||||
}
|
||||
let subagent_history_start_ordinal = codex_rollout::read_session_meta_line(rollout_path)
|
||||
let session_meta = codex_rollout::read_session_meta_line(rollout_path)
|
||||
.await
|
||||
.map_err(thread_store_io_error)?
|
||||
.meta
|
||||
.subagent_history_start_ordinal;
|
||||
.meta;
|
||||
let initial_ordinal = session_meta
|
||||
.history_base
|
||||
.map_or(0, |base| base.end_ordinal_exclusive);
|
||||
let subagent_history_start_ordinal = session_meta.subagent_history_start_ordinal;
|
||||
|
||||
let projections = lines
|
||||
.iter()
|
||||
@@ -67,6 +72,7 @@ pub(super) async fn materialize_to_sqlite(
|
||||
thread_id,
|
||||
start_offset,
|
||||
next_offset,
|
||||
initial_ordinal,
|
||||
projections,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
@@ -10,8 +11,11 @@ use codex_protocol::items::AgentMessageItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::UserMessageItem;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::RolloutLine;
|
||||
@@ -34,8 +38,11 @@ use super::super::test_support::test_config;
|
||||
use crate::AppendThreadItemsParams;
|
||||
use crate::CreateThreadParams;
|
||||
use crate::DeleteThreadParams;
|
||||
use crate::ForkBoundary;
|
||||
use crate::ListThreadsParams;
|
||||
use crate::ListTurnsParams;
|
||||
use crate::PrepareForkParams;
|
||||
use crate::PreparedFork;
|
||||
use crate::ResumeThreadParams;
|
||||
use crate::SortDirection;
|
||||
use crate::StoredTurnItemsView;
|
||||
@@ -373,6 +380,224 @@ WHERE thread_id = ?
|
||||
assert_eq!(projection_state, (rollout_len, 5));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn referenced_paginated_rollout_projects_inherited_ordinal_range() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let source_id = ThreadId::default();
|
||||
create_paginated_thread(&store, source_id).await;
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: source_id,
|
||||
items: vec![
|
||||
turn_started("source-turn"),
|
||||
user_message("source message"),
|
||||
turn_completed("source-turn"),
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append source history");
|
||||
let source_path = store
|
||||
.live_rollout_path(source_id)
|
||||
.await
|
||||
.expect("source rollout path");
|
||||
let (_, source_end_byte_offset) =
|
||||
rollout_line_byte_offsets(source_path.as_path(), /*ordinal*/ 3);
|
||||
let child_id = ThreadId::default();
|
||||
let history_base = HistoryPosition {
|
||||
thread_id: source_id,
|
||||
end_ordinal_exclusive: 4,
|
||||
end_byte_offset: u64::try_from(source_end_byte_offset).expect("source byte offset"),
|
||||
};
|
||||
create_paginated_subagent_thread(
|
||||
&store,
|
||||
child_id,
|
||||
Some(history_base),
|
||||
/*subagent_history_start_ordinal*/ None,
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.persist_thread(child_id)
|
||||
.await
|
||||
.expect("persist child metadata");
|
||||
assert_eq!(
|
||||
prepare_paginated_fork(&store, child_id, ForkBoundary::Latest)
|
||||
.await
|
||||
.history_base,
|
||||
Some(history_base)
|
||||
);
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: child_id,
|
||||
items: vec![
|
||||
turn_started("child-turn"),
|
||||
user_message("child message"),
|
||||
turn_completed("child-turn"),
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append child history");
|
||||
let latest_history_base = prepare_paginated_fork(&store, child_id, ForkBoundary::Latest)
|
||||
.await
|
||||
.history_base;
|
||||
for (boundary, expected_base) in [
|
||||
(ForkBoundary::Latest, latest_history_base),
|
||||
(
|
||||
ForkBoundary::ThroughTurn("source-turn".to_string()),
|
||||
Some(history_base),
|
||||
),
|
||||
(
|
||||
ForkBoundary::BeforeTurn("child-turn".to_string()),
|
||||
Some(history_base),
|
||||
),
|
||||
(
|
||||
ForkBoundary::ThroughTurn("child-turn".to_string()),
|
||||
latest_history_base,
|
||||
),
|
||||
(ForkBoundary::BeforeTurn("source-turn".to_string()), None),
|
||||
] {
|
||||
let prepared = prepare_paginated_fork(&store, child_id, boundary).await;
|
||||
assert_eq!(prepared.history_base, expected_base);
|
||||
assert!(matches!(
|
||||
prepared.model_context.first(),
|
||||
Some(RolloutItem::SessionMeta(meta)) if meta.meta.id == child_id
|
||||
));
|
||||
assert_eq!(
|
||||
contains_user_message(&prepared.model_context, "source message"),
|
||||
expected_base.is_some()
|
||||
);
|
||||
assert_eq!(
|
||||
contains_user_message(&prepared.model_context, "child message"),
|
||||
expected_base == latest_history_base
|
||||
);
|
||||
}
|
||||
|
||||
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
|
||||
home.path().abs(),
|
||||
))
|
||||
.await
|
||||
.expect("open thread history db");
|
||||
let turn_ordinal = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT rollout_ordinal FROM thread_turns WHERE thread_id = ? AND turn_id = ?",
|
||||
)
|
||||
.bind(child_id.to_string())
|
||||
.bind("child-turn")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("read child turn ordinal");
|
||||
let next_ordinal = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT next_rollout_ordinal FROM thread_history_projection_state WHERE thread_id = ?",
|
||||
)
|
||||
.bind(child_id.to_string())
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("read child projection ordinal");
|
||||
assert_eq!((turn_ordinal, next_ordinal), (5, 8));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn named_fork_boundaries_reject_invisible_and_noncanonical_turns() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let source_id = ThreadId::default();
|
||||
create_paginated_thread(&store, source_id).await;
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: source_id,
|
||||
items: vec![turn_started("inherited-turn"), user_message("before fork")],
|
||||
})
|
||||
.await
|
||||
.expect("append inherited active turn");
|
||||
|
||||
let history_base = prepare_paginated_fork(&store, source_id, ForkBoundary::Latest)
|
||||
.await
|
||||
.history_base;
|
||||
let child_id = ThreadId::default();
|
||||
create_paginated_subagent_thread(
|
||||
&store,
|
||||
child_id,
|
||||
history_base,
|
||||
/*subagent_history_start_ordinal*/ None,
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.persist_thread(child_id)
|
||||
.await
|
||||
.expect("persist child metadata");
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: child_id,
|
||||
items: vec![turn_started("child-turn"), turn_completed("child-turn")],
|
||||
})
|
||||
.await
|
||||
.expect("append child turn after inherited active turn");
|
||||
let error = store
|
||||
.prepare_fork(PrepareForkParams {
|
||||
thread_id: child_id,
|
||||
boundary: ForkBoundary::ThroughTurn("inherited-turn".to_string()),
|
||||
})
|
||||
.await
|
||||
.expect_err("cannot fork through an inherited active turn");
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::ThreadStoreError::InvalidRequest { message }
|
||||
if message == "lastTurnId 'inherited-turn' identifies an in-progress turn"
|
||||
));
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: source_id,
|
||||
items: vec![
|
||||
user_message("after fork"),
|
||||
turn_completed("inherited-turn"),
|
||||
turn_started("stale-turn"),
|
||||
turn_started("replacement-turn"),
|
||||
turn_completed("replacement-turn"),
|
||||
completed_item(
|
||||
source_id,
|
||||
"review-turn",
|
||||
TurnItem::UserMessage(UserMessageItem {
|
||||
id: "review-message".to_string(),
|
||||
client_id: None,
|
||||
content: Vec::new(),
|
||||
}),
|
||||
),
|
||||
turn_completed("review-turn"),
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append invisible, stale, and terminal-only turns");
|
||||
|
||||
for (thread_id, boundary, expected_error) in [
|
||||
(
|
||||
child_id,
|
||||
ForkBoundary::ThroughTurn("inherited-turn".to_string()),
|
||||
"fork boundary exceeds inherited source history",
|
||||
),
|
||||
(
|
||||
source_id,
|
||||
ForkBoundary::ThroughTurn("stale-turn".to_string()),
|
||||
"lastTurnId 'stale-turn' identifies an in-progress turn",
|
||||
),
|
||||
(
|
||||
source_id,
|
||||
ForkBoundary::BeforeTurn("review-turn".to_string()),
|
||||
"turn review-turn does not have a persisted start boundary",
|
||||
),
|
||||
] {
|
||||
let error = store
|
||||
.prepare_fork(PrepareForkParams {
|
||||
thread_id,
|
||||
boundary,
|
||||
})
|
||||
.await
|
||||
.expect_err("reject an invalid fork boundary");
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::ThreadStoreError::InvalidRequest { message } if message == expected_error
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_turn_stores_only_its_start_position() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
@@ -408,6 +633,326 @@ async fn active_turn_stores_only_its_start_position() {
|
||||
.await
|
||||
.expect("read active turn position");
|
||||
assert_eq!(turn_position, (Some(turn_start_byte_offset), None, None));
|
||||
|
||||
let (latest_byte_offset, latest_ordinal) = projection_state(&pool, thread_id).await;
|
||||
let prepared = prepare_paginated_fork(&store, thread_id, ForkBoundary::Latest).await;
|
||||
assert_eq!(
|
||||
prepared.history_base,
|
||||
Some(HistoryPosition {
|
||||
thread_id,
|
||||
end_ordinal_exclusive: u64::try_from(latest_ordinal).expect("latest ordinal"),
|
||||
end_byte_offset: u64::try_from(latest_byte_offset).expect("latest byte offset"),
|
||||
})
|
||||
);
|
||||
assert!(prepared.model_context.iter().any(|item| {
|
||||
matches!(item, RolloutItem::EventMsg(EventMsg::TurnStarted(event)) if event.turn_id == "turn-1")
|
||||
}));
|
||||
assert_eq!(
|
||||
prepare_paginated_fork(
|
||||
&store,
|
||||
thread_id,
|
||||
ForkBoundary::BeforeTurn("turn-1".to_string()),
|
||||
)
|
||||
.await
|
||||
.history_base,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paginated_fork_persists_empty_source() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
create_paginated_thread(&store, thread_id).await;
|
||||
let rollout_path = store
|
||||
.live_rollout_path(thread_id)
|
||||
.await
|
||||
.expect("rollout path");
|
||||
assert!(!rollout_path.exists());
|
||||
|
||||
let prepared = prepare_paginated_fork(&store, thread_id, ForkBoundary::Latest).await;
|
||||
|
||||
assert!(rollout_path.exists());
|
||||
assert_eq!(prepared.history_base, None);
|
||||
assert!(matches!(
|
||||
prepared.model_context.as_slice(),
|
||||
[RolloutItem::SessionMeta(meta)] if meta.meta.id == thread_id
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paginated_fork_materializes_compressed_source_and_ancestor() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let ancestor_thread_id = ThreadId::default();
|
||||
create_paginated_thread(&store, ancestor_thread_id).await;
|
||||
store
|
||||
.persist_thread(ancestor_thread_id)
|
||||
.await
|
||||
.expect("persist ancestor meta");
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: ancestor_thread_id,
|
||||
items: vec![
|
||||
turn_started("ancestor-turn"),
|
||||
user_message("inherited ancestor message"),
|
||||
turn_completed("ancestor-turn"),
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append ancestor turn");
|
||||
let ancestor_path = store
|
||||
.live_rollout_path(ancestor_thread_id)
|
||||
.await
|
||||
.expect("ancestor rollout path");
|
||||
let ancestor_base = prepare_paginated_fork(&store, ancestor_thread_id, ForkBoundary::Latest)
|
||||
.await
|
||||
.history_base
|
||||
.expect("ancestor prefix");
|
||||
store
|
||||
.shutdown_thread(ancestor_thread_id)
|
||||
.await
|
||||
.expect("shutdown ancestor");
|
||||
|
||||
let source_thread_id = ThreadId::default();
|
||||
create_paginated_subagent_thread(
|
||||
&store,
|
||||
source_thread_id,
|
||||
Some(ancestor_base),
|
||||
/*subagent_history_start_ordinal*/ None,
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.persist_thread(source_thread_id)
|
||||
.await
|
||||
.expect("persist source meta");
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: source_thread_id,
|
||||
items: vec![
|
||||
turn_started("source-turn"),
|
||||
user_message("inherited source message"),
|
||||
turn_completed("source-turn"),
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append source turn");
|
||||
let source_path = store
|
||||
.live_rollout_path(source_thread_id)
|
||||
.await
|
||||
.expect("source rollout path");
|
||||
store
|
||||
.shutdown_thread(source_thread_id)
|
||||
.await
|
||||
.expect("shutdown source");
|
||||
let ancestor_compressed_path = ancestor_path.with_extension("jsonl.zst");
|
||||
let source_compressed_path = source_path.with_extension("jsonl.zst");
|
||||
compress_rollout(ancestor_path.as_path());
|
||||
compress_rollout(source_path.as_path());
|
||||
let ancestor_modified = fs::metadata(&ancestor_compressed_path)
|
||||
.and_then(|metadata| metadata.modified())
|
||||
.expect("read compressed ancestor timestamp");
|
||||
let source_modified = fs::metadata(&source_compressed_path)
|
||||
.and_then(|metadata| metadata.modified())
|
||||
.expect("read compressed source timestamp");
|
||||
|
||||
let (first, second) = tokio::join!(
|
||||
prepare_paginated_fork(&store, source_thread_id, ForkBoundary::Latest),
|
||||
prepare_paginated_fork(&store, source_thread_id, ForkBoundary::Latest),
|
||||
);
|
||||
assert!(ancestor_path.exists());
|
||||
assert!(source_path.exists());
|
||||
assert!(!ancestor_compressed_path.exists());
|
||||
assert!(!source_compressed_path.exists());
|
||||
assert_eq!(
|
||||
fs::metadata(&ancestor_path)
|
||||
.and_then(|metadata| metadata.modified())
|
||||
.expect("read materialized ancestor timestamp"),
|
||||
ancestor_modified
|
||||
);
|
||||
assert_eq!(
|
||||
fs::metadata(&source_path)
|
||||
.and_then(|metadata| metadata.modified())
|
||||
.expect("read materialized source timestamp"),
|
||||
source_modified
|
||||
);
|
||||
for prepared in [first, second] {
|
||||
assert!(matches!(
|
||||
prepared.model_context.first(),
|
||||
Some(RolloutItem::SessionMeta(meta)) if meta.meta.id == source_thread_id
|
||||
));
|
||||
for message in ["inherited ancestor message", "inherited source message"] {
|
||||
assert!(contains_user_message(
|
||||
prepared.model_context.as_slice(),
|
||||
message
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_fork_keeps_source_reserved_until_lineage_materialization_finishes() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let ancestor_thread_id = ThreadId::default();
|
||||
create_paginated_thread(&store, ancestor_thread_id).await;
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: ancestor_thread_id,
|
||||
items: vec![
|
||||
turn_started("ancestor-turn"),
|
||||
turn_completed("ancestor-turn"),
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append ancestor history");
|
||||
let ancestor_base = prepare_paginated_fork(&store, ancestor_thread_id, ForkBoundary::Latest)
|
||||
.await
|
||||
.history_base
|
||||
.expect("ancestor prefix");
|
||||
|
||||
let source_thread_id = ThreadId::default();
|
||||
create_paginated_subagent_thread(
|
||||
&store,
|
||||
source_thread_id,
|
||||
Some(ancestor_base),
|
||||
/*subagent_history_start_ordinal*/ None,
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.persist_thread(source_thread_id)
|
||||
.await
|
||||
.expect("persist source metadata");
|
||||
let source_path = store
|
||||
.live_rollout_path(source_thread_id)
|
||||
.await
|
||||
.expect("source rollout path");
|
||||
store
|
||||
.shutdown_thread(source_thread_id)
|
||||
.await
|
||||
.expect("shutdown source");
|
||||
compress_rollout(source_path.as_path());
|
||||
|
||||
let ancestor_writer_guard = store.live_writer_locks.lock(ancestor_thread_id).await;
|
||||
let preparation_store = store.clone();
|
||||
let preparation = tokio::spawn(async move {
|
||||
preparation_store
|
||||
.prepare_fork(PrepareForkParams {
|
||||
thread_id: source_thread_id,
|
||||
boundary: ForkBoundary::Latest,
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !source_path.exists() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("detached lineage task should materialize the source");
|
||||
preparation.abort();
|
||||
assert!(
|
||||
preparation
|
||||
.await
|
||||
.expect_err("fork preparation should be cancelled")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let mut delete = Box::pin(store.delete_thread(DeleteThreadParams {
|
||||
thread_id: source_thread_id,
|
||||
}));
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut delete => {
|
||||
panic!("source deletion completed while lineage materialization was active: {result:?}")
|
||||
}
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
drop(ancestor_writer_guard);
|
||||
delete
|
||||
.await
|
||||
.expect("delete source after lineage materialization finishes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_fork_reserves_source_until_child_reference_is_durable() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let source_thread_id = ThreadId::default();
|
||||
create_paginated_thread(&store, source_thread_id).await;
|
||||
store
|
||||
.persist_thread(source_thread_id)
|
||||
.await
|
||||
.expect("persist source metadata");
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id: source_thread_id,
|
||||
items: vec![turn_started("source-turn"), turn_completed("source-turn")],
|
||||
})
|
||||
.await
|
||||
.expect("append source turn");
|
||||
let prepared = store
|
||||
.prepare_fork(PrepareForkParams {
|
||||
thread_id: source_thread_id,
|
||||
boundary: ForkBoundary::Latest,
|
||||
})
|
||||
.await
|
||||
.expect("prepare referenced fork");
|
||||
let history_base = prepared.history_base.expect("source history base");
|
||||
|
||||
let mut delete = Box::pin(store.delete_thread(DeleteThreadParams {
|
||||
thread_id: source_thread_id,
|
||||
}));
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut delete => {
|
||||
panic!("source deletion completed before its child reference was durable: {result:?}")
|
||||
}
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
store.append_items(AppendThreadItemsParams {
|
||||
thread_id: source_thread_id,
|
||||
items: vec![
|
||||
turn_started("later-turn"),
|
||||
user_message("later source message"),
|
||||
turn_completed("later-turn"),
|
||||
],
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("source write should not wait behind deletion")
|
||||
.expect("source writes remain available during fork preparation");
|
||||
assert_eq!(prepared.history_base, Some(history_base));
|
||||
assert!(!contains_user_message(
|
||||
prepared.model_context.as_slice(),
|
||||
"later source message"
|
||||
));
|
||||
|
||||
let child_thread_id = ThreadId::default();
|
||||
create_paginated_subagent_thread(
|
||||
&store,
|
||||
child_thread_id,
|
||||
Some(history_base),
|
||||
/*subagent_history_start_ordinal*/ None,
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.persist_thread(child_thread_id)
|
||||
.await
|
||||
.expect("persist child history reference");
|
||||
drop(prepared);
|
||||
|
||||
let error = delete
|
||||
.await
|
||||
.expect_err("durable child reference protects its source");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("forked history still references")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -418,6 +963,7 @@ async fn subagent_prefix_advances_projection_without_materializing_history() {
|
||||
create_paginated_subagent_thread(
|
||||
&store,
|
||||
thread_id,
|
||||
/*history_base*/ None,
|
||||
/*subagent_history_start_ordinal*/ Some(4),
|
||||
)
|
||||
.await;
|
||||
@@ -491,7 +1037,7 @@ async fn subagent_prefix_advances_projection_without_materializing_history() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replayed_item_snapshot_preserves_creation_ordinal_and_advances_update_ordinal() {
|
||||
async fn unexpected_duplicate_item_completion_does_not_poison_projection() {
|
||||
let home = TempDir::new().expect("temp dir");
|
||||
let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None);
|
||||
let thread_id = ThreadId::default();
|
||||
@@ -514,7 +1060,7 @@ async fn replayed_item_snapshot_preserves_creation_ordinal_and_advances_update_o
|
||||
],
|
||||
})
|
||||
.await
|
||||
.expect("append first item snapshot");
|
||||
.expect("append completed item");
|
||||
let pool = codex_state::open_thread_history_db(&codex_state::SqliteConfig::new_for_testing(
|
||||
home.path().abs(),
|
||||
))
|
||||
@@ -543,7 +1089,7 @@ async fn replayed_item_snapshot_preserves_creation_ordinal_and_advances_update_o
|
||||
)],
|
||||
})
|
||||
.await
|
||||
.expect("append replayed item snapshot");
|
||||
.expect("append unexpected duplicate item completion");
|
||||
|
||||
let item = sqlx::query_as::<_, (i64, i64, i64, String)>(
|
||||
r#"
|
||||
@@ -1273,14 +1819,29 @@ SELECT
|
||||
|
||||
async fn create_paginated_thread(store: &LocalThreadStore, thread_id: ThreadId) {
|
||||
create_paginated_subagent_thread(
|
||||
store, thread_id, /*subagent_history_start_ordinal*/ None,
|
||||
store, thread_id, /*history_base*/ None, /*subagent_history_start_ordinal*/ None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn prepare_paginated_fork(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
boundary: ForkBoundary,
|
||||
) -> PreparedFork {
|
||||
store
|
||||
.prepare_fork(PrepareForkParams {
|
||||
thread_id,
|
||||
boundary,
|
||||
})
|
||||
.await
|
||||
.expect("prepare paginated fork")
|
||||
}
|
||||
|
||||
async fn create_paginated_subagent_thread(
|
||||
store: &LocalThreadStore,
|
||||
thread_id: ThreadId,
|
||||
history_base: Option<HistoryPosition>,
|
||||
subagent_history_start_ordinal: Option<u64>,
|
||||
) {
|
||||
store
|
||||
@@ -1298,6 +1859,7 @@ async fn create_paginated_subagent_thread(
|
||||
selected_capability_roots: Vec::new(),
|
||||
multi_agent_version: None,
|
||||
history_mode: ThreadHistoryMode::Paginated,
|
||||
history_base,
|
||||
subagent_history_start_ordinal,
|
||||
initial_window_id: "window-1".to_string(),
|
||||
metadata: ThreadPersistenceMetadata {
|
||||
@@ -1332,6 +1894,30 @@ fn turn_completed(turn_id: &str) -> RolloutItem {
|
||||
}))
|
||||
}
|
||||
|
||||
fn user_message(message: &str) -> RolloutItem {
|
||||
RolloutItem::ResponseItem(ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputText {
|
||||
text: message.to_string(),
|
||||
}],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn contains_user_message(items: &[RolloutItem], expected: &str) -> bool {
|
||||
items.iter().any(|item| {
|
||||
matches!(
|
||||
item,
|
||||
RolloutItem::ResponseItem(ResponseItem::Message { content, .. })
|
||||
if content.iter().any(|content| {
|
||||
matches!(content, ContentItem::InputText { text } if text == expected)
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn completed_item(thread_id: ThreadId, turn_id: &str, item: TurnItem) -> RolloutItem {
|
||||
RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
|
||||
thread_id,
|
||||
@@ -1372,6 +1958,18 @@ fn rollout_line_byte_offsets(path: &std::path::Path, ordinal: u64) -> (i64, i64)
|
||||
panic!("missing rollout ordinal {ordinal}");
|
||||
}
|
||||
|
||||
fn compress_rollout(path: &Path) {
|
||||
let mut compressed_path = path.as_os_str().to_os_string();
|
||||
compressed_path.push(".zst");
|
||||
let compressed = zstd::stream::encode_all(
|
||||
fs::File::open(path).expect("open rollout for compression"),
|
||||
/*level*/ 0,
|
||||
)
|
||||
.expect("compress rollout");
|
||||
fs::write(compressed_path, compressed).expect("write compressed rollout");
|
||||
fs::remove_file(path).expect("remove plain rollout");
|
||||
}
|
||||
|
||||
async fn projection_state(pool: &sqlx::SqlitePool, thread_id: ThreadId) -> (i64, i64) {
|
||||
sqlx::query_as::<_, (i64, i64)>(
|
||||
r#"
|
||||
|
||||
@@ -15,6 +15,7 @@ pub(super) async fn unarchive_thread(
|
||||
params: ArchiveThreadParams,
|
||||
) -> ThreadStoreResult<StoredThread> {
|
||||
let thread_id = params.thread_id;
|
||||
let _lifecycle_guard = store.live_writer_locks.lock_lifecycle(thread_id).await;
|
||||
let state_db_ctx = store.state_db().await;
|
||||
let archived_path = find_archived_thread_path_by_id_str(
|
||||
store.config.codex_home.as_path(),
|
||||
|
||||
@@ -15,6 +15,8 @@ use crate::ListItemsParams;
|
||||
use crate::ListThreadsParams;
|
||||
use crate::ListTurnsParams;
|
||||
use crate::LoadThreadHistoryParams;
|
||||
use crate::PrepareForkParams;
|
||||
use crate::PreparedFork;
|
||||
use crate::ReadThreadByRolloutPathParams;
|
||||
use crate::ReadThreadParams;
|
||||
use crate::ResumeThreadParams;
|
||||
@@ -95,6 +97,17 @@ pub trait ThreadStore: Any + Send + Sync {
|
||||
})
|
||||
}
|
||||
|
||||
/// Freezes source history and model context used to initialize a referenced fork.
|
||||
///
|
||||
/// Stores without reference-backed fork support can retain this default implementation.
|
||||
fn prepare_fork(&self, _params: PrepareForkParams) -> ThreadStoreFuture<'_, PreparedFork> {
|
||||
Box::pin(async {
|
||||
Err(ThreadStoreError::Unsupported {
|
||||
operation: "prepare_fork",
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads a thread summary and optionally its persisted history.
|
||||
fn read_thread(&self, params: ReadThreadParams) -> ThreadStoreFuture<'_, StoredThread>;
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ use codex_protocol::models::PermissionProfile;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::GitInfo;
|
||||
use codex_protocol::protocol::HistoryPosition;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
@@ -95,6 +96,8 @@ pub struct CreateThreadParams {
|
||||
pub multi_agent_version: Option<MultiAgentVersion>,
|
||||
/// Persisted thread history contract selected when the thread was created.
|
||||
pub history_mode: ThreadHistoryMode,
|
||||
/// Exclusive prefix of another paginated rollout inherited by this thread.
|
||||
pub history_base: Option<HistoryPosition>,
|
||||
/// First rollout ordinal that belongs to this subagent's projected history.
|
||||
pub subagent_history_start_ordinal: Option<u64>,
|
||||
/// Initial context-window identity captured when the thread was created.
|
||||
@@ -175,6 +178,56 @@ pub struct StoredModelContext {
|
||||
pub items: Vec<RolloutItem>,
|
||||
}
|
||||
|
||||
/// Requested boundary for inheriting a paginated thread's history.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ForkBoundary {
|
||||
/// Inherit the source thread's latest durable state.
|
||||
Latest,
|
||||
/// Inherit history through the newest visible occurrence of this turn.
|
||||
ThroughTurn(String),
|
||||
/// Inherit history preceding the original visible occurrence of this turn.
|
||||
BeforeTurn(String),
|
||||
}
|
||||
|
||||
/// Parameters for freezing the source history used to initialize a fork.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PrepareForkParams {
|
||||
/// Immediate source thread whose metadata and approval settings are inherited.
|
||||
pub thread_id: ThreadId,
|
||||
/// Requested inclusive or exclusive fork boundary.
|
||||
pub boundary: ForkBoundary,
|
||||
}
|
||||
|
||||
/// Frozen source history and model context for a reference-backed fork.
|
||||
#[derive(Debug)]
|
||||
pub struct PreparedFork {
|
||||
/// Immediate source thread, even when the normalized history base names an ancestor.
|
||||
pub source_thread_id: ThreadId,
|
||||
/// Frozen physical rollout prefix inherited by the child.
|
||||
pub history_base: Option<HistoryPosition>,
|
||||
/// Bounded model context selected by the requested fork boundary.
|
||||
pub model_context: Arc<Vec<RolloutItem>>,
|
||||
/// Blocks source deletion until the child's history reference is durable.
|
||||
_source_reservation: Box<dyn std::fmt::Debug + Send>,
|
||||
}
|
||||
|
||||
impl PreparedFork {
|
||||
/// Creates a frozen fork snapshot while retaining a backend-owned source reservation.
|
||||
pub fn new(
|
||||
source_thread_id: ThreadId,
|
||||
history_base: Option<HistoryPosition>,
|
||||
model_context: Arc<Vec<RolloutItem>>,
|
||||
source_reservation: impl std::fmt::Debug + Send + 'static,
|
||||
) -> Self {
|
||||
Self {
|
||||
source_thread_id,
|
||||
history_base,
|
||||
model_context,
|
||||
_source_reservation: Box::new(source_reservation),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for reading a thread summary and optionally its replay history.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ReadThreadParams {
|
||||
|
||||
Reference in New Issue
Block a user