mirror of
https://github.com/openai/codex.git
synced 2026-11-16 06:54:36 +00:00
## What changed - Resolve the full rollout lineage when loading model context for paginated threads. - Reverse-scan each lineage segment up to its recorded byte boundary, stopping at segment metadata and preserving the requested thread's canonical session metadata. - Add a bounded reverse-scanner constructor for scanning a frozen JSONL prefix. ## Testing - Cover frozen-prefix reverse scans and nested lineages whose root rollout has been archived. GitOrigin-RevId: 8b97da43a263b7bd09cb7883450bb19eba5f195f
131 lines
4.8 KiB
Rust
131 lines
4.8 KiB
Rust
use std::fs::File;
|
|
use std::io;
|
|
|
|
use codex_protocol::protocol::RolloutItem;
|
|
use codex_protocol::protocol::RolloutLine;
|
|
use codex_protocol::protocol::SessionMetaLine;
|
|
use codex_protocol::protocol::ThreadHistoryMode;
|
|
use codex_rollout::ModelContextScan;
|
|
use codex_rollout::ModelContextScanProgress;
|
|
use codex_rollout::ReverseJsonlScanner;
|
|
use codex_rollout::ScanOutcome;
|
|
|
|
use super::LocalThreadStore;
|
|
use super::read_thread;
|
|
use super::rollout_lineage::RolloutLineage;
|
|
use crate::LoadThreadHistoryParams;
|
|
use crate::StoredModelContext;
|
|
use crate::ThreadStoreError;
|
|
use crate::ThreadStoreResult;
|
|
|
|
#[cfg(test)]
|
|
#[path = "model_context_tests.rs"]
|
|
mod tests;
|
|
|
|
/// Loads rollout items needed to reconstruct the latest model-visible context.
|
|
///
|
|
/// Plain paginated JSONL rollouts use a reverse scan. When it finds both a usable replacement-
|
|
/// history checkpoint and the completed user-turn context needed for resume metadata, the returned
|
|
/// replay starts with the canonical head `SessionMeta` followed by that newest suffix. When no
|
|
/// bounded cutoff is available, the scan continues to the beginning and returns the complete
|
|
/// replay it already accumulated.
|
|
///
|
|
/// Legacy and compressed rollout shapes keep the existing full-history path.
|
|
pub(super) async fn load_latest_model_context(
|
|
store: &LocalThreadStore,
|
|
params: LoadThreadHistoryParams,
|
|
) -> ThreadStoreResult<StoredModelContext> {
|
|
let path = read_thread::resolve_rollout_path(store, params.thread_id, params.include_archived)
|
|
.await?
|
|
.ok_or_else(|| ThreadStoreError::InvalidRequest {
|
|
message: format!("no rollout found for thread id {}", params.thread_id),
|
|
})?;
|
|
|
|
let session_meta = codex_rollout::read_session_meta_line(path.as_path())
|
|
.await
|
|
.map_err(|err| ThreadStoreError::Internal {
|
|
message: format!("failed to read session metadata {}: {err}", path.display()),
|
|
})?;
|
|
if session_meta.meta.id != params.thread_id {
|
|
return Err(ThreadStoreError::InvalidRequest {
|
|
message: format!(
|
|
"rollout at {} belongs to thread {}, not {}",
|
|
path.display(),
|
|
session_meta.meta.id,
|
|
params.thread_id
|
|
),
|
|
});
|
|
}
|
|
|
|
let items = if matches!(session_meta.meta.history_mode, ThreadHistoryMode::Paginated)
|
|
&& !path
|
|
.file_name()
|
|
.and_then(|file_name| file_name.to_str())
|
|
.is_some_and(|file_name| file_name.ends_with(".jsonl.zst"))
|
|
{
|
|
let lineage = store.resolve_rollout_lineage(params.thread_id).await?;
|
|
scan_model_context_from_lineage(lineage, session_meta).await?
|
|
} else {
|
|
read_thread::load_history_items(path.as_path()).await?
|
|
};
|
|
|
|
Ok(StoredModelContext {
|
|
thread_id: params.thread_id,
|
|
items,
|
|
})
|
|
}
|
|
|
|
async fn scan_model_context_from_lineage(
|
|
lineage: RolloutLineage,
|
|
session_meta: SessionMetaLine,
|
|
) -> ThreadStoreResult<Vec<RolloutItem>> {
|
|
let scan = tokio::task::spawn_blocking(move || {
|
|
scan_model_context_from_lineage_blocking(&lineage, session_meta)
|
|
})
|
|
.await
|
|
.map_err(|err| ThreadStoreError::Internal {
|
|
message: format!("failed to join model context scan: {err}"),
|
|
})?;
|
|
match scan {
|
|
Ok(items) => Ok(items),
|
|
Err(err) => Err(ThreadStoreError::Internal {
|
|
message: format!("failed to scan paginated model context lineage: {err}"),
|
|
}),
|
|
}
|
|
}
|
|
|
|
fn scan_model_context_from_lineage_blocking(
|
|
lineage: &RolloutLineage,
|
|
session_meta: SessionMetaLine,
|
|
) -> io::Result<Vec<RolloutItem>> {
|
|
let mut scan = ModelContextScan::default();
|
|
'segments: for segment in lineage.segments().iter().rev() {
|
|
let file = File::open(segment.rollout_path.as_path())?;
|
|
let mut scanner = match segment.end.map(|end| end.end_byte_offset) {
|
|
Some(end_byte_offset) => ReverseJsonlScanner::new_at(file, end_byte_offset)?,
|
|
None => ReverseJsonlScanner::new(file)?,
|
|
};
|
|
while let Some(outcome) = scanner.scan_next::<RolloutLine>()? {
|
|
let ScanOutcome::Parsed(line) = outcome else {
|
|
continue;
|
|
};
|
|
// Each physical segment contributes only its local delta. Its head metadata is
|
|
// replaced with the requested thread's canonical SessionMeta after replay.
|
|
if matches!(&line.item, RolloutItem::SessionMeta(_)) {
|
|
break;
|
|
}
|
|
match scan.push(line.item) {
|
|
ModelContextScanProgress::Continue => {}
|
|
ModelContextScanProgress::Complete => break 'segments,
|
|
}
|
|
}
|
|
}
|
|
|
|
let canonical_meta = session_meta.clone();
|
|
let mut items = scan.finish(session_meta);
|
|
if !matches!(items.first(), Some(RolloutItem::SessionMeta(_))) {
|
|
items.insert(0, RolloutItem::SessionMeta(canonical_meta));
|
|
}
|
|
Ok(items)
|
|
}
|