mirror of
https://github.com/openai/codex.git
synced 2026-11-17 06:57:31 +00:00
Support compression for shared rollout lineages (#41357)
## Why Rollout compression skips files that participate in paginated fork lineages because lineage readers rely on byte offsets into the original JSONL. This prevents shared rollout history from being compressed. ## What changed - Add a seekable rollout reader that preserves logical JSONL offsets for plain and zstd-compressed files without materializing compressed data beside the rollout. - Use the reader for lineage validation, model-context replay, and SQLite history projection, including when compression races with a read. - Add the opt-in `local_thread_store_shared_compression` feature. The default mode continues to leave shared lineages uncompressed for compatibility, while the new mode includes referenced sources and fork pointers. - Record the uncompressed size in new zstd frames so common prefix-bound checks can avoid decoding the full file. ## Testing Add coverage for seekable compressed reads, prefix validation across zstd frame shapes, compressed lineage replay and projection, concurrent forks, and compression-mode behavior. GitOrigin-RevId: 83e519f7eabf7448977eff37c55899858cc17a07
This commit is contained in:
1 parent
39507eea53
commit
1cc81ca89a
14 files changed
+493
-111
No files matched your search
@@ -20,6 +20,7 @@ mod reverse_jsonl_scanner;
|
||||
mod rollout_file_name;
|
||||
mod rollout_reference_index;
|
||||
pub(crate) mod search;
|
||||
mod seekable_reader;
|
||||
pub(crate) mod session_index;
|
||||
mod sqlite_metrics;
|
||||
pub mod state_db;
|
||||
@@ -76,11 +77,14 @@ pub static INTERACTIVE_SESSION_SOURCES: LazyLock<Vec<SessionSource>> = LazyLock:
|
||||
});
|
||||
|
||||
pub use codex_protocol::protocol::SessionMeta;
|
||||
pub use compression::RolloutCompressionMode;
|
||||
pub use compression::RolloutLineReader;
|
||||
pub use compression::existing_rollout_path;
|
||||
pub use compression::open_rollout_line_reader;
|
||||
pub use compression::plain_rollout_path;
|
||||
pub use compression::spawn_rollout_compression_worker;
|
||||
pub use seekable_reader::open_rollout_seekable_reader;
|
||||
pub use seekable_reader::rollout_contains_prefix;
|
||||
|
||||
/// Materializes a compressed rollout as plain JSONL before another rollout references it.
|
||||
pub async fn materialize_rollout_for_reference(
|
||||
|
||||
Reference in new issue
Block a user