Add process-scoped SQLite telemetry (#22154)

## Summary
- add SQLite init, backfill-gate, and fallback telemetry without
introducing a cross-cutting state-db access wrapper
- install one process-scoped telemetry sink after OTEL startup and let
low-level state/rollout paths emit through it directly
- add process-start metrics for the process owners that initialize
SQLite

---------

Co-authored-by: Owen Lin <owen@openai.com>
This commit is contained in:
jif-oaiandOwen Lin authored and GitHub committed 2026-05-11 11:32:40 -07:00
1 parent cf6342b75b
commit b401666ca5
22 files changed
+771 -134

No files matched your search

+65 -13
View File
@@ -4,6 +4,7 @@ use crate::list::Cursor;
use crate::list::SortDirection;
use crate::list::ThreadSortKey;
use crate::metadata;
use crate::sqlite_metrics;
use chrono::DateTime;
use chrono::Utc;
use codex_protocol::ThreadId;
@@ -115,6 +116,29 @@ async fn try_init_with_roots_inner(
sqlite_home.display()
)
})?;
let backfill_gate_started = Instant::now();
let backfill_gate_result = wait_for_backfill_gate(
runtime.as_ref(),
codex_home.as_path(),
default_model_provider_id.as_str(),
backfill_lease_seconds,
)
.await;
codex_state::record_backfill_gate(
/*telemetry*/ None,
backfill_gate_started.elapsed(),
&backfill_gate_result,
);
backfill_gate_result?;
Ok(runtime)
}
async fn wait_for_backfill_gate(
runtime: &codex_state::StateRuntime,
codex_home: &Path,
default_model_provider_id: &str,
backfill_lease_seconds: Option<i64>,
) -> anyhow::Result<()> {
let wait_started = Instant::now();
let mut reported_wait = false;
loop {
@@ -125,24 +149,19 @@ async fn try_init_with_roots_inner(
)
})?;
if backfill_state.status == codex_state::BackfillStatus::Complete {
return Ok(runtime);
return Ok(());
}
if let Some(backfill_lease_seconds) = backfill_lease_seconds {
metadata::backfill_sessions_with_lease(
runtime.as_ref(),
codex_home.as_path(),
default_model_provider_id.as_str(),
runtime,
codex_home,
default_model_provider_id,
backfill_lease_seconds,
)
.await;
} else {
metadata::backfill_sessions(
runtime.as_ref(),
codex_home.as_path(),
default_model_provider_id.as_str(),
)
.await;
metadata::backfill_sessions(runtime, codex_home, default_model_provider_id).await;
}
let backfill_state = runtime.get_backfill_state().await.map_err(|err| {
anyhow::anyhow!(
@@ -151,7 +170,7 @@ async fn try_init_with_roots_inner(
)
})?;
if backfill_state.status == codex_state::BackfillStatus::Complete {
return Ok(runtime);
return Ok(());
}
if wait_started.elapsed() >= STARTUP_BACKFILL_WAIT_TIMEOUT {
return Err(anyhow::anyhow!(
@@ -195,17 +214,40 @@ fn emit_startup_warning(message: &str) {
pub async fn get_state_db(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
let state_path = codex_state::state_db_path(config.sqlite_home());
if !tokio::fs::try_exists(&state_path).await.unwrap_or(false) {
codex_state::record_fallback(
"get_state_db",
"db_unavailable",
/*telemetry_override*/ None,
);
return None;
}
let runtime = codex_state::StateRuntime::init(
let runtime = match codex_state::StateRuntime::init(
config.sqlite_home().to_path_buf(),
config.model_provider_id().to_string(),
)
.await
.ok()?;
{
Ok(runtime) => runtime,
Err(_) => {
codex_state::record_fallback(
"get_state_db",
"db_error",
/*telemetry_override*/ None,
);
return None;
}
};
require_backfill_complete(runtime, config.sqlite_home()).await
}
/// Build a SQLite telemetry recorder backed by an OTEL metrics client.
pub fn sqlite_telemetry_recorder(
metrics: codex_otel::MetricsClient,
originator: &str,
) -> codex_state::DbTelemetryHandle {
sqlite_metrics::recorder(metrics, originator)
}
async fn require_backfill_complete(
runtime: StateDbHandle,
codex_home: &Path,
@@ -218,6 +260,11 @@ async fn require_backfill_complete(
codex_home.display(),
state.status.as_str()
);
codex_state::record_fallback(
"get_state_db",
"backfill_incomplete",
/*telemetry_override*/ None,
);
None
}
Err(err) => {
@@ -225,6 +272,11 @@ async fn require_backfill_complete(
"failed to read backfill state at {}: {err}",
codex_home.display()
);
codex_state::record_fallback(
"get_state_db",
"db_error",
/*telemetry_override*/ None,
);
None
}
}