mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Add process-scoped SQLite telemetry (#22154)
## Summary - add SQLite init, backfill-gate, and fallback telemetry without introducing a cross-cutting state-db access wrapper - install one process-scoped telemetry sink after OTEL startup and let low-level state/rollout paths emit through it directly - add process-start metrics for the process owners that initialize SQLite --------- Co-authored-by: Owen Lin <owen@openai.com>
This commit is contained in:
1 parent
cf6342b75b
commit
b401666ca5
22 files changed
+771
-134
No files matched your search
@@ -4,6 +4,7 @@ use crate::list::Cursor;
|
||||
use crate::list::SortDirection;
|
||||
use crate::list::ThreadSortKey;
|
||||
use crate::metadata;
|
||||
use crate::sqlite_metrics;
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
@@ -115,6 +116,29 @@ async fn try_init_with_roots_inner(
|
||||
sqlite_home.display()
|
||||
)
|
||||
})?;
|
||||
let backfill_gate_started = Instant::now();
|
||||
let backfill_gate_result = wait_for_backfill_gate(
|
||||
runtime.as_ref(),
|
||||
codex_home.as_path(),
|
||||
default_model_provider_id.as_str(),
|
||||
backfill_lease_seconds,
|
||||
)
|
||||
.await;
|
||||
codex_state::record_backfill_gate(
|
||||
/*telemetry*/ None,
|
||||
backfill_gate_started.elapsed(),
|
||||
&backfill_gate_result,
|
||||
);
|
||||
backfill_gate_result?;
|
||||
Ok(runtime)
|
||||
}
|
||||
|
||||
async fn wait_for_backfill_gate(
|
||||
runtime: &codex_state::StateRuntime,
|
||||
codex_home: &Path,
|
||||
default_model_provider_id: &str,
|
||||
backfill_lease_seconds: Option<i64>,
|
||||
) -> anyhow::Result<()> {
|
||||
let wait_started = Instant::now();
|
||||
let mut reported_wait = false;
|
||||
loop {
|
||||
@@ -125,24 +149,19 @@ async fn try_init_with_roots_inner(
|
||||
)
|
||||
})?;
|
||||
if backfill_state.status == codex_state::BackfillStatus::Complete {
|
||||
return Ok(runtime);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(backfill_lease_seconds) = backfill_lease_seconds {
|
||||
metadata::backfill_sessions_with_lease(
|
||||
runtime.as_ref(),
|
||||
codex_home.as_path(),
|
||||
default_model_provider_id.as_str(),
|
||||
runtime,
|
||||
codex_home,
|
||||
default_model_provider_id,
|
||||
backfill_lease_seconds,
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
metadata::backfill_sessions(
|
||||
runtime.as_ref(),
|
||||
codex_home.as_path(),
|
||||
default_model_provider_id.as_str(),
|
||||
)
|
||||
.await;
|
||||
metadata::backfill_sessions(runtime, codex_home, default_model_provider_id).await;
|
||||
}
|
||||
let backfill_state = runtime.get_backfill_state().await.map_err(|err| {
|
||||
anyhow::anyhow!(
|
||||
@@ -151,7 +170,7 @@ async fn try_init_with_roots_inner(
|
||||
)
|
||||
})?;
|
||||
if backfill_state.status == codex_state::BackfillStatus::Complete {
|
||||
return Ok(runtime);
|
||||
return Ok(());
|
||||
}
|
||||
if wait_started.elapsed() >= STARTUP_BACKFILL_WAIT_TIMEOUT {
|
||||
return Err(anyhow::anyhow!(
|
||||
@@ -195,17 +214,40 @@ fn emit_startup_warning(message: &str) {
|
||||
pub async fn get_state_db(config: &impl RolloutConfigView) -> Option<StateDbHandle> {
|
||||
let state_path = codex_state::state_db_path(config.sqlite_home());
|
||||
if !tokio::fs::try_exists(&state_path).await.unwrap_or(false) {
|
||||
codex_state::record_fallback(
|
||||
"get_state_db",
|
||||
"db_unavailable",
|
||||
/*telemetry_override*/ None,
|
||||
);
|
||||
return None;
|
||||
}
|
||||
let runtime = codex_state::StateRuntime::init(
|
||||
let runtime = match codex_state::StateRuntime::init(
|
||||
config.sqlite_home().to_path_buf(),
|
||||
config.model_provider_id().to_string(),
|
||||
)
|
||||
.await
|
||||
.ok()?;
|
||||
{
|
||||
Ok(runtime) => runtime,
|
||||
Err(_) => {
|
||||
codex_state::record_fallback(
|
||||
"get_state_db",
|
||||
"db_error",
|
||||
/*telemetry_override*/ None,
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
require_backfill_complete(runtime, config.sqlite_home()).await
|
||||
}
|
||||
|
||||
/// Build a SQLite telemetry recorder backed by an OTEL metrics client.
|
||||
pub fn sqlite_telemetry_recorder(
|
||||
metrics: codex_otel::MetricsClient,
|
||||
originator: &str,
|
||||
) -> codex_state::DbTelemetryHandle {
|
||||
sqlite_metrics::recorder(metrics, originator)
|
||||
}
|
||||
|
||||
async fn require_backfill_complete(
|
||||
runtime: StateDbHandle,
|
||||
codex_home: &Path,
|
||||
@@ -218,6 +260,11 @@ async fn require_backfill_complete(
|
||||
codex_home.display(),
|
||||
state.status.as_str()
|
||||
);
|
||||
codex_state::record_fallback(
|
||||
"get_state_db",
|
||||
"backfill_incomplete",
|
||||
/*telemetry_override*/ None,
|
||||
);
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -225,6 +272,11 @@ async fn require_backfill_complete(
|
||||
"failed to read backfill state at {}: {err}",
|
||||
codex_home.display()
|
||||
);
|
||||
codex_state::record_fallback(
|
||||
"get_state_db",
|
||||
"db_error",
|
||||
/*telemetry_override*/ None,
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user