fix: Auto-recover from corrupted sqlite databases (#26859)

Further investigation of the sqlite incidents showed that the problems
are due to corruption from the older version of SQLite that we recently
upgraded, and that the data is truly corrupted in the root database --
recovery of all data is not possible. Given that the data is
reconstructable from the rollouts on disk, we should just auto-backup
the database and let codex rebuild the rollout info from the disk
rollouts.

The new behavior is that appserver auto-backs-up and rebuilds (with logs
reflecting that behavior). The CLI now pops a message letting you know
this happened and the paths of the backed-up corrupt db and the new
database. There is also context added so that the desktop app can read
the rebuild info from it and inform the user with it.
This commit is contained in:
David de Regt
2026-06-10 11:24:29 -07:00
committed by GitHub
parent a19d43a40a
commit 3691fe5b76
13 changed files with 693 additions and 139 deletions
+9 -8
View File
@@ -38,6 +38,7 @@ use codex_utils_cli::ProfileV2Name;
use codex_utils_cli::SharedCliOptions;
use codex_utils_cli::resume_hint;
use owo_colors::OwoColorize;
use std::collections::HashSet;
use std::io::IsTerminal;
use std::path::PathBuf;
use supports_color::Stream;
@@ -2188,7 +2189,7 @@ async fn run_interactive_tui(
remote_endpoint.clone(),
)
};
let mut attempted_repair = false;
let mut attempted_backups = HashSet::new();
loop {
let err = match start_tui().await {
Ok(exit_info) => return Ok(exit_info),
@@ -2201,25 +2202,25 @@ async fn run_interactive_tui(
local_state_db::print_locked_guidance(startup_error);
return Ok(AppExitInfo::fatal(startup_error.to_string()));
}
if attempted_repair {
if !local_state_db::is_auto_backup_recoverable(startup_error) {
local_state_db::print_diagnostic_guidance(startup_error);
return Ok(AppExitInfo::fatal(startup_error.to_string()));
}
if !local_state_db::confirm_repair(startup_error)? {
if !attempted_backups.insert(startup_error.database_path().to_path_buf()) {
local_state_db::print_diagnostic_guidance(startup_error);
return Ok(AppExitInfo::fatal(startup_error.to_string()));
}
match local_state_db::repair_files(startup_error).await {
Ok(backups) => local_state_db::print_repair_backups(&backups),
Err(repair_err) => {
local_state_db::print_auto_backup_start(startup_error);
match local_state_db::backup_files_for_fresh_start(startup_error).await {
Ok(backups) => local_state_db::confirm_fresh_start_rebuild(startup_error, &backups)?,
Err(backup_err) => {
local_state_db::print_diagnostic_guidance(startup_error);
return Ok(AppExitInfo::fatal(format!(
"failed to repair Codex local data automatically: {repair_err}"
"failed to move damaged Codex local database files into a backup folder automatically: {backup_err}"
)));
}
}
attempted_repair = true;
}
}