fix(migration): preserve credential errors and recover failure states

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-09-30 16:04:28 +08:00
1 parent 1b9f65f260
commit 961dce99e8
13 files changed
+204 -72

No files matched your search

+191 -61
View File
@@ -207,6 +207,7 @@ pub struct Storage {
/// round-trips to the OS credential store, so hydrating N stored secrets
/// used to mean N credential-store accesses on the startup path.
secret_codec_cache: Arc<Mutex<Option<CachedSecretCodec>>>,
migration_failure: Arc<Mutex<Option<MigrationFailure>>>,
}
/// Key material plus the digest of every key file it was resolved from.
@@ -215,6 +216,14 @@ struct CachedSecretCodec {
key_files: Vec<(PathBuf, Option<[u8; 32]>)>,
}
#[derive(Clone)]
struct MigrationFailure {
code: String,
backup_path: Option<String>,
}
const MIGRATION_LOCK_FILE: &str = ".dbx-secret-migration.lock";
pub const SECRET_STORE_MIGRATION_ID: &str = "secret-store-v1";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -1182,7 +1191,15 @@ fn migration_backup_paths(counts: &serde_json::Value) -> Result<Vec<String>, Str
}
fn migration_error_code(error: &str) -> &'static str {
if error.contains("MISSING_MANAGED_KEY") {
if error.contains("MIGRATION_STATE_WRITE_FAILED") {
"MIGRATION_STATE_WRITE_FAILED"
} else if error.contains("MIGRATION_RESTORE_FAILED") {
"MIGRATION_RESTORE_FAILED"
} else if error.contains("MIGRATION_INTERRUPTED") {
"MIGRATION_INTERRUPTED"
} else if error.contains("BACKUP_FAILED") {
"BACKUP_FAILED"
} else if error.contains("MISSING_MANAGED_KEY") {
"MISSING_MANAGED_KEY"
} else if error.contains("ENCRYPTED_DATA_KEY_MISSING") {
"ENCRYPTED_DATA_KEY_MISSING"
@@ -1192,6 +1209,10 @@ fn migration_error_code(error: &str) -> &'static str {
"SECRET_KEY_INVALID"
} else if error.contains("KEY_FILE_UNAVAILABLE") {
"KEY_FILE_UNAVAILABLE"
} else if error.contains("KEYRING_ACCESS_FAILED") {
"KEYRING_ACCESS_FAILED"
} else if error.contains("KEYRING_WRITE_FAILED") {
"KEYRING_WRITE_FAILED"
} else if error.contains("MISSING_EXTERNAL_KEY") || error.contains("MISSING_PERSISTENT_KEY") {
"MISSING_EXTERNAL_KEY"
} else if error.contains("KEY_PROVIDER_UNAVAILABLE") {
@@ -1209,11 +1230,22 @@ fn migration_error_code(error: &str) -> &'static str {
fn migration_safe_message(code: &str, _error: &str) -> String {
match code {
"MIGRATION_STATE_WRITE_FAILED" => {
"Could not save the migration failure status; keep the backup and retry".to_string()
}
"MIGRATION_RESTORE_FAILED" => "Could not restore the database backup; keep the backup for recovery".to_string(),
"MIGRATION_INTERRUPTED" => "The previous migration was interrupted; keep the backup and retry".to_string(),
"MISSING_MANAGED_KEY" => "A managed data-directory secret key is required".to_string(),
"ENCRYPTED_DATA_KEY_MISSING" => "The key for existing encrypted data is missing".to_string(),
"SECRET_KEY_MISMATCH" => "The configured secret key cannot decrypt existing data".to_string(),
"SECRET_KEY_INVALID" => "The configured secret key is invalid".to_string(),
"KEY_FILE_UNAVAILABLE" => "The configured secret key file is unavailable".to_string(),
"KEYRING_ACCESS_FAILED" => {
"The platform credential store refused access to the DBX secret-store key".to_string()
}
"KEYRING_WRITE_FAILED" => {
"DBX could not save the secret-store key in the platform credential store".to_string()
}
"MISSING_EXTERNAL_KEY" => "An external secret key is required".to_string(),
"BACKUP_FAILED" => "Could not create a migration backup".to_string(),
"LEGACY_JSON_INVALID" => "A legacy configuration file could not be read".to_string(),
@@ -1269,6 +1301,7 @@ impl Storage {
secret_key_policy: SecretKeyPolicy::PlatformDefault,
secret_key_creation_allowed: true,
secret_codec_cache: Arc::new(Mutex::new(None)),
migration_failure: Arc::new(Mutex::new(None)),
};
// Best-effort: switching journal mode is itself a lock-sensitive
// operation, so a transient failure here (e.g. another process
@@ -1324,9 +1357,7 @@ impl Storage {
// files are unchanged across the resolve. A pair taken after resolving
// could combine a stale codec with fresh digests, which the use-time
// digest check would then never reject.
if self.key_file_digests() == key_files {
self.cache_secret_codec(codec, key_files);
}
self.cache_secret_codec_if_unchanged(codec, key_files);
Ok(codec)
}
@@ -1348,6 +1379,14 @@ impl Storage {
*self.secret_codec_cache.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(entry);
}
fn cache_secret_codec_if_unchanged(&self, codec: SecretCodec, before: Vec<(PathBuf, Option<[u8; 32]>)>) {
if self.key_file_digests() == before {
self.cache_secret_codec(codec, before);
} else {
self.invalidate_secret_codec();
}
}
fn invalidate_secret_codec(&self) {
*self.secret_codec_cache.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
}
@@ -1489,7 +1528,20 @@ impl Storage {
pub async fn inspect_data_migration(&self) -> Result<MigrationPreflight, String> {
let files = self.legacy_json_files().await?;
let stored = self.load_migration_state().await?;
let mut stored = self.load_migration_state().await?;
let failure = self.migration_failure.lock().unwrap_or_else(|p| p.into_inner()).clone();
if let Some(failure) = failure {
stored.state = MigrationState::Failed;
stored.error_message = Some(migration_safe_message(&failure.code, ""));
stored.error_code = Some(failure.code);
stored.backup_path = failure.backup_path.or(stored.backup_path);
} else if stored.state == MigrationState::Running && !self.migration_is_active()? {
// A process exit releases the lock. Report interruption without
// modifying the database during a status request.
stored.state = MigrationState::Failed;
stored.error_code = Some("MIGRATION_INTERRUPTED".to_string());
stored.error_message = Some(migration_safe_message("MIGRATION_INTERRUPTED", ""));
}
let current_fingerprint = self.migration_source_fingerprint().await?;
let cached_scan: Option<(i64, i64, i64, i64, i64, i64, i64)> =
if stored.source_fingerprint.as_deref() == Some(current_fingerprint.as_str()) {
@@ -1581,6 +1633,7 @@ impl Storage {
};
// This probe is deliberately read-only. It must not create a keyring
// entry, key file, or change permissions while displaying status.
let key_files_before = self.key_file_digests();
let key_probe = self.resolve_secret_key(false);
let mut key_provider_available = key_probe.is_ok();
let database_plaintext_count = (plaintext + ai + tunnels).max(0) as usize;
@@ -1593,7 +1646,7 @@ impl Storage {
&& !matches!(self.secret_key_policy, SecretKeyPolicy::ExternalOnly)
&& encrypted == 0
&& (database_plaintext_count > 0 || sync_credentials > 0 || files.iter().any(|file| file.exists));
let mut key_error_code = key_probe.as_ref().err().cloned();
let mut key_error_code = key_probe.as_ref().err().map(|error| migration_error_code(error).to_string());
if let Ok(resolved) = key_probe.as_ref() {
if encrypted > 0 && self.validate_existing_encrypted_data(resolved.codec).await.is_err() {
key_provider_available = false;
@@ -1619,6 +1672,8 @@ impl Storage {
"SECRET_KEY_INVALID"
| "SECRET_KEY_MISMATCH"
| "KEY_FILE_UNAVAILABLE"
| "KEYRING_ACCESS_FAILED"
| "KEYRING_WRITE_FAILED"
| "MISSING_EXTERNAL_KEY"
| "ENCRYPTED_DATA_KEY_MISSING"
)
@@ -1635,7 +1690,9 @@ impl Storage {
None => MigrationKeyStatus::Unavailable,
};
let source_changed = stored.source_fingerprint.as_deref().is_some_and(|value| value != current_fingerprint);
let state = if fatal_key_error
let state = if matches!(stored.state, MigrationState::Failed | MigrationState::Running) {
stored.state.clone()
} else if fatal_key_error
|| missing_required_key
|| missing_existing_key
|| (has_legacy_data
@@ -1661,10 +1718,16 @@ impl Storage {
"SECRET_KEY_INVALID" => "The configured secret key is invalid",
"SECRET_KEY_MISMATCH" => "The configured secret key cannot decrypt existing data",
"KEY_FILE_UNAVAILABLE" => "The configured secret key file is unavailable",
"KEYRING_ACCESS_FAILED" => "The platform credential store refused access to the DBX secret-store key",
"KEYRING_WRITE_FAILED" => "DBX could not save the secret-store key in the platform credential store",
"MISSING_EXTERNAL_KEY" => "An external secret key is required",
_ => "The local secret provider is unavailable",
};
(Some(code), Some(message.to_string()))
} else if stored.state == MigrationState::Failed && stored.error_code.is_some() {
let code = migration_error_code(stored.error_code.as_deref().unwrap()).to_string();
let message = migration_safe_message(&code, "");
(Some(code), Some(message))
} else if missing_required_key {
(Some("MISSING_EXTERNAL_KEY".to_string()), Some("An external secret key is required".to_string()))
} else if missing_existing_key {
@@ -1680,6 +1743,14 @@ impl Storage {
.map(|resolved| resolved.source.as_str())
.unwrap_or(SecretKeySource::Unavailable.as_str())
.to_string();
// Reuse a successful read during the subsequent migration start. This
// keeps the read-only preflight and the first migration write from
// prompting the platform credential store twice in one action.
if key_provider_available {
if let Ok(resolved) = key_probe.as_ref() {
self.cache_secret_codec_if_unchanged(resolved.codec, key_files_before);
}
}
let preflight = MigrationPreflight {
migration_id: SECRET_STORE_MIGRATION_ID.to_string(),
state,
@@ -1730,43 +1801,77 @@ impl Storage {
.await
}
pub async fn start_data_migration(&self) -> Result<MigrationReport, String> {
let lock = DATA_MIGRATION_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await;
// Migration may create or replace key material, so the codec resolved
// for earlier reads must not be reused here.
self.invalidate_secret_codec();
let preflight = self.inspect_data_migration().await?;
if preflight.is_ready() {
return Ok(MigrationReport {
migration_id: SECRET_STORE_MIGRATION_ID.to_string(),
state: preflight.state,
backup_path: preflight.backup_path,
database_plaintext_count: 0,
legacy_json_files: Vec::new(),
verified_secret_count: 0,
error_code: None,
error_message: None,
});
fn migration_is_active(&self) -> Result<bool, String> {
let file =
match std::fs::OpenOptions::new().read(true).write(true).open(self.data_dir().join(MIGRATION_LOCK_FILE)) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(_) => return Err("MIGRATION_LOCK_UNAVAILABLE".to_string()),
};
match fs2::FileExt::try_lock_exclusive(&file) {
Ok(()) => Ok(false), // Closing the handle releases the probe lock.
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => Ok(true),
Err(_) => Err("MIGRATION_LOCK_UNAVAILABLE".to_string()),
}
if !preflight.key_provider_available && !preflight.key_creation_allowed {
let code = preflight.error_code.as_deref().unwrap_or("KEY_PROVIDER_UNAVAILABLE");
let error = code.to_string();
self.set_migration_state(MigrationState::Failed, None, Some(code), Some(&error), None).await?;
return Err(error);
}
fn lock_data_migration(&self) -> Result<std::fs::File, String> {
let mut options = std::fs::OpenOptions::new();
options.read(true).write(true).create(true).truncate(false);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
self.set_migration_state(MigrationState::Running, None, None, None, Some(&migration_counts_json(&preflight)))
.await?;
let backup_path = match self.create_migration_backup().await {
Ok(path) => path,
Err(error) => {
let _ = self
.set_migration_state(MigrationState::Failed, None, Some("BACKUP_FAILED"), Some(&error), None)
.await;
return Err(error);
let file = options
.open(self.data_dir().join(MIGRATION_LOCK_FILE))
.map_err(|_| "MIGRATION_LOCK_UNAVAILABLE".to_string())?;
fs2::FileExt::try_lock_exclusive(&file).map_err(|error| {
if error.kind() == std::io::ErrorKind::WouldBlock {
"MIGRATION_IN_PROGRESS".to_string()
} else {
"MIGRATION_LOCK_UNAVAILABLE".to_string()
}
};
self.set_migration_state(MigrationState::Running, Some(&backup_path), None, None, None).await?;
})?;
Ok(file)
}
pub async fn start_data_migration(&self) -> Result<MigrationReport, String> {
let _lock = DATA_MIGRATION_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await;
let _file_lock = self.lock_data_migration()?;
*self.migration_failure.lock().unwrap_or_else(|p| p.into_inner()) = None;
self.invalidate_secret_codec();
let mut backup_path: Option<String> = None;
let mut data_started = false;
let result = async {
let preflight = self.inspect_data_migration().await?;
if preflight.is_ready() {
return Ok(MigrationReport {
migration_id: SECRET_STORE_MIGRATION_ID.to_string(),
state: preflight.state,
backup_path: preflight.backup_path,
database_plaintext_count: 0,
legacy_json_files: Vec::new(),
verified_secret_count: 0,
error_code: None,
error_message: None,
});
}
if !preflight.key_provider_available && !preflight.key_creation_allowed {
return Err(preflight.error_code.unwrap_or_else(|| "KEY_PROVIDER_UNAVAILABLE".to_string()));
}
self.set_migration_state(
MigrationState::Running,
None,
None,
None,
Some(&migration_counts_json(&preflight)),
)
.await?;
backup_path = Some(self.create_migration_backup().await.map_err(|_| "BACKUP_FAILED".to_string())?);
let backup_path = backup_path.as_deref().unwrap();
self.set_migration_state(MigrationState::Running, Some(backup_path), None, None, None).await?;
data_started = true;
let codec = self.secret_codec(preflight.key_creation_allowed)?;
self.run_database_legacy_migrations(&codec).await?;
self.migrate_from_json_staged(self.data_dir()).await?;
@@ -1774,7 +1879,7 @@ impl Storage {
let renamed = self.finalize_legacy_json_files().await?;
self.set_migration_state(
MigrationState::Succeeded,
Some(&backup_path),
Some(backup_path),
None,
None,
Some(&{
@@ -1796,7 +1901,7 @@ impl Storage {
Ok::<MigrationReport, String>(MigrationReport {
migration_id: SECRET_STORE_MIGRATION_ID.to_string(),
state: MigrationState::Succeeded,
backup_path: Some(backup_path.clone()),
backup_path: Some(backup_path.to_string()),
database_plaintext_count: preflight.database_plaintext_count,
legacy_json_files: renamed,
verified_secret_count: verified,
@@ -1805,24 +1910,37 @@ impl Storage {
})
}
.await;
if let Err(error) = &result {
// Every database migration helper uses its own transaction. Restore
// the pre-migration SQLite snapshot if a later JSON parse, write,
// or verification step fails, so the live database is never left
// half-upgraded.
let _ = self.restore_database_backup(&backup_path).await;
let _ = self
.set_migration_state(
MigrationState::Failed,
Some(&backup_path),
Some(migration_error_code(error)),
Some(error),
None,
)
.await;
match result {
Ok(report) => Ok(report),
Err(error) => {
self.invalidate_secret_codec();
let mut code = migration_error_code(&error).to_string();
if data_started {
if let Some(path) = backup_path.as_deref() {
if self.restore_database_backup(path).await.is_err() {
code = "MIGRATION_RESTORE_FAILED".to_string();
}
}
}
if self
.set_migration_state(MigrationState::Failed, backup_path.as_deref(), Some(&code), Some(&code), None)
.await
.is_err()
{
code = "MIGRATION_STATE_WRITE_FAILED".to_string();
// A read-only status request must still report the failed
// action if SQLite cannot persist it. After process exit,
// a retained running record is detected via the file lock.
*self.migration_failure.lock().unwrap_or_else(|p| p.into_inner()) =
Some(MigrationFailure { code: code.clone(), backup_path });
}
if code == "MIGRATION_STATE_WRITE_FAILED" || code == "MIGRATION_RESTORE_FAILED" {
Err(code)
} else {
Err(error)
}
}
}
drop(lock);
result
}
pub async fn retry_data_migration(&self) -> Result<MigrationReport, String> {
@@ -1994,7 +2112,8 @@ impl Storage {
}
let counts_json = counts.to_string();
let backup_path = backup_path.or(previous.backup_path);
let source_fingerprint = Some(self.migration_source_fingerprint().await?);
let source_fingerprint =
if state == "failed" { None } else { Some(self.migration_source_fingerprint().await?) };
self.with_conn(move |conn| {
conn.execute(
"INSERT INTO data_migrations (migration_id,state,source_fingerprint,counts_json,backup_path,error_code,error_message,started_at,completed_at)
@@ -9176,7 +9295,8 @@ mod tests {
.unwrap()
.with_secret_key_policy(SecretKeyPolicy::ManagedDataDir);
let status = reopened.inspect_data_migration().await.unwrap();
assert_eq!(status.state, super::MigrationState::Running);
assert_eq!(status.state, super::MigrationState::Failed);
assert_eq!(status.error_code.as_deref(), Some("MIGRATION_INTERRUPTED"));
assert!(!status.is_ready());
reopened.retry_data_migration().await.unwrap();
assert!(reopened.inspect_data_migration().await.unwrap().is_ready());
@@ -13836,6 +13956,16 @@ mod tests {
assert_eq!(super::migration_backup_paths(&valid).unwrap(), vec!["/tmp/dbx-secret-migration-a"]);
}
#[test]
fn migration_errors_preserve_keyring_failure_classification() {
assert_eq!(super::migration_error_code("KEYRING_ACCESS_FAILED: item denied"), "KEYRING_ACCESS_FAILED");
assert_eq!(super::migration_error_code("KEYRING_WRITE_FAILED: item denied"), "KEYRING_WRITE_FAILED");
assert_eq!(
super::migration_safe_message("KEYRING_ACCESS_FAILED", "ignored"),
"The platform credential store refused access to the DBX secret-store key"
);
}
#[tokio::test]
async fn stored_connection_count_reads_the_table_without_loading_secrets() {
let dir = tempfile::tempdir().unwrap();