feat(security): encrypt local credentials and guide data migration

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-09-23 11:02:52 +08:00
1 parent 19f36557b9
commit 250e0c0a10
58 files changed
+8831 -435

No files matched your search

+2 -2
View File
@@ -48,7 +48,7 @@ fn api_path_suffix<'a>(path: &'a str, public_base_path: &str) -> Option<&'a str>
path.strip_prefix(base)?.strip_prefix("/api/")
}
fn middleware_api_path_suffix<'a>(path: &'a str, public_base_path: &str) -> Option<&'a str> {
pub(crate) fn middleware_api_path_suffix<'a>(path: &'a str, public_base_path: &str) -> Option<&'a str> {
if let Some(suffix) = api_path_suffix(path, public_base_path) {
return Some(suffix);
}
@@ -281,7 +281,7 @@ mod tests {
}
#[test]
fn middleware_api_path_suffix_handles_nested_router_paths() {
pub(crate) fn middleware_api_path_suffix_handles_nested_router_paths() {
assert_eq!(middleware_api_path_suffix("/auth/check", "/"), Some("auth/check"));
assert_eq!(middleware_api_path_suffix("/connection/list", "/"), Some("connection/list"));
assert_eq!(middleware_api_path_suffix("/api/connection/list", "/"), Some("connection/list"));
+84 -5
View File
@@ -13,17 +13,19 @@ use argon2::password_hash::rand_core::OsRng;
use argon2::password_hash::SaltString;
use argon2::{Argon2, PasswordHasher};
use axum::extract::DefaultBodyLimit;
use axum::http::Uri;
use axum::http::{Request, StatusCode, Uri};
use axum::middleware;
use axum::response::Redirect;
use axum::response::{IntoResponse, Redirect, Response};
use axum::routing::{delete, get, post};
use axum::Router;
use dbx_core::connection::AppState;
use dbx_core::persistence::secret_codec::SecretKeyPolicy;
use dbx_core::sql_dialect::dialect_loader::{register_core_dialects, DialectPluginLoader, DialectRegistry};
use dbx_core::sql_dialect::hot_reload::DialectHotReload;
use dbx_core::storage::Storage;
use dbx_mcp::{streamable_http_router, DbxBackend, HttpAuth, LocalBackend};
use state::WebState;
use std::sync::atomic::{AtomicBool, Ordering};
use tokio::sync::RwLock;
use tower_http::compression::predicate::{DefaultPredicate, NotForContentType, Predicate};
use tower_http::compression::CompressionLayer;
@@ -78,6 +80,35 @@ fn web_body_limit_bytes_from_value(value: Option<&str>) -> usize {
mb.saturating_mul(1024 * 1024)
}
async fn migration_gate(
state: axum::extract::State<Arc<WebState>>,
request: Request<axum::body::Body>,
next: axum::middleware::Next,
) -> Response {
let suffix = auth::middleware_api_path_suffix(request.uri().path(), &state.public_base_path);
let allowed = suffix.is_some_and(|path| {
matches!(
path,
"migration/status" | "migration/start" | "migration/retry" | "migration/cleanup-backups" | "ping"
) || path.starts_with("auth/")
});
if !allowed && !state.migration_ready.load(Ordering::Acquire) {
return (
StatusCode::LOCKED,
axum::Json(serde_json::json!({
"code": "DATA_MIGRATION_REQUIRED",
"message": "Complete the data security migration before using DBX"
})),
)
.into_response();
}
next.run(request).await
}
async fn storage_migration_ready(app: &Arc<AppState>) -> bool {
app.storage.inspect_data_migration().await.map(|status| status.is_ready()).unwrap_or(false)
}
fn web_agent_dir(data_dir: &std::path::Path) -> std::path::PathBuf {
web_agent_dir_from_env(data_dir, std::env::var("DBX_AGENT_DIR").ok())
}
@@ -301,8 +332,10 @@ async fn main() {
let app_state = {
let db_path = data_dir.join("dbx.db");
let storage = Storage::open(&db_path).await.expect("Failed to open storage");
storage.migrate_from_json(&data_dir).await.expect("Failed to migrate JSON data");
let storage = Storage::open_unmigrated(&db_path)
.await
.expect("Failed to open storage")
.with_secret_key_policy(SecretKeyPolicy::ManagedDataDir);
// Initialize core dialect registry and load external plugin dialects
register_core_dialects();
@@ -349,6 +382,7 @@ async fn main() {
let public_base_path = normalize_public_base_path(std::env::var("DBX_PUBLIC_BASE_PATH").ok());
let migration_ready = storage_migration_ready(&app_state).await;
let web_state = Arc::new(WebState {
app: app_state,
data_dir,
@@ -364,12 +398,17 @@ async fn main() {
login_rate_limit: tokio::sync::Mutex::new(state::LoginRateLimit { fail_count: 0, locked_until: None }),
export_files: RwLock::new(HashMap::new()),
ssh_prompts: Arc::new(ssh_prompt::SshPromptHub::new()),
migration_ready: Arc::new(AtomicBool::new(migration_ready)),
});
ssh_prompt::install_web_ssh_prompt_bridge(web_state.ssh_prompts.clone());
// API routes
let api = Router::new()
.route("/migration/status", get(routes::migration::status))
.route("/migration/start", post(routes::migration::start))
.route("/migration/retry", post(routes::migration::retry))
.route("/migration/cleanup-backups", post(routes::migration::cleanup_backups))
// Auth
.route("/auth/login", post(auth::login))
.route("/auth/check", get(auth::check))
@@ -1188,6 +1227,7 @@ async fn main() {
api.route("/query/build-duckdb-attach-database-sql", post(routes::query::build_duckdb_attach_database_sql));
let api = add_mq_routes(api)
.layer(middleware::from_fn_with_state(web_state.clone(), migration_gate))
.layer(middleware::from_fn_with_state(web_state.clone(), auth::auth_middleware))
.with_state(web_state.clone());
@@ -1199,7 +1239,7 @@ async fn main() {
.layer(tower_http::trace::TraceLayer::new_for_http());
if let Some(mcp_router) = web_mcp_router(&web_state).expect("Invalid DBX Web MCP configuration") {
app = app.merge(mcp_router);
app = app.merge(mcp_router.layer(middleware::from_fn_with_state(web_state.clone(), migration_gate)));
tracing::info!("DBX Web MCP is enabled at /mcp");
}
@@ -1248,6 +1288,45 @@ mod tests {
use axum::Router;
use tower_http::compression::predicate::Predicate;
#[tokio::test]
async fn migration_http_gate_blocks_business_until_ready_but_allows_cleanup_handler() {
use std::sync::{atomic::Ordering, Arc};
let directory = tempfile::tempdir().unwrap();
let storage = dbx_core::storage::Storage::open_unmigrated(&directory.path().join("dbx.db")).await.unwrap();
let app = Arc::new(dbx_core::connection::AppState::new(storage));
let state = Arc::new(crate::state::WebState::for_tests(app, directory.path().to_path_buf()));
state.migration_ready.store(false, Ordering::Release);
let router = Router::new()
.route("/api/migration/status", get(|| async { "status" }))
.route("/api/migration/cleanup-backups", post(|| async { "cleanup" }))
.route("/api/connection/list", get(|| async { "connections" }))
.route("/mcp", post(|| async { "mcp" }))
.layer(axum::middleware::from_fn_with_state(state.clone(), super::migration_gate));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
let client = reqwest::Client::new();
assert_eq!(
client.get(format!("http://{address}/api/migration/status")).send().await.unwrap().status(),
StatusCode::OK
);
for (method, path) in [(reqwest::Method::GET, "/api/connection/list"), (reqwest::Method::POST, "/mcp")] {
let response = client.request(method, format!("http://{address}{path}")).send().await.unwrap();
assert_eq!(response.status(), StatusCode::LOCKED);
assert_eq!(response.json::<serde_json::Value>().await.unwrap()["code"], "DATA_MIGRATION_REQUIRED");
}
assert_eq!(
client.post(format!("http://{address}/api/migration/cleanup-backups")).send().await.unwrap().status(),
StatusCode::OK
);
state.migration_ready.store(true, Ordering::Release);
assert_eq!(
client.get(format!("http://{address}/api/connection/list")).send().await.unwrap().status(),
StatusCode::OK
);
server.abort();
}
fn compression_response(content_type: &str) -> Response<Body> {
Response::builder().header(CONTENT_TYPE, content_type).body(Body::from(vec![b'x'; 64])).unwrap()
}
+56 -17
View File
@@ -3,16 +3,16 @@ use std::sync::Arc;
use axum::extract::State;
use axum::Json;
use dbx_core::cloud_sync::{
apply_sync_snapshot, build_sync_snapshot, build_sync_snapshot_with_saved_secrets, finalize_snippet_migration,
forget_snippet_token, forget_webdav_password,
forget_webdav_sync_secrets_passphrase as core_forget_webdav_sync_secrets_passphrase, resolve_snippet_token,
resolve_webdav_password, resolve_webdav_sync_secrets_passphrase, retry_pending_snippet_cleanup,
save_snippet_sync_id_for_instance as core_save_snippet_sync_id, save_snippet_token, save_webdav_password,
save_webdav_sync_secrets_preference as core_save_webdav_sync_secrets_preference, snippet_saved_token_status,
snippet_sync_settings_for_instance as core_snippet_sync_settings, webdav_saved_password_status,
webdav_sync_secrets_status as core_webdav_sync_secrets_status, ApplySnapshotOptions, ApplySnapshotSummary,
SnippetProvider, SnippetSyncClient, SnippetSyncConfig, SnippetSyncSettings, SnippetSyncSummary, SnippetTokenStatus,
WebDavClient, WebDavConfig, WebDavPasswordStatus, WebDavSyncSecretsStatus, WebDavSyncSummary,
apply_sync_snapshot, build_sync_snapshot_with_options, finalize_snippet_migration, forget_snippet_token,
forget_webdav_password, forget_webdav_sync_secrets_passphrase as core_forget_webdav_sync_secrets_passphrase,
resolve_snippet_token, resolve_webdav_password, resolve_webdav_sync_secrets_passphrase,
retry_pending_snippet_cleanup, save_snippet_sync_id_for_instance as core_save_snippet_sync_id, save_snippet_token,
save_webdav_password, save_webdav_sync_secrets_preference as core_save_webdav_sync_secrets_preference,
snippet_saved_token_status, snippet_sync_settings_for_instance as core_snippet_sync_settings,
webdav_saved_password_status, webdav_sync_secrets_status as core_webdav_sync_secrets_status, ApplySnapshotOptions,
ApplySnapshotSummary, SnippetProvider, SnippetSyncClient, SnippetSyncConfig, SnippetSyncSettings,
SnippetSyncSummary, SnippetTokenStatus, WebDavClient, WebDavConfig, WebDavPasswordStatus, WebDavSyncSecretsStatus,
WebDavSyncSummary,
};
use dbx_core::storage::DesktopSettings;
use serde::{Deserialize, Serialize};
@@ -57,6 +57,9 @@ pub struct WebDavUploadRequest {
pub config: WebDavConfig,
pub editor_settings: Option<serde_json::Value>,
pub secrets_passphrase: Option<String>,
/// Explicit opt-in for exporting credentials.
#[serde(default)]
pub include_secrets: bool,
}
#[derive(Debug, Clone, Deserialize)]
@@ -64,6 +67,15 @@ pub struct WebDavUploadRequest {
pub struct WebDavDownloadRequest {
pub config: WebDavConfig,
pub secrets_passphrase: Option<String>,
/// Explicitly controls whether encrypted or legacy plaintext secrets may
/// be restored. Older clients omitted this field and retain the historical
/// restore behavior.
#[serde(default = "default_restore_secrets")]
pub restore_secrets: bool,
}
fn default_restore_secrets() -> bool {
true
}
#[derive(Debug, Clone, Deserialize)]
@@ -180,11 +192,27 @@ pub async fn webdav_sync_upload(
Json(mut req): Json<WebDavUploadRequest>,
) -> Result<Json<WebDavSyncSummary>, AppError> {
resolve_webdav_password(&state.app.storage, &mut req.config).await.map_err(AppError::from)?;
let snapshot = build_sync_snapshot_with_saved_secrets(
let explicit_passphrase = req.secrets_passphrase.as_deref().and_then(|value| {
let value = value.trim();
(!value.is_empty()).then_some(value)
});
let saved_passphrase = if explicit_passphrase.is_none() {
resolve_webdav_sync_secrets_passphrase(&state.app.storage).await.map_err(AppError::from)?
} else {
None
};
let passphrase = explicit_passphrase.or(saved_passphrase.as_deref());
let snapshot = build_sync_snapshot_with_options(
&state.app.storage,
env!("CARGO_PKG_VERSION"),
req.editor_settings,
req.secrets_passphrase.as_deref(),
dbx_core::cloud_sync::SyncExportOptions {
include_secrets: req.include_secrets,
sync_passphrase: passphrase,
include_ai_secrets: req.include_secrets,
include_tunnel_secrets: req.include_secrets,
include_plugin_secrets: req.include_secrets,
},
)
.await
.map_err(AppError::from)?;
@@ -208,7 +236,7 @@ pub async fn webdav_sync_download(
&snapshot,
ApplySnapshotOptions {
secrets_passphrase: explicit_passphrase.or(saved_passphrase.as_deref()),
restore_secrets: true,
restore_secrets: req.restore_secrets,
},
)
.await
@@ -299,10 +327,21 @@ pub async fn snippet_sync_upload(
} else {
None
};
let snapshot =
build_sync_snapshot(&state.app.storage, env!("CARGO_PKG_VERSION"), req.editor_settings, secrets_passphrase)
.await
.map_err(AppError::from)?;
let include_secrets = req.include_secrets;
let snapshot = build_sync_snapshot_with_options(
&state.app.storage,
env!("CARGO_PKG_VERSION"),
req.editor_settings,
dbx_core::cloud_sync::SyncExportOptions {
include_secrets,
sync_passphrase: secrets_passphrase,
include_ai_secrets: include_secrets,
include_tunnel_secrets: include_secrets,
include_plugin_secrets: include_secrets,
},
)
.await
.map_err(AppError::from)?;
let client = SnippetSyncClient::new(req.config).map_err(AppError::from)?;
let mut summary = client
.put_snapshot(&snapshot, req.snippet_passphrase.as_deref(), secrets_passphrase)
+47
View File
@@ -0,0 +1,47 @@
use axum::{extract::State, http::StatusCode, response::IntoResponse, Json};
use dbx_core::storage::{MigrationPreflight, MigrationReport};
use serde_json::json;
use std::sync::Arc;
use crate::state::WebState;
pub async fn status(
State(state): State<Arc<WebState>>,
) -> Result<Json<MigrationPreflight>, (StatusCode, Json<serde_json::Value>)> {
state.app.storage.inspect_data_migration().await.map(Json).map_err(error_response)
}
pub async fn start(
State(state): State<Arc<WebState>>,
) -> Result<Json<MigrationReport>, (StatusCode, Json<serde_json::Value>)> {
state.migration_ready.store(false, std::sync::atomic::Ordering::Release);
match state.app.storage.start_data_migration().await {
Ok(report) => {
let ready =
state.app.storage.inspect_data_migration().await.map(|status| status.is_ready()).unwrap_or(false);
state.migration_ready.store(ready, std::sync::atomic::Ordering::Release);
Ok(Json(report))
}
Err(error) => {
state.migration_ready.store(false, std::sync::atomic::Ordering::Release);
Err(error_response(error))
}
}
}
pub async fn retry(
State(state): State<Arc<WebState>>,
) -> Result<Json<MigrationReport>, (StatusCode, Json<serde_json::Value>)> {
start(State(state)).await
}
pub async fn cleanup_backups(
State(state): State<Arc<WebState>>,
) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {
state.app.storage.cleanup_migration_backups().await.map_err(error_response)?;
Ok(StatusCode::NO_CONTENT)
}
fn error_response(error: String) -> (StatusCode, Json<serde_json::Value>) {
(StatusCode::BAD_REQUEST, Json(json!({"error": error})))
}
+1
View File
@@ -16,6 +16,7 @@ pub mod history;
pub mod jdbc;
pub mod layout;
pub mod mcp_policy;
pub mod migration;
pub mod mongo;
pub mod mongodb_dump;
pub mod mongodb_import_export;
+1
View File
@@ -1571,6 +1571,7 @@ mod tests {
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
export_files: RwLock::new(HashMap::new()),
ssh_prompts: Arc::new(crate::ssh_prompt::SshPromptHub::new()),
migration_ready: Arc::new(std::sync::atomic::AtomicBool::new(true)),
});
(state, dir)
}
+3
View File
@@ -1,6 +1,7 @@
use dbx_core::connection::AppState;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use tokio::sync::{broadcast, watch, Mutex, RwLock};
use tokio_util::sync::CancellationToken;
@@ -43,6 +44,7 @@ pub struct WebState {
/// Completed Web export temp files waiting for the browser download.
pub export_files: RwLock<HashMap<String, WebExportFile>>,
pub ssh_prompts: Arc<crate::ssh_prompt::SshPromptHub>,
pub migration_ready: Arc<AtomicBool>,
}
impl WebState {
@@ -68,6 +70,7 @@ impl WebState {
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
export_files: RwLock::new(HashMap::new()),
ssh_prompts: Arc::new(crate::ssh_prompt::SshPromptHub::new()),
migration_ready: Arc::new(AtomicBool::new(true)),
}
}
}