feat(security): encrypt local credentials and guide data migration
This commit is contained in:
58 files changed
+8831
-435
No files matched your search
@@ -48,7 +48,7 @@ fn api_path_suffix<'a>(path: &'a str, public_base_path: &str) -> Option<&'a str>
|
||||
path.strip_prefix(base)?.strip_prefix("/api/")
|
||||
}
|
||||
|
||||
fn middleware_api_path_suffix<'a>(path: &'a str, public_base_path: &str) -> Option<&'a str> {
|
||||
pub(crate) fn middleware_api_path_suffix<'a>(path: &'a str, public_base_path: &str) -> Option<&'a str> {
|
||||
if let Some(suffix) = api_path_suffix(path, public_base_path) {
|
||||
return Some(suffix);
|
||||
}
|
||||
@@ -281,7 +281,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn middleware_api_path_suffix_handles_nested_router_paths() {
|
||||
pub(crate) fn middleware_api_path_suffix_handles_nested_router_paths() {
|
||||
assert_eq!(middleware_api_path_suffix("/auth/check", "/"), Some("auth/check"));
|
||||
assert_eq!(middleware_api_path_suffix("/connection/list", "/"), Some("connection/list"));
|
||||
assert_eq!(middleware_api_path_suffix("/api/connection/list", "/"), Some("connection/list"));
|
||||
|
||||
@@ -13,17 +13,19 @@ use argon2::password_hash::rand_core::OsRng;
|
||||
use argon2::password_hash::SaltString;
|
||||
use argon2::{Argon2, PasswordHasher};
|
||||
use axum::extract::DefaultBodyLimit;
|
||||
use axum::http::Uri;
|
||||
use axum::http::{Request, StatusCode, Uri};
|
||||
use axum::middleware;
|
||||
use axum::response::Redirect;
|
||||
use axum::response::{IntoResponse, Redirect, Response};
|
||||
use axum::routing::{delete, get, post};
|
||||
use axum::Router;
|
||||
use dbx_core::connection::AppState;
|
||||
use dbx_core::persistence::secret_codec::SecretKeyPolicy;
|
||||
use dbx_core::sql_dialect::dialect_loader::{register_core_dialects, DialectPluginLoader, DialectRegistry};
|
||||
use dbx_core::sql_dialect::hot_reload::DialectHotReload;
|
||||
use dbx_core::storage::Storage;
|
||||
use dbx_mcp::{streamable_http_router, DbxBackend, HttpAuth, LocalBackend};
|
||||
use state::WebState;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::sync::RwLock;
|
||||
use tower_http::compression::predicate::{DefaultPredicate, NotForContentType, Predicate};
|
||||
use tower_http::compression::CompressionLayer;
|
||||
@@ -78,6 +80,35 @@ fn web_body_limit_bytes_from_value(value: Option<&str>) -> usize {
|
||||
mb.saturating_mul(1024 * 1024)
|
||||
}
|
||||
|
||||
async fn migration_gate(
|
||||
state: axum::extract::State<Arc<WebState>>,
|
||||
request: Request<axum::body::Body>,
|
||||
next: axum::middleware::Next,
|
||||
) -> Response {
|
||||
let suffix = auth::middleware_api_path_suffix(request.uri().path(), &state.public_base_path);
|
||||
let allowed = suffix.is_some_and(|path| {
|
||||
matches!(
|
||||
path,
|
||||
"migration/status" | "migration/start" | "migration/retry" | "migration/cleanup-backups" | "ping"
|
||||
) || path.starts_with("auth/")
|
||||
});
|
||||
if !allowed && !state.migration_ready.load(Ordering::Acquire) {
|
||||
return (
|
||||
StatusCode::LOCKED,
|
||||
axum::Json(serde_json::json!({
|
||||
"code": "DATA_MIGRATION_REQUIRED",
|
||||
"message": "Complete the data security migration before using DBX"
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
next.run(request).await
|
||||
}
|
||||
|
||||
async fn storage_migration_ready(app: &Arc<AppState>) -> bool {
|
||||
app.storage.inspect_data_migration().await.map(|status| status.is_ready()).unwrap_or(false)
|
||||
}
|
||||
|
||||
fn web_agent_dir(data_dir: &std::path::Path) -> std::path::PathBuf {
|
||||
web_agent_dir_from_env(data_dir, std::env::var("DBX_AGENT_DIR").ok())
|
||||
}
|
||||
@@ -301,8 +332,10 @@ async fn main() {
|
||||
|
||||
let app_state = {
|
||||
let db_path = data_dir.join("dbx.db");
|
||||
let storage = Storage::open(&db_path).await.expect("Failed to open storage");
|
||||
storage.migrate_from_json(&data_dir).await.expect("Failed to migrate JSON data");
|
||||
let storage = Storage::open_unmigrated(&db_path)
|
||||
.await
|
||||
.expect("Failed to open storage")
|
||||
.with_secret_key_policy(SecretKeyPolicy::ManagedDataDir);
|
||||
|
||||
// Initialize core dialect registry and load external plugin dialects
|
||||
register_core_dialects();
|
||||
@@ -349,6 +382,7 @@ async fn main() {
|
||||
|
||||
let public_base_path = normalize_public_base_path(std::env::var("DBX_PUBLIC_BASE_PATH").ok());
|
||||
|
||||
let migration_ready = storage_migration_ready(&app_state).await;
|
||||
let web_state = Arc::new(WebState {
|
||||
app: app_state,
|
||||
data_dir,
|
||||
@@ -364,12 +398,17 @@ async fn main() {
|
||||
login_rate_limit: tokio::sync::Mutex::new(state::LoginRateLimit { fail_count: 0, locked_until: None }),
|
||||
export_files: RwLock::new(HashMap::new()),
|
||||
ssh_prompts: Arc::new(ssh_prompt::SshPromptHub::new()),
|
||||
migration_ready: Arc::new(AtomicBool::new(migration_ready)),
|
||||
});
|
||||
|
||||
ssh_prompt::install_web_ssh_prompt_bridge(web_state.ssh_prompts.clone());
|
||||
|
||||
// API routes
|
||||
let api = Router::new()
|
||||
.route("/migration/status", get(routes::migration::status))
|
||||
.route("/migration/start", post(routes::migration::start))
|
||||
.route("/migration/retry", post(routes::migration::retry))
|
||||
.route("/migration/cleanup-backups", post(routes::migration::cleanup_backups))
|
||||
// Auth
|
||||
.route("/auth/login", post(auth::login))
|
||||
.route("/auth/check", get(auth::check))
|
||||
@@ -1188,6 +1227,7 @@ async fn main() {
|
||||
api.route("/query/build-duckdb-attach-database-sql", post(routes::query::build_duckdb_attach_database_sql));
|
||||
|
||||
let api = add_mq_routes(api)
|
||||
.layer(middleware::from_fn_with_state(web_state.clone(), migration_gate))
|
||||
.layer(middleware::from_fn_with_state(web_state.clone(), auth::auth_middleware))
|
||||
.with_state(web_state.clone());
|
||||
|
||||
@@ -1199,7 +1239,7 @@ async fn main() {
|
||||
.layer(tower_http::trace::TraceLayer::new_for_http());
|
||||
|
||||
if let Some(mcp_router) = web_mcp_router(&web_state).expect("Invalid DBX Web MCP configuration") {
|
||||
app = app.merge(mcp_router);
|
||||
app = app.merge(mcp_router.layer(middleware::from_fn_with_state(web_state.clone(), migration_gate)));
|
||||
tracing::info!("DBX Web MCP is enabled at /mcp");
|
||||
}
|
||||
|
||||
@@ -1248,6 +1288,45 @@ mod tests {
|
||||
use axum::Router;
|
||||
use tower_http::compression::predicate::Predicate;
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_http_gate_blocks_business_until_ready_but_allows_cleanup_handler() {
|
||||
use std::sync::{atomic::Ordering, Arc};
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let storage = dbx_core::storage::Storage::open_unmigrated(&directory.path().join("dbx.db")).await.unwrap();
|
||||
let app = Arc::new(dbx_core::connection::AppState::new(storage));
|
||||
let state = Arc::new(crate::state::WebState::for_tests(app, directory.path().to_path_buf()));
|
||||
state.migration_ready.store(false, Ordering::Release);
|
||||
let router = Router::new()
|
||||
.route("/api/migration/status", get(|| async { "status" }))
|
||||
.route("/api/migration/cleanup-backups", post(|| async { "cleanup" }))
|
||||
.route("/api/connection/list", get(|| async { "connections" }))
|
||||
.route("/mcp", post(|| async { "mcp" }))
|
||||
.layer(axum::middleware::from_fn_with_state(state.clone(), super::migration_gate));
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
|
||||
let client = reqwest::Client::new();
|
||||
assert_eq!(
|
||||
client.get(format!("http://{address}/api/migration/status")).send().await.unwrap().status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
for (method, path) in [(reqwest::Method::GET, "/api/connection/list"), (reqwest::Method::POST, "/mcp")] {
|
||||
let response = client.request(method, format!("http://{address}{path}")).send().await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::LOCKED);
|
||||
assert_eq!(response.json::<serde_json::Value>().await.unwrap()["code"], "DATA_MIGRATION_REQUIRED");
|
||||
}
|
||||
assert_eq!(
|
||||
client.post(format!("http://{address}/api/migration/cleanup-backups")).send().await.unwrap().status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
state.migration_ready.store(true, Ordering::Release);
|
||||
assert_eq!(
|
||||
client.get(format!("http://{address}/api/connection/list")).send().await.unwrap().status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
fn compression_response(content_type: &str) -> Response<Body> {
|
||||
Response::builder().header(CONTENT_TYPE, content_type).body(Body::from(vec![b'x'; 64])).unwrap()
|
||||
}
|
||||
|
||||
@@ -3,16 +3,16 @@ use std::sync::Arc;
|
||||
use axum::extract::State;
|
||||
use axum::Json;
|
||||
use dbx_core::cloud_sync::{
|
||||
apply_sync_snapshot, build_sync_snapshot, build_sync_snapshot_with_saved_secrets, finalize_snippet_migration,
|
||||
forget_snippet_token, forget_webdav_password,
|
||||
forget_webdav_sync_secrets_passphrase as core_forget_webdav_sync_secrets_passphrase, resolve_snippet_token,
|
||||
resolve_webdav_password, resolve_webdav_sync_secrets_passphrase, retry_pending_snippet_cleanup,
|
||||
save_snippet_sync_id_for_instance as core_save_snippet_sync_id, save_snippet_token, save_webdav_password,
|
||||
save_webdav_sync_secrets_preference as core_save_webdav_sync_secrets_preference, snippet_saved_token_status,
|
||||
snippet_sync_settings_for_instance as core_snippet_sync_settings, webdav_saved_password_status,
|
||||
webdav_sync_secrets_status as core_webdav_sync_secrets_status, ApplySnapshotOptions, ApplySnapshotSummary,
|
||||
SnippetProvider, SnippetSyncClient, SnippetSyncConfig, SnippetSyncSettings, SnippetSyncSummary, SnippetTokenStatus,
|
||||
WebDavClient, WebDavConfig, WebDavPasswordStatus, WebDavSyncSecretsStatus, WebDavSyncSummary,
|
||||
apply_sync_snapshot, build_sync_snapshot_with_options, finalize_snippet_migration, forget_snippet_token,
|
||||
forget_webdav_password, forget_webdav_sync_secrets_passphrase as core_forget_webdav_sync_secrets_passphrase,
|
||||
resolve_snippet_token, resolve_webdav_password, resolve_webdav_sync_secrets_passphrase,
|
||||
retry_pending_snippet_cleanup, save_snippet_sync_id_for_instance as core_save_snippet_sync_id, save_snippet_token,
|
||||
save_webdav_password, save_webdav_sync_secrets_preference as core_save_webdav_sync_secrets_preference,
|
||||
snippet_saved_token_status, snippet_sync_settings_for_instance as core_snippet_sync_settings,
|
||||
webdav_saved_password_status, webdav_sync_secrets_status as core_webdav_sync_secrets_status, ApplySnapshotOptions,
|
||||
ApplySnapshotSummary, SnippetProvider, SnippetSyncClient, SnippetSyncConfig, SnippetSyncSettings,
|
||||
SnippetSyncSummary, SnippetTokenStatus, WebDavClient, WebDavConfig, WebDavPasswordStatus, WebDavSyncSecretsStatus,
|
||||
WebDavSyncSummary,
|
||||
};
|
||||
use dbx_core::storage::DesktopSettings;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -57,6 +57,9 @@ pub struct WebDavUploadRequest {
|
||||
pub config: WebDavConfig,
|
||||
pub editor_settings: Option<serde_json::Value>,
|
||||
pub secrets_passphrase: Option<String>,
|
||||
/// Explicit opt-in for exporting credentials.
|
||||
#[serde(default)]
|
||||
pub include_secrets: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
@@ -64,6 +67,15 @@ pub struct WebDavUploadRequest {
|
||||
pub struct WebDavDownloadRequest {
|
||||
pub config: WebDavConfig,
|
||||
pub secrets_passphrase: Option<String>,
|
||||
/// Explicitly controls whether encrypted or legacy plaintext secrets may
|
||||
/// be restored. Older clients omitted this field and retain the historical
|
||||
/// restore behavior.
|
||||
#[serde(default = "default_restore_secrets")]
|
||||
pub restore_secrets: bool,
|
||||
}
|
||||
|
||||
fn default_restore_secrets() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
@@ -180,11 +192,27 @@ pub async fn webdav_sync_upload(
|
||||
Json(mut req): Json<WebDavUploadRequest>,
|
||||
) -> Result<Json<WebDavSyncSummary>, AppError> {
|
||||
resolve_webdav_password(&state.app.storage, &mut req.config).await.map_err(AppError::from)?;
|
||||
let snapshot = build_sync_snapshot_with_saved_secrets(
|
||||
let explicit_passphrase = req.secrets_passphrase.as_deref().and_then(|value| {
|
||||
let value = value.trim();
|
||||
(!value.is_empty()).then_some(value)
|
||||
});
|
||||
let saved_passphrase = if explicit_passphrase.is_none() {
|
||||
resolve_webdav_sync_secrets_passphrase(&state.app.storage).await.map_err(AppError::from)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let passphrase = explicit_passphrase.or(saved_passphrase.as_deref());
|
||||
let snapshot = build_sync_snapshot_with_options(
|
||||
&state.app.storage,
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
req.editor_settings,
|
||||
req.secrets_passphrase.as_deref(),
|
||||
dbx_core::cloud_sync::SyncExportOptions {
|
||||
include_secrets: req.include_secrets,
|
||||
sync_passphrase: passphrase,
|
||||
include_ai_secrets: req.include_secrets,
|
||||
include_tunnel_secrets: req.include_secrets,
|
||||
include_plugin_secrets: req.include_secrets,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(AppError::from)?;
|
||||
@@ -208,7 +236,7 @@ pub async fn webdav_sync_download(
|
||||
&snapshot,
|
||||
ApplySnapshotOptions {
|
||||
secrets_passphrase: explicit_passphrase.or(saved_passphrase.as_deref()),
|
||||
restore_secrets: true,
|
||||
restore_secrets: req.restore_secrets,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -299,10 +327,21 @@ pub async fn snippet_sync_upload(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let snapshot =
|
||||
build_sync_snapshot(&state.app.storage, env!("CARGO_PKG_VERSION"), req.editor_settings, secrets_passphrase)
|
||||
.await
|
||||
.map_err(AppError::from)?;
|
||||
let include_secrets = req.include_secrets;
|
||||
let snapshot = build_sync_snapshot_with_options(
|
||||
&state.app.storage,
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
req.editor_settings,
|
||||
dbx_core::cloud_sync::SyncExportOptions {
|
||||
include_secrets,
|
||||
sync_passphrase: secrets_passphrase,
|
||||
include_ai_secrets: include_secrets,
|
||||
include_tunnel_secrets: include_secrets,
|
||||
include_plugin_secrets: include_secrets,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(AppError::from)?;
|
||||
let client = SnippetSyncClient::new(req.config).map_err(AppError::from)?;
|
||||
let mut summary = client
|
||||
.put_snapshot(&snapshot, req.snippet_passphrase.as_deref(), secrets_passphrase)
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
use axum::{extract::State, http::StatusCode, response::IntoResponse, Json};
|
||||
use dbx_core::storage::{MigrationPreflight, MigrationReport};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::state::WebState;
|
||||
|
||||
pub async fn status(
|
||||
State(state): State<Arc<WebState>>,
|
||||
) -> Result<Json<MigrationPreflight>, (StatusCode, Json<serde_json::Value>)> {
|
||||
state.app.storage.inspect_data_migration().await.map(Json).map_err(error_response)
|
||||
}
|
||||
|
||||
pub async fn start(
|
||||
State(state): State<Arc<WebState>>,
|
||||
) -> Result<Json<MigrationReport>, (StatusCode, Json<serde_json::Value>)> {
|
||||
state.migration_ready.store(false, std::sync::atomic::Ordering::Release);
|
||||
match state.app.storage.start_data_migration().await {
|
||||
Ok(report) => {
|
||||
let ready =
|
||||
state.app.storage.inspect_data_migration().await.map(|status| status.is_ready()).unwrap_or(false);
|
||||
state.migration_ready.store(ready, std::sync::atomic::Ordering::Release);
|
||||
Ok(Json(report))
|
||||
}
|
||||
Err(error) => {
|
||||
state.migration_ready.store(false, std::sync::atomic::Ordering::Release);
|
||||
Err(error_response(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn retry(
|
||||
State(state): State<Arc<WebState>>,
|
||||
) -> Result<Json<MigrationReport>, (StatusCode, Json<serde_json::Value>)> {
|
||||
start(State(state)).await
|
||||
}
|
||||
|
||||
pub async fn cleanup_backups(
|
||||
State(state): State<Arc<WebState>>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {
|
||||
state.app.storage.cleanup_migration_backups().await.map_err(error_response)?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
fn error_response(error: String) -> (StatusCode, Json<serde_json::Value>) {
|
||||
(StatusCode::BAD_REQUEST, Json(json!({"error": error})))
|
||||
}
|
||||
@@ -16,6 +16,7 @@ pub mod history;
|
||||
pub mod jdbc;
|
||||
pub mod layout;
|
||||
pub mod mcp_policy;
|
||||
pub mod migration;
|
||||
pub mod mongo;
|
||||
pub mod mongodb_dump;
|
||||
pub mod mongodb_import_export;
|
||||
|
||||
@@ -1571,6 +1571,7 @@ mod tests {
|
||||
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
|
||||
export_files: RwLock::new(HashMap::new()),
|
||||
ssh_prompts: Arc::new(crate::ssh_prompt::SshPromptHub::new()),
|
||||
migration_ready: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
});
|
||||
(state, dir)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use dbx_core::connection::AppState;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{broadcast, watch, Mutex, RwLock};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -43,6 +44,7 @@ pub struct WebState {
|
||||
/// Completed Web export temp files waiting for the browser download.
|
||||
pub export_files: RwLock<HashMap<String, WebExportFile>>,
|
||||
pub ssh_prompts: Arc<crate::ssh_prompt::SshPromptHub>,
|
||||
pub migration_ready: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WebState {
|
||||
@@ -68,6 +70,7 @@ impl WebState {
|
||||
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
|
||||
export_files: RwLock::new(HashMap::new()),
|
||||
ssh_prompts: Arc::new(crate::ssh_prompt::SshPromptHub::new()),
|
||||
migration_ready: Arc::new(AtomicBool::new(true)),
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user