refactor(workspace): split SQL and database driver crates

This commit is contained in:
t8y2 committed 2026-09-25 15:51:58 +08:00
1 parent b99c60aff6
commit e0da2d09d3
180 files changed
+1422 -538

No files matched your search

+27
View File
@@ -0,0 +1,27 @@
[package]
name = "dbx-driver-support"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
publish = false
[features]
default = []
test-support = ["dbx-platform/test-support"]
[dependencies]
dbx-platform = { path = "../dbx-platform" }
dbx-types = { path = "../dbx-types", default-features = false }
bytes = "1"
portpicker = "0.1.1"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
russh = "0.60"
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] }
sqlparser = { version = "0.62.0", features = ["visitor"] }
tokio = { version = "1", features = ["io-util", "net", "rt", "sync", "time"] }
tokio-util = { version = "0.7", features = ["rt"] }
uuid = { version = "1", features = ["v4"] }
[dev-dependencies]
tokio = { version = "1", features = ["macros", "rt-multi-thread", "test-util"] }
+318
View File
@@ -0,0 +1,318 @@
use crate::path_utils::expand_tilde;
use reqwest::ClientBuilder;
use reqwest::{Certificate, Identity};
use std::fmt;
use std::future::Future;
use std::time::Duration;
// Re-export types so that `db::QueryResult` etc. work within dbx-core
pub use crate::file_validator::validate_file_path;
pub use crate::types::*;
pub const CONNECTION_TIMEOUT_SECS: u64 = 5;
pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PoolCheckoutStage {
Wait,
Create,
Recycle,
Unknown,
}
impl PoolCheckoutStage {
pub fn as_str(self) -> &'static str {
match self {
Self::Wait => "wait",
Self::Create => "create",
Self::Recycle => "recycle",
Self::Unknown => "unknown",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PoolCheckoutError {
Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration },
Failed { database: &'static str, stage: PoolCheckoutStage, detail: String },
Canceled,
}
impl PoolCheckoutError {
pub fn stage(&self) -> Option<PoolCheckoutStage> {
match self {
Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage),
Self::Canceled => None,
}
}
pub fn is_pool_saturation(&self) -> bool {
matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. })
}
}
impl fmt::Display for PoolCheckoutError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Timeout { database, stage, timeout } => write!(
formatter,
"{database} connection pool checkout timed out [stage={}, timeout_ms={}]",
stage.as_str(),
timeout.as_millis()
),
Self::Failed { database, stage, detail } => {
write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str())
}
Self::Canceled => formatter.write_str(crate::execution::QUERY_CANCELED),
}
}
}
pub fn connection_timeout() -> Duration {
Duration::from_secs(CONNECTION_TIMEOUT_SECS)
}
pub fn http_client_builder(timeout: Duration) -> ClientBuilder {
reqwest::Client::builder().connect_timeout(timeout).no_proxy()
}
/// Augments an HTTP client builder with custom TLS material for drivers that
/// need to trust a non-system CA or present a client certificate (mTLS).
///
/// * `ca_cert_path` adds an extra root certificate (PEM bundle) alongside the
/// system roots, so self-signed server certificates are trusted.
/// * `client_cert_path` + `client_key_path` (required together) are concatenated
/// into a single PEM identity, as expected by reqwest's rustls backend.
///
/// Returns the augmented builder, or a descriptive error if a configured
/// certificate/key file cannot be read or parsed. Callers surface that error
/// instead of silently falling back to the system trust store, so a typo in a
/// cert path fails the connection loudly rather than being ignored.
pub fn apply_tls_certificates(
mut builder: ClientBuilder,
ca_cert_path: Option<&str>,
client_cert_path: Option<&str>,
client_key_path: Option<&str>,
) -> Result<ClientBuilder, String> {
// The desktop build unifies reqwest with both `default-tls` and `rustls-tls`,
// and `TlsBackend` defaults to native-tls there, which rejects the PEM identity
// below with `incompatible TLS identity type`. Pin rustls like the Consul and
// etcd-metrics clients so custom CA roots and PEM client identities work in
// every target.
builder = builder.use_rustls_tls();
if let Some(ca) = ca_cert_path.filter(|path| !path.is_empty()) {
let ca = expand_tilde(ca);
let contents = std::fs::read(&ca)
.map_err(|error| format!("Failed to read Elasticsearch CA certificate at {ca}: {error}"))?;
let certificates = Certificate::from_pem_bundle(&contents)
.map_err(|error| format!("Failed to parse Elasticsearch CA certificate at {ca}: {error}"))?;
for certificate in certificates {
builder = builder.add_root_certificate(certificate);
}
}
if let (Some(cert_path), Some(key_path)) =
(client_cert_path.filter(|path| !path.is_empty()), client_key_path.filter(|path| !path.is_empty()))
{
let cert_path = expand_tilde(cert_path);
let key_path = expand_tilde(key_path);
let mut pem = std::fs::read(&cert_path)
.map_err(|error| format!("Failed to read Elasticsearch client certificate at {cert_path}: {error}"))?;
// reqwest's rustls backend concatenates the cert and key as two PEM
// blocks; ensure they are newline-separated so parsing succeeds even
// when the cert file does not end with a trailing newline.
if !pem.ends_with(b"\n") {
pem.push(b'\n');
}
pem.extend(
std::fs::read(&key_path)
.map_err(|error| format!("Failed to read Elasticsearch client key at {key_path}: {error}"))?,
);
let identity = Identity::from_pem(&pem)
.map_err(|error| format!("Failed to parse Elasticsearch client identity: {error}"))?;
builder = builder.identity(identity);
}
Ok(builder)
}
pub use dbx_types::json::JS_MAX_SAFE_INTEGER;
pub use dbx_types::json::{json_value_for_js, safe_i64_to_json, safe_u64_to_json};
pub fn hex_encode(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for &byte in bytes {
out.push(HEX[(byte >> 4) as usize] as char);
out.push(HEX[(byte & 0x0f) as usize] as char);
}
out
}
pub fn binary_value_to_json(bytes: &[u8]) -> serde_json::Value {
serde_json::Value::String(format!("0x{}", hex_encode(bytes)))
}
pub fn tcp_probe_timeout() -> Duration {
Duration::from_secs(TCP_PROBE_TIMEOUT_SECS)
}
pub fn parse_connect_timeout(url: &str) -> Duration {
parse_connect_timeout_with_fallback(url, connection_timeout())
}
pub fn parse_connect_timeout_with_fallback(url: &str, fallback: Duration) -> Duration {
let Some(query) = url.split('?').nth(1) else {
return fallback;
};
for param in query.split('&') {
let trimmed = param.trim();
if trimmed.is_empty() {
continue;
}
let (key, value) = match trimmed.split_once('=') {
Some(pair) => pair,
None => continue,
};
if key.eq_ignore_ascii_case("connect_timeout")
|| key.eq_ignore_ascii_case("connectTimeout")
|| key.eq_ignore_ascii_case("connection_timeout")
|| key.eq_ignore_ascii_case("connectionTimeout")
{
if let Ok(v) = value.parse::<u64>() {
if (1..=300).contains(&v) {
return Duration::from_secs(v);
}
}
}
}
fallback
}
pub async fn with_connection_timeout<T, F>(label: &str, timeout: Duration, future: F) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
tokio::time::timeout(timeout, future)
.await
.map_err(|_| format!("{label} connection timed out ({}s)", timeout.as_secs()))?
}
pub async fn probe_tcp_endpoint(label: &str, host: &str, port: u16, timeout: Duration) -> Result<(), String> {
tokio::time::timeout(timeout, tokio::net::TcpStream::connect((host, port)))
.await
.map_err(|_| format!("{label} TCP connection timed out ({}s)", timeout.as_secs()))?
.map(|_| ())
.map_err(|e| format!("{label} TCP connection failed: {e}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn binary_values_are_displayed_as_prefixed_hex() {
assert_eq!(binary_value_to_json(&[0x00, 0x01, 0xab, 0xff]), serde_json::json!("0x0001abff"));
}
}
#[cfg(test)]
mod tls_tests {
use super::{apply_tls_certificates, http_client_builder};
use std::path::Path;
use std::process::Command;
use std::time::Duration;
/// Generates a throwaway CA + client certificate/key (PEM) via openssl and
/// returns their paths. If openssl is unavailable the files stay empty, which
/// makes `apply_tls_certificates` surface a read error instead of degrading.
fn gen_certs(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) {
let ca_key = dir.join("ca.key");
let ca_pem = dir.join("ca.pem");
let csr = dir.join("client.csr");
let client_key = dir.join("client.key");
let client_crt = dir.join("client.crt");
// Extensions force an X.509 v3 client cert; some toolchains (e.g.
// OpenSSL 3.0 on CI) emit v1 without one, which rustls rejects.
let client_ext = dir.join("client.ext");
let _ = std::fs::write(&client_ext, "basicConstraints=critical,CA:FALSE\n");
let _ = Command::new("openssl")
.args([
"req",
"-x509",
"-newkey",
"rsa:2048",
"-nodes",
"-keyout",
ca_key.to_str().unwrap(),
"-out",
ca_pem.to_str().unwrap(),
"-days",
"1",
"-subj",
"/CN=dbx-test-ca",
])
.output();
let _ = Command::new("openssl")
.args([
"req",
"-newkey",
"rsa:2048",
"-nodes",
"-keyout",
client_key.to_str().unwrap(),
"-out",
csr.to_str().unwrap(),
"-subj",
"/CN=dbx-test-client",
])
.output();
let _ = Command::new("openssl")
.args([
"x509",
"-req",
"-in",
csr.to_str().unwrap(),
"-CA",
ca_pem.to_str().unwrap(),
"-CAkey",
ca_key.to_str().unwrap(),
"-CAcreateserial",
"-out",
client_crt.to_str().unwrap(),
"-days",
"1",
"-extfile",
client_ext.to_str().unwrap(),
])
.output();
(ca_pem, client_crt, client_key)
}
#[test]
fn apply_tls_certificates_builds_with_ca_and_client_cert() {
let dir = std::env::temp_dir().join(format!("dbx-tls-test-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let (ca, client_crt, client_key) = gen_certs(&dir);
let built = apply_tls_certificates(
http_client_builder(Duration::from_secs(5)),
ca.to_str(),
client_crt.to_str(),
client_key.to_str(),
)
.expect("client should build with CA + client cert")
.build();
assert!(built.is_ok(), "built client should be usable with CA + client cert: {:?}", built.err());
// A misconfigured CA path must now surface a descriptive error instead of
// silently degrading to the system trust store.
let missing = apply_tls_certificates(
http_client_builder(Duration::from_secs(5)),
Some("/nonexistent/ca.pem"),
None,
None,
);
assert!(missing.is_err(), "missing CA path must surface a read error");
let _ = std::fs::remove_dir_all(&dir);
}
}
+223
View File
@@ -0,0 +1,223 @@
//! Structural, quote-and-parenthesis-aware scanning helpers for
//! post-processing already-rendered `CREATE TABLE` DDL text.
//!
//! A naive `str::find` for a literal marker (e.g. `" SERVER \""` or a bare
//! `;`) can be fooled by that same text appearing inside a quoted string
//! literal, a quoted identifier, or a nested parenthesized sub-expression —
//! a column's `DEFAULT`/`CHECK` expression can legally contain any of those.
//! These helpers track quote and parenthesis depth so callers only ever
//! match at the top level.
use std::ops::Range;
fn char_at(sql: &str, pos: usize) -> Option<char> {
sql.get(pos..)?.chars().next()
}
/// Skips a `'...'` string literal (`''`-escaped, with backslash-escape
/// tolerance) or a `"..."` quoted identifier (`""`-escaped) starting at
/// `pos`, which must point at the opening `quote` character. Returns the
/// index just past the closing quote (or `sql.len()` if unterminated).
fn skip_quoted(sql: &str, pos: usize, quote: char) -> usize {
let mut i = pos + quote.len_utf8();
while i < sql.len() {
let Some(ch) = char_at(sql, i) else { break };
let next = char_at(sql, i + ch.len_utf8());
if ch == quote {
if next == Some(quote) {
i += ch.len_utf8() + quote.len_utf8();
continue;
}
return i + ch.len_utf8();
}
if quote == '\'' && ch == '\\' {
i += ch.len_utf8();
if let Some(escaped) = char_at(sql, i) {
i += escaped.len_utf8();
}
continue;
}
i += ch.len_utf8();
}
sql.len()
}
/// Skips a PostgreSQL `$tag$...$tag$` dollar-quoted body starting at `pos`
/// (pointing at the first `$`). Returns `None` if `pos` isn't actually the
/// start of a valid dollar-quote tag (e.g. a stray `$` in an expression), in
/// which case the caller should treat `$` as an ordinary character.
fn skip_dollar_quoted(sql: &str, pos: usize) -> Option<usize> {
let tag_end_offset = sql.get(pos + 1..)?.find('$')?;
let tag_end = pos + 1 + tag_end_offset;
let tag = &sql[pos + 1..tag_end];
let valid_tag = tag.is_empty()
|| (tag.chars().next().is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_')
&& tag.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_'));
if !valid_tag {
return None;
}
let delimiter = &sql[pos..=tag_end];
let content_start = tag_end + 1;
sql.get(content_start..)?.find(delimiter).map(|offset| content_start + offset + delimiter.len())
}
/// Finds the byte index just past the first top-level (parenthesis-depth-0)
/// closing paren at or after `from` — the end of the outermost parenthesized
/// group that starts somewhere at or after `from` — treating quoted or
/// dollar-quoted content as opaque so parens or keywords inside a
/// `DEFAULT`/`CHECK` expression can't confuse the count. Returns `None` if no
/// top-level `(` is ever opened, or it's never closed.
pub fn find_top_level_paren_close(sql: &str, from: usize) -> Option<usize> {
let mut i = from;
let mut depth: usize = 0;
let mut opened = false;
while i < sql.len() {
let ch = char_at(sql, i)?;
match ch {
'\'' | '"' => {
i = skip_quoted(sql, i, ch);
continue;
}
'$' => {
if let Some(end) = skip_dollar_quoted(sql, i) {
i = end;
continue;
}
}
'(' => {
depth += 1;
opened = true;
}
')' => {
depth = depth.saturating_sub(1);
i += ch.len_utf8();
if opened && depth == 0 {
return Some(i);
}
continue;
}
_ => {}
}
i += ch.len_utf8();
}
None
}
/// Finds the byte index of the next top-level (depth-0, outside any quoted
/// or dollar-quoted content) `;` at or after `from`.
fn find_top_level_semicolon(sql: &str, from: usize) -> Option<usize> {
let mut i = from;
let mut depth: usize = 0;
while i < sql.len() {
let ch = char_at(sql, i)?;
match ch {
'\'' | '"' => {
i = skip_quoted(sql, i, ch);
continue;
}
'$' => {
if let Some(end) = skip_dollar_quoted(sql, i) {
i = end;
continue;
}
}
'(' => depth += 1,
')' => depth = depth.saturating_sub(1),
';' if depth == 0 => return Some(i),
_ => {}
}
i += ch.len_utf8();
}
None
}
/// Splits `sql` into consecutive top-level statement ranges, each ending
/// just past its own top-level `;`. The last statement may lack a trailing
/// `;`, in which case its range runs to the end of the string.
pub fn top_level_statement_ranges(sql: &str) -> Vec<Range<usize>> {
let mut ranges = Vec::new();
let mut start = 0;
while start < sql.len() {
match find_top_level_semicolon(sql, start) {
Some(semi) => {
ranges.push(start..semi + 1);
start = semi + 1;
}
None => {
ranges.push(start..sql.len());
break;
}
}
}
ranges
}
/// If `stmt` (ignoring leading whitespace) begins with `CREATE TABLE
/// "schema"."table"` or `CREATE FOREIGN TABLE "schema"."table"`, returns the
/// unescaped `(schema, table)` pair.
pub fn parse_create_table_relation(stmt: &str) -> Option<(String, String)> {
let trimmed = stmt.trim_start();
let rest =
trimmed.strip_prefix("CREATE FOREIGN TABLE ").or_else(|| trimmed.strip_prefix("CREATE TABLE "))?.trim_start();
let (schema, rest) = unquote_ident(rest)?;
let (table, _) = unquote_ident(rest.strip_prefix('.')?)?;
Some((schema, table))
}
fn unquote_ident(s: &str) -> Option<(String, &str)> {
if !s.starts_with('"') {
return None;
}
let end = skip_quoted(s, 0, '"');
if end < 2 {
return None;
}
let raw = &s[1..end - 1];
Some((raw.replace("\"\"", "\""), &s[end..]))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn paren_close_skips_nested_parens_and_quotes() {
let ddl =
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n) SERVER \"s\"";
let close = find_top_level_paren_close(ddl, 0).unwrap();
assert_eq!(
&ddl[..close],
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n)"
);
}
#[test]
fn paren_close_ignores_literal_server_text_inside_quotes() {
let ddl = "\"public\".\"t\" (\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"real_server\"";
let close = find_top_level_paren_close(ddl, 0).unwrap();
assert!(ddl[close..].starts_with(" SERVER \"real_server\""));
}
#[test]
fn top_level_statement_ranges_split_on_depth_zero_semicolons() {
let ddl = "CREATE TABLE \"s\".\"t\" (\n \"a\" text DEFAULT 'x;y'\n);\nALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (a <> '');";
let ranges = top_level_statement_ranges(ddl);
assert_eq!(ranges.len(), 2);
assert!(ddl[ranges[0].clone()].starts_with("CREATE TABLE"));
assert!(ddl[ranges[0].clone()].ends_with(";"));
assert!(ddl[ranges[1].clone()].trim_start().starts_with("ALTER TABLE"));
}
#[test]
fn parse_create_table_relation_unescapes_doubled_quotes() {
let (schema, table) =
parse_create_table_relation("CREATE TABLE \"pub\"\"lic\".\"t\" (\n \"a\" int\n);").unwrap();
assert_eq!(schema, "pub\"lic");
assert_eq!(table, "t");
}
#[test]
fn parse_create_table_relation_rejects_non_create_table() {
assert!(parse_create_table_relation("ALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (true);").is_none());
}
}
@@ -0,0 +1,69 @@
use serde::{Deserialize, Serialize};
/// Common result shape returned by document-store queries.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DocumentQueryResult {
pub documents: Vec<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub raw_documents: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub extended_documents: Option<Vec<serde_json::Value>>,
pub total: u64,
// Older document stores always return an exact total. Keep that wire shape
// unchanged and only send the flag when Elasticsearch reports a lower bound.
#[serde(default = "default_total_is_exact", skip_serializing_if = "is_true")]
pub total_is_exact: bool,
/// Opaque document-store cursor for APIs such as DynamoDB that do not
/// support offset pagination.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_cursor: Option<String>,
}
fn default_total_is_exact() -> bool {
true
}
fn is_true(value: &bool) -> bool {
*value
}
#[cfg(test)]
mod tests {
use super::DocumentQueryResult;
#[test]
fn exact_results_keep_the_existing_wire_shape() {
let result = DocumentQueryResult {
documents: Vec::new(),
raw_documents: None,
extended_documents: None,
total: 1,
total_is_exact: true,
next_cursor: None,
};
let serialized = serde_json::to_value(result).unwrap();
assert!(serialized.get("total_is_exact").is_none());
let deserialized: DocumentQueryResult = serde_json::from_value(serde_json::json!({
"documents": [],
"total": 1,
}))
.unwrap();
assert!(deserialized.total_is_exact);
}
#[test]
fn inexact_results_preserve_the_explicit_flag() {
let deserialized: DocumentQueryResult = serde_json::from_value(serde_json::json!({
"documents": [],
"total": 10_000_000,
"total_is_exact": false,
}))
.unwrap();
assert!(!deserialized.total_is_exact);
let serialized = serde_json::to_value(deserialized).unwrap();
assert_eq!(serialized.get("total_is_exact"), Some(&serde_json::json!(false)));
}
}
@@ -0,0 +1,426 @@
// SQL → Elasticsearch `_search` DSL translator.
//
// ES SQL's `_sql` endpoint refuses several common shapes — `LIKE` on a `text`
// field with no `.keyword` sub-field, `SELECT *` returning an array field
// like `host.ip`, ... — even though the same intent is easy to express with
// the raw query DSL. So instead of delegating to ES SQL (or its
// `_sql/translate` cousin, which lives in the same engine), we parse the
// SQL ourselves with `sqlparser-rs` and emit a `_search` body.
use serde_json::{json, Map, Value as Json};
use sqlparser::ast::{
BinaryOperator, Expr, OrderByExpr, OrderByKind, Query, Select, SelectItem, SetExpr, Statement, TableFactor,
UnaryOperator, Value as SqlValue,
};
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
const AUTO_PAGED_SELECT_STAR_SIZE: usize = 100;
pub struct TranslatedSelectStar {
pub index: String,
pub body: Json,
/// User wrote `LIMIT N` (no OFFSET) — explicit row cap.
pub user_limited: bool,
/// SQL had OFFSET — signals this came from the pagination plan.
pub from_plan_pagination: bool,
}
/// Try to translate a `SELECT * FROM <index> [WHERE …] [ORDER BY …]
/// [LIMIT N] [OFFSET M]` statement into a `_search` body.
///
/// Returns `Ok(None)` if the statement isn't a recognised `SELECT *` shape
/// (caller should fall back to a different path). Returns `Err` if it IS a
/// `SELECT *` but a clause is unsupported (so the caller surfaces a clear
/// error rather than silently downgrading).
pub fn translate_select_star(sql: &str) -> Result<Option<TranslatedSelectStar>, String> {
let dialect = GenericDialect {};
let statements = Parser::parse_sql(&dialect, sql).map_err(|e| format!("SQL parse error: {e}"))?;
let Some(stmt) = statements.first() else {
return Ok(None);
};
let Statement::Query(query) = stmt else {
return Ok(None);
};
let SetExpr::Select(select) = query.body.as_ref() else {
return Ok(None);
};
if !is_select_star(select) {
return Ok(None);
}
let Some(source) = single_table_source(select)? else {
return Ok(None);
};
let mut body: Map<String, Json> = Map::new();
if let Some(where_expr) = &select.selection {
body.insert("query".to_string(), translate_predicate(where_expr, source.alias.as_deref())?);
}
if let Some(order_by) = &query.order_by {
if let OrderByKind::Expressions(items) = &order_by.kind {
if let Some(sort) = translate_order_by(items, source.alias.as_deref())? {
body.insert("sort".to_string(), sort);
}
}
}
let (limit, offset) = extract_limit_offset(query)?;
let user_limited = limit.is_some() && offset.is_none();
let from_plan_pagination = offset.is_some();
let effective_size = limit.unwrap_or(AUTO_PAGED_SELECT_STAR_SIZE);
let effective_from = offset.unwrap_or(0);
body.insert("size".to_string(), json!(effective_size));
if effective_from > 0 {
body.insert("from".to_string(), json!(effective_from));
}
Ok(Some(TranslatedSelectStar { index: source.index, body: Json::Object(body), user_limited, from_plan_pagination }))
}
fn is_select_star(select: &Select) -> bool {
select.projection.len() == 1 && matches!(select.projection.first(), Some(SelectItem::Wildcard(_)))
}
struct SelectSource {
index: String,
alias: Option<String>,
}
fn single_table_source(select: &Select) -> Result<Option<SelectSource>, String> {
if select.from.len() != 1 {
return Ok(None);
}
let table = &select.from[0];
if !table.joins.is_empty() {
return Err("JOINs are not supported in Elasticsearch SQL translation".to_string());
}
let TableFactor::Table { name, alias, .. } = &table.relation else {
return Ok(None);
};
let joined = name
.0
.iter()
.map(|part| match part {
sqlparser::ast::ObjectNamePart::Identifier(ident) => ident.value.clone(),
other => format!("{other}"),
})
.collect::<Vec<_>>()
.join(".");
Ok(Some(SelectSource { index: joined, alias: alias.as_ref().map(|alias| alias.name.value.clone()) }))
}
fn translate_predicate(expr: &Expr, table_alias: Option<&str>) -> Result<Json, String> {
match expr {
Expr::Nested(inner) => translate_predicate(inner, table_alias),
Expr::BinaryOp { left, op, right } => translate_binary_op(left, op, right, table_alias),
Expr::Like { negated, expr, pattern, .. } | Expr::ILike { negated, expr, pattern, .. } => {
translate_like(*negated, expr, pattern, table_alias)
}
Expr::InList { expr, list, negated } => translate_in_list(expr, list, *negated, table_alias),
Expr::IsNull(inner) => translate_is_null(inner, false, table_alias),
Expr::IsNotNull(inner) => translate_is_null(inner, true, table_alias),
Expr::Between { expr, negated, low, high } => translate_between(expr, *negated, low, high, table_alias),
Expr::UnaryOp { op: UnaryOperator::Not, expr } => {
let inner = translate_predicate(expr, table_alias)?;
Ok(json!({ "bool": { "must_not": [inner] } }))
}
other => Err(format!("Unsupported WHERE expression: {other}")),
}
}
fn translate_binary_op(
left: &Expr,
op: &BinaryOperator,
right: &Expr,
table_alias: Option<&str>,
) -> Result<Json, String> {
match op {
BinaryOperator::And => {
let mut must = Vec::new();
collect_and(left, &mut must, table_alias)?;
collect_and(right, &mut must, table_alias)?;
Ok(json!({ "bool": { "must": must } }))
}
BinaryOperator::Or => {
let mut should = Vec::new();
collect_or(left, &mut should, table_alias)?;
collect_or(right, &mut should, table_alias)?;
Ok(json!({ "bool": { "should": should, "minimum_should_match": 1 } }))
}
BinaryOperator::Eq => term_clause(left, right, table_alias),
BinaryOperator::NotEq => {
let inner = term_clause(left, right, table_alias)?;
Ok(json!({ "bool": { "must_not": [inner] } }))
}
BinaryOperator::Gt => range_clause(left, right, "gt", table_alias),
BinaryOperator::Lt => range_clause(left, right, "lt", table_alias),
BinaryOperator::GtEq => range_clause(left, right, "gte", table_alias),
BinaryOperator::LtEq => range_clause(left, right, "lte", table_alias),
other => Err(format!("Unsupported operator: {other}")),
}
}
fn collect_and(expr: &Expr, out: &mut Vec<Json>, table_alias: Option<&str>) -> Result<(), String> {
if let Expr::BinaryOp { left, op: BinaryOperator::And, right } = expr {
collect_and(left, out, table_alias)?;
collect_and(right, out, table_alias)?;
} else {
out.push(translate_predicate(expr, table_alias)?);
}
Ok(())
}
fn collect_or(expr: &Expr, out: &mut Vec<Json>, table_alias: Option<&str>) -> Result<(), String> {
if let Expr::BinaryOp { left, op: BinaryOperator::Or, right } = expr {
collect_or(left, out, table_alias)?;
collect_or(right, out, table_alias)?;
} else {
out.push(translate_predicate(expr, table_alias)?);
}
Ok(())
}
fn term_clause(left: &Expr, right: &Expr, table_alias: Option<&str>) -> Result<Json, String> {
let field = field_path(left, table_alias)?;
let value = literal_to_json(right)?;
Ok(json!({ "term": { field: value } }))
}
fn range_clause(left: &Expr, right: &Expr, key: &str, table_alias: Option<&str>) -> Result<Json, String> {
let field = field_path(left, table_alias)?;
let value = literal_to_json(right)?;
Ok(json!({ "range": { field: { key: value } } }))
}
fn translate_like(negated: bool, expr: &Expr, pattern: &Expr, table_alias: Option<&str>) -> Result<Json, String> {
let field = field_path(expr, table_alias)?;
let pattern_str = match literal_string(pattern)? {
Some(s) => s,
None => return Err("LIKE pattern must be a string literal".to_string()),
};
// Optimise the common `prefix%` shape into a `prefix` query — cheaper than
// `wildcard` and supported on any analyzed text field.
let is_prefix = !pattern_str.starts_with('%')
&& pattern_str.ends_with('%')
&& !pattern_str[..pattern_str.len() - 1].contains(['%', '_']);
let query = if is_prefix {
let prefix = &pattern_str[..pattern_str.len() - 1];
json!({ "prefix": { field: prefix } })
} else {
let es_pattern = sql_like_to_wildcard(&pattern_str);
json!({
"wildcard": {
field: { "value": es_pattern, "case_insensitive": true }
}
})
};
if negated {
Ok(json!({ "bool": { "must_not": [query] } }))
} else {
Ok(query)
}
}
fn translate_in_list(expr: &Expr, list: &[Expr], negated: bool, table_alias: Option<&str>) -> Result<Json, String> {
let field = field_path(expr, table_alias)?;
let values: Vec<Json> = list.iter().map(literal_to_json).collect::<Result<_, _>>()?;
let query = json!({ "terms": { field: values } });
if negated {
Ok(json!({ "bool": { "must_not": [query] } }))
} else {
Ok(query)
}
}
fn translate_is_null(expr: &Expr, negated: bool, table_alias: Option<&str>) -> Result<Json, String> {
let field = field_path(expr, table_alias)?;
let exists = json!({ "exists": { "field": field } });
if negated {
// IS NOT NULL → field exists
Ok(exists)
} else {
Ok(json!({ "bool": { "must_not": [exists] } }))
}
}
fn translate_between(
expr: &Expr,
negated: bool,
low: &Expr,
high: &Expr,
table_alias: Option<&str>,
) -> Result<Json, String> {
let field = field_path(expr, table_alias)?;
let lo = literal_to_json(low)?;
let hi = literal_to_json(high)?;
let query = json!({ "range": { field: { "gte": lo, "lte": hi } } });
if negated {
Ok(json!({ "bool": { "must_not": [query] } }))
} else {
Ok(query)
}
}
fn translate_order_by(items: &[OrderByExpr], table_alias: Option<&str>) -> Result<Option<Json>, String> {
let mut sort = Vec::new();
for item in items {
let field = field_path(&item.expr, table_alias)?;
let order = if item.options.asc.unwrap_or(true) { "asc" } else { "desc" };
sort.push(json!({ field: { "order": order } }));
}
if sort.is_empty() {
Ok(None)
} else {
Ok(Some(Json::Array(sort)))
}
}
fn extract_limit_offset(query: &Query) -> Result<(Option<usize>, Option<usize>), String> {
let limit = match &query.limit_clause {
Some(sqlparser::ast::LimitClause::LimitOffset { limit, .. }) => {
limit.as_ref().map(usize_literal).transpose()?
}
_ => None,
};
let offset = match &query.limit_clause {
Some(sqlparser::ast::LimitClause::LimitOffset { offset: Some(off), .. }) => Some(usize_literal(&off.value)?),
_ => None,
};
Ok((limit, offset))
}
fn usize_literal(expr: &Expr) -> Result<usize, String> {
match expr {
Expr::Value(v) => match &v.value {
SqlValue::Number(n, _) => n.parse::<usize>().map_err(|e| format!("Invalid number: {e}")),
other => Err(format!("Expected integer literal, got {other:?}")),
},
other => Err(format!("Expected integer literal, got {other}")),
}
}
fn field_path(expr: &Expr, table_alias: Option<&str>) -> Result<String, String> {
match expr {
Expr::Identifier(ident) => Ok(ident.value.clone()),
Expr::CompoundIdentifier(parts) => {
let mut field_parts = parts.iter().map(|p| p.value.clone()).collect::<Vec<_>>();
if let (Some(alias), Some(first)) = (table_alias, field_parts.first()) {
// Dotted Elasticsearch fields are valid document paths; only
// strip a known SQL table alias such as `ass.agencyCode`.
if first.eq_ignore_ascii_case(alias) {
field_parts.remove(0);
}
}
Ok(field_parts.join("."))
}
other => Err(format!("Expected column name, got {other}")),
}
}
fn literal_to_json(expr: &Expr) -> Result<Json, String> {
let Expr::Value(v) = expr else {
return Err(format!("Expected literal value, got {expr}"));
};
match &v.value {
SqlValue::SingleQuotedString(s) | SqlValue::DoubleQuotedString(s) => Ok(Json::String(s.clone())),
SqlValue::Number(n, _) => serde_json::from_str::<Json>(n).map_err(|e| format!("Invalid number {n}: {e}")),
SqlValue::Boolean(b) => Ok(Json::Bool(*b)),
SqlValue::Null => Ok(Json::Null),
other => Err(format!("Unsupported literal: {other:?}")),
}
}
fn literal_string(expr: &Expr) -> Result<Option<String>, String> {
let Expr::Value(v) = expr else {
return Err(format!("Expected string literal, got {expr}"));
};
Ok(match &v.value {
SqlValue::SingleQuotedString(s) | SqlValue::DoubleQuotedString(s) => Some(s.clone()),
_ => None,
})
}
/// Convert SQL LIKE pattern to ES wildcard pattern.
/// SQL: `%` = any chars, `_` = single char. Backslash escapes both.
/// ES wildcard: `*` = any chars, `?` = single char. `\` escapes `*` / `?` / `\`.
fn sql_like_to_wildcard(pattern: &str) -> String {
let mut out = String::with_capacity(pattern.len());
let mut chars = pattern.chars().peekable();
while let Some(ch) = chars.next() {
match ch {
'\\' => {
if let Some(&next) = chars.peek() {
out.push(next);
chars.next();
}
}
'%' => out.push('*'),
'_' => out.push('?'),
'*' | '?' => {
// Escape ES wildcard metacharacters that the user wrote literally.
out.push('\\');
out.push(ch);
}
_ => out.push(ch),
}
}
out
}
#[cfg(test)]
mod tests {
use super::translate_select_star;
use serde_json::json;
#[test]
fn translate_select_star_strips_table_alias_from_where_fields() {
let translated = translate_select_star(
"SELECT * FROM assets002 AS ass WHERE ass.agencyCode = '035021' AND ass.mofDivCode = '420108000'",
)
.unwrap()
.unwrap();
assert_eq!(translated.index, "assets002");
assert_eq!(
translated.body["query"],
json!({
"bool": {
"must": [
{ "term": { "agencyCode": "035021" } },
{ "term": { "mofDivCode": "420108000" } }
]
}
})
);
}
#[test]
fn translate_select_star_strips_table_alias_before_nested_field_path() {
let translated =
translate_select_star("SELECT * FROM assets002 ass WHERE ass.department.code = 'A01'").unwrap().unwrap();
assert_eq!(translated.body["query"], json!({ "term": { "department.code": "A01" } }));
}
#[test]
fn translate_select_star_preserves_unaliased_dotted_document_fields() {
let translated = translate_select_star("SELECT * FROM logs WHERE host.ip = '127.0.0.1'").unwrap().unwrap();
assert_eq!(translated.body["query"], json!({ "term": { "host.ip": "127.0.0.1" } }));
}
#[test]
fn translate_select_star_strips_alias_in_like_and_order_by() {
let translated = translate_select_star(
"SELECT * FROM assets002 ass WHERE ass.assetName LIKE 'car%' ORDER BY ass.createdAt DESC LIMIT 10",
)
.unwrap()
.unwrap();
assert_eq!(translated.body["query"], json!({ "prefix": { "assetName": "car" } }));
assert_eq!(translated.body["sort"], json!([{ "createdAt": { "order": "desc" } }]));
assert_eq!(translated.body["size"], json!(10));
}
}
+340
View File
@@ -0,0 +1,340 @@
use std::future::Future;
use std::sync::{
atomic::{AtomicU64, Ordering},
Arc,
};
use std::time::Duration;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use crate::db;
use crate::models::connection::ConnectionConfig;
pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
pub const MAX_ROWS: usize = 10000;
pub const QUERY_CANCELED: &str = "Query canceled";
/// Unified database operation execution budget.
/// query_timeout = None only means SQL execution has no upper limit;
/// checkout/connect/recycle/cancel/cleanup always have hard upper limits and cannot be disabled.
#[derive(Debug, Clone)]
pub struct DbOperationBudget {
pub checkout_timeout: Duration,
pub connect_timeout: Duration,
pub recycle_timeout: Duration,
pub query_timeout: Option<Duration>,
pub cancel_timeout: Duration,
pub cleanup_timeout: Duration,
}
impl DbOperationBudget {
/// Build an execution budget from connection config.
/// checkout/connect/recycle use connect_timeout_secs (clamped to 1s minimum, 300s maximum).
/// query_timeout follows resolve_query_timeout semantics (Some(0) -> None).
/// cancel/cleanup are fixed values and cannot be disabled.
pub fn from_config(connect_timeout_secs: u64, query_timeout_secs: Option<u64>) -> Self {
let infra_timeout = Duration::from_secs(connect_timeout_secs.clamp(1, 300));
Self {
checkout_timeout: infra_timeout,
connect_timeout: infra_timeout,
recycle_timeout: infra_timeout,
query_timeout: resolve_query_timeout(query_timeout_secs),
cancel_timeout: Duration::from_secs(5),
cleanup_timeout: Duration::from_secs(3),
}
}
pub fn from_connection_config(config: &ConnectionConfig) -> Self {
// effective_query_timeout_secs, not the raw field: it preserves 0 ("no limit")
// and applies the per-database-type floor Cloud Spanner needs for DDL.
Self::from_config(config.effective_connect_timeout_secs(), Some(config.effective_query_timeout_secs()))
}
/// Use global default values (when no connection config is available).
pub fn with_defaults() -> Self {
let default_infra = db::connection_timeout();
Self {
checkout_timeout: default_infra,
connect_timeout: default_infra,
recycle_timeout: default_infra,
query_timeout: Some(QUERY_TIMEOUT),
cancel_timeout: Duration::from_secs(5),
cleanup_timeout: Duration::from_secs(3),
}
}
}
pub struct StreamProgressClock {
started_at: tokio::time::Instant,
last_progress_ms: AtomicU64,
#[cfg(any(test, feature = "test-support"))]
marked: std::sync::atomic::AtomicBool,
}
impl Default for StreamProgressClock {
fn default() -> Self {
Self::new()
}
}
impl StreamProgressClock {
pub fn new() -> Self {
Self {
started_at: tokio::time::Instant::now(),
last_progress_ms: AtomicU64::new(0),
#[cfg(any(test, feature = "test-support"))]
marked: std::sync::atomic::AtomicBool::new(false),
}
}
pub fn mark(&self) {
self.last_progress_ms.store(self.started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
#[cfg(any(test, feature = "test-support"))]
self.marked.store(true, Ordering::Relaxed);
}
/// Whether any progress has been recorded yet. Test-only: production code
/// only needs the derived inactivity window. A row read within the first
/// millisecond records a zero timestamp, so this cannot be derived from
/// `last_progress_ms`.
#[cfg(any(test, feature = "test-support"))]
pub fn marked(&self) -> bool {
self.marked.load(Ordering::Relaxed)
}
fn elapsed_since_progress(&self) -> Duration {
let last_progress_ms = self.last_progress_ms.load(Ordering::Relaxed);
let elapsed_ms = self.started_at.elapsed().as_millis() as u64;
Duration::from_millis(elapsed_ms.saturating_sub(last_progress_ms))
}
}
pub fn timeout_error() -> String {
timeout_error_for(QUERY_TIMEOUT)
}
pub fn timeout_error_for(timeout_duration: Duration) -> String {
let seconds = timeout_duration.as_secs().max(1);
format!("Query timed out after {seconds} seconds")
}
pub fn canceled_error() -> String {
QUERY_CANCELED.to_string()
}
pub async fn await_stream_with_progress_timeout<F, T>(
stream_future: F,
timeout: Option<Duration>,
progress_clock: Arc<StreamProgressClock>,
cancel_token: Option<&CancellationToken>,
timeout_message: String,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
let Some(timeout) = timeout else {
return match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => Err(canceled_error()),
result = stream_future => result,
}
}
None => stream_future.await,
};
};
tokio::pin!(stream_future);
loop {
// Query timeout is an inactivity budget, not a cap on total stream duration.
let remaining = timeout.saturating_sub(progress_clock.elapsed_since_progress());
if remaining.is_zero() {
return Err(timeout_message.clone());
}
let sleep = tokio::time::sleep(remaining);
tokio::pin!(sleep);
match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => return Err(canceled_error()),
result = &mut stream_future => return result,
_ = &mut sleep => {},
}
}
None => {
tokio::select! {
biased;
result = &mut stream_future => return result,
_ = &mut sleep => {},
}
}
}
if progress_clock.elapsed_since_progress() >= timeout {
return Err(timeout_message);
}
}
}
pub fn is_canceled(cancel_token: &Option<CancellationToken>) -> bool {
cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false)
}
pub async fn wait_for_query<F>(cancel_token: Option<CancellationToken>, future: F) -> Result<db::QueryResult, String>
where
F: Future<Output = Result<db::QueryResult, String>>,
{
wait_for_query_with_timeout(cancel_token, QUERY_TIMEOUT, future).await
}
pub async fn wait_for_query_with_timeout<F>(
cancel_token: Option<CancellationToken>,
timeout_duration: Duration,
future: F,
) -> Result<db::QueryResult, String>
where
F: Future<Output = Result<db::QueryResult, String>>,
{
wait_for_result_with_timeout(cancel_token, timeout_duration, future).await
}
pub async fn wait_for_result_with_timeout<T, F>(
cancel_token: Option<CancellationToken>,
timeout_duration: Duration,
future: F,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
if let Some(token) = cancel_token {
tokio::select! {
biased;
_ = token.cancelled() => Err(canceled_error()),
result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration))?,
}
} else {
timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration))?
}
}
/// Like `wait_for_query_with_timeout` but with an optional timeout.
/// `None` means no timeout (only cancellation can stop the query).
pub async fn wait_for_query_opt<F>(
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
future: F,
) -> Result<db::QueryResult, String>
where
F: Future<Output = Result<db::QueryResult, String>>,
{
wait_for_result_opt(cancel_token, timeout_duration, future).await
}
pub async fn wait_for_result_opt<T, F>(
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
future: F,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
match timeout_duration {
Some(d) => wait_for_result_with_timeout(cancel_token, d, future).await,
None => match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => Err(canceled_error()),
result = future => result,
}
}
None => future.await,
},
}
}
pub async fn wait_for_value_opt<T, F>(
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
future: F,
) -> Result<T, String>
where
F: Future<Output = T>,
{
match timeout_duration {
Some(timeout_duration) => {
if let Some(token) = cancel_token {
tokio::select! {
biased;
_ = token.cancelled() => Err(canceled_error()),
result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration)),
}
} else {
timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration))
}
}
None => match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => Err(canceled_error()),
result = future => Ok(result),
}
}
None => Ok(future.await),
},
}
}
/// Locks a mutex-guarded shared connection (e.g. SQL Server's single connection
/// per pool key) and reports how long the caller waited for the lock alongside
/// the guard. Callers should fold the returned wait time into any execution-time
/// metric they report, since a driver-level timer that only starts once the lock
/// is held cannot see time spent queued behind another operation on the same
/// connection.
pub async fn lock_shared_client_with_wait<'a, T>(
client: &'a Arc<tokio::sync::Mutex<T>>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
) -> Result<(tokio::sync::MutexGuard<'a, T>, u128), String> {
let started_at = std::time::Instant::now();
let guard = wait_for_value_opt(cancel_token, timeout_duration, client.lock()).await?;
Ok((guard, started_at.elapsed().as_millis()))
}
pub fn query_timeout_duration(timeout_secs: Option<u64>) -> Option<Duration> {
match timeout_secs {
Some(0) => None,
Some(n) => Some(Duration::from_secs(n)),
None => Some(QUERY_TIMEOUT),
}
}
pub fn resolve_query_timeout(timeout_secs: Option<u64>) -> Option<Duration> {
query_timeout_duration(timeout_secs)
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(start_paused = true)]
async fn default_progress_clock_tracks_inactivity_from_creation_and_marks() {
let clock = StreamProgressClock::default();
assert_eq!(clock.elapsed_since_progress(), Duration::ZERO);
assert!(!clock.marked());
tokio::time::advance(Duration::from_secs(2)).await;
assert_eq!(clock.elapsed_since_progress(), Duration::from_secs(2));
clock.mark();
assert!(clock.marked());
assert_eq!(clock.elapsed_since_progress(), Duration::ZERO);
tokio::time::advance(Duration::from_secs(1)).await;
assert_eq!(clock.elapsed_since_progress(), Duration::from_secs(1));
}
}
@@ -0,0 +1,130 @@
use std::path::Path;
use crate::path_utils::expand_tilde;
/// Validates a file path for database connections.
///
/// Performs comprehensive checks including:
/// - Empty path validation
/// - Null character detection
/// - File existence (for local paths)
/// - File type validation (must be a file, not directory)
/// - Network path detection (skips validation for network paths)
///
/// # Arguments
/// * `path` - The file path to validate
/// * `is_network_path` - Closure to determine if path is a network path
///
/// # Returns
/// * `Ok(())` if validation passes
/// * `Err(String)` with descriptive error message if validation fails
pub fn validate_file_path<F>(path: &str, is_network_path: F) -> Result<(), String>
where
F: Fn(&str) -> bool,
{
// Check if path is empty
if path.is_empty() {
return Err("File path cannot be empty".to_string());
}
// Check if path contains invalid characters
if path.contains('\0') {
return Err("File path contains null characters".to_string());
}
let expanded = expand_tilde(path);
let path_obj = Path::new(&expanded);
// For non-network paths, perform file system checks
if !is_network_path(path) {
if !path_obj.exists() {
return Err(format!("File does not exist: {}", path));
}
// Check if path is actually a file, not a directory
if path_obj.is_dir() {
return Err(format!("File path is a directory, not a file: {}", path));
}
// Check if path is a valid file
if !path_obj.is_file() {
return Err(format!("File path is not a valid file: {}", path));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn is_network_path_test(path: &str) -> bool {
path.starts_with("\\\\") || path.starts_with("//")
}
#[test]
fn test_empty_path() {
let result = validate_file_path("", is_network_path_test);
assert!(result.is_err());
assert!(result.unwrap_err().contains("empty"));
}
#[test]
fn test_null_character() {
let result = validate_file_path("path\0invalid", is_network_path_test);
assert!(result.is_err());
assert!(result.unwrap_err().contains("null"));
}
#[test]
fn test_network_path_skips_validation() {
let result = validate_file_path("//network/path/nonexistent.db", is_network_path_test);
assert!(result.is_ok());
}
#[test]
fn test_nonexistent_local_file() {
let result = validate_file_path("/nonexistent/path/to/file.db", is_network_path_test);
assert!(result.is_err());
assert!(result.unwrap_err().contains("does not exist"));
}
#[test]
fn test_tilde_expands_to_home() {
let home = std::env::var("HOME").unwrap_or_default();
if home.is_empty() {
return; // skip if no HOME
}
let result = validate_file_path(&format!("{}/.bashrc", home), is_network_path_test);
// $HOME/.bashrc may or may not exist, but the path shouldn't contain literal ~
// We just verify no error mentioning tilde
if let Err(e) = &result {
assert!(!e.contains('~'), "Error should not contain literal tilde: {e}");
}
}
#[test]
fn test_tilde_only() {
let home = std::env::var("HOME").unwrap_or_default();
if home.is_empty() {
return;
}
let result = validate_file_path("~", |_| true); // treat as network to skip fs check
assert!(result.is_ok());
}
#[test]
fn test_no_tilde_unchanged() {
let result = validate_file_path("/absolute/path", is_network_path_test);
assert!(result.is_err());
assert!(result.unwrap_err().contains("/absolute/path"));
}
#[test]
fn test_unknown_named_user_path_remains_literal() {
let result = validate_file_path("~dbx-user-that-must-not-exist-7088", is_network_path_test);
assert!(result.is_err());
assert!(result.unwrap_err().contains('~'));
}
}
@@ -0,0 +1,441 @@
use bytes::Bytes;
use reqwest::header::{HeaderValue, AUTHORIZATION, CONTENT_TYPE};
use reqwest::{Client, StatusCode, Url};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
use tokio::time::{timeout, Duration};
use uuid::Uuid;
const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
const TCP_READ_BUFFER_SIZE: usize = 16 * 1024;
const HTTP_READ_WAIT_MS: u64 = 1_000;
const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(15);
#[derive(Default)]
pub struct HttpTunnelManager {
tunnels: Mutex<HashMap<String, (JoinHandle<()>, u16)>>,
}
impl HttpTunnelManager {
pub fn new() -> Self {
Self { tunnels: Mutex::new(HashMap::new()) }
}
pub async fn start_tunnel(
&self,
connection_id: &str,
tunnel_url: &str,
token: &str,
connect_timeout_secs: u64,
remote_host: &str,
remote_port: u16,
) -> Result<u16, String> {
if let Some(local_port) = self.local_port(connection_id).await {
return Ok(local_port);
}
let local_port = portpicker::pick_unused_port().ok_or("No available port")?;
let listener = TcpListener::bind(("127.0.0.1", local_port))
.await
.map_err(|e| format!("Failed to bind HTTP tunnel local port: {e}"))?;
let endpoint =
Arc::new(HttpTunnelEndpoint::new(tunnel_url, token, connect_timeout_secs, remote_host, remote_port)?);
let handle = tokio::spawn(http_tunnel_forward_loop(listener, endpoint));
let mut tunnels = self.tunnels.lock().await;
if let Some((_, existing_port)) = tunnels.get(connection_id) {
handle.abort();
return Ok(*existing_port);
}
tunnels.insert(connection_id.to_string(), (handle, local_port));
Ok(local_port)
}
pub async fn local_port(&self, connection_id: &str) -> Option<u16> {
self.tunnels.lock().await.get(connection_id).map(|(_, port)| *port)
}
pub async fn stop_tunnel(&self, connection_id: &str) {
if let Some((handle, _)) = self.tunnels.lock().await.remove(connection_id) {
handle.abort();
}
}
pub async fn stop_tunnels_with_prefix(&self, connection_id_prefix: &str) {
let mut tunnels = self.tunnels.lock().await;
let keys: Vec<String> = tunnels.keys().filter(|key| key.starts_with(connection_id_prefix)).cloned().collect();
for key in keys {
if let Some((handle, _)) = tunnels.remove(&key) {
handle.abort();
}
}
}
pub async fn stop_all_tunnels(&self) {
let tunnels = std::mem::take(&mut *self.tunnels.lock().await);
for (handle, _) in tunnels.into_values() {
handle.abort();
}
}
}
struct HttpTunnelEndpoint {
url: String,
token: String,
connect_timeout: Duration,
target_host: String,
target_port: u16,
client: Client,
}
impl HttpTunnelEndpoint {
fn new(
tunnel_url: &str,
token: &str,
connect_timeout_secs: u64,
remote_host: &str,
remote_port: u16,
) -> Result<Self, String> {
validate_script_url(tunnel_url)?;
Ok(Self {
url: tunnel_url.trim().to_string(),
token: token.trim().to_string(),
connect_timeout: effective_connect_timeout(connect_timeout_secs),
target_host: remote_host.to_string(),
target_port: remote_port,
client: Client::new(),
})
}
}
async fn http_tunnel_forward_loop(listener: TcpListener, endpoint: Arc<HttpTunnelEndpoint>) {
loop {
let (inbound, _) = match listener.accept().await {
Ok(pair) => pair,
Err(_) => break,
};
let endpoint = endpoint.clone();
tokio::spawn(async move {
let _ = bridge_tcp_to_http_script(inbound, endpoint).await;
});
}
}
async fn bridge_tcp_to_http_script(mut inbound: TcpStream, endpoint: Arc<HttpTunnelEndpoint>) -> Result<(), String> {
let session = Uuid::new_v4().simple().to_string();
open_session(&endpoint, &session).await?;
let (mut tcp_reader, mut tcp_writer) = inbound.split();
let tcp_to_http = async {
let mut buf = vec![0_u8; TCP_READ_BUFFER_SIZE];
loop {
let n = tcp_reader.read(&mut buf).await.map_err(|e| format!("Failed to read local TCP stream: {e}"))?;
if n == 0 {
return Ok::<(), String>(());
}
write_session(&endpoint, &session, Bytes::copy_from_slice(&buf[..n])).await?;
}
};
let http_to_tcp = async {
loop {
let bytes = read_session(&endpoint, &session).await?;
if !bytes.is_empty() {
tcp_writer.write_all(&bytes).await.map_err(|e| format!("Failed to write local TCP stream: {e}"))?;
}
}
};
let result = tokio::select! {
result = tcp_to_http => result,
result = http_to_tcp => result,
};
let _ = close_session(&endpoint, &session).await;
result
}
async fn open_session(endpoint: &HttpTunnelEndpoint, session: &str) -> Result<(), String> {
let url = script_url(
&endpoint.url,
"open",
session,
Some((&endpoint.target_host, endpoint.target_port)),
Some(endpoint.connect_timeout.as_secs()),
None,
)?;
let response = timeout(
endpoint.connect_timeout + HTTP_REQUEST_TIMEOUT,
authorized_request(endpoint, endpoint.client.post(url)).send(),
)
.await
.map_err(|_| "HTTP tunnel script open timed out".to_string())?
.map_err(|e| format!("Failed to open HTTP tunnel script session: {e}"))?;
ensure_success(response, "open HTTP tunnel script session").await
}
async fn write_session(endpoint: &HttpTunnelEndpoint, session: &str, data: Bytes) -> Result<(), String> {
let url = script_url(&endpoint.url, "write", session, None, None, None)?;
let response = timeout(
HTTP_REQUEST_TIMEOUT,
authorized_request(endpoint, endpoint.client.post(url))
.header(CONTENT_TYPE, "application/octet-stream")
.body(data)
.send(),
)
.await
.map_err(|_| "HTTP tunnel script write timed out".to_string())?
.map_err(|e| format!("Failed to write HTTP tunnel script session: {e}"))?;
ensure_success(response, "write HTTP tunnel script session").await
}
async fn read_session(endpoint: &HttpTunnelEndpoint, session: &str) -> Result<Bytes, String> {
let url = script_url(&endpoint.url, "read", session, None, None, Some(HTTP_READ_WAIT_MS))?;
let response = timeout(HTTP_REQUEST_TIMEOUT, authorized_request(endpoint, endpoint.client.post(url)).send())
.await
.map_err(|_| "HTTP tunnel script read timed out".to_string())?
.map_err(|e| format!("Failed to read HTTP tunnel script session: {e}"))?;
let status = response.status();
if status == StatusCode::NO_CONTENT {
return Ok(Bytes::new());
}
if status == StatusCode::OK {
return response.bytes().await.map_err(|e| format!("Failed to read HTTP tunnel script response: {e}"));
}
if status == StatusCode::GONE {
return Err("HTTP tunnel script session closed".to_string());
}
Err(error_response_message(response, "read HTTP tunnel script session").await)
}
async fn close_session(endpoint: &HttpTunnelEndpoint, session: &str) -> Result<(), String> {
let url = script_url(&endpoint.url, "close", session, None, None, None)?;
let response = timeout(HTTP_REQUEST_TIMEOUT, authorized_request(endpoint, endpoint.client.post(url)).send())
.await
.map_err(|_| "HTTP tunnel script close timed out".to_string())?
.map_err(|e| format!("Failed to close HTTP tunnel script session: {e}"))?;
ensure_success(response, "close HTTP tunnel script session").await
}
fn authorized_request(endpoint: &HttpTunnelEndpoint, builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
if endpoint.token.is_empty() {
return builder;
}
match HeaderValue::from_str(&endpoint.token) {
Ok(token) => builder
.header("X-DBX-Tunnel-Token", token.clone())
.header(AUTHORIZATION, format!("Bearer {}", endpoint.token)),
Err(_) => builder,
}
}
async fn ensure_success(response: reqwest::Response, action: &str) -> Result<(), String> {
if response.status().is_success() {
Ok(())
} else {
Err(error_response_message(response, action).await)
}
}
async fn error_response_message(response: reqwest::Response, action: &str) -> String {
let status = response.status();
let body = response.text().await.unwrap_or_default();
if body.trim().is_empty() {
format!("Failed to {action}: HTTP {status}")
} else {
format!("Failed to {action}: HTTP {status}: {}", body.trim())
}
}
fn script_url(
tunnel_url: &str,
action: &str,
session: &str,
target: Option<(&str, u16)>,
connect_timeout_secs: Option<u64>,
wait_ms: Option<u64>,
) -> Result<Url, String> {
let mut url = validate_script_url(tunnel_url)?;
url.query_pairs_mut().append_pair("dbx_action", action).append_pair("dbx_session", session);
if let Some((host, port)) = target {
url.query_pairs_mut().append_pair("dbx_target_host", host).append_pair("dbx_target_port", &port.to_string());
}
if let Some(connect_timeout_secs) = connect_timeout_secs {
url.query_pairs_mut()
.append_pair("dbx_connect_timeout", &effective_connect_timeout_secs(connect_timeout_secs).to_string());
}
if let Some(wait_ms) = wait_ms {
url.query_pairs_mut().append_pair("dbx_wait_ms", &wait_ms.to_string());
}
Ok(url)
}
fn validate_script_url(tunnel_url: &str) -> Result<Url, String> {
let trimmed = tunnel_url.trim();
if trimmed.is_empty() {
return Err("HTTP tunnel script URL is required".to_string());
}
let url = Url::parse(trimmed).map_err(|e| format!("HTTP tunnel script URL is invalid: {e}"))?;
match url.scheme() {
"http" | "https" => Ok(url),
other => Err(format!("Unsupported HTTP tunnel script URL scheme: {other}")),
}
}
fn effective_connect_timeout(value: u64) -> Duration {
Duration::from_secs(effective_connect_timeout_secs(value))
}
fn effective_connect_timeout_secs(value: u64) -> u64 {
if value == 0 {
DEFAULT_CONNECT_TIMEOUT.as_secs()
} else {
value.clamp(1, 300)
}
}
#[cfg(test)]
mod tests {
use super::{script_url, validate_script_url, HttpTunnelManager};
use std::sync::Arc;
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{Mutex, Notify};
use tokio::time::{timeout, Duration};
#[test]
fn script_url_preserves_existing_query_and_appends_action() {
let url = script_url(
"https://gateway.example.com/dbx_tunnel.php?keep=1",
"open",
"session-1",
Some(("mysql.internal", 3306)),
Some(10),
None,
)
.unwrap();
assert_eq!(
url.as_str(),
"https://gateway.example.com/dbx_tunnel.php?keep=1&dbx_action=open&dbx_session=session-1&dbx_target_host=mysql.internal&dbx_target_port=3306&dbx_connect_timeout=10"
);
}
#[test]
fn script_url_rejects_non_http_schemes() {
let err = validate_script_url("file:///tmp/tunnel.php").unwrap_err();
assert!(err.contains("Unsupported HTTP tunnel script URL scheme"));
}
#[tokio::test]
async fn manager_forwards_sequential_round_trips_and_closes_session() {
let script = MockScript::start().await;
let manager = HttpTunnelManager::new();
let local_port = manager.start_tunnel("test", &script.url, "secret", 5, "mysql.internal", 3306).await.unwrap();
let mut client = TcpStream::connect(("127.0.0.1", local_port)).await.unwrap();
let close_notification = script.closed.notified();
timeout(Duration::from_secs(10), async {
for index in 0..100_u32 {
let payload = format!("request-{index:03}-{}", "x".repeat((index % 17) as usize));
client.write_all(payload.as_bytes()).await.unwrap();
let mut response = vec![0_u8; payload.len()];
client.read_exact(&mut response).await.unwrap();
assert_eq!(response, payload.as_bytes());
}
})
.await
.expect("100 sequential HTTP tunnel round trips timed out");
drop(client);
timeout(Duration::from_secs(2), close_notification).await.expect("HTTP tunnel session was not closed");
manager.stop_tunnel("test").await;
script.handle.abort();
}
struct MockScript {
url: String,
closed: Arc<Notify>,
handle: tokio::task::JoinHandle<()>,
}
impl MockScript {
async fn start() -> Self {
let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
let addr = listener.local_addr().unwrap();
let buffered = Arc::new(Mutex::new(Vec::<u8>::new()));
let closed = Arc::new(Notify::new());
let handle = {
let buffered = buffered.clone();
let closed = closed.clone();
tokio::spawn(async move {
loop {
let Ok((stream, _)) = listener.accept().await else {
break;
};
let buffered = buffered.clone();
let closed = closed.clone();
tokio::spawn(async move {
handle_mock_http_request(stream, buffered, closed).await;
});
}
})
};
Self { url: format!("http://{addr}/dbx_tunnel.php"), closed, handle }
}
}
async fn handle_mock_http_request(stream: TcpStream, buffered: Arc<Mutex<Vec<u8>>>, closed: Arc<Notify>) {
let mut reader = BufReader::new(stream);
let mut request_line = String::new();
reader.read_line(&mut request_line).await.unwrap();
let mut content_length = 0_usize;
loop {
let mut line = String::new();
reader.read_line(&mut line).await.unwrap();
let trimmed = line.trim_end();
if trimmed.is_empty() {
break;
}
let lower = trimmed.to_ascii_lowercase();
if let Some(value) = lower.strip_prefix("content-length:") {
content_length = value.trim().parse().unwrap();
}
}
let mut body = vec![0_u8; content_length];
reader.read_exact(&mut body).await.unwrap();
let mut stream = reader.into_inner();
if request_line.contains("dbx_action=write") {
buffered.lock().await.extend_from_slice(&body);
write_http_response(&mut stream, "200 OK", b"OK").await;
} else if request_line.contains("dbx_action=read") {
let bytes = {
let mut buffered = buffered.lock().await;
std::mem::take(&mut *buffered)
};
if bytes.is_empty() {
write_http_response(&mut stream, "204 No Content", b"").await;
} else {
write_http_response(&mut stream, "200 OK", &bytes).await;
}
} else if request_line.contains("dbx_action=close") {
write_http_response(&mut stream, "200 OK", b"OK").await;
closed.notify_one();
} else {
write_http_response(&mut stream, "200 OK", b"OK").await;
}
}
async fn write_http_response(stream: &mut TcpStream, status: &str, body: &[u8]) {
let header = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
stream.write_all(header.as_bytes()).await.unwrap();
stream.write_all(body).await.unwrap();
}
}
+16
View File
@@ -0,0 +1,16 @@
#![recursion_limit = "256"]
pub use dbx_platform::path_utils;
pub use dbx_types::{models, types};
pub mod db;
pub mod ddl_scan;
pub mod document_result;
pub mod elasticsearch_sql;
pub mod execution;
pub mod file_validator;
pub mod http_tunnel;
pub mod runtime_config;
pub mod ssh_config;
pub mod ssh_host_key;
pub mod wkb;
@@ -0,0 +1,7 @@
pub const DUCKDB_WORKER_MAX_PROCESSES_MIN: usize = 1;
pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16;
pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4;
pub fn normalize_duckdb_worker_max_processes(value: usize) -> usize {
value.clamp(DUCKDB_WORKER_MAX_PROCESSES_MIN, DUCKDB_WORKER_MAX_PROCESSES_MAX)
}
+498
View File
@@ -0,0 +1,498 @@
use serde::Serialize;
use crate::models::connection::SshTunnelConfig;
use crate::path_utils::expand_tilde;
/// Legacy sentinel values used by connection payloads that predate the
/// explicit SSH authentication method field. Current connection forms always
/// set `auth_method`, so their user and port values must be treated as
/// explicit, including the valid defaults `root` and `22`.
const DEFAULT_USER_SENTINEL: &str = "root";
const DEFAULT_PORT_SENTINEL: u16 = 22;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct SshConfigHostEntry {
pub alias: String,
pub host_name: Option<String>,
pub port: Option<u16>,
pub user: Option<String>,
pub identity_file: Option<String>,
/// The `ProxyJump` alias, if any. Only a single hop is parsed (OpenSSH's
/// comma-separated multi-jump form is not supported); the jump itself
/// may declare its own `ProxyJump`, which `resolve_ssh_tunnel_chain`
/// follows recursively.
pub proxy_jump: Option<String>,
}
/// Reads and parses `~/.ssh/config`. Returns an empty list (not an error) if
/// the file does not exist, since that's a normal state for users without
/// an SSH config.
pub fn list_hosts() -> Result<Vec<SshConfigHostEntry>, String> {
let path = expand_tilde("~/.ssh/config");
match std::fs::read_to_string(&path) {
Ok(content) => Ok(parse_ssh_config(&content)),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()),
Err(err) => Err(format!("Failed to read {path}: {err}")),
}
}
pub fn find_host(alias: &str) -> Option<SshConfigHostEntry> {
list_hosts().ok()?.into_iter().find(|entry| entry.alias == alias)
}
/// Fills in `host`, `user`, `port`, and `key_path` from a matching `~/.ssh/config`
/// `Host` block, without overwriting values the user has explicitly set.
///
/// Only `ssh.host` is matched against config aliases; `user`/`port`/`key_path`
/// are filled in from that same matched entry. Older payloads without an
/// `auth_method` retain the legacy default-filling behavior so imported
/// connections continue to resolve their aliases. Payloads produced by the
/// current connection form keep all user and port values as entered.
pub fn resolve_ssh_tunnel_config(ssh: &SshTunnelConfig) -> SshTunnelConfig {
match find_host(&ssh.host) {
Some(entry) => apply_host_entry(ssh, entry),
None => ssh.clone(),
}
}
/// Resolves `ssh.host` plus, when its `~/.ssh/config` entry (or an ancestor
/// reached by following `ProxyJump`) declares `ProxyJump`, every jump host
/// leading to it. The returned list is in connection order — outermost jump
/// first, `ssh`'s own resolved config last — and always has at least one
/// element. A chain of length 1 means there is no `ProxyJump` to honor, and
/// behaves exactly like [`resolve_ssh_tunnel_config`].
///
/// Jump hosts have no dedicated credentials field in DBX's connection form,
/// so each one inherits `ssh`'s password/key/agent settings; only the
/// host/port/user/identity file come from its own `~/.ssh/config` entry.
pub fn resolve_ssh_tunnel_chain(ssh: &SshTunnelConfig) -> Vec<SshTunnelConfig> {
resolve_chain_from_entries(ssh, &list_hosts().unwrap_or_default())
}
fn resolve_chain_from_entries(ssh: &SshTunnelConfig, entries: &[SshConfigHostEntry]) -> Vec<SshTunnelConfig> {
let find = |alias: &str| entries.iter().find(|entry| entry.alias == alias).cloned();
let Some(target_entry) = find(&ssh.host) else {
return vec![ssh.clone()];
};
let resolved_target = apply_host_entry(ssh, target_entry.clone());
let mut jump_hops = Vec::new();
let mut seen_aliases = std::collections::HashSet::new();
seen_aliases.insert(ssh.host.clone());
let mut next_jump = target_entry.proxy_jump.clone();
while let Some(alias) = next_jump {
// Guard against a cycle in `~/.ssh/config` (e.g. two hosts naming
// each other as ProxyJump) rather than looping forever.
if !seen_aliases.insert(alias.clone()) {
break;
}
let Some(entry) = find(&alias) else { break };
next_jump = entry.proxy_jump.clone();
jump_hops.push(jump_hop_from_entry(ssh, entry));
}
jump_hops.reverse();
jump_hops.push(resolved_target);
jump_hops
}
/// Builds a synthetic hop for a `ProxyJump` host. Unlike the leaf host in
/// `ssh`, there's no user-filled form for this host, so its identity comes
/// entirely from `~/.ssh/config` (falling back to the leaf's own settings
/// for anything the config entry doesn't specify).
fn jump_hop_from_entry(leaf: &SshTunnelConfig, entry: SshConfigHostEntry) -> SshTunnelConfig {
SshTunnelConfig {
id: format!("{}::proxyjump::{}", leaf.id, entry.alias),
name: entry.alias.clone(),
enabled: true,
host: entry.host_name.unwrap_or_else(|| entry.alias.clone()),
port: entry.port.unwrap_or(DEFAULT_PORT_SENTINEL),
user: entry.user.unwrap_or_else(|| leaf.user.clone()),
password: leaf.password.clone(),
key_path: entry.identity_file.unwrap_or_else(|| leaf.key_path.clone()),
key_passphrase: leaf.key_passphrase.clone(),
connect_timeout_secs: leaf.connect_timeout_secs,
expose_lan: false,
use_ssh_agent: leaf.use_ssh_agent,
ssh_agent_sock_path: leaf.ssh_agent_sock_path.clone(),
auth_method: leaf.auth_method.clone(),
allow_exec_channel_proxy: leaf.allow_exec_channel_proxy,
profile_id: String::new(),
}
}
/// Applies a resolved `~/.ssh/config` entry onto `ssh`, without overwriting
/// values the user has explicitly set. The empty `auth_method` identifies
/// legacy payloads where default values cannot be distinguished from fields
/// that were left blank in the old form.
fn apply_host_entry(ssh: &SshTunnelConfig, entry: SshConfigHostEntry) -> SshTunnelConfig {
let mut resolved = ssh.clone();
let is_legacy_payload = ssh.auth_method.is_empty();
if let Some(host_name) = entry.host_name {
resolved.host = host_name;
}
if is_legacy_payload && resolved.user == DEFAULT_USER_SENTINEL {
if let Some(user) = entry.user {
resolved.user = user;
}
}
if is_legacy_payload && resolved.port == DEFAULT_PORT_SENTINEL {
if let Some(port) = entry.port {
resolved.port = port;
}
}
if resolved.key_path.is_empty() {
if let Some(identity_file) = entry.identity_file {
resolved.key_path = identity_file;
// If the SSH config supplied the only usable credential, make the
// backend use it even when an older/default UI payload still says
// "password" with an empty password.
if resolved.auth_method.is_empty() || (resolved.auth_method == "password" && resolved.password.is_empty()) {
resolved.auth_method = "key".to_string();
}
}
}
resolved
}
/// Parses a minimal subset of OpenSSH client config syntax: `Host`, `HostName`,
/// `Port`, `User`, `IdentityFile`, `ProxyJump`. Wildcard host patterns
/// (containing `*` or `?`) are skipped since they aren't usable as a literal
/// alias in the host field. `Include` and other directives are not
/// supported, and `ProxyJump`'s comma-separated multi-hop form is read as a
/// single alias (its first hop).
fn parse_ssh_config(content: &str) -> Vec<SshConfigHostEntry> {
let mut entries: Vec<SshConfigHostEntry> = Vec::new();
let mut current_aliases: Vec<String> = Vec::new();
for raw_line in content.lines() {
let line = strip_comment(raw_line).trim();
if line.is_empty() {
continue;
}
let Some((keyword, value)) = split_directive(line) else {
continue;
};
match keyword.to_ascii_lowercase().as_str() {
"host" => {
current_aliases = value
.split_whitespace()
.filter(|alias| !alias.contains('*') && !alias.contains('?'))
.map(str::to_string)
.collect();
for alias in &current_aliases {
entries.push(SshConfigHostEntry {
alias: alias.clone(),
host_name: None,
port: None,
user: None,
identity_file: None,
proxy_jump: None,
});
}
}
"hostname" => set_current_field(&mut entries, &current_aliases, |entry| {
entry.host_name = Some(value.to_string());
}),
"port" => {
if let Ok(port) = value.parse::<u16>() {
set_current_field(&mut entries, &current_aliases, |entry| {
entry.port = Some(port);
});
}
}
"user" => set_current_field(&mut entries, &current_aliases, |entry| {
entry.user = Some(value.to_string());
}),
"identityfile" => set_current_field(&mut entries, &current_aliases, |entry| {
entry.identity_file = Some(value.to_string());
}),
"proxyjump" => {
if let Some(first_hop) = value.split(',').next().map(str::trim).filter(|hop| !hop.is_empty()) {
// A `user@host:port` jump target names an alias-incompatible
// literal host; only a plain alias is resolvable here.
if !first_hop.contains('@') && !first_hop.contains(':') {
let first_hop = first_hop.to_string();
set_current_field(&mut entries, &current_aliases, |entry| {
entry.proxy_jump = Some(first_hop.clone());
});
}
}
}
_ => {}
}
}
entries
}
fn set_current_field(
entries: &mut [SshConfigHostEntry],
current_aliases: &[String],
apply: impl Fn(&mut SshConfigHostEntry),
) {
for entry in entries.iter_mut() {
if current_aliases.contains(&entry.alias) {
apply(entry);
}
}
}
fn strip_comment(line: &str) -> &str {
match line.find('#') {
Some(index) => &line[..index],
None => line,
}
}
/// Splits a config line into `(keyword, value)`. OpenSSH allows the keyword
/// and value to be separated by whitespace or a single `=`.
fn split_directive(line: &str) -> Option<(&str, &str)> {
let line = line.trim();
let split_index = line.find(|c: char| c.is_whitespace() || c == '=')?;
let keyword = &line[..split_index];
let value = line[split_index..].trim_start_matches(|c: char| c.is_whitespace() || c == '=').trim();
if keyword.is_empty() || value.is_empty() {
return None;
}
Some((keyword, value))
}
#[cfg(test)]
mod tests {
use super::*;
fn config(host: &str) -> SshTunnelConfig {
SshTunnelConfig {
profile_id: String::new(),
id: "1".to_string(),
name: String::new(),
enabled: true,
host: host.to_string(),
port: DEFAULT_PORT_SENTINEL,
user: DEFAULT_USER_SENTINEL.to_string(),
password: String::new(),
key_path: String::new(),
key_passphrase: String::new(),
connect_timeout_secs: 5,
expose_lan: false,
use_ssh_agent: false,
ssh_agent_sock_path: String::new(),
auth_method: String::new(),
allow_exec_channel_proxy: false,
}
}
#[test]
fn parses_basic_host_block() {
let entries = parse_ssh_config(
"Host myserver\n HostName 10.0.0.5\n Port 2222\n User deploy\n IdentityFile ~/.ssh/id_ed25519\n",
);
assert_eq!(entries.len(), 1);
let entry = &entries[0];
assert_eq!(entry.alias, "myserver");
assert_eq!(entry.host_name, Some("10.0.0.5".to_string()));
assert_eq!(entry.port, Some(2222));
assert_eq!(entry.user, Some("deploy".to_string()));
assert_eq!(entry.identity_file, Some("~/.ssh/id_ed25519".to_string()));
}
#[test]
fn one_line_can_declare_multiple_aliases() {
let entries = parse_ssh_config("Host prod prod-alias\n HostName 10.0.0.9\n");
assert_eq!(entries.len(), 2);
assert!(entries.iter().all(|entry| entry.host_name == Some("10.0.0.9".to_string())));
assert_eq!(entries[0].alias, "prod");
assert_eq!(entries[1].alias, "prod-alias");
}
#[test]
fn skips_wildcard_host_patterns() {
let entries = parse_ssh_config("Host *.example.com\n User git\nHost real\n User deploy\n");
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].alias, "real");
}
#[test]
fn ignores_comments_and_blank_lines() {
let entries = parse_ssh_config("# a comment\n\nHost myserver # inline comment\n User deploy\n");
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].user, Some("deploy".to_string()));
}
#[test]
fn parses_proxy_jump() {
let entries = parse_ssh_config("Host target\n HostName 1.1.1.2\n ProxyJump gateway\n");
assert_eq!(entries[0].proxy_jump, Some("gateway".to_string()));
}
#[test]
fn proxy_jump_multi_hop_form_reads_only_the_first_hop() {
let entries = parse_ssh_config("Host target\n ProxyJump gw1,gw2\n");
assert_eq!(entries[0].proxy_jump, Some("gw1".to_string()));
}
#[test]
fn proxy_jump_user_at_host_form_is_not_resolvable_as_an_alias() {
let entries = parse_ssh_config("Host target\n ProxyJump jumper@1.1.1.1\n");
assert_eq!(entries[0].proxy_jump, None);
}
fn entry(alias: &str) -> SshConfigHostEntry {
SshConfigHostEntry {
alias: alias.to_string(),
host_name: Some("10.0.0.5".to_string()),
port: Some(2222),
user: Some("deploy".to_string()),
identity_file: Some("~/.ssh/id_ed25519".to_string()),
proxy_jump: None,
}
}
#[test]
fn resolve_fills_unset_fields_from_matching_alias() {
let ssh = config("myserver");
let resolved = apply_host_entry(&ssh, entry("myserver"));
assert_eq!(resolved.host, "10.0.0.5");
assert_eq!(resolved.port, 2222);
assert_eq!(resolved.user, "deploy");
assert_eq!(resolved.key_path, "~/.ssh/id_ed25519");
assert_eq!(resolved.auth_method, "key");
}
#[test]
fn resolve_keeps_password_auth_when_password_is_present() {
let mut ssh = config("myserver");
ssh.password = "secret".to_string();
ssh.auth_method = "password".to_string();
let resolved = apply_host_entry(&ssh, entry("myserver"));
assert_eq!(resolved.key_path, "~/.ssh/id_ed25519");
assert_eq!(resolved.auth_method, "password");
}
#[test]
fn resolve_keeps_default_user_and_port_for_current_payloads() {
let mut ssh = config("myserver");
ssh.auth_method = "password".to_string();
let resolved = apply_host_entry(&ssh, entry("myserver"));
assert_eq!(resolved.host, "10.0.0.5");
assert_eq!(resolved.user, DEFAULT_USER_SENTINEL);
assert_eq!(resolved.port, DEFAULT_PORT_SENTINEL);
}
#[test]
fn resolve_preserves_key_plus_password_method() {
let mut ssh = config("myserver");
ssh.auth_method = "key+password".to_string();
ssh.password = "secret".to_string();
let resolved = apply_host_entry(&ssh, entry("myserver"));
assert_eq!(resolved.key_path, "~/.ssh/id_ed25519");
assert_eq!(resolved.auth_method, "key+password");
}
#[test]
fn resolve_does_not_override_explicit_values() {
let mut ssh = config("myserver");
ssh.user = "alice".to_string();
ssh.port = 9999;
ssh.key_path = "/explicit/key".to_string();
let resolved = apply_host_entry(&ssh, entry("myserver"));
assert_eq!(resolved.host, "10.0.0.5");
assert_eq!(resolved.user, "alice");
assert_eq!(resolved.port, 9999);
assert_eq!(resolved.key_path, "/explicit/key");
}
#[test]
fn resolve_is_noop_when_host_does_not_match_any_alias() {
// `resolve_ssh_tunnel_config` looks up the real `~/.ssh/config`; an
// alias this unlikely to exist on a test machine exercises the
// "no match found" branch without needing to mock the filesystem.
let ssh = config("dbx-test-alias-that-should-never-exist-anywhere");
let resolved = resolve_ssh_tunnel_config(&ssh);
assert_eq!(resolved, ssh);
}
fn jump_entry(alias: &str, host_name: &str, proxy_jump: Option<&str>) -> SshConfigHostEntry {
SshConfigHostEntry {
alias: alias.to_string(),
host_name: Some(host_name.to_string()),
port: None,
user: None,
identity_file: None,
proxy_jump: proxy_jump.map(str::to_string),
}
}
#[test]
fn chain_is_single_hop_without_proxy_jump() {
let ssh = config("myserver");
let chain = resolve_chain_from_entries(&ssh, std::slice::from_ref(&entry("myserver")));
assert_eq!(chain.len(), 1);
assert_eq!(chain[0].host, "10.0.0.5");
}
#[test]
fn chain_prepends_the_proxy_jump_hop_in_connection_order() {
// Mirrors issue #7706: `target` (ProxyJump gateway) resolves to
// [gateway, target], gateway dialed first.
let ssh = config("target");
let entries = [jump_entry("gateway", "1.1.1.1", None), jump_entry("target", "1.1.1.2", Some("gateway"))];
let chain = resolve_chain_from_entries(&ssh, &entries);
assert_eq!(chain.len(), 2);
assert_eq!(chain[0].host, "1.1.1.1");
assert_eq!(chain[0].name, "gateway");
assert_eq!(chain[1].host, "1.1.1.2");
}
#[test]
fn chain_jump_hop_inherits_leaf_credentials_when_config_has_no_identity_file() {
let mut ssh = config("target");
ssh.auth_method = "key".to_string();
ssh.key_path = "/home/user/.ssh/id_rsa".to_string();
let entries = [jump_entry("gateway", "1.1.1.1", None), jump_entry("target", "1.1.1.2", Some("gateway"))];
let chain = resolve_chain_from_entries(&ssh, &entries);
assert_eq!(chain[0].auth_method, "key");
assert_eq!(chain[0].key_path, "/home/user/.ssh/id_rsa");
}
#[test]
fn chain_follows_multiple_proxy_jump_hops_recursively() {
let ssh = config("target");
let entries = [
jump_entry("gateway-outer", "1.1.1.1", None),
jump_entry("gateway-inner", "1.1.1.2", Some("gateway-outer")),
jump_entry("target", "1.1.1.3", Some("gateway-inner")),
];
let chain = resolve_chain_from_entries(&ssh, &entries);
assert_eq!(
chain.iter().map(|hop| hop.host.as_str()).collect::<Vec<_>>(),
vec!["1.1.1.1", "1.1.1.2", "1.1.1.3"]
);
}
#[test]
fn chain_breaks_a_proxy_jump_cycle_instead_of_looping_forever() {
let ssh = config("a");
let entries = [jump_entry("a", "1.1.1.1", Some("b")), jump_entry("b", "1.1.1.2", Some("a"))];
let chain = resolve_chain_from_entries(&ssh, &entries);
// Must terminate; the exact truncation point is an implementation
// detail of cycle detection, not a behavior callers rely on.
assert!(chain.len() <= entries.len());
assert_eq!(chain.last().unwrap().host, "1.1.1.1");
}
}
@@ -0,0 +1,186 @@
//! Host-key verification for the SSH tunnel client.
//!
//! Before any credential is sent, russh invokes
//! [`russh::client::Handler::check_server_key`]. The original implementation
//! accepted *every* server key, which let a man-in-the-middle endpoint reject
//! the public key and trick the client into sending the password instead.
//! This module makes that callback actually verify the server identity.
//!
//! Design
//! ------
//! Verification is layered and **fail-closed**, reusing russh's own OpenSSH-
//! format `known_hosts` module instead of a hand-rolled store:
//!
//! 1. The user's system `~/.ssh/known_hosts` is checked **read-only**. Reusing
//! it means a host already trusted in a terminal is also trusted here, and a
//! key change recorded there is detected here too. dbx never *writes* to it.
//! 2. dbx's own store at `<data_dir>/known_hosts` (OpenSSH format, managed by
//! russh's `known_hosts` module) is checked. This is where dbx-accepted host
//! keys live — NOT `~/.ssh` — so we never pollute the user's SSH config.
//! 3. An unknown host is reported as [`HostKeyState::Unknown`]; the caller
//! (the `check_server_key` handler) then performs an **explicit TOFU** flow:
//! it asks the UI to confirm the key and, on acceptance, records it via
//! [`HostKeyVerifier::learn`].
//! 4. A *changed* key in the dbx store is [`HostKeyState::Changed`]. The caller
//! prompts the user to compare fingerprints and, if they choose to update,
//! replaces the old entry via [`HostKeyVerifier::replace`]. A changed key
//! in `~/.ssh/known_hosts` stays a hard reject — dbx does not edit that file.
//!
//! Hardening rules:
//! * A changed key is never trusted silently. Credentials are sent only after
//! the user explicitly continues (session-only) or updates the saved key.
//! * If an unknown host's key **cannot be persisted** (e.g. no write permission
//! on `<data_dir>`), `learn` fails — but that failure is surfaced by the
//! caller, which still refuses to trust the host silently.
use std::io;
use std::path::{Path, PathBuf};
use russh::keys::known_hosts::{
check_known_hosts, check_known_hosts_path, known_host_keys_path, learn_known_hosts_path,
};
use russh::keys::parse_public_key_base64;
use russh::keys::ssh_key::{HashAlg, PublicKey};
/// Outcome of checking a server key against the known-hosts stores.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HostKeyState {
/// The key matches a previously trusted entry (system or dbx store).
Trusted,
/// The key is not present in either store (candidate for explicit TOFU).
Unknown,
/// The dbx store has a different key of the same algorithm. The caller
/// should prompt before replacing it; `previous_fingerprint` is the saved
/// SHA256 fingerprint (or a fallback locator if the line could not be parsed).
Changed { previous_fingerprint: String },
}
/// Host-key verifier backed by russh's OpenSSH-format known-hosts store.
///
/// `known_hosts_path` is the dbx-managed store (typically
/// `<data_dir>/known_hosts`). The system `~/.ssh/known_hosts` is also consulted
/// read-only via [`check_known_hosts`].
pub struct HostKeyVerifier {
known_hosts_path: PathBuf,
}
impl HostKeyVerifier {
/// Creates a verifier that records/reads accepted host keys at `known_hosts_path`.
pub fn new(known_hosts_path: PathBuf) -> Self {
Self { known_hosts_path }
}
/// Layered, fail-closed check. A *changed* key in `~/.ssh` is an `Err`. A
/// changed key in the dbx store is [`HostKeyState::Changed`] so the UI can
/// offer a one-click replace.
pub fn check(&self, host: &str, port: u16, key: &PublicKey) -> Result<HostKeyState, io::Error> {
// 1. System known_hosts (read-only). A changed key there is a hard reject.
match check_known_hosts(host, port, key) {
Ok(true) => return Ok(HostKeyState::Trusted),
Err(russh::keys::Error::KeyChanged { line }) => {
return Err(host_key_changed_error(host, port, line, "~/.ssh/known_hosts"));
}
// Missing system file, no home dir, or an unreadable system file:
// don't fail the whole connection on it; fall through to the dbx
// store / TOFU, which still protects against MITM.
_ => {}
}
// 2. dbx-managed known_hosts.
match check_known_hosts_path(host, port, key, &self.known_hosts_path) {
Ok(true) => return Ok(HostKeyState::Trusted),
Err(russh::keys::Error::KeyChanged { line }) => {
let previous_fingerprint = previous_fingerprint_for(&self.known_hosts_path, host, port, key)
.unwrap_or_else(|| format!("(known_hosts line {line})"));
return Ok(HostKeyState::Changed { previous_fingerprint });
}
// Unknown (or an unreadable dbx store): report as a candidate for TOFU.
_ => {}
}
Ok(HostKeyState::Unknown)
}
/// Records a host key into the dbx store (TOFU persistence). Called by the
/// caller only after the user explicitly accepts the key. A write failure
/// is reported (so the caller knows persistence did not happen) but does
/// not by itself abort the session — the host may simply be trusted for
/// this session only.
pub fn learn(&self, host: &str, port: u16, key: &PublicKey) -> Result<(), io::Error> {
learn_known_hosts_path(host, port, key, &self.known_hosts_path).map_err(|e| {
io::Error::new(
io::ErrorKind::PermissionDenied,
format!(
"Failed to persist host key for {host}:{port} to {} ({e}). \
The host is trusted for this session only.",
self.known_hosts_path.display()
),
)
})
}
/// Drops the previous same-algorithm key for this host in the dbx store,
/// then records `key`. Used after the user confirms a fingerprint change.
pub fn replace(&self, host: &str, port: u16, key: &PublicKey) -> Result<(), io::Error> {
self.forget_matching_algorithm(host, port, key)?;
self.learn(host, port, key)
}
fn forget_matching_algorithm(&self, host: &str, port: u16, key: &PublicKey) -> Result<(), io::Error> {
let recorded = match known_host_keys_path(host, port, &self.known_hosts_path) {
Ok(keys) => keys,
Err(_) => return Ok(()),
};
let drop_keys: Vec<PublicKey> = recorded
.into_iter()
.filter(|(_, recorded)| recorded.algorithm() == key.algorithm())
.map(|(_, recorded)| recorded)
.collect();
if drop_keys.is_empty() {
return Ok(());
}
let contents = match std::fs::read_to_string(&self.known_hosts_path) {
Ok(contents) => contents,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error),
};
let mut kept = Vec::new();
for line in contents.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
kept.push(line);
continue;
}
let should_drop = trimmed
.split_whitespace()
.nth(2)
.and_then(|encoded| parse_public_key_base64(encoded).ok())
.map(|parsed| drop_keys.iter().any(|recorded| recorded == &parsed))
.unwrap_or(false);
if !should_drop {
kept.push(line);
}
}
let mut out = kept.join("\n");
if !out.is_empty() {
out.push('\n');
}
std::fs::write(&self.known_hosts_path, out)
}
}
fn previous_fingerprint_for(path: &Path, host: &str, port: u16, key: &PublicKey) -> Option<String> {
let recorded = known_host_keys_path(host, port, path).ok()?;
recorded
.into_iter()
.find(|(_, recorded)| recorded.algorithm() == key.algorithm() && recorded != key)
.map(|(_, recorded)| recorded.fingerprint(HashAlg::Sha256).to_string())
}
fn host_key_changed_error(host: &str, port: u16, line: usize, store: &str) -> io::Error {
io::Error::other(format!(
"Host key for {host}:{port} changed (recorded at {store}, line {line}). \
This may indicate a man-in-the-middle attack. Remove the old entry and reconnect only if you expect this change."
))
}
+375
View File
@@ -0,0 +1,375 @@
/// Shared WKB (Well-Known Binary) geometry parsing shared by PostgreSQL (EWKB)
/// and MySQL (standard WKB). Parses WKB bytes into WKT while retaining the
/// top-level EWKB SRID when present.
///
/// Supports: POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING,
/// MULTIPOLYGON, GEOMETRYCOLLECTION, and their Z/M/ZM variants.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct WkbDimensions {
has_z: bool,
has_m: bool,
}
impl WkbDimensions {
fn suffix(self) -> &'static str {
match (self.has_z, self.has_m) {
(false, false) => "",
(true, false) => " Z",
(false, true) => " M",
(true, true) => " ZM",
}
}
fn coordinate_len(self) -> usize {
2 + usize::from(self.has_z) + usize::from(self.has_m)
}
}
#[derive(Debug, Clone, PartialEq)]
enum WkbGeometry {
Point { dims: WkbDimensions, coords: Option<Vec<f64>> },
LineString { dims: WkbDimensions, points: Vec<Vec<f64>> },
Polygon { dims: WkbDimensions, rings: Vec<Vec<Vec<f64>>> },
MultiPoint { dims: WkbDimensions, points: Vec<Option<Vec<f64>>> },
MultiLineString { dims: WkbDimensions, lines: Vec<Vec<Vec<f64>>> },
MultiPolygon { dims: WkbDimensions, polygons: Vec<Vec<Vec<Vec<f64>>>> },
GeometryCollection { dims: WkbDimensions, geometries: Vec<WkbGeometry> },
}
#[derive(Debug, Clone, PartialEq)]
pub struct DecodedGeometry {
pub wkt: String,
pub srid: Option<u32>,
}
impl WkbGeometry {
fn to_wkt(&self) -> String {
match self {
Self::Point { dims, coords } => match coords {
Some(coords) => format!("POINT{}({})", dims.suffix(), format_wkb_coordinate(coords)),
None => format!("POINT{} EMPTY", dims.suffix()),
},
Self::LineString { dims, points } => {
if points.is_empty() {
format!("LINESTRING{} EMPTY", dims.suffix())
} else {
format!("LINESTRING{}({})", dims.suffix(), format_wkb_coordinate_sequence(points))
}
}
Self::Polygon { dims, rings } => {
if rings.is_empty() {
format!("POLYGON{} EMPTY", dims.suffix())
} else {
format!(
"POLYGON{}({})",
dims.suffix(),
rings
.iter()
.map(|ring| format!("({})", format_wkb_coordinate_sequence(ring)))
.collect::<Vec<_>>()
.join(",")
)
}
}
Self::MultiPoint { dims, points } => {
if points.is_empty() {
format!("MULTIPOINT{} EMPTY", dims.suffix())
} else {
format!(
"MULTIPOINT{}({})",
dims.suffix(),
points
.iter()
.map(|point| match point {
Some(coords) => format!("({})", format_wkb_coordinate(coords)),
None => "EMPTY".to_string(),
})
.collect::<Vec<_>>()
.join(",")
)
}
}
Self::MultiLineString { dims, lines } => {
if lines.is_empty() {
format!("MULTILINESTRING{} EMPTY", dims.suffix())
} else {
format!(
"MULTILINESTRING{}({})",
dims.suffix(),
lines
.iter()
.map(|line| format!("({})", format_wkb_coordinate_sequence(line)))
.collect::<Vec<_>>()
.join(",")
)
}
}
Self::MultiPolygon { dims, polygons } => {
if polygons.is_empty() {
format!("MULTIPOLYGON{} EMPTY", dims.suffix())
} else {
format!(
"MULTIPOLYGON{}({})",
dims.suffix(),
polygons
.iter()
.map(|polygon| {
format!(
"({})",
polygon
.iter()
.map(|ring| format!("({})", format_wkb_coordinate_sequence(ring)))
.collect::<Vec<_>>()
.join(",")
)
})
.collect::<Vec<_>>()
.join(",")
)
}
}
Self::GeometryCollection { dims, geometries } => {
if geometries.is_empty() {
format!("GEOMETRYCOLLECTION{} EMPTY", dims.suffix())
} else {
format!(
"GEOMETRYCOLLECTION{}({})",
dims.suffix(),
geometries.iter().map(WkbGeometry::to_wkt).collect::<Vec<_>>().join(",")
)
}
}
}
}
}
fn format_wkb_coordinate(coords: &[f64]) -> String {
coords.iter().map(|value| value.to_string()).collect::<Vec<_>>().join(" ")
}
fn format_wkb_coordinate_sequence(points: &[Vec<f64>]) -> String {
points.iter().map(|point| format_wkb_coordinate(point)).collect::<Vec<_>>().join(",")
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum WkbEndian {
Big,
Little,
}
struct WkbReader<'a> {
raw: &'a [u8],
pos: usize,
}
impl<'a> WkbReader<'a> {
fn new(raw: &'a [u8]) -> Self {
Self { raw, pos: 0 }
}
fn read_u8(&mut self) -> Option<u8> {
let value = *self.raw.get(self.pos)?;
self.pos += 1;
Some(value)
}
fn read_array<const N: usize>(&mut self) -> Option<[u8; N]> {
let end = self.pos.checked_add(N)?;
let bytes: [u8; N] = self.raw.get(self.pos..end)?.try_into().ok()?;
self.pos = end;
Some(bytes)
}
fn read_u32(&mut self, endian: WkbEndian) -> Option<u32> {
let bytes = self.read_array::<4>()?;
Some(match endian {
WkbEndian::Big => u32::from_be_bytes(bytes),
WkbEndian::Little => u32::from_le_bytes(bytes),
})
}
fn read_f64(&mut self, endian: WkbEndian) -> Option<f64> {
let bytes = self.read_array::<8>()?;
Some(match endian {
WkbEndian::Big => f64::from_be_bytes(bytes),
WkbEndian::Little => f64::from_le_bytes(bytes),
})
}
}
fn parse_wkb_dimensions(type_word: u32) -> (u32, WkbDimensions, bool) {
let mut base_type = type_word & 0x1FFF_FFFF;
let mut dims = WkbDimensions { has_z: (type_word & 0x8000_0000) != 0, has_m: (type_word & 0x4000_0000) != 0 };
let has_srid = (type_word & 0x2000_0000) != 0;
if base_type >= 3000 {
dims.has_z = true;
dims.has_m = true;
base_type -= 3000;
} else if base_type >= 2000 {
dims.has_m = true;
base_type -= 2000;
} else if base_type >= 1000 {
dims.has_z = true;
base_type -= 1000;
}
(base_type, dims, has_srid)
}
fn read_wkb_point_coords(reader: &mut WkbReader<'_>, dims: WkbDimensions) -> Option<Option<Vec<f64>>> {
let endian = match reader.read_u8()? {
0 => WkbEndian::Big,
1 => WkbEndian::Little,
_ => return None,
};
let type_word = reader.read_u32(endian)?;
let (base_type, parsed_dims, has_srid) = parse_wkb_dimensions(type_word);
if base_type != 1 || parsed_dims != dims {
return None;
}
if has_srid {
reader.read_u32(endian)?;
}
let coords = (0..parsed_dims.coordinate_len()).map(|_| reader.read_f64(endian)).collect::<Option<Vec<_>>>()?;
Some((!coords.iter().all(|value| value.is_nan())).then_some(coords))
}
fn parse_wkb_points(reader: &mut WkbReader<'_>, endian: WkbEndian, dims: WkbDimensions) -> Option<Vec<Vec<f64>>> {
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
(0..count)
.map(|_| (0..dims.coordinate_len()).map(|_| reader.read_f64(endian)).collect::<Option<Vec<_>>>())
.collect::<Option<Vec<_>>>()
}
fn parse_wkb_geometry(reader: &mut WkbReader<'_>) -> Option<(WkbGeometry, Option<u32>)> {
let endian = match reader.read_u8()? {
0 => WkbEndian::Big,
1 => WkbEndian::Little,
_ => return None,
};
let type_word = reader.read_u32(endian)?;
let (base_type, dims, has_srid) = parse_wkb_dimensions(type_word);
let srid = has_srid.then(|| reader.read_u32(endian)).flatten().filter(|value| *value != 0);
let geometry = match base_type {
1 => {
let coords = (0..dims.coordinate_len()).map(|_| reader.read_f64(endian)).collect::<Option<Vec<_>>>()?;
let coords = if coords.iter().all(|value| value.is_nan()) { None } else { Some(coords) };
Some(WkbGeometry::Point { dims, coords })
}
2 => Some(WkbGeometry::LineString { dims, points: parse_wkb_points(reader, endian, dims)? }),
3 => {
let ring_count = usize::try_from(reader.read_u32(endian)?).ok()?;
let rings = (0..ring_count).map(|_| parse_wkb_points(reader, endian, dims)).collect::<Option<Vec<_>>>()?;
Some(WkbGeometry::Polygon { dims, rings })
}
4 => {
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
let points = (0..count).map(|_| read_wkb_point_coords(reader, dims)).collect::<Option<Vec<_>>>()?;
Some(WkbGeometry::MultiPoint { dims, points })
}
5 => {
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
let lines = (0..count)
.map(|_| match parse_wkb_geometry(reader)?.0 {
WkbGeometry::LineString { points, .. } => Some(points),
_ => None,
})
.collect::<Option<Vec<_>>>()?;
Some(WkbGeometry::MultiLineString { dims, lines })
}
6 => {
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
let polygons = (0..count)
.map(|_| match parse_wkb_geometry(reader)?.0 {
WkbGeometry::Polygon { rings, .. } => Some(rings),
_ => None,
})
.collect::<Option<Vec<_>>>()?;
Some(WkbGeometry::MultiPolygon { dims, polygons })
}
7 => {
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
let geometries =
(0..count).map(|_| parse_wkb_geometry(reader).map(|parsed| parsed.0)).collect::<Option<Vec<_>>>()?;
Some(WkbGeometry::GeometryCollection { dims, geometries })
}
_ => None,
}?;
Some((geometry, srid))
}
pub fn decode_wkb_geometry(raw: &[u8]) -> Option<DecodedGeometry> {
let mut reader = WkbReader::new(raw);
let (geometry, srid) = parse_wkb_geometry(&mut reader)?;
if reader.pos != raw.len() {
return None;
}
Some(DecodedGeometry { wkt: geometry.to_wkt(), srid })
}
/// Parse WKB/EWKB bytes into a WKT (Well-Known Text) string.
///
/// Supports both standard WKB (MySQL) and Extended WKB (PostgreSQL/PostGIS).
/// Returns `None` if the bytes cannot be parsed as valid WKB.
pub fn wkb_to_wkt(raw: &[u8]) -> Option<String> {
decode_wkb_geometry(raw).map(|geometry| geometry.wkt)
}
#[cfg(test)]
mod tests {
use super::*;
fn decode_hex(value: &str) -> Vec<u8> {
value
.as_bytes()
.chunks_exact(2)
.map(|pair| {
let text = std::str::from_utf8(pair).unwrap();
u8::from_str_radix(text, 16).unwrap()
})
.collect()
}
#[test]
fn ewkb_decoder_retains_top_level_srid() {
let decoded = decode_wkb_geometry(&decode_hex("0101000020E6100000C520B07268195D404E62105839F44340")).unwrap();
assert_eq!(decoded.wkt, "POINT(116.397 39.908)");
assert_eq!(decoded.srid, Some(4326));
}
#[test]
fn standard_wkb_has_no_srid() {
let decoded = decode_wkb_geometry(&decode_hex("0101000000000000000000F03F0000000000000040")).unwrap();
assert_eq!(decoded.wkt, "POINT(1 2)");
assert_eq!(decoded.srid, None);
}
#[test]
fn big_endian_ewkb_retains_srid() {
let decoded = decode_wkb_geometry(&decode_hex("002000000100000F113FF00000000000004000000000000000")).unwrap();
assert_eq!(decoded.wkt, "POINT(1 2)");
assert_eq!(decoded.srid, Some(3857));
}
#[test]
fn ewkb_srid_zero_is_unknown() {
let decoded = decode_wkb_geometry(&decode_hex("010100002000000000000000000000F03F0000000000000040")).unwrap();
assert_eq!(decoded.wkt, "POINT(1 2)");
assert_eq!(decoded.srid, None);
}
#[test]
fn multipoint_preserves_valid_empty_points() {
let decoded =
decode_wkb_geometry(&decode_hex("0104000000010000000101000000000000000000F87F000000000000F87F")).unwrap();
assert_eq!(decoded.wkt, "MULTIPOINT(EMPTY)");
}
#[test]
fn multipoint_rejects_truncated_child_points() {
assert!(decode_wkb_geometry(&decode_hex("0104000000010000000101000000")).is_none());
}
}