refactor(workspace): split SQL and database driver crates
This commit is contained in:
180 files changed
+1422
-538
No files matched your search
@@ -0,0 +1,27 @@
|
||||
[package]
|
||||
name = "dbx-driver-support"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "Apache-2.0"
|
||||
publish = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
test-support = ["dbx-platform/test-support"]
|
||||
|
||||
[dependencies]
|
||||
dbx-platform = { path = "../dbx-platform" }
|
||||
dbx-types = { path = "../dbx-types", default-features = false }
|
||||
bytes = "1"
|
||||
portpicker = "0.1.1"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
||||
russh = "0.60"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sqlparser = { version = "0.62.0", features = ["visitor"] }
|
||||
tokio = { version = "1", features = ["io-util", "net", "rt", "sync", "time"] }
|
||||
tokio-util = { version = "0.7", features = ["rt"] }
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["macros", "rt-multi-thread", "test-util"] }
|
||||
@@ -0,0 +1,318 @@
|
||||
use crate::path_utils::expand_tilde;
|
||||
use reqwest::ClientBuilder;
|
||||
use reqwest::{Certificate, Identity};
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
// Re-export types so that `db::QueryResult` etc. work within dbx-core
|
||||
pub use crate::file_validator::validate_file_path;
|
||||
pub use crate::types::*;
|
||||
|
||||
pub const CONNECTION_TIMEOUT_SECS: u64 = 5;
|
||||
pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PoolCheckoutStage {
|
||||
Wait,
|
||||
Create,
|
||||
Recycle,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl PoolCheckoutStage {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Wait => "wait",
|
||||
Self::Create => "create",
|
||||
Self::Recycle => "recycle",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PoolCheckoutError {
|
||||
Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration },
|
||||
Failed { database: &'static str, stage: PoolCheckoutStage, detail: String },
|
||||
Canceled,
|
||||
}
|
||||
|
||||
impl PoolCheckoutError {
|
||||
pub fn stage(&self) -> Option<PoolCheckoutStage> {
|
||||
match self {
|
||||
Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage),
|
||||
Self::Canceled => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_pool_saturation(&self) -> bool {
|
||||
matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. })
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PoolCheckoutError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Timeout { database, stage, timeout } => write!(
|
||||
formatter,
|
||||
"{database} connection pool checkout timed out [stage={}, timeout_ms={}]",
|
||||
stage.as_str(),
|
||||
timeout.as_millis()
|
||||
),
|
||||
Self::Failed { database, stage, detail } => {
|
||||
write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str())
|
||||
}
|
||||
Self::Canceled => formatter.write_str(crate::execution::QUERY_CANCELED),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn connection_timeout() -> Duration {
|
||||
Duration::from_secs(CONNECTION_TIMEOUT_SECS)
|
||||
}
|
||||
|
||||
pub fn http_client_builder(timeout: Duration) -> ClientBuilder {
|
||||
reqwest::Client::builder().connect_timeout(timeout).no_proxy()
|
||||
}
|
||||
|
||||
/// Augments an HTTP client builder with custom TLS material for drivers that
|
||||
/// need to trust a non-system CA or present a client certificate (mTLS).
|
||||
///
|
||||
/// * `ca_cert_path` adds an extra root certificate (PEM bundle) alongside the
|
||||
/// system roots, so self-signed server certificates are trusted.
|
||||
/// * `client_cert_path` + `client_key_path` (required together) are concatenated
|
||||
/// into a single PEM identity, as expected by reqwest's rustls backend.
|
||||
///
|
||||
/// Returns the augmented builder, or a descriptive error if a configured
|
||||
/// certificate/key file cannot be read or parsed. Callers surface that error
|
||||
/// instead of silently falling back to the system trust store, so a typo in a
|
||||
/// cert path fails the connection loudly rather than being ignored.
|
||||
pub fn apply_tls_certificates(
|
||||
mut builder: ClientBuilder,
|
||||
ca_cert_path: Option<&str>,
|
||||
client_cert_path: Option<&str>,
|
||||
client_key_path: Option<&str>,
|
||||
) -> Result<ClientBuilder, String> {
|
||||
// The desktop build unifies reqwest with both `default-tls` and `rustls-tls`,
|
||||
// and `TlsBackend` defaults to native-tls there, which rejects the PEM identity
|
||||
// below with `incompatible TLS identity type`. Pin rustls like the Consul and
|
||||
// etcd-metrics clients so custom CA roots and PEM client identities work in
|
||||
// every target.
|
||||
builder = builder.use_rustls_tls();
|
||||
if let Some(ca) = ca_cert_path.filter(|path| !path.is_empty()) {
|
||||
let ca = expand_tilde(ca);
|
||||
let contents = std::fs::read(&ca)
|
||||
.map_err(|error| format!("Failed to read Elasticsearch CA certificate at {ca}: {error}"))?;
|
||||
let certificates = Certificate::from_pem_bundle(&contents)
|
||||
.map_err(|error| format!("Failed to parse Elasticsearch CA certificate at {ca}: {error}"))?;
|
||||
for certificate in certificates {
|
||||
builder = builder.add_root_certificate(certificate);
|
||||
}
|
||||
}
|
||||
if let (Some(cert_path), Some(key_path)) =
|
||||
(client_cert_path.filter(|path| !path.is_empty()), client_key_path.filter(|path| !path.is_empty()))
|
||||
{
|
||||
let cert_path = expand_tilde(cert_path);
|
||||
let key_path = expand_tilde(key_path);
|
||||
let mut pem = std::fs::read(&cert_path)
|
||||
.map_err(|error| format!("Failed to read Elasticsearch client certificate at {cert_path}: {error}"))?;
|
||||
// reqwest's rustls backend concatenates the cert and key as two PEM
|
||||
// blocks; ensure they are newline-separated so parsing succeeds even
|
||||
// when the cert file does not end with a trailing newline.
|
||||
if !pem.ends_with(b"\n") {
|
||||
pem.push(b'\n');
|
||||
}
|
||||
pem.extend(
|
||||
std::fs::read(&key_path)
|
||||
.map_err(|error| format!("Failed to read Elasticsearch client key at {key_path}: {error}"))?,
|
||||
);
|
||||
let identity = Identity::from_pem(&pem)
|
||||
.map_err(|error| format!("Failed to parse Elasticsearch client identity: {error}"))?;
|
||||
builder = builder.identity(identity);
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
pub use dbx_types::json::JS_MAX_SAFE_INTEGER;
|
||||
pub use dbx_types::json::{json_value_for_js, safe_i64_to_json, safe_u64_to_json};
|
||||
|
||||
pub fn hex_encode(bytes: &[u8]) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut out = String::with_capacity(bytes.len() * 2);
|
||||
for &byte in bytes {
|
||||
out.push(HEX[(byte >> 4) as usize] as char);
|
||||
out.push(HEX[(byte & 0x0f) as usize] as char);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn binary_value_to_json(bytes: &[u8]) -> serde_json::Value {
|
||||
serde_json::Value::String(format!("0x{}", hex_encode(bytes)))
|
||||
}
|
||||
|
||||
pub fn tcp_probe_timeout() -> Duration {
|
||||
Duration::from_secs(TCP_PROBE_TIMEOUT_SECS)
|
||||
}
|
||||
|
||||
pub fn parse_connect_timeout(url: &str) -> Duration {
|
||||
parse_connect_timeout_with_fallback(url, connection_timeout())
|
||||
}
|
||||
|
||||
pub fn parse_connect_timeout_with_fallback(url: &str, fallback: Duration) -> Duration {
|
||||
let Some(query) = url.split('?').nth(1) else {
|
||||
return fallback;
|
||||
};
|
||||
for param in query.split('&') {
|
||||
let trimmed = param.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (key, value) = match trimmed.split_once('=') {
|
||||
Some(pair) => pair,
|
||||
None => continue,
|
||||
};
|
||||
if key.eq_ignore_ascii_case("connect_timeout")
|
||||
|| key.eq_ignore_ascii_case("connectTimeout")
|
||||
|| key.eq_ignore_ascii_case("connection_timeout")
|
||||
|| key.eq_ignore_ascii_case("connectionTimeout")
|
||||
{
|
||||
if let Ok(v) = value.parse::<u64>() {
|
||||
if (1..=300).contains(&v) {
|
||||
return Duration::from_secs(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fallback
|
||||
}
|
||||
|
||||
pub async fn with_connection_timeout<T, F>(label: &str, timeout: Duration, future: F) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
{
|
||||
tokio::time::timeout(timeout, future)
|
||||
.await
|
||||
.map_err(|_| format!("{label} connection timed out ({}s)", timeout.as_secs()))?
|
||||
}
|
||||
|
||||
pub async fn probe_tcp_endpoint(label: &str, host: &str, port: u16, timeout: Duration) -> Result<(), String> {
|
||||
tokio::time::timeout(timeout, tokio::net::TcpStream::connect((host, port)))
|
||||
.await
|
||||
.map_err(|_| format!("{label} TCP connection timed out ({}s)", timeout.as_secs()))?
|
||||
.map(|_| ())
|
||||
.map_err(|e| format!("{label} TCP connection failed: {e}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn binary_values_are_displayed_as_prefixed_hex() {
|
||||
assert_eq!(binary_value_to_json(&[0x00, 0x01, 0xab, 0xff]), serde_json::json!("0x0001abff"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tls_tests {
|
||||
use super::{apply_tls_certificates, http_client_builder};
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Generates a throwaway CA + client certificate/key (PEM) via openssl and
|
||||
/// returns their paths. If openssl is unavailable the files stay empty, which
|
||||
/// makes `apply_tls_certificates` surface a read error instead of degrading.
|
||||
fn gen_certs(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) {
|
||||
let ca_key = dir.join("ca.key");
|
||||
let ca_pem = dir.join("ca.pem");
|
||||
let csr = dir.join("client.csr");
|
||||
let client_key = dir.join("client.key");
|
||||
let client_crt = dir.join("client.crt");
|
||||
// Extensions force an X.509 v3 client cert; some toolchains (e.g.
|
||||
// OpenSSL 3.0 on CI) emit v1 without one, which rustls rejects.
|
||||
let client_ext = dir.join("client.ext");
|
||||
let _ = std::fs::write(&client_ext, "basicConstraints=critical,CA:FALSE\n");
|
||||
let _ = Command::new("openssl")
|
||||
.args([
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
ca_key.to_str().unwrap(),
|
||||
"-out",
|
||||
ca_pem.to_str().unwrap(),
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
"/CN=dbx-test-ca",
|
||||
])
|
||||
.output();
|
||||
let _ = Command::new("openssl")
|
||||
.args([
|
||||
"req",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
client_key.to_str().unwrap(),
|
||||
"-out",
|
||||
csr.to_str().unwrap(),
|
||||
"-subj",
|
||||
"/CN=dbx-test-client",
|
||||
])
|
||||
.output();
|
||||
let _ = Command::new("openssl")
|
||||
.args([
|
||||
"x509",
|
||||
"-req",
|
||||
"-in",
|
||||
csr.to_str().unwrap(),
|
||||
"-CA",
|
||||
ca_pem.to_str().unwrap(),
|
||||
"-CAkey",
|
||||
ca_key.to_str().unwrap(),
|
||||
"-CAcreateserial",
|
||||
"-out",
|
||||
client_crt.to_str().unwrap(),
|
||||
"-days",
|
||||
"1",
|
||||
"-extfile",
|
||||
client_ext.to_str().unwrap(),
|
||||
])
|
||||
.output();
|
||||
(ca_pem, client_crt, client_key)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_tls_certificates_builds_with_ca_and_client_cert() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-tls-test-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let (ca, client_crt, client_key) = gen_certs(&dir);
|
||||
|
||||
let built = apply_tls_certificates(
|
||||
http_client_builder(Duration::from_secs(5)),
|
||||
ca.to_str(),
|
||||
client_crt.to_str(),
|
||||
client_key.to_str(),
|
||||
)
|
||||
.expect("client should build with CA + client cert")
|
||||
.build();
|
||||
assert!(built.is_ok(), "built client should be usable with CA + client cert: {:?}", built.err());
|
||||
|
||||
// A misconfigured CA path must now surface a descriptive error instead of
|
||||
// silently degrading to the system trust store.
|
||||
let missing = apply_tls_certificates(
|
||||
http_client_builder(Duration::from_secs(5)),
|
||||
Some("/nonexistent/ca.pem"),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
assert!(missing.is_err(), "missing CA path must surface a read error");
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
//! Structural, quote-and-parenthesis-aware scanning helpers for
|
||||
//! post-processing already-rendered `CREATE TABLE` DDL text.
|
||||
//!
|
||||
//! A naive `str::find` for a literal marker (e.g. `" SERVER \""` or a bare
|
||||
//! `;`) can be fooled by that same text appearing inside a quoted string
|
||||
//! literal, a quoted identifier, or a nested parenthesized sub-expression —
|
||||
//! a column's `DEFAULT`/`CHECK` expression can legally contain any of those.
|
||||
//! These helpers track quote and parenthesis depth so callers only ever
|
||||
//! match at the top level.
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
fn char_at(sql: &str, pos: usize) -> Option<char> {
|
||||
sql.get(pos..)?.chars().next()
|
||||
}
|
||||
|
||||
/// Skips a `'...'` string literal (`''`-escaped, with backslash-escape
|
||||
/// tolerance) or a `"..."` quoted identifier (`""`-escaped) starting at
|
||||
/// `pos`, which must point at the opening `quote` character. Returns the
|
||||
/// index just past the closing quote (or `sql.len()` if unterminated).
|
||||
fn skip_quoted(sql: &str, pos: usize, quote: char) -> usize {
|
||||
let mut i = pos + quote.len_utf8();
|
||||
while i < sql.len() {
|
||||
let Some(ch) = char_at(sql, i) else { break };
|
||||
let next = char_at(sql, i + ch.len_utf8());
|
||||
if ch == quote {
|
||||
if next == Some(quote) {
|
||||
i += ch.len_utf8() + quote.len_utf8();
|
||||
continue;
|
||||
}
|
||||
return i + ch.len_utf8();
|
||||
}
|
||||
if quote == '\'' && ch == '\\' {
|
||||
i += ch.len_utf8();
|
||||
if let Some(escaped) = char_at(sql, i) {
|
||||
i += escaped.len_utf8();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
i += ch.len_utf8();
|
||||
}
|
||||
sql.len()
|
||||
}
|
||||
|
||||
/// Skips a PostgreSQL `$tag$...$tag$` dollar-quoted body starting at `pos`
|
||||
/// (pointing at the first `$`). Returns `None` if `pos` isn't actually the
|
||||
/// start of a valid dollar-quote tag (e.g. a stray `$` in an expression), in
|
||||
/// which case the caller should treat `$` as an ordinary character.
|
||||
fn skip_dollar_quoted(sql: &str, pos: usize) -> Option<usize> {
|
||||
let tag_end_offset = sql.get(pos + 1..)?.find('$')?;
|
||||
let tag_end = pos + 1 + tag_end_offset;
|
||||
let tag = &sql[pos + 1..tag_end];
|
||||
let valid_tag = tag.is_empty()
|
||||
|| (tag.chars().next().is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_')
|
||||
&& tag.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '_'));
|
||||
if !valid_tag {
|
||||
return None;
|
||||
}
|
||||
let delimiter = &sql[pos..=tag_end];
|
||||
let content_start = tag_end + 1;
|
||||
sql.get(content_start..)?.find(delimiter).map(|offset| content_start + offset + delimiter.len())
|
||||
}
|
||||
|
||||
/// Finds the byte index just past the first top-level (parenthesis-depth-0)
|
||||
/// closing paren at or after `from` — the end of the outermost parenthesized
|
||||
/// group that starts somewhere at or after `from` — treating quoted or
|
||||
/// dollar-quoted content as opaque so parens or keywords inside a
|
||||
/// `DEFAULT`/`CHECK` expression can't confuse the count. Returns `None` if no
|
||||
/// top-level `(` is ever opened, or it's never closed.
|
||||
pub fn find_top_level_paren_close(sql: &str, from: usize) -> Option<usize> {
|
||||
let mut i = from;
|
||||
let mut depth: usize = 0;
|
||||
let mut opened = false;
|
||||
while i < sql.len() {
|
||||
let ch = char_at(sql, i)?;
|
||||
match ch {
|
||||
'\'' | '"' => {
|
||||
i = skip_quoted(sql, i, ch);
|
||||
continue;
|
||||
}
|
||||
'$' => {
|
||||
if let Some(end) = skip_dollar_quoted(sql, i) {
|
||||
i = end;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
'(' => {
|
||||
depth += 1;
|
||||
opened = true;
|
||||
}
|
||||
')' => {
|
||||
depth = depth.saturating_sub(1);
|
||||
i += ch.len_utf8();
|
||||
if opened && depth == 0 {
|
||||
return Some(i);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
i += ch.len_utf8();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Finds the byte index of the next top-level (depth-0, outside any quoted
|
||||
/// or dollar-quoted content) `;` at or after `from`.
|
||||
fn find_top_level_semicolon(sql: &str, from: usize) -> Option<usize> {
|
||||
let mut i = from;
|
||||
let mut depth: usize = 0;
|
||||
while i < sql.len() {
|
||||
let ch = char_at(sql, i)?;
|
||||
match ch {
|
||||
'\'' | '"' => {
|
||||
i = skip_quoted(sql, i, ch);
|
||||
continue;
|
||||
}
|
||||
'$' => {
|
||||
if let Some(end) = skip_dollar_quoted(sql, i) {
|
||||
i = end;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
'(' => depth += 1,
|
||||
')' => depth = depth.saturating_sub(1),
|
||||
';' if depth == 0 => return Some(i),
|
||||
_ => {}
|
||||
}
|
||||
i += ch.len_utf8();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Splits `sql` into consecutive top-level statement ranges, each ending
|
||||
/// just past its own top-level `;`. The last statement may lack a trailing
|
||||
/// `;`, in which case its range runs to the end of the string.
|
||||
pub fn top_level_statement_ranges(sql: &str) -> Vec<Range<usize>> {
|
||||
let mut ranges = Vec::new();
|
||||
let mut start = 0;
|
||||
while start < sql.len() {
|
||||
match find_top_level_semicolon(sql, start) {
|
||||
Some(semi) => {
|
||||
ranges.push(start..semi + 1);
|
||||
start = semi + 1;
|
||||
}
|
||||
None => {
|
||||
ranges.push(start..sql.len());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
ranges
|
||||
}
|
||||
|
||||
/// If `stmt` (ignoring leading whitespace) begins with `CREATE TABLE
|
||||
/// "schema"."table"` or `CREATE FOREIGN TABLE "schema"."table"`, returns the
|
||||
/// unescaped `(schema, table)` pair.
|
||||
pub fn parse_create_table_relation(stmt: &str) -> Option<(String, String)> {
|
||||
let trimmed = stmt.trim_start();
|
||||
let rest =
|
||||
trimmed.strip_prefix("CREATE FOREIGN TABLE ").or_else(|| trimmed.strip_prefix("CREATE TABLE "))?.trim_start();
|
||||
let (schema, rest) = unquote_ident(rest)?;
|
||||
let (table, _) = unquote_ident(rest.strip_prefix('.')?)?;
|
||||
Some((schema, table))
|
||||
}
|
||||
|
||||
fn unquote_ident(s: &str) -> Option<(String, &str)> {
|
||||
if !s.starts_with('"') {
|
||||
return None;
|
||||
}
|
||||
let end = skip_quoted(s, 0, '"');
|
||||
if end < 2 {
|
||||
return None;
|
||||
}
|
||||
let raw = &s[1..end - 1];
|
||||
Some((raw.replace("\"\"", "\""), &s[end..]))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn paren_close_skips_nested_parens_and_quotes() {
|
||||
let ddl =
|
||||
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n) SERVER \"s\"";
|
||||
let close = find_top_level_paren_close(ddl, 0).unwrap();
|
||||
assert_eq!(
|
||||
&ddl[..close],
|
||||
"\"public\".\"t\" (\n \"a\" text DEFAULT 'x(y)'::text,\n \"b\" text CHECK ((b <> 'z'))\n)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paren_close_ignores_literal_server_text_inside_quotes() {
|
||||
let ddl = "\"public\".\"t\" (\n \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"real_server\"";
|
||||
let close = find_top_level_paren_close(ddl, 0).unwrap();
|
||||
assert!(ddl[close..].starts_with(" SERVER \"real_server\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn top_level_statement_ranges_split_on_depth_zero_semicolons() {
|
||||
let ddl = "CREATE TABLE \"s\".\"t\" (\n \"a\" text DEFAULT 'x;y'\n);\nALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (a <> '');";
|
||||
let ranges = top_level_statement_ranges(ddl);
|
||||
assert_eq!(ranges.len(), 2);
|
||||
assert!(ddl[ranges[0].clone()].starts_with("CREATE TABLE"));
|
||||
assert!(ddl[ranges[0].clone()].ends_with(";"));
|
||||
assert!(ddl[ranges[1].clone()].trim_start().starts_with("ALTER TABLE"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_create_table_relation_unescapes_doubled_quotes() {
|
||||
let (schema, table) =
|
||||
parse_create_table_relation("CREATE TABLE \"pub\"\"lic\".\"t\" (\n \"a\" int\n);").unwrap();
|
||||
assert_eq!(schema, "pub\"lic");
|
||||
assert_eq!(table, "t");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_create_table_relation_rejects_non_create_table() {
|
||||
assert!(parse_create_table_relation("ALTER TABLE \"s\".\"t\" ADD CONSTRAINT c CHECK (true);").is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Common result shape returned by document-store queries.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DocumentQueryResult {
|
||||
pub documents: Vec<serde_json::Value>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub raw_documents: Option<Vec<String>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub extended_documents: Option<Vec<serde_json::Value>>,
|
||||
pub total: u64,
|
||||
// Older document stores always return an exact total. Keep that wire shape
|
||||
// unchanged and only send the flag when Elasticsearch reports a lower bound.
|
||||
#[serde(default = "default_total_is_exact", skip_serializing_if = "is_true")]
|
||||
pub total_is_exact: bool,
|
||||
/// Opaque document-store cursor for APIs such as DynamoDB that do not
|
||||
/// support offset pagination.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
|
||||
fn default_total_is_exact() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn is_true(value: &bool) -> bool {
|
||||
*value
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::DocumentQueryResult;
|
||||
|
||||
#[test]
|
||||
fn exact_results_keep_the_existing_wire_shape() {
|
||||
let result = DocumentQueryResult {
|
||||
documents: Vec::new(),
|
||||
raw_documents: None,
|
||||
extended_documents: None,
|
||||
total: 1,
|
||||
total_is_exact: true,
|
||||
next_cursor: None,
|
||||
};
|
||||
|
||||
let serialized = serde_json::to_value(result).unwrap();
|
||||
assert!(serialized.get("total_is_exact").is_none());
|
||||
|
||||
let deserialized: DocumentQueryResult = serde_json::from_value(serde_json::json!({
|
||||
"documents": [],
|
||||
"total": 1,
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(deserialized.total_is_exact);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inexact_results_preserve_the_explicit_flag() {
|
||||
let deserialized: DocumentQueryResult = serde_json::from_value(serde_json::json!({
|
||||
"documents": [],
|
||||
"total": 10_000_000,
|
||||
"total_is_exact": false,
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(!deserialized.total_is_exact);
|
||||
|
||||
let serialized = serde_json::to_value(deserialized).unwrap();
|
||||
assert_eq!(serialized.get("total_is_exact"), Some(&serde_json::json!(false)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
// SQL → Elasticsearch `_search` DSL translator.
|
||||
//
|
||||
// ES SQL's `_sql` endpoint refuses several common shapes — `LIKE` on a `text`
|
||||
// field with no `.keyword` sub-field, `SELECT *` returning an array field
|
||||
// like `host.ip`, ... — even though the same intent is easy to express with
|
||||
// the raw query DSL. So instead of delegating to ES SQL (or its
|
||||
// `_sql/translate` cousin, which lives in the same engine), we parse the
|
||||
// SQL ourselves with `sqlparser-rs` and emit a `_search` body.
|
||||
|
||||
use serde_json::{json, Map, Value as Json};
|
||||
use sqlparser::ast::{
|
||||
BinaryOperator, Expr, OrderByExpr, OrderByKind, Query, Select, SelectItem, SetExpr, Statement, TableFactor,
|
||||
UnaryOperator, Value as SqlValue,
|
||||
};
|
||||
use sqlparser::dialect::GenericDialect;
|
||||
use sqlparser::parser::Parser;
|
||||
|
||||
const AUTO_PAGED_SELECT_STAR_SIZE: usize = 100;
|
||||
|
||||
pub struct TranslatedSelectStar {
|
||||
pub index: String,
|
||||
pub body: Json,
|
||||
/// User wrote `LIMIT N` (no OFFSET) — explicit row cap.
|
||||
pub user_limited: bool,
|
||||
/// SQL had OFFSET — signals this came from the pagination plan.
|
||||
pub from_plan_pagination: bool,
|
||||
}
|
||||
|
||||
/// Try to translate a `SELECT * FROM <index> [WHERE …] [ORDER BY …]
|
||||
/// [LIMIT N] [OFFSET M]` statement into a `_search` body.
|
||||
///
|
||||
/// Returns `Ok(None)` if the statement isn't a recognised `SELECT *` shape
|
||||
/// (caller should fall back to a different path). Returns `Err` if it IS a
|
||||
/// `SELECT *` but a clause is unsupported (so the caller surfaces a clear
|
||||
/// error rather than silently downgrading).
|
||||
pub fn translate_select_star(sql: &str) -> Result<Option<TranslatedSelectStar>, String> {
|
||||
let dialect = GenericDialect {};
|
||||
let statements = Parser::parse_sql(&dialect, sql).map_err(|e| format!("SQL parse error: {e}"))?;
|
||||
let Some(stmt) = statements.first() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Statement::Query(query) = stmt else {
|
||||
return Ok(None);
|
||||
};
|
||||
let SetExpr::Select(select) = query.body.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !is_select_star(select) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(source) = single_table_source(select)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut body: Map<String, Json> = Map::new();
|
||||
if let Some(where_expr) = &select.selection {
|
||||
body.insert("query".to_string(), translate_predicate(where_expr, source.alias.as_deref())?);
|
||||
}
|
||||
if let Some(order_by) = &query.order_by {
|
||||
if let OrderByKind::Expressions(items) = &order_by.kind {
|
||||
if let Some(sort) = translate_order_by(items, source.alias.as_deref())? {
|
||||
body.insert("sort".to_string(), sort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (limit, offset) = extract_limit_offset(query)?;
|
||||
let user_limited = limit.is_some() && offset.is_none();
|
||||
let from_plan_pagination = offset.is_some();
|
||||
let effective_size = limit.unwrap_or(AUTO_PAGED_SELECT_STAR_SIZE);
|
||||
let effective_from = offset.unwrap_or(0);
|
||||
body.insert("size".to_string(), json!(effective_size));
|
||||
if effective_from > 0 {
|
||||
body.insert("from".to_string(), json!(effective_from));
|
||||
}
|
||||
|
||||
Ok(Some(TranslatedSelectStar { index: source.index, body: Json::Object(body), user_limited, from_plan_pagination }))
|
||||
}
|
||||
|
||||
fn is_select_star(select: &Select) -> bool {
|
||||
select.projection.len() == 1 && matches!(select.projection.first(), Some(SelectItem::Wildcard(_)))
|
||||
}
|
||||
|
||||
struct SelectSource {
|
||||
index: String,
|
||||
alias: Option<String>,
|
||||
}
|
||||
|
||||
fn single_table_source(select: &Select) -> Result<Option<SelectSource>, String> {
|
||||
if select.from.len() != 1 {
|
||||
return Ok(None);
|
||||
}
|
||||
let table = &select.from[0];
|
||||
if !table.joins.is_empty() {
|
||||
return Err("JOINs are not supported in Elasticsearch SQL translation".to_string());
|
||||
}
|
||||
let TableFactor::Table { name, alias, .. } = &table.relation else {
|
||||
return Ok(None);
|
||||
};
|
||||
let joined = name
|
||||
.0
|
||||
.iter()
|
||||
.map(|part| match part {
|
||||
sqlparser::ast::ObjectNamePart::Identifier(ident) => ident.value.clone(),
|
||||
other => format!("{other}"),
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(".");
|
||||
Ok(Some(SelectSource { index: joined, alias: alias.as_ref().map(|alias| alias.name.value.clone()) }))
|
||||
}
|
||||
|
||||
fn translate_predicate(expr: &Expr, table_alias: Option<&str>) -> Result<Json, String> {
|
||||
match expr {
|
||||
Expr::Nested(inner) => translate_predicate(inner, table_alias),
|
||||
Expr::BinaryOp { left, op, right } => translate_binary_op(left, op, right, table_alias),
|
||||
Expr::Like { negated, expr, pattern, .. } | Expr::ILike { negated, expr, pattern, .. } => {
|
||||
translate_like(*negated, expr, pattern, table_alias)
|
||||
}
|
||||
Expr::InList { expr, list, negated } => translate_in_list(expr, list, *negated, table_alias),
|
||||
Expr::IsNull(inner) => translate_is_null(inner, false, table_alias),
|
||||
Expr::IsNotNull(inner) => translate_is_null(inner, true, table_alias),
|
||||
Expr::Between { expr, negated, low, high } => translate_between(expr, *negated, low, high, table_alias),
|
||||
Expr::UnaryOp { op: UnaryOperator::Not, expr } => {
|
||||
let inner = translate_predicate(expr, table_alias)?;
|
||||
Ok(json!({ "bool": { "must_not": [inner] } }))
|
||||
}
|
||||
other => Err(format!("Unsupported WHERE expression: {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_binary_op(
|
||||
left: &Expr,
|
||||
op: &BinaryOperator,
|
||||
right: &Expr,
|
||||
table_alias: Option<&str>,
|
||||
) -> Result<Json, String> {
|
||||
match op {
|
||||
BinaryOperator::And => {
|
||||
let mut must = Vec::new();
|
||||
collect_and(left, &mut must, table_alias)?;
|
||||
collect_and(right, &mut must, table_alias)?;
|
||||
Ok(json!({ "bool": { "must": must } }))
|
||||
}
|
||||
BinaryOperator::Or => {
|
||||
let mut should = Vec::new();
|
||||
collect_or(left, &mut should, table_alias)?;
|
||||
collect_or(right, &mut should, table_alias)?;
|
||||
Ok(json!({ "bool": { "should": should, "minimum_should_match": 1 } }))
|
||||
}
|
||||
BinaryOperator::Eq => term_clause(left, right, table_alias),
|
||||
BinaryOperator::NotEq => {
|
||||
let inner = term_clause(left, right, table_alias)?;
|
||||
Ok(json!({ "bool": { "must_not": [inner] } }))
|
||||
}
|
||||
BinaryOperator::Gt => range_clause(left, right, "gt", table_alias),
|
||||
BinaryOperator::Lt => range_clause(left, right, "lt", table_alias),
|
||||
BinaryOperator::GtEq => range_clause(left, right, "gte", table_alias),
|
||||
BinaryOperator::LtEq => range_clause(left, right, "lte", table_alias),
|
||||
other => Err(format!("Unsupported operator: {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_and(expr: &Expr, out: &mut Vec<Json>, table_alias: Option<&str>) -> Result<(), String> {
|
||||
if let Expr::BinaryOp { left, op: BinaryOperator::And, right } = expr {
|
||||
collect_and(left, out, table_alias)?;
|
||||
collect_and(right, out, table_alias)?;
|
||||
} else {
|
||||
out.push(translate_predicate(expr, table_alias)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_or(expr: &Expr, out: &mut Vec<Json>, table_alias: Option<&str>) -> Result<(), String> {
|
||||
if let Expr::BinaryOp { left, op: BinaryOperator::Or, right } = expr {
|
||||
collect_or(left, out, table_alias)?;
|
||||
collect_or(right, out, table_alias)?;
|
||||
} else {
|
||||
out.push(translate_predicate(expr, table_alias)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn term_clause(left: &Expr, right: &Expr, table_alias: Option<&str>) -> Result<Json, String> {
|
||||
let field = field_path(left, table_alias)?;
|
||||
let value = literal_to_json(right)?;
|
||||
Ok(json!({ "term": { field: value } }))
|
||||
}
|
||||
|
||||
fn range_clause(left: &Expr, right: &Expr, key: &str, table_alias: Option<&str>) -> Result<Json, String> {
|
||||
let field = field_path(left, table_alias)?;
|
||||
let value = literal_to_json(right)?;
|
||||
Ok(json!({ "range": { field: { key: value } } }))
|
||||
}
|
||||
|
||||
fn translate_like(negated: bool, expr: &Expr, pattern: &Expr, table_alias: Option<&str>) -> Result<Json, String> {
|
||||
let field = field_path(expr, table_alias)?;
|
||||
let pattern_str = match literal_string(pattern)? {
|
||||
Some(s) => s,
|
||||
None => return Err("LIKE pattern must be a string literal".to_string()),
|
||||
};
|
||||
|
||||
// Optimise the common `prefix%` shape into a `prefix` query — cheaper than
|
||||
// `wildcard` and supported on any analyzed text field.
|
||||
let is_prefix = !pattern_str.starts_with('%')
|
||||
&& pattern_str.ends_with('%')
|
||||
&& !pattern_str[..pattern_str.len() - 1].contains(['%', '_']);
|
||||
let query = if is_prefix {
|
||||
let prefix = &pattern_str[..pattern_str.len() - 1];
|
||||
json!({ "prefix": { field: prefix } })
|
||||
} else {
|
||||
let es_pattern = sql_like_to_wildcard(&pattern_str);
|
||||
json!({
|
||||
"wildcard": {
|
||||
field: { "value": es_pattern, "case_insensitive": true }
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
if negated {
|
||||
Ok(json!({ "bool": { "must_not": [query] } }))
|
||||
} else {
|
||||
Ok(query)
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_in_list(expr: &Expr, list: &[Expr], negated: bool, table_alias: Option<&str>) -> Result<Json, String> {
|
||||
let field = field_path(expr, table_alias)?;
|
||||
let values: Vec<Json> = list.iter().map(literal_to_json).collect::<Result<_, _>>()?;
|
||||
let query = json!({ "terms": { field: values } });
|
||||
if negated {
|
||||
Ok(json!({ "bool": { "must_not": [query] } }))
|
||||
} else {
|
||||
Ok(query)
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_is_null(expr: &Expr, negated: bool, table_alias: Option<&str>) -> Result<Json, String> {
|
||||
let field = field_path(expr, table_alias)?;
|
||||
let exists = json!({ "exists": { "field": field } });
|
||||
if negated {
|
||||
// IS NOT NULL → field exists
|
||||
Ok(exists)
|
||||
} else {
|
||||
Ok(json!({ "bool": { "must_not": [exists] } }))
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_between(
|
||||
expr: &Expr,
|
||||
negated: bool,
|
||||
low: &Expr,
|
||||
high: &Expr,
|
||||
table_alias: Option<&str>,
|
||||
) -> Result<Json, String> {
|
||||
let field = field_path(expr, table_alias)?;
|
||||
let lo = literal_to_json(low)?;
|
||||
let hi = literal_to_json(high)?;
|
||||
let query = json!({ "range": { field: { "gte": lo, "lte": hi } } });
|
||||
if negated {
|
||||
Ok(json!({ "bool": { "must_not": [query] } }))
|
||||
} else {
|
||||
Ok(query)
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_order_by(items: &[OrderByExpr], table_alias: Option<&str>) -> Result<Option<Json>, String> {
|
||||
let mut sort = Vec::new();
|
||||
for item in items {
|
||||
let field = field_path(&item.expr, table_alias)?;
|
||||
let order = if item.options.asc.unwrap_or(true) { "asc" } else { "desc" };
|
||||
sort.push(json!({ field: { "order": order } }));
|
||||
}
|
||||
if sort.is_empty() {
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(Some(Json::Array(sort)))
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_limit_offset(query: &Query) -> Result<(Option<usize>, Option<usize>), String> {
|
||||
let limit = match &query.limit_clause {
|
||||
Some(sqlparser::ast::LimitClause::LimitOffset { limit, .. }) => {
|
||||
limit.as_ref().map(usize_literal).transpose()?
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let offset = match &query.limit_clause {
|
||||
Some(sqlparser::ast::LimitClause::LimitOffset { offset: Some(off), .. }) => Some(usize_literal(&off.value)?),
|
||||
_ => None,
|
||||
};
|
||||
Ok((limit, offset))
|
||||
}
|
||||
|
||||
fn usize_literal(expr: &Expr) -> Result<usize, String> {
|
||||
match expr {
|
||||
Expr::Value(v) => match &v.value {
|
||||
SqlValue::Number(n, _) => n.parse::<usize>().map_err(|e| format!("Invalid number: {e}")),
|
||||
other => Err(format!("Expected integer literal, got {other:?}")),
|
||||
},
|
||||
other => Err(format!("Expected integer literal, got {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn field_path(expr: &Expr, table_alias: Option<&str>) -> Result<String, String> {
|
||||
match expr {
|
||||
Expr::Identifier(ident) => Ok(ident.value.clone()),
|
||||
Expr::CompoundIdentifier(parts) => {
|
||||
let mut field_parts = parts.iter().map(|p| p.value.clone()).collect::<Vec<_>>();
|
||||
if let (Some(alias), Some(first)) = (table_alias, field_parts.first()) {
|
||||
// Dotted Elasticsearch fields are valid document paths; only
|
||||
// strip a known SQL table alias such as `ass.agencyCode`.
|
||||
if first.eq_ignore_ascii_case(alias) {
|
||||
field_parts.remove(0);
|
||||
}
|
||||
}
|
||||
Ok(field_parts.join("."))
|
||||
}
|
||||
other => Err(format!("Expected column name, got {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn literal_to_json(expr: &Expr) -> Result<Json, String> {
|
||||
let Expr::Value(v) = expr else {
|
||||
return Err(format!("Expected literal value, got {expr}"));
|
||||
};
|
||||
match &v.value {
|
||||
SqlValue::SingleQuotedString(s) | SqlValue::DoubleQuotedString(s) => Ok(Json::String(s.clone())),
|
||||
SqlValue::Number(n, _) => serde_json::from_str::<Json>(n).map_err(|e| format!("Invalid number {n}: {e}")),
|
||||
SqlValue::Boolean(b) => Ok(Json::Bool(*b)),
|
||||
SqlValue::Null => Ok(Json::Null),
|
||||
other => Err(format!("Unsupported literal: {other:?}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn literal_string(expr: &Expr) -> Result<Option<String>, String> {
|
||||
let Expr::Value(v) = expr else {
|
||||
return Err(format!("Expected string literal, got {expr}"));
|
||||
};
|
||||
Ok(match &v.value {
|
||||
SqlValue::SingleQuotedString(s) | SqlValue::DoubleQuotedString(s) => Some(s.clone()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert SQL LIKE pattern to ES wildcard pattern.
|
||||
/// SQL: `%` = any chars, `_` = single char. Backslash escapes both.
|
||||
/// ES wildcard: `*` = any chars, `?` = single char. `\` escapes `*` / `?` / `\`.
|
||||
fn sql_like_to_wildcard(pattern: &str) -> String {
|
||||
let mut out = String::with_capacity(pattern.len());
|
||||
let mut chars = pattern.chars().peekable();
|
||||
while let Some(ch) = chars.next() {
|
||||
match ch {
|
||||
'\\' => {
|
||||
if let Some(&next) = chars.peek() {
|
||||
out.push(next);
|
||||
chars.next();
|
||||
}
|
||||
}
|
||||
'%' => out.push('*'),
|
||||
'_' => out.push('?'),
|
||||
'*' | '?' => {
|
||||
// Escape ES wildcard metacharacters that the user wrote literally.
|
||||
out.push('\\');
|
||||
out.push(ch);
|
||||
}
|
||||
_ => out.push(ch),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::translate_select_star;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn translate_select_star_strips_table_alias_from_where_fields() {
|
||||
let translated = translate_select_star(
|
||||
"SELECT * FROM assets002 AS ass WHERE ass.agencyCode = '035021' AND ass.mofDivCode = '420108000'",
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(translated.index, "assets002");
|
||||
assert_eq!(
|
||||
translated.body["query"],
|
||||
json!({
|
||||
"bool": {
|
||||
"must": [
|
||||
{ "term": { "agencyCode": "035021" } },
|
||||
{ "term": { "mofDivCode": "420108000" } }
|
||||
]
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translate_select_star_strips_table_alias_before_nested_field_path() {
|
||||
let translated =
|
||||
translate_select_star("SELECT * FROM assets002 ass WHERE ass.department.code = 'A01'").unwrap().unwrap();
|
||||
|
||||
assert_eq!(translated.body["query"], json!({ "term": { "department.code": "A01" } }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translate_select_star_preserves_unaliased_dotted_document_fields() {
|
||||
let translated = translate_select_star("SELECT * FROM logs WHERE host.ip = '127.0.0.1'").unwrap().unwrap();
|
||||
|
||||
assert_eq!(translated.body["query"], json!({ "term": { "host.ip": "127.0.0.1" } }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translate_select_star_strips_alias_in_like_and_order_by() {
|
||||
let translated = translate_select_star(
|
||||
"SELECT * FROM assets002 ass WHERE ass.assetName LIKE 'car%' ORDER BY ass.createdAt DESC LIMIT 10",
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(translated.body["query"], json!({ "prefix": { "assetName": "car" } }));
|
||||
assert_eq!(translated.body["sort"], json!([{ "createdAt": { "order": "desc" } }]));
|
||||
assert_eq!(translated.body["size"], json!(10));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
use std::future::Future;
|
||||
use std::sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
};
|
||||
use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::db;
|
||||
use crate::models::connection::ConnectionConfig;
|
||||
|
||||
pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
pub const MAX_ROWS: usize = 10000;
|
||||
pub const QUERY_CANCELED: &str = "Query canceled";
|
||||
|
||||
/// Unified database operation execution budget.
|
||||
/// query_timeout = None only means SQL execution has no upper limit;
|
||||
/// checkout/connect/recycle/cancel/cleanup always have hard upper limits and cannot be disabled.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DbOperationBudget {
|
||||
pub checkout_timeout: Duration,
|
||||
pub connect_timeout: Duration,
|
||||
pub recycle_timeout: Duration,
|
||||
pub query_timeout: Option<Duration>,
|
||||
pub cancel_timeout: Duration,
|
||||
pub cleanup_timeout: Duration,
|
||||
}
|
||||
|
||||
impl DbOperationBudget {
|
||||
/// Build an execution budget from connection config.
|
||||
/// checkout/connect/recycle use connect_timeout_secs (clamped to 1s minimum, 300s maximum).
|
||||
/// query_timeout follows resolve_query_timeout semantics (Some(0) -> None).
|
||||
/// cancel/cleanup are fixed values and cannot be disabled.
|
||||
pub fn from_config(connect_timeout_secs: u64, query_timeout_secs: Option<u64>) -> Self {
|
||||
let infra_timeout = Duration::from_secs(connect_timeout_secs.clamp(1, 300));
|
||||
Self {
|
||||
checkout_timeout: infra_timeout,
|
||||
connect_timeout: infra_timeout,
|
||||
recycle_timeout: infra_timeout,
|
||||
query_timeout: resolve_query_timeout(query_timeout_secs),
|
||||
cancel_timeout: Duration::from_secs(5),
|
||||
cleanup_timeout: Duration::from_secs(3),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_connection_config(config: &ConnectionConfig) -> Self {
|
||||
// effective_query_timeout_secs, not the raw field: it preserves 0 ("no limit")
|
||||
// and applies the per-database-type floor Cloud Spanner needs for DDL.
|
||||
Self::from_config(config.effective_connect_timeout_secs(), Some(config.effective_query_timeout_secs()))
|
||||
}
|
||||
|
||||
/// Use global default values (when no connection config is available).
|
||||
pub fn with_defaults() -> Self {
|
||||
let default_infra = db::connection_timeout();
|
||||
Self {
|
||||
checkout_timeout: default_infra,
|
||||
connect_timeout: default_infra,
|
||||
recycle_timeout: default_infra,
|
||||
query_timeout: Some(QUERY_TIMEOUT),
|
||||
cancel_timeout: Duration::from_secs(5),
|
||||
cleanup_timeout: Duration::from_secs(3),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct StreamProgressClock {
|
||||
started_at: tokio::time::Instant,
|
||||
last_progress_ms: AtomicU64,
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
marked: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl Default for StreamProgressClock {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamProgressClock {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
started_at: tokio::time::Instant::now(),
|
||||
last_progress_ms: AtomicU64::new(0),
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
marked: std::sync::atomic::AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mark(&self) {
|
||||
self.last_progress_ms.store(self.started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
self.marked.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Whether any progress has been recorded yet. Test-only: production code
|
||||
/// only needs the derived inactivity window. A row read within the first
|
||||
/// millisecond records a zero timestamp, so this cannot be derived from
|
||||
/// `last_progress_ms`.
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub fn marked(&self) -> bool {
|
||||
self.marked.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn elapsed_since_progress(&self) -> Duration {
|
||||
let last_progress_ms = self.last_progress_ms.load(Ordering::Relaxed);
|
||||
let elapsed_ms = self.started_at.elapsed().as_millis() as u64;
|
||||
Duration::from_millis(elapsed_ms.saturating_sub(last_progress_ms))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn timeout_error() -> String {
|
||||
timeout_error_for(QUERY_TIMEOUT)
|
||||
}
|
||||
|
||||
pub fn timeout_error_for(timeout_duration: Duration) -> String {
|
||||
let seconds = timeout_duration.as_secs().max(1);
|
||||
format!("Query timed out after {seconds} seconds")
|
||||
}
|
||||
|
||||
pub fn canceled_error() -> String {
|
||||
QUERY_CANCELED.to_string()
|
||||
}
|
||||
|
||||
pub async fn await_stream_with_progress_timeout<F, T>(
|
||||
stream_future: F,
|
||||
timeout: Option<Duration>,
|
||||
progress_clock: Arc<StreamProgressClock>,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
timeout_message: String,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
{
|
||||
let Some(timeout) = timeout else {
|
||||
return match cancel_token {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => Err(canceled_error()),
|
||||
result = stream_future => result,
|
||||
}
|
||||
}
|
||||
None => stream_future.await,
|
||||
};
|
||||
};
|
||||
|
||||
tokio::pin!(stream_future);
|
||||
loop {
|
||||
// Query timeout is an inactivity budget, not a cap on total stream duration.
|
||||
let remaining = timeout.saturating_sub(progress_clock.elapsed_since_progress());
|
||||
if remaining.is_zero() {
|
||||
return Err(timeout_message.clone());
|
||||
}
|
||||
let sleep = tokio::time::sleep(remaining);
|
||||
tokio::pin!(sleep);
|
||||
|
||||
match cancel_token {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(canceled_error()),
|
||||
result = &mut stream_future => return result,
|
||||
_ = &mut sleep => {},
|
||||
}
|
||||
}
|
||||
None => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut stream_future => return result,
|
||||
_ = &mut sleep => {},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if progress_clock.elapsed_since_progress() >= timeout {
|
||||
return Err(timeout_message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_canceled(cancel_token: &Option<CancellationToken>) -> bool {
|
||||
cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false)
|
||||
}
|
||||
|
||||
pub async fn wait_for_query<F>(cancel_token: Option<CancellationToken>, future: F) -> Result<db::QueryResult, String>
|
||||
where
|
||||
F: Future<Output = Result<db::QueryResult, String>>,
|
||||
{
|
||||
wait_for_query_with_timeout(cancel_token, QUERY_TIMEOUT, future).await
|
||||
}
|
||||
|
||||
pub async fn wait_for_query_with_timeout<F>(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Duration,
|
||||
future: F,
|
||||
) -> Result<db::QueryResult, String>
|
||||
where
|
||||
F: Future<Output = Result<db::QueryResult, String>>,
|
||||
{
|
||||
wait_for_result_with_timeout(cancel_token, timeout_duration, future).await
|
||||
}
|
||||
|
||||
pub async fn wait_for_result_with_timeout<T, F>(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Duration,
|
||||
future: F,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
{
|
||||
if let Some(token) = cancel_token {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => Err(canceled_error()),
|
||||
result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration))?,
|
||||
}
|
||||
} else {
|
||||
timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration))?
|
||||
}
|
||||
}
|
||||
|
||||
/// Like `wait_for_query_with_timeout` but with an optional timeout.
|
||||
/// `None` means no timeout (only cancellation can stop the query).
|
||||
pub async fn wait_for_query_opt<F>(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
future: F,
|
||||
) -> Result<db::QueryResult, String>
|
||||
where
|
||||
F: Future<Output = Result<db::QueryResult, String>>,
|
||||
{
|
||||
wait_for_result_opt(cancel_token, timeout_duration, future).await
|
||||
}
|
||||
|
||||
pub async fn wait_for_result_opt<T, F>(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
future: F,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
{
|
||||
match timeout_duration {
|
||||
Some(d) => wait_for_result_with_timeout(cancel_token, d, future).await,
|
||||
None => match cancel_token {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => Err(canceled_error()),
|
||||
result = future => result,
|
||||
}
|
||||
}
|
||||
None => future.await,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn wait_for_value_opt<T, F>(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
future: F,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
match timeout_duration {
|
||||
Some(timeout_duration) => {
|
||||
if let Some(token) = cancel_token {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => Err(canceled_error()),
|
||||
result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration)),
|
||||
}
|
||||
} else {
|
||||
timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration))
|
||||
}
|
||||
}
|
||||
None => match cancel_token {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => Err(canceled_error()),
|
||||
result = future => Ok(result),
|
||||
}
|
||||
}
|
||||
None => Ok(future.await),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Locks a mutex-guarded shared connection (e.g. SQL Server's single connection
|
||||
/// per pool key) and reports how long the caller waited for the lock alongside
|
||||
/// the guard. Callers should fold the returned wait time into any execution-time
|
||||
/// metric they report, since a driver-level timer that only starts once the lock
|
||||
/// is held cannot see time spent queued behind another operation on the same
|
||||
/// connection.
|
||||
pub async fn lock_shared_client_with_wait<'a, T>(
|
||||
client: &'a Arc<tokio::sync::Mutex<T>>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
) -> Result<(tokio::sync::MutexGuard<'a, T>, u128), String> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let guard = wait_for_value_opt(cancel_token, timeout_duration, client.lock()).await?;
|
||||
Ok((guard, started_at.elapsed().as_millis()))
|
||||
}
|
||||
|
||||
pub fn query_timeout_duration(timeout_secs: Option<u64>) -> Option<Duration> {
|
||||
match timeout_secs {
|
||||
Some(0) => None,
|
||||
Some(n) => Some(Duration::from_secs(n)),
|
||||
None => Some(QUERY_TIMEOUT),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve_query_timeout(timeout_secs: Option<u64>) -> Option<Duration> {
|
||||
query_timeout_duration(timeout_secs)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn default_progress_clock_tracks_inactivity_from_creation_and_marks() {
|
||||
let clock = StreamProgressClock::default();
|
||||
assert_eq!(clock.elapsed_since_progress(), Duration::ZERO);
|
||||
assert!(!clock.marked());
|
||||
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
assert_eq!(clock.elapsed_since_progress(), Duration::from_secs(2));
|
||||
|
||||
clock.mark();
|
||||
assert!(clock.marked());
|
||||
assert_eq!(clock.elapsed_since_progress(), Duration::ZERO);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(clock.elapsed_since_progress(), Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
use std::path::Path;
|
||||
|
||||
use crate::path_utils::expand_tilde;
|
||||
|
||||
/// Validates a file path for database connections.
|
||||
///
|
||||
/// Performs comprehensive checks including:
|
||||
/// - Empty path validation
|
||||
/// - Null character detection
|
||||
/// - File existence (for local paths)
|
||||
/// - File type validation (must be a file, not directory)
|
||||
/// - Network path detection (skips validation for network paths)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `path` - The file path to validate
|
||||
/// * `is_network_path` - Closure to determine if path is a network path
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(())` if validation passes
|
||||
/// * `Err(String)` with descriptive error message if validation fails
|
||||
pub fn validate_file_path<F>(path: &str, is_network_path: F) -> Result<(), String>
|
||||
where
|
||||
F: Fn(&str) -> bool,
|
||||
{
|
||||
// Check if path is empty
|
||||
if path.is_empty() {
|
||||
return Err("File path cannot be empty".to_string());
|
||||
}
|
||||
|
||||
// Check if path contains invalid characters
|
||||
if path.contains('\0') {
|
||||
return Err("File path contains null characters".to_string());
|
||||
}
|
||||
|
||||
let expanded = expand_tilde(path);
|
||||
let path_obj = Path::new(&expanded);
|
||||
|
||||
// For non-network paths, perform file system checks
|
||||
if !is_network_path(path) {
|
||||
if !path_obj.exists() {
|
||||
return Err(format!("File does not exist: {}", path));
|
||||
}
|
||||
|
||||
// Check if path is actually a file, not a directory
|
||||
if path_obj.is_dir() {
|
||||
return Err(format!("File path is a directory, not a file: {}", path));
|
||||
}
|
||||
|
||||
// Check if path is a valid file
|
||||
if !path_obj.is_file() {
|
||||
return Err(format!("File path is not a valid file: {}", path));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn is_network_path_test(path: &str) -> bool {
|
||||
path.starts_with("\\\\") || path.starts_with("//")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_path() {
|
||||
let result = validate_file_path("", is_network_path_test);
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("empty"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_null_character() {
|
||||
let result = validate_file_path("path\0invalid", is_network_path_test);
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("null"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_network_path_skips_validation() {
|
||||
let result = validate_file_path("//network/path/nonexistent.db", is_network_path_test);
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nonexistent_local_file() {
|
||||
let result = validate_file_path("/nonexistent/path/to/file.db", is_network_path_test);
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("does not exist"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tilde_expands_to_home() {
|
||||
let home = std::env::var("HOME").unwrap_or_default();
|
||||
if home.is_empty() {
|
||||
return; // skip if no HOME
|
||||
}
|
||||
let result = validate_file_path(&format!("{}/.bashrc", home), is_network_path_test);
|
||||
// $HOME/.bashrc may or may not exist, but the path shouldn't contain literal ~
|
||||
// We just verify no error mentioning tilde
|
||||
if let Err(e) = &result {
|
||||
assert!(!e.contains('~'), "Error should not contain literal tilde: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tilde_only() {
|
||||
let home = std::env::var("HOME").unwrap_or_default();
|
||||
if home.is_empty() {
|
||||
return;
|
||||
}
|
||||
let result = validate_file_path("~", |_| true); // treat as network to skip fs check
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_tilde_unchanged() {
|
||||
let result = validate_file_path("/absolute/path", is_network_path_test);
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("/absolute/path"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_named_user_path_remains_literal() {
|
||||
let result = validate_file_path("~dbx-user-that-must-not-exist-7088", is_network_path_test);
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains('~'));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
use bytes::Bytes;
|
||||
use reqwest::header::{HeaderValue, AUTHORIZATION, CONTENT_TYPE};
|
||||
use reqwest::{Client, StatusCode, Url};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{timeout, Duration};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const TCP_READ_BUFFER_SIZE: usize = 16 * 1024;
|
||||
const HTTP_READ_WAIT_MS: u64 = 1_000;
|
||||
const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct HttpTunnelManager {
|
||||
tunnels: Mutex<HashMap<String, (JoinHandle<()>, u16)>>,
|
||||
}
|
||||
|
||||
impl HttpTunnelManager {
|
||||
pub fn new() -> Self {
|
||||
Self { tunnels: Mutex::new(HashMap::new()) }
|
||||
}
|
||||
|
||||
pub async fn start_tunnel(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
tunnel_url: &str,
|
||||
token: &str,
|
||||
connect_timeout_secs: u64,
|
||||
remote_host: &str,
|
||||
remote_port: u16,
|
||||
) -> Result<u16, String> {
|
||||
if let Some(local_port) = self.local_port(connection_id).await {
|
||||
return Ok(local_port);
|
||||
}
|
||||
let local_port = portpicker::pick_unused_port().ok_or("No available port")?;
|
||||
let listener = TcpListener::bind(("127.0.0.1", local_port))
|
||||
.await
|
||||
.map_err(|e| format!("Failed to bind HTTP tunnel local port: {e}"))?;
|
||||
|
||||
let endpoint =
|
||||
Arc::new(HttpTunnelEndpoint::new(tunnel_url, token, connect_timeout_secs, remote_host, remote_port)?);
|
||||
let handle = tokio::spawn(http_tunnel_forward_loop(listener, endpoint));
|
||||
|
||||
let mut tunnels = self.tunnels.lock().await;
|
||||
if let Some((_, existing_port)) = tunnels.get(connection_id) {
|
||||
handle.abort();
|
||||
return Ok(*existing_port);
|
||||
}
|
||||
|
||||
tunnels.insert(connection_id.to_string(), (handle, local_port));
|
||||
Ok(local_port)
|
||||
}
|
||||
|
||||
pub async fn local_port(&self, connection_id: &str) -> Option<u16> {
|
||||
self.tunnels.lock().await.get(connection_id).map(|(_, port)| *port)
|
||||
}
|
||||
|
||||
pub async fn stop_tunnel(&self, connection_id: &str) {
|
||||
if let Some((handle, _)) = self.tunnels.lock().await.remove(connection_id) {
|
||||
handle.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn stop_tunnels_with_prefix(&self, connection_id_prefix: &str) {
|
||||
let mut tunnels = self.tunnels.lock().await;
|
||||
let keys: Vec<String> = tunnels.keys().filter(|key| key.starts_with(connection_id_prefix)).cloned().collect();
|
||||
for key in keys {
|
||||
if let Some((handle, _)) = tunnels.remove(&key) {
|
||||
handle.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn stop_all_tunnels(&self) {
|
||||
let tunnels = std::mem::take(&mut *self.tunnels.lock().await);
|
||||
for (handle, _) in tunnels.into_values() {
|
||||
handle.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct HttpTunnelEndpoint {
|
||||
url: String,
|
||||
token: String,
|
||||
connect_timeout: Duration,
|
||||
target_host: String,
|
||||
target_port: u16,
|
||||
client: Client,
|
||||
}
|
||||
|
||||
impl HttpTunnelEndpoint {
|
||||
fn new(
|
||||
tunnel_url: &str,
|
||||
token: &str,
|
||||
connect_timeout_secs: u64,
|
||||
remote_host: &str,
|
||||
remote_port: u16,
|
||||
) -> Result<Self, String> {
|
||||
validate_script_url(tunnel_url)?;
|
||||
Ok(Self {
|
||||
url: tunnel_url.trim().to_string(),
|
||||
token: token.trim().to_string(),
|
||||
connect_timeout: effective_connect_timeout(connect_timeout_secs),
|
||||
target_host: remote_host.to_string(),
|
||||
target_port: remote_port,
|
||||
client: Client::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn http_tunnel_forward_loop(listener: TcpListener, endpoint: Arc<HttpTunnelEndpoint>) {
|
||||
loop {
|
||||
let (inbound, _) = match listener.accept().await {
|
||||
Ok(pair) => pair,
|
||||
Err(_) => break,
|
||||
};
|
||||
let endpoint = endpoint.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = bridge_tcp_to_http_script(inbound, endpoint).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn bridge_tcp_to_http_script(mut inbound: TcpStream, endpoint: Arc<HttpTunnelEndpoint>) -> Result<(), String> {
|
||||
let session = Uuid::new_v4().simple().to_string();
|
||||
open_session(&endpoint, &session).await?;
|
||||
|
||||
let (mut tcp_reader, mut tcp_writer) = inbound.split();
|
||||
let tcp_to_http = async {
|
||||
let mut buf = vec![0_u8; TCP_READ_BUFFER_SIZE];
|
||||
loop {
|
||||
let n = tcp_reader.read(&mut buf).await.map_err(|e| format!("Failed to read local TCP stream: {e}"))?;
|
||||
if n == 0 {
|
||||
return Ok::<(), String>(());
|
||||
}
|
||||
write_session(&endpoint, &session, Bytes::copy_from_slice(&buf[..n])).await?;
|
||||
}
|
||||
};
|
||||
|
||||
let http_to_tcp = async {
|
||||
loop {
|
||||
let bytes = read_session(&endpoint, &session).await?;
|
||||
if !bytes.is_empty() {
|
||||
tcp_writer.write_all(&bytes).await.map_err(|e| format!("Failed to write local TCP stream: {e}"))?;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let result = tokio::select! {
|
||||
result = tcp_to_http => result,
|
||||
result = http_to_tcp => result,
|
||||
};
|
||||
let _ = close_session(&endpoint, &session).await;
|
||||
result
|
||||
}
|
||||
|
||||
async fn open_session(endpoint: &HttpTunnelEndpoint, session: &str) -> Result<(), String> {
|
||||
let url = script_url(
|
||||
&endpoint.url,
|
||||
"open",
|
||||
session,
|
||||
Some((&endpoint.target_host, endpoint.target_port)),
|
||||
Some(endpoint.connect_timeout.as_secs()),
|
||||
None,
|
||||
)?;
|
||||
let response = timeout(
|
||||
endpoint.connect_timeout + HTTP_REQUEST_TIMEOUT,
|
||||
authorized_request(endpoint, endpoint.client.post(url)).send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "HTTP tunnel script open timed out".to_string())?
|
||||
.map_err(|e| format!("Failed to open HTTP tunnel script session: {e}"))?;
|
||||
ensure_success(response, "open HTTP tunnel script session").await
|
||||
}
|
||||
|
||||
async fn write_session(endpoint: &HttpTunnelEndpoint, session: &str, data: Bytes) -> Result<(), String> {
|
||||
let url = script_url(&endpoint.url, "write", session, None, None, None)?;
|
||||
let response = timeout(
|
||||
HTTP_REQUEST_TIMEOUT,
|
||||
authorized_request(endpoint, endpoint.client.post(url))
|
||||
.header(CONTENT_TYPE, "application/octet-stream")
|
||||
.body(data)
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "HTTP tunnel script write timed out".to_string())?
|
||||
.map_err(|e| format!("Failed to write HTTP tunnel script session: {e}"))?;
|
||||
ensure_success(response, "write HTTP tunnel script session").await
|
||||
}
|
||||
|
||||
async fn read_session(endpoint: &HttpTunnelEndpoint, session: &str) -> Result<Bytes, String> {
|
||||
let url = script_url(&endpoint.url, "read", session, None, None, Some(HTTP_READ_WAIT_MS))?;
|
||||
let response = timeout(HTTP_REQUEST_TIMEOUT, authorized_request(endpoint, endpoint.client.post(url)).send())
|
||||
.await
|
||||
.map_err(|_| "HTTP tunnel script read timed out".to_string())?
|
||||
.map_err(|e| format!("Failed to read HTTP tunnel script session: {e}"))?;
|
||||
let status = response.status();
|
||||
if status == StatusCode::NO_CONTENT {
|
||||
return Ok(Bytes::new());
|
||||
}
|
||||
if status == StatusCode::OK {
|
||||
return response.bytes().await.map_err(|e| format!("Failed to read HTTP tunnel script response: {e}"));
|
||||
}
|
||||
if status == StatusCode::GONE {
|
||||
return Err("HTTP tunnel script session closed".to_string());
|
||||
}
|
||||
Err(error_response_message(response, "read HTTP tunnel script session").await)
|
||||
}
|
||||
|
||||
async fn close_session(endpoint: &HttpTunnelEndpoint, session: &str) -> Result<(), String> {
|
||||
let url = script_url(&endpoint.url, "close", session, None, None, None)?;
|
||||
let response = timeout(HTTP_REQUEST_TIMEOUT, authorized_request(endpoint, endpoint.client.post(url)).send())
|
||||
.await
|
||||
.map_err(|_| "HTTP tunnel script close timed out".to_string())?
|
||||
.map_err(|e| format!("Failed to close HTTP tunnel script session: {e}"))?;
|
||||
ensure_success(response, "close HTTP tunnel script session").await
|
||||
}
|
||||
|
||||
fn authorized_request(endpoint: &HttpTunnelEndpoint, builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
if endpoint.token.is_empty() {
|
||||
return builder;
|
||||
}
|
||||
match HeaderValue::from_str(&endpoint.token) {
|
||||
Ok(token) => builder
|
||||
.header("X-DBX-Tunnel-Token", token.clone())
|
||||
.header(AUTHORIZATION, format!("Bearer {}", endpoint.token)),
|
||||
Err(_) => builder,
|
||||
}
|
||||
}
|
||||
|
||||
async fn ensure_success(response: reqwest::Response, action: &str) -> Result<(), String> {
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(error_response_message(response, action).await)
|
||||
}
|
||||
}
|
||||
|
||||
async fn error_response_message(response: reqwest::Response, action: &str) -> String {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
if body.trim().is_empty() {
|
||||
format!("Failed to {action}: HTTP {status}")
|
||||
} else {
|
||||
format!("Failed to {action}: HTTP {status}: {}", body.trim())
|
||||
}
|
||||
}
|
||||
|
||||
fn script_url(
|
||||
tunnel_url: &str,
|
||||
action: &str,
|
||||
session: &str,
|
||||
target: Option<(&str, u16)>,
|
||||
connect_timeout_secs: Option<u64>,
|
||||
wait_ms: Option<u64>,
|
||||
) -> Result<Url, String> {
|
||||
let mut url = validate_script_url(tunnel_url)?;
|
||||
url.query_pairs_mut().append_pair("dbx_action", action).append_pair("dbx_session", session);
|
||||
if let Some((host, port)) = target {
|
||||
url.query_pairs_mut().append_pair("dbx_target_host", host).append_pair("dbx_target_port", &port.to_string());
|
||||
}
|
||||
if let Some(connect_timeout_secs) = connect_timeout_secs {
|
||||
url.query_pairs_mut()
|
||||
.append_pair("dbx_connect_timeout", &effective_connect_timeout_secs(connect_timeout_secs).to_string());
|
||||
}
|
||||
if let Some(wait_ms) = wait_ms {
|
||||
url.query_pairs_mut().append_pair("dbx_wait_ms", &wait_ms.to_string());
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn validate_script_url(tunnel_url: &str) -> Result<Url, String> {
|
||||
let trimmed = tunnel_url.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err("HTTP tunnel script URL is required".to_string());
|
||||
}
|
||||
let url = Url::parse(trimmed).map_err(|e| format!("HTTP tunnel script URL is invalid: {e}"))?;
|
||||
match url.scheme() {
|
||||
"http" | "https" => Ok(url),
|
||||
other => Err(format!("Unsupported HTTP tunnel script URL scheme: {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn effective_connect_timeout(value: u64) -> Duration {
|
||||
Duration::from_secs(effective_connect_timeout_secs(value))
|
||||
}
|
||||
|
||||
fn effective_connect_timeout_secs(value: u64) -> u64 {
|
||||
if value == 0 {
|
||||
DEFAULT_CONNECT_TIMEOUT.as_secs()
|
||||
} else {
|
||||
value.clamp(1, 300)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{script_url, validate_script_url, HttpTunnelManager};
|
||||
use std::sync::Arc;
|
||||
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
use tokio::time::{timeout, Duration};
|
||||
|
||||
#[test]
|
||||
fn script_url_preserves_existing_query_and_appends_action() {
|
||||
let url = script_url(
|
||||
"https://gateway.example.com/dbx_tunnel.php?keep=1",
|
||||
"open",
|
||||
"session-1",
|
||||
Some(("mysql.internal", 3306)),
|
||||
Some(10),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
url.as_str(),
|
||||
"https://gateway.example.com/dbx_tunnel.php?keep=1&dbx_action=open&dbx_session=session-1&dbx_target_host=mysql.internal&dbx_target_port=3306&dbx_connect_timeout=10"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_url_rejects_non_http_schemes() {
|
||||
let err = validate_script_url("file:///tmp/tunnel.php").unwrap_err();
|
||||
|
||||
assert!(err.contains("Unsupported HTTP tunnel script URL scheme"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manager_forwards_sequential_round_trips_and_closes_session() {
|
||||
let script = MockScript::start().await;
|
||||
let manager = HttpTunnelManager::new();
|
||||
let local_port = manager.start_tunnel("test", &script.url, "secret", 5, "mysql.internal", 3306).await.unwrap();
|
||||
let mut client = TcpStream::connect(("127.0.0.1", local_port)).await.unwrap();
|
||||
let close_notification = script.closed.notified();
|
||||
|
||||
timeout(Duration::from_secs(10), async {
|
||||
for index in 0..100_u32 {
|
||||
let payload = format!("request-{index:03}-{}", "x".repeat((index % 17) as usize));
|
||||
client.write_all(payload.as_bytes()).await.unwrap();
|
||||
let mut response = vec![0_u8; payload.len()];
|
||||
client.read_exact(&mut response).await.unwrap();
|
||||
|
||||
assert_eq!(response, payload.as_bytes());
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("100 sequential HTTP tunnel round trips timed out");
|
||||
|
||||
drop(client);
|
||||
timeout(Duration::from_secs(2), close_notification).await.expect("HTTP tunnel session was not closed");
|
||||
|
||||
manager.stop_tunnel("test").await;
|
||||
script.handle.abort();
|
||||
}
|
||||
|
||||
struct MockScript {
|
||||
url: String,
|
||||
closed: Arc<Notify>,
|
||||
handle: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl MockScript {
|
||||
async fn start() -> Self {
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let buffered = Arc::new(Mutex::new(Vec::<u8>::new()));
|
||||
let closed = Arc::new(Notify::new());
|
||||
let handle = {
|
||||
let buffered = buffered.clone();
|
||||
let closed = closed.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
break;
|
||||
};
|
||||
let buffered = buffered.clone();
|
||||
let closed = closed.clone();
|
||||
tokio::spawn(async move {
|
||||
handle_mock_http_request(stream, buffered, closed).await;
|
||||
});
|
||||
}
|
||||
})
|
||||
};
|
||||
Self { url: format!("http://{addr}/dbx_tunnel.php"), closed, handle }
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_mock_http_request(stream: TcpStream, buffered: Arc<Mutex<Vec<u8>>>, closed: Arc<Notify>) {
|
||||
let mut reader = BufReader::new(stream);
|
||||
let mut request_line = String::new();
|
||||
reader.read_line(&mut request_line).await.unwrap();
|
||||
let mut content_length = 0_usize;
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).await.unwrap();
|
||||
let trimmed = line.trim_end();
|
||||
if trimmed.is_empty() {
|
||||
break;
|
||||
}
|
||||
let lower = trimmed.to_ascii_lowercase();
|
||||
if let Some(value) = lower.strip_prefix("content-length:") {
|
||||
content_length = value.trim().parse().unwrap();
|
||||
}
|
||||
}
|
||||
let mut body = vec![0_u8; content_length];
|
||||
reader.read_exact(&mut body).await.unwrap();
|
||||
let mut stream = reader.into_inner();
|
||||
if request_line.contains("dbx_action=write") {
|
||||
buffered.lock().await.extend_from_slice(&body);
|
||||
write_http_response(&mut stream, "200 OK", b"OK").await;
|
||||
} else if request_line.contains("dbx_action=read") {
|
||||
let bytes = {
|
||||
let mut buffered = buffered.lock().await;
|
||||
std::mem::take(&mut *buffered)
|
||||
};
|
||||
if bytes.is_empty() {
|
||||
write_http_response(&mut stream, "204 No Content", b"").await;
|
||||
} else {
|
||||
write_http_response(&mut stream, "200 OK", &bytes).await;
|
||||
}
|
||||
} else if request_line.contains("dbx_action=close") {
|
||||
write_http_response(&mut stream, "200 OK", b"OK").await;
|
||||
closed.notify_one();
|
||||
} else {
|
||||
write_http_response(&mut stream, "200 OK", b"OK").await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_http_response(stream: &mut TcpStream, status: &str, body: &[u8]) {
|
||||
let header = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
|
||||
stream.write_all(header.as_bytes()).await.unwrap();
|
||||
stream.write_all(body).await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
pub use dbx_platform::path_utils;
|
||||
pub use dbx_types::{models, types};
|
||||
|
||||
pub mod db;
|
||||
pub mod ddl_scan;
|
||||
pub mod document_result;
|
||||
pub mod elasticsearch_sql;
|
||||
pub mod execution;
|
||||
pub mod file_validator;
|
||||
pub mod http_tunnel;
|
||||
pub mod runtime_config;
|
||||
pub mod ssh_config;
|
||||
pub mod ssh_host_key;
|
||||
pub mod wkb;
|
||||
@@ -0,0 +1,7 @@
|
||||
pub const DUCKDB_WORKER_MAX_PROCESSES_MIN: usize = 1;
|
||||
pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16;
|
||||
pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4;
|
||||
|
||||
pub fn normalize_duckdb_worker_max_processes(value: usize) -> usize {
|
||||
value.clamp(DUCKDB_WORKER_MAX_PROCESSES_MIN, DUCKDB_WORKER_MAX_PROCESSES_MAX)
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::models::connection::SshTunnelConfig;
|
||||
use crate::path_utils::expand_tilde;
|
||||
|
||||
/// Legacy sentinel values used by connection payloads that predate the
|
||||
/// explicit SSH authentication method field. Current connection forms always
|
||||
/// set `auth_method`, so their user and port values must be treated as
|
||||
/// explicit, including the valid defaults `root` and `22`.
|
||||
const DEFAULT_USER_SENTINEL: &str = "root";
|
||||
const DEFAULT_PORT_SENTINEL: u16 = 22;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct SshConfigHostEntry {
|
||||
pub alias: String,
|
||||
pub host_name: Option<String>,
|
||||
pub port: Option<u16>,
|
||||
pub user: Option<String>,
|
||||
pub identity_file: Option<String>,
|
||||
/// The `ProxyJump` alias, if any. Only a single hop is parsed (OpenSSH's
|
||||
/// comma-separated multi-jump form is not supported); the jump itself
|
||||
/// may declare its own `ProxyJump`, which `resolve_ssh_tunnel_chain`
|
||||
/// follows recursively.
|
||||
pub proxy_jump: Option<String>,
|
||||
}
|
||||
|
||||
/// Reads and parses `~/.ssh/config`. Returns an empty list (not an error) if
|
||||
/// the file does not exist, since that's a normal state for users without
|
||||
/// an SSH config.
|
||||
pub fn list_hosts() -> Result<Vec<SshConfigHostEntry>, String> {
|
||||
let path = expand_tilde("~/.ssh/config");
|
||||
match std::fs::read_to_string(&path) {
|
||||
Ok(content) => Ok(parse_ssh_config(&content)),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()),
|
||||
Err(err) => Err(format!("Failed to read {path}: {err}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_host(alias: &str) -> Option<SshConfigHostEntry> {
|
||||
list_hosts().ok()?.into_iter().find(|entry| entry.alias == alias)
|
||||
}
|
||||
|
||||
/// Fills in `host`, `user`, `port`, and `key_path` from a matching `~/.ssh/config`
|
||||
/// `Host` block, without overwriting values the user has explicitly set.
|
||||
///
|
||||
/// Only `ssh.host` is matched against config aliases; `user`/`port`/`key_path`
|
||||
/// are filled in from that same matched entry. Older payloads without an
|
||||
/// `auth_method` retain the legacy default-filling behavior so imported
|
||||
/// connections continue to resolve their aliases. Payloads produced by the
|
||||
/// current connection form keep all user and port values as entered.
|
||||
pub fn resolve_ssh_tunnel_config(ssh: &SshTunnelConfig) -> SshTunnelConfig {
|
||||
match find_host(&ssh.host) {
|
||||
Some(entry) => apply_host_entry(ssh, entry),
|
||||
None => ssh.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves `ssh.host` plus, when its `~/.ssh/config` entry (or an ancestor
|
||||
/// reached by following `ProxyJump`) declares `ProxyJump`, every jump host
|
||||
/// leading to it. The returned list is in connection order — outermost jump
|
||||
/// first, `ssh`'s own resolved config last — and always has at least one
|
||||
/// element. A chain of length 1 means there is no `ProxyJump` to honor, and
|
||||
/// behaves exactly like [`resolve_ssh_tunnel_config`].
|
||||
///
|
||||
/// Jump hosts have no dedicated credentials field in DBX's connection form,
|
||||
/// so each one inherits `ssh`'s password/key/agent settings; only the
|
||||
/// host/port/user/identity file come from its own `~/.ssh/config` entry.
|
||||
pub fn resolve_ssh_tunnel_chain(ssh: &SshTunnelConfig) -> Vec<SshTunnelConfig> {
|
||||
resolve_chain_from_entries(ssh, &list_hosts().unwrap_or_default())
|
||||
}
|
||||
|
||||
fn resolve_chain_from_entries(ssh: &SshTunnelConfig, entries: &[SshConfigHostEntry]) -> Vec<SshTunnelConfig> {
|
||||
let find = |alias: &str| entries.iter().find(|entry| entry.alias == alias).cloned();
|
||||
|
||||
let Some(target_entry) = find(&ssh.host) else {
|
||||
return vec![ssh.clone()];
|
||||
};
|
||||
let resolved_target = apply_host_entry(ssh, target_entry.clone());
|
||||
|
||||
let mut jump_hops = Vec::new();
|
||||
let mut seen_aliases = std::collections::HashSet::new();
|
||||
seen_aliases.insert(ssh.host.clone());
|
||||
let mut next_jump = target_entry.proxy_jump.clone();
|
||||
while let Some(alias) = next_jump {
|
||||
// Guard against a cycle in `~/.ssh/config` (e.g. two hosts naming
|
||||
// each other as ProxyJump) rather than looping forever.
|
||||
if !seen_aliases.insert(alias.clone()) {
|
||||
break;
|
||||
}
|
||||
let Some(entry) = find(&alias) else { break };
|
||||
next_jump = entry.proxy_jump.clone();
|
||||
jump_hops.push(jump_hop_from_entry(ssh, entry));
|
||||
}
|
||||
|
||||
jump_hops.reverse();
|
||||
jump_hops.push(resolved_target);
|
||||
jump_hops
|
||||
}
|
||||
|
||||
/// Builds a synthetic hop for a `ProxyJump` host. Unlike the leaf host in
|
||||
/// `ssh`, there's no user-filled form for this host, so its identity comes
|
||||
/// entirely from `~/.ssh/config` (falling back to the leaf's own settings
|
||||
/// for anything the config entry doesn't specify).
|
||||
fn jump_hop_from_entry(leaf: &SshTunnelConfig, entry: SshConfigHostEntry) -> SshTunnelConfig {
|
||||
SshTunnelConfig {
|
||||
id: format!("{}::proxyjump::{}", leaf.id, entry.alias),
|
||||
name: entry.alias.clone(),
|
||||
enabled: true,
|
||||
host: entry.host_name.unwrap_or_else(|| entry.alias.clone()),
|
||||
port: entry.port.unwrap_or(DEFAULT_PORT_SENTINEL),
|
||||
user: entry.user.unwrap_or_else(|| leaf.user.clone()),
|
||||
password: leaf.password.clone(),
|
||||
key_path: entry.identity_file.unwrap_or_else(|| leaf.key_path.clone()),
|
||||
key_passphrase: leaf.key_passphrase.clone(),
|
||||
connect_timeout_secs: leaf.connect_timeout_secs,
|
||||
expose_lan: false,
|
||||
use_ssh_agent: leaf.use_ssh_agent,
|
||||
ssh_agent_sock_path: leaf.ssh_agent_sock_path.clone(),
|
||||
auth_method: leaf.auth_method.clone(),
|
||||
allow_exec_channel_proxy: leaf.allow_exec_channel_proxy,
|
||||
profile_id: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies a resolved `~/.ssh/config` entry onto `ssh`, without overwriting
|
||||
/// values the user has explicitly set. The empty `auth_method` identifies
|
||||
/// legacy payloads where default values cannot be distinguished from fields
|
||||
/// that were left blank in the old form.
|
||||
fn apply_host_entry(ssh: &SshTunnelConfig, entry: SshConfigHostEntry) -> SshTunnelConfig {
|
||||
let mut resolved = ssh.clone();
|
||||
let is_legacy_payload = ssh.auth_method.is_empty();
|
||||
|
||||
if let Some(host_name) = entry.host_name {
|
||||
resolved.host = host_name;
|
||||
}
|
||||
if is_legacy_payload && resolved.user == DEFAULT_USER_SENTINEL {
|
||||
if let Some(user) = entry.user {
|
||||
resolved.user = user;
|
||||
}
|
||||
}
|
||||
if is_legacy_payload && resolved.port == DEFAULT_PORT_SENTINEL {
|
||||
if let Some(port) = entry.port {
|
||||
resolved.port = port;
|
||||
}
|
||||
}
|
||||
if resolved.key_path.is_empty() {
|
||||
if let Some(identity_file) = entry.identity_file {
|
||||
resolved.key_path = identity_file;
|
||||
// If the SSH config supplied the only usable credential, make the
|
||||
// backend use it even when an older/default UI payload still says
|
||||
// "password" with an empty password.
|
||||
if resolved.auth_method.is_empty() || (resolved.auth_method == "password" && resolved.password.is_empty()) {
|
||||
resolved.auth_method = "key".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolved
|
||||
}
|
||||
|
||||
/// Parses a minimal subset of OpenSSH client config syntax: `Host`, `HostName`,
|
||||
/// `Port`, `User`, `IdentityFile`, `ProxyJump`. Wildcard host patterns
|
||||
/// (containing `*` or `?`) are skipped since they aren't usable as a literal
|
||||
/// alias in the host field. `Include` and other directives are not
|
||||
/// supported, and `ProxyJump`'s comma-separated multi-hop form is read as a
|
||||
/// single alias (its first hop).
|
||||
fn parse_ssh_config(content: &str) -> Vec<SshConfigHostEntry> {
|
||||
let mut entries: Vec<SshConfigHostEntry> = Vec::new();
|
||||
let mut current_aliases: Vec<String> = Vec::new();
|
||||
|
||||
for raw_line in content.lines() {
|
||||
let line = strip_comment(raw_line).trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some((keyword, value)) = split_directive(line) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match keyword.to_ascii_lowercase().as_str() {
|
||||
"host" => {
|
||||
current_aliases = value
|
||||
.split_whitespace()
|
||||
.filter(|alias| !alias.contains('*') && !alias.contains('?'))
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
for alias in ¤t_aliases {
|
||||
entries.push(SshConfigHostEntry {
|
||||
alias: alias.clone(),
|
||||
host_name: None,
|
||||
port: None,
|
||||
user: None,
|
||||
identity_file: None,
|
||||
proxy_jump: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
"hostname" => set_current_field(&mut entries, ¤t_aliases, |entry| {
|
||||
entry.host_name = Some(value.to_string());
|
||||
}),
|
||||
"port" => {
|
||||
if let Ok(port) = value.parse::<u16>() {
|
||||
set_current_field(&mut entries, ¤t_aliases, |entry| {
|
||||
entry.port = Some(port);
|
||||
});
|
||||
}
|
||||
}
|
||||
"user" => set_current_field(&mut entries, ¤t_aliases, |entry| {
|
||||
entry.user = Some(value.to_string());
|
||||
}),
|
||||
"identityfile" => set_current_field(&mut entries, ¤t_aliases, |entry| {
|
||||
entry.identity_file = Some(value.to_string());
|
||||
}),
|
||||
"proxyjump" => {
|
||||
if let Some(first_hop) = value.split(',').next().map(str::trim).filter(|hop| !hop.is_empty()) {
|
||||
// A `user@host:port` jump target names an alias-incompatible
|
||||
// literal host; only a plain alias is resolvable here.
|
||||
if !first_hop.contains('@') && !first_hop.contains(':') {
|
||||
let first_hop = first_hop.to_string();
|
||||
set_current_field(&mut entries, ¤t_aliases, |entry| {
|
||||
entry.proxy_jump = Some(first_hop.clone());
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
entries
|
||||
}
|
||||
|
||||
fn set_current_field(
|
||||
entries: &mut [SshConfigHostEntry],
|
||||
current_aliases: &[String],
|
||||
apply: impl Fn(&mut SshConfigHostEntry),
|
||||
) {
|
||||
for entry in entries.iter_mut() {
|
||||
if current_aliases.contains(&entry.alias) {
|
||||
apply(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_comment(line: &str) -> &str {
|
||||
match line.find('#') {
|
||||
Some(index) => &line[..index],
|
||||
None => line,
|
||||
}
|
||||
}
|
||||
|
||||
/// Splits a config line into `(keyword, value)`. OpenSSH allows the keyword
|
||||
/// and value to be separated by whitespace or a single `=`.
|
||||
fn split_directive(line: &str) -> Option<(&str, &str)> {
|
||||
let line = line.trim();
|
||||
let split_index = line.find(|c: char| c.is_whitespace() || c == '=')?;
|
||||
let keyword = &line[..split_index];
|
||||
let value = line[split_index..].trim_start_matches(|c: char| c.is_whitespace() || c == '=').trim();
|
||||
if keyword.is_empty() || value.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some((keyword, value))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config(host: &str) -> SshTunnelConfig {
|
||||
SshTunnelConfig {
|
||||
profile_id: String::new(),
|
||||
id: "1".to_string(),
|
||||
name: String::new(),
|
||||
enabled: true,
|
||||
host: host.to_string(),
|
||||
port: DEFAULT_PORT_SENTINEL,
|
||||
user: DEFAULT_USER_SENTINEL.to_string(),
|
||||
password: String::new(),
|
||||
key_path: String::new(),
|
||||
key_passphrase: String::new(),
|
||||
connect_timeout_secs: 5,
|
||||
expose_lan: false,
|
||||
use_ssh_agent: false,
|
||||
ssh_agent_sock_path: String::new(),
|
||||
auth_method: String::new(),
|
||||
allow_exec_channel_proxy: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_basic_host_block() {
|
||||
let entries = parse_ssh_config(
|
||||
"Host myserver\n HostName 10.0.0.5\n Port 2222\n User deploy\n IdentityFile ~/.ssh/id_ed25519\n",
|
||||
);
|
||||
assert_eq!(entries.len(), 1);
|
||||
let entry = &entries[0];
|
||||
assert_eq!(entry.alias, "myserver");
|
||||
assert_eq!(entry.host_name, Some("10.0.0.5".to_string()));
|
||||
assert_eq!(entry.port, Some(2222));
|
||||
assert_eq!(entry.user, Some("deploy".to_string()));
|
||||
assert_eq!(entry.identity_file, Some("~/.ssh/id_ed25519".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_line_can_declare_multiple_aliases() {
|
||||
let entries = parse_ssh_config("Host prod prod-alias\n HostName 10.0.0.9\n");
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert!(entries.iter().all(|entry| entry.host_name == Some("10.0.0.9".to_string())));
|
||||
assert_eq!(entries[0].alias, "prod");
|
||||
assert_eq!(entries[1].alias, "prod-alias");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_wildcard_host_patterns() {
|
||||
let entries = parse_ssh_config("Host *.example.com\n User git\nHost real\n User deploy\n");
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].alias, "real");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_comments_and_blank_lines() {
|
||||
let entries = parse_ssh_config("# a comment\n\nHost myserver # inline comment\n User deploy\n");
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].user, Some("deploy".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_proxy_jump() {
|
||||
let entries = parse_ssh_config("Host target\n HostName 1.1.1.2\n ProxyJump gateway\n");
|
||||
assert_eq!(entries[0].proxy_jump, Some("gateway".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_jump_multi_hop_form_reads_only_the_first_hop() {
|
||||
let entries = parse_ssh_config("Host target\n ProxyJump gw1,gw2\n");
|
||||
assert_eq!(entries[0].proxy_jump, Some("gw1".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_jump_user_at_host_form_is_not_resolvable_as_an_alias() {
|
||||
let entries = parse_ssh_config("Host target\n ProxyJump jumper@1.1.1.1\n");
|
||||
assert_eq!(entries[0].proxy_jump, None);
|
||||
}
|
||||
|
||||
fn entry(alias: &str) -> SshConfigHostEntry {
|
||||
SshConfigHostEntry {
|
||||
alias: alias.to_string(),
|
||||
host_name: Some("10.0.0.5".to_string()),
|
||||
port: Some(2222),
|
||||
user: Some("deploy".to_string()),
|
||||
identity_file: Some("~/.ssh/id_ed25519".to_string()),
|
||||
proxy_jump: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fills_unset_fields_from_matching_alias() {
|
||||
let ssh = config("myserver");
|
||||
let resolved = apply_host_entry(&ssh, entry("myserver"));
|
||||
assert_eq!(resolved.host, "10.0.0.5");
|
||||
assert_eq!(resolved.port, 2222);
|
||||
assert_eq!(resolved.user, "deploy");
|
||||
assert_eq!(resolved.key_path, "~/.ssh/id_ed25519");
|
||||
assert_eq!(resolved.auth_method, "key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_keeps_password_auth_when_password_is_present() {
|
||||
let mut ssh = config("myserver");
|
||||
ssh.password = "secret".to_string();
|
||||
ssh.auth_method = "password".to_string();
|
||||
let resolved = apply_host_entry(&ssh, entry("myserver"));
|
||||
assert_eq!(resolved.key_path, "~/.ssh/id_ed25519");
|
||||
assert_eq!(resolved.auth_method, "password");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_keeps_default_user_and_port_for_current_payloads() {
|
||||
let mut ssh = config("myserver");
|
||||
ssh.auth_method = "password".to_string();
|
||||
|
||||
let resolved = apply_host_entry(&ssh, entry("myserver"));
|
||||
|
||||
assert_eq!(resolved.host, "10.0.0.5");
|
||||
assert_eq!(resolved.user, DEFAULT_USER_SENTINEL);
|
||||
assert_eq!(resolved.port, DEFAULT_PORT_SENTINEL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_preserves_key_plus_password_method() {
|
||||
let mut ssh = config("myserver");
|
||||
ssh.auth_method = "key+password".to_string();
|
||||
ssh.password = "secret".to_string();
|
||||
let resolved = apply_host_entry(&ssh, entry("myserver"));
|
||||
assert_eq!(resolved.key_path, "~/.ssh/id_ed25519");
|
||||
assert_eq!(resolved.auth_method, "key+password");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_does_not_override_explicit_values() {
|
||||
let mut ssh = config("myserver");
|
||||
ssh.user = "alice".to_string();
|
||||
ssh.port = 9999;
|
||||
ssh.key_path = "/explicit/key".to_string();
|
||||
let resolved = apply_host_entry(&ssh, entry("myserver"));
|
||||
assert_eq!(resolved.host, "10.0.0.5");
|
||||
assert_eq!(resolved.user, "alice");
|
||||
assert_eq!(resolved.port, 9999);
|
||||
assert_eq!(resolved.key_path, "/explicit/key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_is_noop_when_host_does_not_match_any_alias() {
|
||||
// `resolve_ssh_tunnel_config` looks up the real `~/.ssh/config`; an
|
||||
// alias this unlikely to exist on a test machine exercises the
|
||||
// "no match found" branch without needing to mock the filesystem.
|
||||
let ssh = config("dbx-test-alias-that-should-never-exist-anywhere");
|
||||
let resolved = resolve_ssh_tunnel_config(&ssh);
|
||||
assert_eq!(resolved, ssh);
|
||||
}
|
||||
|
||||
fn jump_entry(alias: &str, host_name: &str, proxy_jump: Option<&str>) -> SshConfigHostEntry {
|
||||
SshConfigHostEntry {
|
||||
alias: alias.to_string(),
|
||||
host_name: Some(host_name.to_string()),
|
||||
port: None,
|
||||
user: None,
|
||||
identity_file: None,
|
||||
proxy_jump: proxy_jump.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_is_single_hop_without_proxy_jump() {
|
||||
let ssh = config("myserver");
|
||||
let chain = resolve_chain_from_entries(&ssh, std::slice::from_ref(&entry("myserver")));
|
||||
assert_eq!(chain.len(), 1);
|
||||
assert_eq!(chain[0].host, "10.0.0.5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_prepends_the_proxy_jump_hop_in_connection_order() {
|
||||
// Mirrors issue #7706: `target` (ProxyJump gateway) resolves to
|
||||
// [gateway, target], gateway dialed first.
|
||||
let ssh = config("target");
|
||||
let entries = [jump_entry("gateway", "1.1.1.1", None), jump_entry("target", "1.1.1.2", Some("gateway"))];
|
||||
|
||||
let chain = resolve_chain_from_entries(&ssh, &entries);
|
||||
|
||||
assert_eq!(chain.len(), 2);
|
||||
assert_eq!(chain[0].host, "1.1.1.1");
|
||||
assert_eq!(chain[0].name, "gateway");
|
||||
assert_eq!(chain[1].host, "1.1.1.2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_jump_hop_inherits_leaf_credentials_when_config_has_no_identity_file() {
|
||||
let mut ssh = config("target");
|
||||
ssh.auth_method = "key".to_string();
|
||||
ssh.key_path = "/home/user/.ssh/id_rsa".to_string();
|
||||
let entries = [jump_entry("gateway", "1.1.1.1", None), jump_entry("target", "1.1.1.2", Some("gateway"))];
|
||||
|
||||
let chain = resolve_chain_from_entries(&ssh, &entries);
|
||||
|
||||
assert_eq!(chain[0].auth_method, "key");
|
||||
assert_eq!(chain[0].key_path, "/home/user/.ssh/id_rsa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_follows_multiple_proxy_jump_hops_recursively() {
|
||||
let ssh = config("target");
|
||||
let entries = [
|
||||
jump_entry("gateway-outer", "1.1.1.1", None),
|
||||
jump_entry("gateway-inner", "1.1.1.2", Some("gateway-outer")),
|
||||
jump_entry("target", "1.1.1.3", Some("gateway-inner")),
|
||||
];
|
||||
|
||||
let chain = resolve_chain_from_entries(&ssh, &entries);
|
||||
|
||||
assert_eq!(
|
||||
chain.iter().map(|hop| hop.host.as_str()).collect::<Vec<_>>(),
|
||||
vec!["1.1.1.1", "1.1.1.2", "1.1.1.3"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_breaks_a_proxy_jump_cycle_instead_of_looping_forever() {
|
||||
let ssh = config("a");
|
||||
let entries = [jump_entry("a", "1.1.1.1", Some("b")), jump_entry("b", "1.1.1.2", Some("a"))];
|
||||
|
||||
let chain = resolve_chain_from_entries(&ssh, &entries);
|
||||
|
||||
// Must terminate; the exact truncation point is an implementation
|
||||
// detail of cycle detection, not a behavior callers rely on.
|
||||
assert!(chain.len() <= entries.len());
|
||||
assert_eq!(chain.last().unwrap().host, "1.1.1.1");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
//! Host-key verification for the SSH tunnel client.
|
||||
//!
|
||||
//! Before any credential is sent, russh invokes
|
||||
//! [`russh::client::Handler::check_server_key`]. The original implementation
|
||||
//! accepted *every* server key, which let a man-in-the-middle endpoint reject
|
||||
//! the public key and trick the client into sending the password instead.
|
||||
//! This module makes that callback actually verify the server identity.
|
||||
//!
|
||||
//! Design
|
||||
//! ------
|
||||
//! Verification is layered and **fail-closed**, reusing russh's own OpenSSH-
|
||||
//! format `known_hosts` module instead of a hand-rolled store:
|
||||
//!
|
||||
//! 1. The user's system `~/.ssh/known_hosts` is checked **read-only**. Reusing
|
||||
//! it means a host already trusted in a terminal is also trusted here, and a
|
||||
//! key change recorded there is detected here too. dbx never *writes* to it.
|
||||
//! 2. dbx's own store at `<data_dir>/known_hosts` (OpenSSH format, managed by
|
||||
//! russh's `known_hosts` module) is checked. This is where dbx-accepted host
|
||||
//! keys live — NOT `~/.ssh` — so we never pollute the user's SSH config.
|
||||
//! 3. An unknown host is reported as [`HostKeyState::Unknown`]; the caller
|
||||
//! (the `check_server_key` handler) then performs an **explicit TOFU** flow:
|
||||
//! it asks the UI to confirm the key and, on acceptance, records it via
|
||||
//! [`HostKeyVerifier::learn`].
|
||||
//! 4. A *changed* key in the dbx store is [`HostKeyState::Changed`]. The caller
|
||||
//! prompts the user to compare fingerprints and, if they choose to update,
|
||||
//! replaces the old entry via [`HostKeyVerifier::replace`]. A changed key
|
||||
//! in `~/.ssh/known_hosts` stays a hard reject — dbx does not edit that file.
|
||||
//!
|
||||
//! Hardening rules:
|
||||
//! * A changed key is never trusted silently. Credentials are sent only after
|
||||
//! the user explicitly continues (session-only) or updates the saved key.
|
||||
//! * If an unknown host's key **cannot be persisted** (e.g. no write permission
|
||||
//! on `<data_dir>`), `learn` fails — but that failure is surfaced by the
|
||||
//! caller, which still refuses to trust the host silently.
|
||||
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use russh::keys::known_hosts::{
|
||||
check_known_hosts, check_known_hosts_path, known_host_keys_path, learn_known_hosts_path,
|
||||
};
|
||||
use russh::keys::parse_public_key_base64;
|
||||
use russh::keys::ssh_key::{HashAlg, PublicKey};
|
||||
|
||||
/// Outcome of checking a server key against the known-hosts stores.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HostKeyState {
|
||||
/// The key matches a previously trusted entry (system or dbx store).
|
||||
Trusted,
|
||||
/// The key is not present in either store (candidate for explicit TOFU).
|
||||
Unknown,
|
||||
/// The dbx store has a different key of the same algorithm. The caller
|
||||
/// should prompt before replacing it; `previous_fingerprint` is the saved
|
||||
/// SHA256 fingerprint (or a fallback locator if the line could not be parsed).
|
||||
Changed { previous_fingerprint: String },
|
||||
}
|
||||
|
||||
/// Host-key verifier backed by russh's OpenSSH-format known-hosts store.
|
||||
///
|
||||
/// `known_hosts_path` is the dbx-managed store (typically
|
||||
/// `<data_dir>/known_hosts`). The system `~/.ssh/known_hosts` is also consulted
|
||||
/// read-only via [`check_known_hosts`].
|
||||
pub struct HostKeyVerifier {
|
||||
known_hosts_path: PathBuf,
|
||||
}
|
||||
|
||||
impl HostKeyVerifier {
|
||||
/// Creates a verifier that records/reads accepted host keys at `known_hosts_path`.
|
||||
pub fn new(known_hosts_path: PathBuf) -> Self {
|
||||
Self { known_hosts_path }
|
||||
}
|
||||
|
||||
/// Layered, fail-closed check. A *changed* key in `~/.ssh` is an `Err`. A
|
||||
/// changed key in the dbx store is [`HostKeyState::Changed`] so the UI can
|
||||
/// offer a one-click replace.
|
||||
pub fn check(&self, host: &str, port: u16, key: &PublicKey) -> Result<HostKeyState, io::Error> {
|
||||
// 1. System known_hosts (read-only). A changed key there is a hard reject.
|
||||
match check_known_hosts(host, port, key) {
|
||||
Ok(true) => return Ok(HostKeyState::Trusted),
|
||||
Err(russh::keys::Error::KeyChanged { line }) => {
|
||||
return Err(host_key_changed_error(host, port, line, "~/.ssh/known_hosts"));
|
||||
}
|
||||
// Missing system file, no home dir, or an unreadable system file:
|
||||
// don't fail the whole connection on it; fall through to the dbx
|
||||
// store / TOFU, which still protects against MITM.
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// 2. dbx-managed known_hosts.
|
||||
match check_known_hosts_path(host, port, key, &self.known_hosts_path) {
|
||||
Ok(true) => return Ok(HostKeyState::Trusted),
|
||||
Err(russh::keys::Error::KeyChanged { line }) => {
|
||||
let previous_fingerprint = previous_fingerprint_for(&self.known_hosts_path, host, port, key)
|
||||
.unwrap_or_else(|| format!("(known_hosts line {line})"));
|
||||
return Ok(HostKeyState::Changed { previous_fingerprint });
|
||||
}
|
||||
// Unknown (or an unreadable dbx store): report as a candidate for TOFU.
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Ok(HostKeyState::Unknown)
|
||||
}
|
||||
|
||||
/// Records a host key into the dbx store (TOFU persistence). Called by the
|
||||
/// caller only after the user explicitly accepts the key. A write failure
|
||||
/// is reported (so the caller knows persistence did not happen) but does
|
||||
/// not by itself abort the session — the host may simply be trusted for
|
||||
/// this session only.
|
||||
pub fn learn(&self, host: &str, port: u16, key: &PublicKey) -> Result<(), io::Error> {
|
||||
learn_known_hosts_path(host, port, key, &self.known_hosts_path).map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::PermissionDenied,
|
||||
format!(
|
||||
"Failed to persist host key for {host}:{port} to {} ({e}). \
|
||||
The host is trusted for this session only.",
|
||||
self.known_hosts_path.display()
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Drops the previous same-algorithm key for this host in the dbx store,
|
||||
/// then records `key`. Used after the user confirms a fingerprint change.
|
||||
pub fn replace(&self, host: &str, port: u16, key: &PublicKey) -> Result<(), io::Error> {
|
||||
self.forget_matching_algorithm(host, port, key)?;
|
||||
self.learn(host, port, key)
|
||||
}
|
||||
|
||||
fn forget_matching_algorithm(&self, host: &str, port: u16, key: &PublicKey) -> Result<(), io::Error> {
|
||||
let recorded = match known_host_keys_path(host, port, &self.known_hosts_path) {
|
||||
Ok(keys) => keys,
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
let drop_keys: Vec<PublicKey> = recorded
|
||||
.into_iter()
|
||||
.filter(|(_, recorded)| recorded.algorithm() == key.algorithm())
|
||||
.map(|(_, recorded)| recorded)
|
||||
.collect();
|
||||
if drop_keys.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let contents = match std::fs::read_to_string(&self.known_hosts_path) {
|
||||
Ok(contents) => contents,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mut kept = Vec::new();
|
||||
for line in contents.lines() {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() || trimmed.starts_with('#') {
|
||||
kept.push(line);
|
||||
continue;
|
||||
}
|
||||
let should_drop = trimmed
|
||||
.split_whitespace()
|
||||
.nth(2)
|
||||
.and_then(|encoded| parse_public_key_base64(encoded).ok())
|
||||
.map(|parsed| drop_keys.iter().any(|recorded| recorded == &parsed))
|
||||
.unwrap_or(false);
|
||||
if !should_drop {
|
||||
kept.push(line);
|
||||
}
|
||||
}
|
||||
let mut out = kept.join("\n");
|
||||
if !out.is_empty() {
|
||||
out.push('\n');
|
||||
}
|
||||
std::fs::write(&self.known_hosts_path, out)
|
||||
}
|
||||
}
|
||||
|
||||
fn previous_fingerprint_for(path: &Path, host: &str, port: u16, key: &PublicKey) -> Option<String> {
|
||||
let recorded = known_host_keys_path(host, port, path).ok()?;
|
||||
recorded
|
||||
.into_iter()
|
||||
.find(|(_, recorded)| recorded.algorithm() == key.algorithm() && recorded != key)
|
||||
.map(|(_, recorded)| recorded.fingerprint(HashAlg::Sha256).to_string())
|
||||
}
|
||||
|
||||
fn host_key_changed_error(host: &str, port: u16, line: usize, store: &str) -> io::Error {
|
||||
io::Error::other(format!(
|
||||
"Host key for {host}:{port} changed (recorded at {store}, line {line}). \
|
||||
This may indicate a man-in-the-middle attack. Remove the old entry and reconnect only if you expect this change."
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
/// Shared WKB (Well-Known Binary) geometry parsing shared by PostgreSQL (EWKB)
|
||||
/// and MySQL (standard WKB). Parses WKB bytes into WKT while retaining the
|
||||
/// top-level EWKB SRID when present.
|
||||
///
|
||||
/// Supports: POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING,
|
||||
/// MULTIPOLYGON, GEOMETRYCOLLECTION, and their Z/M/ZM variants.
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct WkbDimensions {
|
||||
has_z: bool,
|
||||
has_m: bool,
|
||||
}
|
||||
|
||||
impl WkbDimensions {
|
||||
fn suffix(self) -> &'static str {
|
||||
match (self.has_z, self.has_m) {
|
||||
(false, false) => "",
|
||||
(true, false) => " Z",
|
||||
(false, true) => " M",
|
||||
(true, true) => " ZM",
|
||||
}
|
||||
}
|
||||
|
||||
fn coordinate_len(self) -> usize {
|
||||
2 + usize::from(self.has_z) + usize::from(self.has_m)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum WkbGeometry {
|
||||
Point { dims: WkbDimensions, coords: Option<Vec<f64>> },
|
||||
LineString { dims: WkbDimensions, points: Vec<Vec<f64>> },
|
||||
Polygon { dims: WkbDimensions, rings: Vec<Vec<Vec<f64>>> },
|
||||
MultiPoint { dims: WkbDimensions, points: Vec<Option<Vec<f64>>> },
|
||||
MultiLineString { dims: WkbDimensions, lines: Vec<Vec<Vec<f64>>> },
|
||||
MultiPolygon { dims: WkbDimensions, polygons: Vec<Vec<Vec<Vec<f64>>>> },
|
||||
GeometryCollection { dims: WkbDimensions, geometries: Vec<WkbGeometry> },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DecodedGeometry {
|
||||
pub wkt: String,
|
||||
pub srid: Option<u32>,
|
||||
}
|
||||
|
||||
impl WkbGeometry {
|
||||
fn to_wkt(&self) -> String {
|
||||
match self {
|
||||
Self::Point { dims, coords } => match coords {
|
||||
Some(coords) => format!("POINT{}({})", dims.suffix(), format_wkb_coordinate(coords)),
|
||||
None => format!("POINT{} EMPTY", dims.suffix()),
|
||||
},
|
||||
Self::LineString { dims, points } => {
|
||||
if points.is_empty() {
|
||||
format!("LINESTRING{} EMPTY", dims.suffix())
|
||||
} else {
|
||||
format!("LINESTRING{}({})", dims.suffix(), format_wkb_coordinate_sequence(points))
|
||||
}
|
||||
}
|
||||
Self::Polygon { dims, rings } => {
|
||||
if rings.is_empty() {
|
||||
format!("POLYGON{} EMPTY", dims.suffix())
|
||||
} else {
|
||||
format!(
|
||||
"POLYGON{}({})",
|
||||
dims.suffix(),
|
||||
rings
|
||||
.iter()
|
||||
.map(|ring| format!("({})", format_wkb_coordinate_sequence(ring)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
)
|
||||
}
|
||||
}
|
||||
Self::MultiPoint { dims, points } => {
|
||||
if points.is_empty() {
|
||||
format!("MULTIPOINT{} EMPTY", dims.suffix())
|
||||
} else {
|
||||
format!(
|
||||
"MULTIPOINT{}({})",
|
||||
dims.suffix(),
|
||||
points
|
||||
.iter()
|
||||
.map(|point| match point {
|
||||
Some(coords) => format!("({})", format_wkb_coordinate(coords)),
|
||||
None => "EMPTY".to_string(),
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
)
|
||||
}
|
||||
}
|
||||
Self::MultiLineString { dims, lines } => {
|
||||
if lines.is_empty() {
|
||||
format!("MULTILINESTRING{} EMPTY", dims.suffix())
|
||||
} else {
|
||||
format!(
|
||||
"MULTILINESTRING{}({})",
|
||||
dims.suffix(),
|
||||
lines
|
||||
.iter()
|
||||
.map(|line| format!("({})", format_wkb_coordinate_sequence(line)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
)
|
||||
}
|
||||
}
|
||||
Self::MultiPolygon { dims, polygons } => {
|
||||
if polygons.is_empty() {
|
||||
format!("MULTIPOLYGON{} EMPTY", dims.suffix())
|
||||
} else {
|
||||
format!(
|
||||
"MULTIPOLYGON{}({})",
|
||||
dims.suffix(),
|
||||
polygons
|
||||
.iter()
|
||||
.map(|polygon| {
|
||||
format!(
|
||||
"({})",
|
||||
polygon
|
||||
.iter()
|
||||
.map(|ring| format!("({})", format_wkb_coordinate_sequence(ring)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
)
|
||||
}
|
||||
}
|
||||
Self::GeometryCollection { dims, geometries } => {
|
||||
if geometries.is_empty() {
|
||||
format!("GEOMETRYCOLLECTION{} EMPTY", dims.suffix())
|
||||
} else {
|
||||
format!(
|
||||
"GEOMETRYCOLLECTION{}({})",
|
||||
dims.suffix(),
|
||||
geometries.iter().map(WkbGeometry::to_wkt).collect::<Vec<_>>().join(",")
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn format_wkb_coordinate(coords: &[f64]) -> String {
|
||||
coords.iter().map(|value| value.to_string()).collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
fn format_wkb_coordinate_sequence(points: &[Vec<f64>]) -> String {
|
||||
points.iter().map(|point| format_wkb_coordinate(point)).collect::<Vec<_>>().join(",")
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum WkbEndian {
|
||||
Big,
|
||||
Little,
|
||||
}
|
||||
|
||||
struct WkbReader<'a> {
|
||||
raw: &'a [u8],
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl<'a> WkbReader<'a> {
|
||||
fn new(raw: &'a [u8]) -> Self {
|
||||
Self { raw, pos: 0 }
|
||||
}
|
||||
|
||||
fn read_u8(&mut self) -> Option<u8> {
|
||||
let value = *self.raw.get(self.pos)?;
|
||||
self.pos += 1;
|
||||
Some(value)
|
||||
}
|
||||
|
||||
fn read_array<const N: usize>(&mut self) -> Option<[u8; N]> {
|
||||
let end = self.pos.checked_add(N)?;
|
||||
let bytes: [u8; N] = self.raw.get(self.pos..end)?.try_into().ok()?;
|
||||
self.pos = end;
|
||||
Some(bytes)
|
||||
}
|
||||
|
||||
fn read_u32(&mut self, endian: WkbEndian) -> Option<u32> {
|
||||
let bytes = self.read_array::<4>()?;
|
||||
Some(match endian {
|
||||
WkbEndian::Big => u32::from_be_bytes(bytes),
|
||||
WkbEndian::Little => u32::from_le_bytes(bytes),
|
||||
})
|
||||
}
|
||||
|
||||
fn read_f64(&mut self, endian: WkbEndian) -> Option<f64> {
|
||||
let bytes = self.read_array::<8>()?;
|
||||
Some(match endian {
|
||||
WkbEndian::Big => f64::from_be_bytes(bytes),
|
||||
WkbEndian::Little => f64::from_le_bytes(bytes),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_wkb_dimensions(type_word: u32) -> (u32, WkbDimensions, bool) {
|
||||
let mut base_type = type_word & 0x1FFF_FFFF;
|
||||
let mut dims = WkbDimensions { has_z: (type_word & 0x8000_0000) != 0, has_m: (type_word & 0x4000_0000) != 0 };
|
||||
let has_srid = (type_word & 0x2000_0000) != 0;
|
||||
|
||||
if base_type >= 3000 {
|
||||
dims.has_z = true;
|
||||
dims.has_m = true;
|
||||
base_type -= 3000;
|
||||
} else if base_type >= 2000 {
|
||||
dims.has_m = true;
|
||||
base_type -= 2000;
|
||||
} else if base_type >= 1000 {
|
||||
dims.has_z = true;
|
||||
base_type -= 1000;
|
||||
}
|
||||
|
||||
(base_type, dims, has_srid)
|
||||
}
|
||||
|
||||
fn read_wkb_point_coords(reader: &mut WkbReader<'_>, dims: WkbDimensions) -> Option<Option<Vec<f64>>> {
|
||||
let endian = match reader.read_u8()? {
|
||||
0 => WkbEndian::Big,
|
||||
1 => WkbEndian::Little,
|
||||
_ => return None,
|
||||
};
|
||||
let type_word = reader.read_u32(endian)?;
|
||||
let (base_type, parsed_dims, has_srid) = parse_wkb_dimensions(type_word);
|
||||
if base_type != 1 || parsed_dims != dims {
|
||||
return None;
|
||||
}
|
||||
if has_srid {
|
||||
reader.read_u32(endian)?;
|
||||
}
|
||||
let coords = (0..parsed_dims.coordinate_len()).map(|_| reader.read_f64(endian)).collect::<Option<Vec<_>>>()?;
|
||||
Some((!coords.iter().all(|value| value.is_nan())).then_some(coords))
|
||||
}
|
||||
|
||||
fn parse_wkb_points(reader: &mut WkbReader<'_>, endian: WkbEndian, dims: WkbDimensions) -> Option<Vec<Vec<f64>>> {
|
||||
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
|
||||
(0..count)
|
||||
.map(|_| (0..dims.coordinate_len()).map(|_| reader.read_f64(endian)).collect::<Option<Vec<_>>>())
|
||||
.collect::<Option<Vec<_>>>()
|
||||
}
|
||||
|
||||
fn parse_wkb_geometry(reader: &mut WkbReader<'_>) -> Option<(WkbGeometry, Option<u32>)> {
|
||||
let endian = match reader.read_u8()? {
|
||||
0 => WkbEndian::Big,
|
||||
1 => WkbEndian::Little,
|
||||
_ => return None,
|
||||
};
|
||||
let type_word = reader.read_u32(endian)?;
|
||||
let (base_type, dims, has_srid) = parse_wkb_dimensions(type_word);
|
||||
let srid = has_srid.then(|| reader.read_u32(endian)).flatten().filter(|value| *value != 0);
|
||||
|
||||
let geometry = match base_type {
|
||||
1 => {
|
||||
let coords = (0..dims.coordinate_len()).map(|_| reader.read_f64(endian)).collect::<Option<Vec<_>>>()?;
|
||||
let coords = if coords.iter().all(|value| value.is_nan()) { None } else { Some(coords) };
|
||||
Some(WkbGeometry::Point { dims, coords })
|
||||
}
|
||||
2 => Some(WkbGeometry::LineString { dims, points: parse_wkb_points(reader, endian, dims)? }),
|
||||
3 => {
|
||||
let ring_count = usize::try_from(reader.read_u32(endian)?).ok()?;
|
||||
let rings = (0..ring_count).map(|_| parse_wkb_points(reader, endian, dims)).collect::<Option<Vec<_>>>()?;
|
||||
Some(WkbGeometry::Polygon { dims, rings })
|
||||
}
|
||||
4 => {
|
||||
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
|
||||
let points = (0..count).map(|_| read_wkb_point_coords(reader, dims)).collect::<Option<Vec<_>>>()?;
|
||||
Some(WkbGeometry::MultiPoint { dims, points })
|
||||
}
|
||||
5 => {
|
||||
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
|
||||
let lines = (0..count)
|
||||
.map(|_| match parse_wkb_geometry(reader)?.0 {
|
||||
WkbGeometry::LineString { points, .. } => Some(points),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
Some(WkbGeometry::MultiLineString { dims, lines })
|
||||
}
|
||||
6 => {
|
||||
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
|
||||
let polygons = (0..count)
|
||||
.map(|_| match parse_wkb_geometry(reader)?.0 {
|
||||
WkbGeometry::Polygon { rings, .. } => Some(rings),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
Some(WkbGeometry::MultiPolygon { dims, polygons })
|
||||
}
|
||||
7 => {
|
||||
let count = usize::try_from(reader.read_u32(endian)?).ok()?;
|
||||
let geometries =
|
||||
(0..count).map(|_| parse_wkb_geometry(reader).map(|parsed| parsed.0)).collect::<Option<Vec<_>>>()?;
|
||||
Some(WkbGeometry::GeometryCollection { dims, geometries })
|
||||
}
|
||||
_ => None,
|
||||
}?;
|
||||
Some((geometry, srid))
|
||||
}
|
||||
|
||||
pub fn decode_wkb_geometry(raw: &[u8]) -> Option<DecodedGeometry> {
|
||||
let mut reader = WkbReader::new(raw);
|
||||
let (geometry, srid) = parse_wkb_geometry(&mut reader)?;
|
||||
if reader.pos != raw.len() {
|
||||
return None;
|
||||
}
|
||||
Some(DecodedGeometry { wkt: geometry.to_wkt(), srid })
|
||||
}
|
||||
|
||||
/// Parse WKB/EWKB bytes into a WKT (Well-Known Text) string.
|
||||
///
|
||||
/// Supports both standard WKB (MySQL) and Extended WKB (PostgreSQL/PostGIS).
|
||||
/// Returns `None` if the bytes cannot be parsed as valid WKB.
|
||||
pub fn wkb_to_wkt(raw: &[u8]) -> Option<String> {
|
||||
decode_wkb_geometry(raw).map(|geometry| geometry.wkt)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn decode_hex(value: &str) -> Vec<u8> {
|
||||
value
|
||||
.as_bytes()
|
||||
.chunks_exact(2)
|
||||
.map(|pair| {
|
||||
let text = std::str::from_utf8(pair).unwrap();
|
||||
u8::from_str_radix(text, 16).unwrap()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ewkb_decoder_retains_top_level_srid() {
|
||||
let decoded = decode_wkb_geometry(&decode_hex("0101000020E6100000C520B07268195D404E62105839F44340")).unwrap();
|
||||
assert_eq!(decoded.wkt, "POINT(116.397 39.908)");
|
||||
assert_eq!(decoded.srid, Some(4326));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_wkb_has_no_srid() {
|
||||
let decoded = decode_wkb_geometry(&decode_hex("0101000000000000000000F03F0000000000000040")).unwrap();
|
||||
assert_eq!(decoded.wkt, "POINT(1 2)");
|
||||
assert_eq!(decoded.srid, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn big_endian_ewkb_retains_srid() {
|
||||
let decoded = decode_wkb_geometry(&decode_hex("002000000100000F113FF00000000000004000000000000000")).unwrap();
|
||||
assert_eq!(decoded.wkt, "POINT(1 2)");
|
||||
assert_eq!(decoded.srid, Some(3857));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ewkb_srid_zero_is_unknown() {
|
||||
let decoded = decode_wkb_geometry(&decode_hex("010100002000000000000000000000F03F0000000000000040")).unwrap();
|
||||
assert_eq!(decoded.wkt, "POINT(1 2)");
|
||||
assert_eq!(decoded.srid, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipoint_preserves_valid_empty_points() {
|
||||
let decoded =
|
||||
decode_wkb_geometry(&decode_hex("0104000000010000000101000000000000000000F87F000000000000F87F")).unwrap();
|
||||
assert_eq!(decoded.wkt, "MULTIPOINT(EMPTY)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipoint_rejects_truncated_child_points() {
|
||||
assert!(decode_wkb_geometry(&decode_hex("0104000000010000000101000000")).is_none());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user