feat(table-import): add DuckDB Parquet imports
This commit is contained in:
1 parent
f61f64f727
commit
02f27dc294
26 files changed
+686
-46
No files matched your search
@@ -50,13 +50,17 @@ impl DuckDbWorkerSession {
|
||||
pub fn execute(&mut self, params: DuckDbWorkerExecuteParams) -> Result<db::QueryResult, String> {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
let locked = connection.lock().map_err(|e| e.to_string())?;
|
||||
let result = crate::query::duckdb_execute_for_database(
|
||||
&locked,
|
||||
&self.attached_names,
|
||||
params.database.as_deref(),
|
||||
¶ms.sql,
|
||||
params.max_rows,
|
||||
)?;
|
||||
let result = if params.preserve_insertion_order {
|
||||
execute_with_preserved_insertion_order(&locked, &self.attached_names, ¶ms)?
|
||||
} else {
|
||||
crate::query::duckdb_execute_for_database(
|
||||
&locked,
|
||||
&self.attached_names,
|
||||
params.database.as_deref(),
|
||||
¶ms.sql,
|
||||
params.max_rows,
|
||||
)?
|
||||
};
|
||||
if let Some(name) = crate::sql::attached_name_from_attach_sql(¶ms.sql) {
|
||||
if !self.attached_names.iter().any(|attached| attached.eq_ignore_ascii_case(&name)) {
|
||||
self.attached_names.push(name);
|
||||
@@ -191,6 +195,53 @@ impl DuckDbWorkerSession {
|
||||
}
|
||||
}
|
||||
|
||||
fn execute_with_preserved_insertion_order(
|
||||
connection: &duckdb::Connection,
|
||||
attached_names: &[String],
|
||||
params: &DuckDbWorkerExecuteParams,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let current: String = connection
|
||||
.query_row("SELECT CAST(current_setting('preserve_insertion_order') AS VARCHAR)", [], |row| row.get(0))
|
||||
.map_err(|error| format!("Failed to read DuckDB preserve_insertion_order setting: {error}"))?;
|
||||
let was_enabled = current
|
||||
.parse::<bool>()
|
||||
.map_err(|error| format!("Invalid DuckDB preserve_insertion_order setting {current:?}: {error}"))?;
|
||||
if was_enabled {
|
||||
return crate::query::duckdb_execute_for_database(
|
||||
connection,
|
||||
attached_names,
|
||||
params.database.as_deref(),
|
||||
¶ms.sql,
|
||||
params.max_rows,
|
||||
);
|
||||
}
|
||||
|
||||
// LIMIT/OFFSET Parquet pages are separate scans. Temporarily enabling this
|
||||
// setting makes every page use the same source order without changing the
|
||||
// user's connection-level setting after the request completes.
|
||||
connection
|
||||
.execute_batch("SET preserve_insertion_order = true")
|
||||
.map_err(|error| format!("Failed to enable DuckDB insertion-order preservation: {error}"))?;
|
||||
let result = crate::query::duckdb_execute_for_database(
|
||||
connection,
|
||||
attached_names,
|
||||
params.database.as_deref(),
|
||||
¶ms.sql,
|
||||
params.max_rows,
|
||||
);
|
||||
let restore = connection.execute_batch("SET preserve_insertion_order = false");
|
||||
|
||||
match (result, restore) {
|
||||
(result, Ok(())) => result,
|
||||
(Ok(_), Err(error)) => {
|
||||
Err(format!("DuckDB query succeeded but restoring preserve_insertion_order failed: {error}"))
|
||||
}
|
||||
(Err(query_error), Err(restore_error)) => {
|
||||
Err(format!("{query_error}; restoring preserve_insertion_order also failed: {restore_error}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn expand_tilde(path: &str) -> String {
|
||||
if !path.starts_with('~') {
|
||||
return path.to_string();
|
||||
@@ -497,13 +548,69 @@ mod tests {
|
||||
.expect("connect");
|
||||
|
||||
let result = session
|
||||
.execute(DuckDbWorkerExecuteParams { sql: "SELECT 1 AS value".to_string(), database: None, max_rows: None })
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: "SELECT 1 AS value".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: false,
|
||||
})
|
||||
.expect("execute");
|
||||
|
||||
assert_eq!(result.columns, vec!["value"]);
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!(1)]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_scopes_insertion_order_preservation() {
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
session
|
||||
.connect(DuckDbWorkerConnectParams {
|
||||
path: ":memory:".to_string(),
|
||||
attached_databases: Vec::new(),
|
||||
init_script: Some("SET preserve_insertion_order = false".to_string()),
|
||||
})
|
||||
.expect("connect");
|
||||
|
||||
let disabled = session
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: false,
|
||||
})
|
||||
.expect("read initial setting");
|
||||
assert_eq!(disabled.rows, vec![vec![serde_json::json!(false)]]);
|
||||
|
||||
let preserved = session
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: true,
|
||||
})
|
||||
.expect("execute with order preservation");
|
||||
assert_eq!(preserved.rows, vec![vec![serde_json::json!(true)]]);
|
||||
|
||||
session
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: "SELECT * FROM missing_table".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: true,
|
||||
})
|
||||
.expect_err("missing table should fail");
|
||||
|
||||
let restored = session
|
||||
.execute(DuckDbWorkerExecuteParams {
|
||||
sql: "SELECT current_setting('preserve_insertion_order') AS setting".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: false,
|
||||
})
|
||||
.expect("read restored setting");
|
||||
assert_eq!(restored.rows, vec![vec![serde_json::json!(false)]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_session_lists_tables_and_columns() {
|
||||
let mut session = DuckDbWorkerSession::default();
|
||||
@@ -522,6 +629,7 @@ mod tests {
|
||||
.to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: false,
|
||||
})
|
||||
.expect("create table");
|
||||
|
||||
@@ -566,6 +674,7 @@ mod tests {
|
||||
sql: "CREATE VIEW active_orders AS SELECT 1 AS id".to_string(),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: false,
|
||||
})
|
||||
.expect("create view");
|
||||
|
||||
@@ -713,6 +822,7 @@ mod tests {
|
||||
sql: format!("ATTACH '{}' AS \"sales db\";", attached_path.to_string_lossy().replace('\'', "''")),
|
||||
database: None,
|
||||
max_rows: None,
|
||||
preserve_insertion_order: false,
|
||||
})
|
||||
.expect("attach sql");
|
||||
|
||||
|
||||
@@ -271,6 +271,9 @@ pub struct DuckDbWorkerExecuteParams {
|
||||
pub database: Option<String>,
|
||||
#[serde(default)]
|
||||
pub max_rows: Option<usize>,
|
||||
/// Temporarily preserve source row order while executing this query.
|
||||
#[serde(default)]
|
||||
pub preserve_insertion_order: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
|
||||
@@ -351,6 +351,7 @@ async fn worker_process_exits_when_stdin_closes_during_active_query() {
|
||||
sql: "SELECT sum(sin(i::DOUBLE) * cos(i::DOUBLE / 3.0)) FROM range(100000000000) AS t(i)".to_string(),
|
||||
database: None,
|
||||
max_rows: Some(10),
|
||||
preserve_insertion_order: false,
|
||||
},
|
||||
)
|
||||
.expect("execute request"),
|
||||
@@ -508,10 +509,14 @@ async fn worker_shutdown_checkpoints_and_removes_wal() {
|
||||
let _ = std::fs::remove_file(&db_path);
|
||||
let _ = std::fs::remove_file(&wal_path);
|
||||
|
||||
let client =
|
||||
DuckDbWorkerClient::open_with_executable(executable.clone(), db_path.to_string_lossy().to_string(), Vec::new(), None)
|
||||
.await
|
||||
.expect("worker process connects");
|
||||
let client = DuckDbWorkerClient::open_with_executable(
|
||||
executable.clone(),
|
||||
db_path.to_string_lossy().to_string(),
|
||||
Vec::new(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("worker process connects");
|
||||
client
|
||||
.execute(
|
||||
None,
|
||||
|
||||
Reference in new issue
Block a user