feat(table-import): add DuckDB Parquet imports
This commit is contained in:
1 parent
f61f64f727
commit
02f27dc294
26 files changed
+686
-46
No files matched your search
@@ -209,6 +209,35 @@ impl DuckDbWorkerClient {
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
query_timeout: Option<Duration>,
|
||||
) -> Result<db::QueryResult, DuckDbWorkerError> {
|
||||
self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, false).await
|
||||
}
|
||||
|
||||
/// Executes a query with DuckDB insertion-order preservation enabled for
|
||||
/// the duration of that request. This is used by Parquet pagination:
|
||||
/// independent LIMIT/OFFSET scans must see the same source order even
|
||||
/// when the connection's init script disabled `preserve_insertion_order`.
|
||||
pub async fn execute_preserving_insertion_order(
|
||||
&self,
|
||||
database: Option<String>,
|
||||
sql: String,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
query_timeout: Option<Duration>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, true)
|
||||
.await
|
||||
.map_err(|error| error.message)
|
||||
}
|
||||
|
||||
async fn execute_typed_with_options(
|
||||
&self,
|
||||
database: Option<String>,
|
||||
sql: String,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
query_timeout: Option<Duration>,
|
||||
preserve_insertion_order: bool,
|
||||
) -> Result<db::QueryResult, DuckDbWorkerError> {
|
||||
let _query_guard = self.inner.query_lock.lock().await;
|
||||
let client = self.clone();
|
||||
@@ -228,7 +257,7 @@ impl DuckDbWorkerClient {
|
||||
match client
|
||||
.send_request_structured::<db::QueryResult>(
|
||||
DuckDbWorkerMethod::Execute,
|
||||
DuckDbWorkerExecuteParams { sql, database, max_rows },
|
||||
DuckDbWorkerExecuteParams { sql, database, max_rows, preserve_insertion_order },
|
||||
None,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -116,6 +116,13 @@ pub struct DuckDbWorkerExecuteParams {
|
||||
pub database: Option<String>,
|
||||
#[serde(default)]
|
||||
pub max_rows: Option<usize>,
|
||||
/// Temporarily preserve source row order while executing this query.
|
||||
///
|
||||
/// Parquet imports page through the same file with LIMIT/OFFSET. The
|
||||
/// worker must opt into stable insertion order for every page when a
|
||||
/// connection init script disabled DuckDB's default.
|
||||
#[serde(default)]
|
||||
pub preserve_insertion_order: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
|
||||
Reference in new issue
Block a user