feat(table-import): add DuckDB Parquet imports

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-09-28 21:38:32 +08:00
1 parent f61f64f727
commit 02f27dc294
26 files changed
+686 -46

No files matched your search

@@ -209,6 +209,35 @@ impl DuckDbWorkerClient {
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
) -> Result<db::QueryResult, DuckDbWorkerError> {
self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, false).await
}
/// Executes a query with DuckDB insertion-order preservation enabled for
/// the duration of that request. This is used by Parquet pagination:
/// independent LIMIT/OFFSET scans must see the same source order even
/// when the connection's init script disabled `preserve_insertion_order`.
pub async fn execute_preserving_insertion_order(
&self,
database: Option<String>,
sql: String,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
) -> Result<db::QueryResult, String> {
self.execute_typed_with_options(database, sql, max_rows, cancel_token, query_timeout, true)
.await
.map_err(|error| error.message)
}
async fn execute_typed_with_options(
&self,
database: Option<String>,
sql: String,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
query_timeout: Option<Duration>,
preserve_insertion_order: bool,
) -> Result<db::QueryResult, DuckDbWorkerError> {
let _query_guard = self.inner.query_lock.lock().await;
let client = self.clone();
@@ -228,7 +257,7 @@ impl DuckDbWorkerClient {
match client
.send_request_structured::<db::QueryResult>(
DuckDbWorkerMethod::Execute,
DuckDbWorkerExecuteParams { sql, database, max_rows },
DuckDbWorkerExecuteParams { sql, database, max_rows, preserve_insertion_order },
None,
)
.await
@@ -116,6 +116,13 @@ pub struct DuckDbWorkerExecuteParams {
pub database: Option<String>,
#[serde(default)]
pub max_rows: Option<usize>,
/// Temporarily preserve source row order while executing this query.
///
/// Parquet imports page through the same file with LIMIT/OFFSET. The
/// worker must opt into stable insertion order for every page when a
/// connection init script disabled DuckDB's default.
#[serde(default)]
pub preserve_insertion_order: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]