feat(table-import): add DuckDB Parquet imports

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-09-28 21:38:32 +08:00
1 parent f61f64f727
commit 02f27dc294
26 files changed
+686 -46

No files matched your search

@@ -127,6 +127,8 @@ async fn benchmark_file(
let preview_started = Instant::now();
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: path_text.to_string(),
connection_id: None,
database: None,
source_ref: None,
source_format: Some(format),
parse_options: parse_options.clone(),
+363 -7
View File
@@ -24,6 +24,7 @@ use sqlparser::dialect::{GenericDialect, MsSqlDialect, MySqlDialect, OracleDiale
use sqlparser::parser::Parser;
use crate::connection::{task_client_session_id, AppState, PoolKind};
use crate::db;
use crate::models::connection::DatabaseType;
use crate::sql::SqlParsingOptions;
use crate::sql_file_import::{SqlFileStreamDecoder, StreamingSqlFileSplitter};
@@ -135,6 +136,7 @@ pub enum TableImportSourceFormat {
Json,
Excel,
Sql,
Parquet,
}
impl TableImportSourceFormat {
@@ -146,6 +148,7 @@ impl TableImportSourceFormat {
TableImportSourceFormat::Json => "json",
TableImportSourceFormat::Excel => "excel",
TableImportSourceFormat::Sql => "sql",
TableImportSourceFormat::Parquet => "parquet",
}
}
@@ -237,6 +240,10 @@ impl Default for TableImportParseOptions {
pub struct TableImportPreviewRequest {
pub file_path: String,
#[serde(default)]
pub connection_id: Option<String>,
#[serde(default)]
pub database: Option<String>,
#[serde(default)]
pub source_ref: Option<String>,
#[serde(default)]
pub source_format: Option<TableImportSourceFormat>,
@@ -447,6 +454,7 @@ pub enum ImportFileKind {
Json,
Xlsx,
Sql,
Parquet,
}
impl ImportFileKind {
@@ -458,6 +466,7 @@ impl ImportFileKind {
ImportFileKind::Json => "json",
ImportFileKind::Xlsx => "xlsx",
ImportFileKind::Sql => "sql",
ImportFileKind::Parquet => "parquet",
}
}
}
@@ -476,6 +485,8 @@ pub fn import_file_kind(path: &str) -> Result<ImportFileKind, String> {
Ok(ImportFileKind::Xlsx)
} else if lower.ends_with(".sql") {
Ok(ImportFileKind::Sql)
} else if lower.ends_with(".parquet") {
Ok(ImportFileKind::Parquet)
} else {
Err("Unsupported import file type".to_string())
}
@@ -489,6 +500,7 @@ pub fn source_format_for_path(path: &str) -> Result<TableImportSourceFormat, Str
ImportFileKind::Json => TableImportSourceFormat::Json,
ImportFileKind::Xlsx => TableImportSourceFormat::Excel,
ImportFileKind::Sql => TableImportSourceFormat::Sql,
ImportFileKind::Parquet => TableImportSourceFormat::Parquet,
})
}
@@ -501,6 +513,117 @@ pub fn effective_source_format(
.ok_or_else(|| "Unsupported import file type".to_string())
}
#[cfg(feature = "duckdb-sidecar")]
#[derive(Clone)]
struct DuckDbImportContext {
client: Arc<db::duckdb_worker_process::DuckDbWorkerClient>,
database: Option<String>,
query_timeout: Option<Duration>,
}
#[cfg(not(feature = "duckdb-sidecar"))]
#[derive(Clone)]
struct DuckDbImportContext;
#[cfg(feature = "duckdb-sidecar")]
impl DuckDbImportContext {
async fn execute(&self, sql: String, max_rows: Option<usize>) -> Result<db::QueryResult, String> {
self.client.execute(self.database.clone(), sql, max_rows, None, self.query_timeout).await
}
async fn execute_preserving_insertion_order(
&self,
sql: String,
max_rows: Option<usize>,
) -> Result<db::QueryResult, String> {
// Each Parquet page is an independent query. Ask the worker to scope
// DuckDB's order-preservation setting to that request so OFFSET refers
// to the same source order even when the connection configured it off.
self.client
.execute_preserving_insertion_order(self.database.clone(), sql, max_rows, None, self.query_timeout)
.await
}
}
#[cfg(feature = "duckdb-sidecar")]
async fn duckdb_import_context_for_pool(
state: &AppState,
pool_key: &str,
database: &str,
) -> Result<DuckDbImportContext, String> {
let pool = state.pool_handle(pool_key).await.ok_or_else(|| "Connection pool not found".to_string())?;
let client = match pool {
PoolKind::DuckDbWorker(client) => client,
_ => return Err("Parquet import requires a DuckDB connection".to_string()),
};
let query_timeout = crate::query::operation_budget_for_pool_key(state, pool_key, None).await.query_timeout;
Ok(DuckDbImportContext {
client,
database: (!database.trim().is_empty()).then_some(database.to_string()),
query_timeout,
})
}
async fn duckdb_import_context_for_source(
state: &AppState,
pool_key: &str,
db_type: &DatabaseType,
database: &str,
source_format: TableImportSourceFormat,
) -> Result<Option<DuckDbImportContext>, String> {
if source_format != TableImportSourceFormat::Parquet {
return Ok(None);
}
if *db_type != DatabaseType::DuckDb {
return Err("Parquet import is only supported for DuckDB connections".to_string());
}
#[cfg(feature = "duckdb-sidecar")]
{
return duckdb_import_context_for_pool(state, pool_key, database).await.map(Some);
}
#[cfg(not(feature = "duckdb-sidecar"))]
{
let _ = (state, pool_key, database);
Err("DuckDB worker support is not compiled in this build".to_string())
}
}
fn duckdb_sql_string_literal(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
fn duckdb_parquet_scan_sql(path: &str, limit: usize, offset: Option<usize>) -> String {
let mut sql = format!("SELECT * FROM read_parquet({}) LIMIT {}", duckdb_sql_string_literal(path), limit);
if let Some(offset) = offset {
sql.push_str(&format!(" OFFSET {offset}"));
}
sql
}
fn duckdb_parquet_count_sql(path: &str) -> String {
format!("SELECT COUNT(*) AS dbx_total_rows FROM read_parquet({})", duckdb_sql_string_literal(path))
}
fn duckdb_count_result(result: &db::QueryResult) -> Result<usize, String> {
let value = result
.rows
.first()
.and_then(|row| row.first())
.ok_or_else(|| "DuckDB Parquet row-count query returned no result".to_string())?;
match value {
serde_json::Value::Number(number) => number
.as_u64()
.and_then(|count| usize::try_from(count).ok())
.ok_or_else(|| "DuckDB Parquet row count is outside the supported range".to_string()),
serde_json::Value::String(value) => value
.parse::<u64>()
.ok()
.and_then(|count| usize::try_from(count).ok())
.ok_or_else(|| "DuckDB Parquet row count is outside the supported range".to_string()),
_ => Err("DuckDB Parquet row-count query returned an invalid value".to_string()),
}
}
pub fn normalize_header(value: &str, index: usize) -> String {
let trimmed = value.trim();
if trimmed.is_empty() {
@@ -2172,10 +2295,99 @@ fn stream_json_rows_to_channel(
///
/// 分隔文本与 Excel 仍然先把行解析进内存;`.sql` 脚本与 JSON 改用流式行来源
/// 增量产出,内存不再随文件体积增长,因此二者都没有 100 MB 的体积上限。
// Keep Parquet decoding in the DuckDB worker so compression and logical/nested type handling
// stay identical to normal DuckDB queries; the import pipeline only owns mapping and writes.
#[cfg(feature = "duckdb-sidecar")]
struct DuckDbParquetRowStream {
context: DuckDbImportContext,
file_path: String,
columns: Vec<String>,
total_rows: usize,
next_offset: usize,
pending: Option<Vec<Vec<serde_json::Value>>>,
}
#[cfg(feature = "duckdb-sidecar")]
impl DuckDbParquetRowStream {
async fn open(context: DuckDbImportContext, file_path: &str, batch_size: usize) -> Result<Self, String> {
let total_rows = duckdb_count_result(&context.execute(duckdb_parquet_count_sql(file_path), Some(1)).await?)?;
if total_rows == 0 {
return Err("Parquet file has no rows".to_string());
}
let first_batch_size = batch_size.max(1);
let first = context
.execute_preserving_insertion_order(
duckdb_parquet_scan_sql(file_path, first_batch_size, None),
Some(first_batch_size),
)
.await?;
if total_rows > 0 && first.rows.is_empty() {
return Err("DuckDB Parquet scan returned no rows for a non-empty file".to_string());
}
Ok(Self {
context,
file_path: file_path.to_string(),
columns: first.columns,
total_rows,
next_offset: first.rows.len(),
pending: (!first.rows.is_empty()).then_some(first.rows),
})
}
fn columns(&self) -> Vec<String> {
self.columns.clone()
}
fn total_rows(&self) -> usize {
self.total_rows
}
async fn next_batch(&mut self, max_rows: usize) -> Result<Option<Vec<Vec<serde_json::Value>>>, String> {
if let Some(rows) = self.pending.take() {
return Ok(Some(rows));
}
if self.next_offset >= self.total_rows {
return Ok(None);
}
let limit = max_rows.max(1);
// The worker protocol exposes bounded query results but no cursor API. LIMIT/OFFSET keeps
// each response bounded and lets the existing import writer retain its batch semantics.
let result = self
.context
.execute_preserving_insertion_order(
duckdb_parquet_scan_sql(&self.file_path, limit, Some(self.next_offset)),
Some(limit),
)
.await?;
if result.rows.is_empty() {
return Err(format!(
"DuckDB Parquet scan ended before the expected row count (read {}, expected {})",
self.next_offset, self.total_rows
));
}
self.next_offset = self.next_offset.saturating_add(result.rows.len());
Ok(Some(result.rows))
}
}
enum ImportRowSource {
Materialized { columns: Vec<String>, rows: std::vec::IntoIter<Vec<serde_json::Value>>, total_rows: usize },
Sql { stream: Box<SqlImportRowStream>, bytes_read: Arc<AtomicU64>, pending: Option<Vec<Vec<serde_json::Value>>> },
Json { stream: Box<JsonImportRowStream> },
Materialized {
columns: Vec<String>,
rows: std::vec::IntoIter<Vec<serde_json::Value>>,
total_rows: usize,
},
Sql {
stream: Box<SqlImportRowStream>,
bytes_read: Arc<AtomicU64>,
pending: Option<Vec<Vec<serde_json::Value>>>,
},
Json {
stream: Box<JsonImportRowStream>,
},
#[cfg(feature = "duckdb-sidecar")]
DuckDb {
stream: Box<DuckDbParquetRowStream>,
},
}
impl ImportRowSource {
@@ -2189,6 +2401,7 @@ impl ImportRowSource {
parse_options: &TableImportParseOptions,
text_source_columns: HashSet<String>,
first_batch_rows: usize,
duckdb_context: Option<DuckDbImportContext>,
) -> Result<Self, String> {
if source_format == TableImportSourceFormat::Sql {
let bytes_read = Arc::new(AtomicU64::new(0));
@@ -2200,6 +2413,12 @@ impl ImportRowSource {
let stream = JsonImportRowStream::open(file_path, parse_options, first_batch_rows).await?;
return Ok(Self::Json { stream: Box::new(stream) });
}
#[cfg(feature = "duckdb-sidecar")]
if source_format == TableImportSourceFormat::Parquet {
let context = duckdb_context.ok_or_else(|| "Parquet import requires a DuckDB connection".to_string())?;
let stream = DuckDbParquetRowStream::open(context, file_path, first_batch_rows).await?;
return Ok(Self::DuckDb { stream: Box::new(stream) });
}
let parsed = parse_import_file_with_options_and_text_columns(
file_path,
Some(source_format),
@@ -2218,6 +2437,8 @@ impl ImportRowSource {
stream.columns().ok_or_else(|| "No INSERT statements found in SQL file".to_string())
}
Self::Json { stream, .. } => Ok(stream.columns.clone()),
#[cfg(feature = "duckdb-sidecar")]
Self::DuckDb { stream } => Ok(stream.columns()),
}
}
@@ -2226,6 +2447,8 @@ impl ImportRowSource {
Self::Materialized { total_rows, .. } => *total_rows,
Self::Sql { stream, .. } => stream.total_rows(),
Self::Json { stream, .. } => stream.total_rows,
#[cfg(feature = "duckdb-sidecar")]
Self::DuckDb { stream } => stream.total_rows(),
}
}
@@ -2243,6 +2466,8 @@ impl ImportRowSource {
Self::Materialized { .. } => total_bytes,
Self::Sql { bytes_read, .. } => bytes_read.load(Ordering::Relaxed).min(total_bytes),
Self::Json { stream, .. } => stream.bytes_read.load(Ordering::Relaxed).min(total_bytes),
#[cfg(feature = "duckdb-sidecar")]
Self::DuckDb { .. } => total_bytes,
}
}
@@ -2259,6 +2484,8 @@ impl ImportRowSource {
stream.next_batch(max_rows).await
}
Self::Json { stream, .. } => stream.receiver.recv().await.transpose(),
#[cfg(feature = "duckdb-sidecar")]
Self::DuckDb { stream } => stream.next_batch(max_rows).await,
}
}
}
@@ -4178,7 +4405,11 @@ pub fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result<ParsedImportF
fn ensure_non_streaming_file_size(path: &str, format: TableImportSourceFormat) -> Result<(), String> {
// 分隔文本、SQL 脚本与 JSON 都是流式解析,内存占用不随文件体积增长,没有体积上限。
if format.is_delimited() || format == TableImportSourceFormat::Sql || format == TableImportSourceFormat::Json {
if format.is_delimited()
|| format == TableImportSourceFormat::Sql
|| format == TableImportSourceFormat::Json
|| format == TableImportSourceFormat::Parquet
{
return Ok(());
}
let metadata = std::fs::metadata(path).map_err(|e| e.to_string())?;
@@ -4247,6 +4478,7 @@ async fn parse_import_file_with_options_and_text_columns(
.await
.map_err(|e| e.to_string())?
}
TableImportSourceFormat::Parquet => Err("Parquet import requires a DuckDB connection".to_string()),
}
}
@@ -4291,6 +4523,52 @@ async fn parse_import_preview_file_with_options(
Ok((parsed, true, sheets))
}
#[cfg(feature = "duckdb-sidecar")]
async fn parse_duckdb_parquet_preview(
context: &DuckDbImportContext,
path: &str,
preview_limit: usize,
) -> Result<ParsedImportFile, String> {
let total_rows = duckdb_count_result(&context.execute(duckdb_parquet_count_sql(path), Some(1)).await?)?;
if total_rows == 0 {
return Err("Parquet file has no rows".to_string());
}
let preview_limit = preview_limit.max(1);
let result = context
.execute_preserving_insertion_order(duckdb_parquet_scan_sql(path, preview_limit, None), Some(preview_limit))
.await?;
if result.columns.is_empty() {
return Err("Parquet file has no columns".to_string());
}
if total_rows > 0 && result.rows.is_empty() {
return Err("DuckDB Parquet scan returned no rows for a non-empty file".to_string());
}
Ok(ParsedImportFile { columns: result.columns, rows: result.rows, total_rows, effective_encoding: None })
}
async fn parse_import_preview_file_with_context(
path: &str,
format: TableImportSourceFormat,
options: &TableImportParseOptions,
preview_limit: usize,
duckdb_context: Option<&DuckDbImportContext>,
) -> Result<(ParsedImportFile, bool, Vec<String>), String> {
if format == TableImportSourceFormat::Parquet {
#[cfg(feature = "duckdb-sidecar")]
{
let context = duckdb_context.ok_or_else(|| "Parquet import requires a DuckDB connection".to_string())?;
let parsed = parse_duckdb_parquet_preview(context, path, preview_limit).await?;
return Ok((parsed, true, Vec::new()));
}
#[cfg(not(feature = "duckdb-sidecar"))]
{
let _ = (path, options, preview_limit, duckdb_context);
return Err("DuckDB worker support is not compiled in this build".to_string());
}
}
parse_import_preview_file_with_options(path, format, options, preview_limit).await
}
pub async fn parse_import_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
parse_import_file_with_options(path, None, &TableImportParseOptions::default(), preview_limit).await
}
@@ -6164,15 +6442,17 @@ fn validated_prepared_import_source(
})
}
pub async fn preview_table_import_file_with_request(
async fn preview_table_import_file_with_context(
request: TableImportPreviewRequest,
duckdb_context: Option<&DuckDbImportContext>,
) -> Result<TableImportPreview, String> {
let format = effective_source_format(&request.file_path, request.source_format)?;
let (parsed, total_rows_exact, sheets) = parse_import_preview_file_with_options(
let (parsed, total_rows_exact, sheets) = parse_import_preview_file_with_context(
&request.file_path,
format,
&request.parse_options,
request.preview_limit.unwrap_or(DEFAULT_PREVIEW_LIMIT),
duckdb_context,
)
.await?;
let metadata = tokio::fs::metadata(&request.file_path).await.map_err(|e| e.to_string())?;
@@ -6198,9 +6478,50 @@ pub async fn preview_table_import_file_with_request(
})
}
pub async fn preview_table_import_file_with_request(
request: TableImportPreviewRequest,
) -> Result<TableImportPreview, String> {
preview_table_import_file_with_context(request, None).await
}
pub async fn preview_table_import_file_with_state(
state: &AppState,
request: TableImportPreviewRequest,
) -> Result<TableImportPreview, String> {
let format = effective_source_format(&request.file_path, request.source_format)?;
if format != TableImportSourceFormat::Parquet {
return preview_table_import_file_with_request(request).await;
}
let connection_id = request
.connection_id
.as_deref()
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| "Parquet import preview requires a DuckDB connection".to_string())?
.to_string();
let db_type = crate::transfer::get_db_type(state, &connection_id).await?;
if db_type != DatabaseType::DuckDb {
return Err("Parquet import is only supported for DuckDB connections".to_string());
}
let database = request.database.clone().filter(|value| !value.trim().is_empty());
let client_session_id = format!("table-import-preview-{}", uuid::Uuid::new_v4());
let pool_key =
state.get_or_create_pool_for_session(&connection_id, database.as_deref(), Some(&client_session_id)).await?;
let context =
duckdb_import_context_for_source(state, &pool_key, &db_type, database.as_deref().unwrap_or_default(), format)
.await;
let result = match context {
Ok(context) => preview_table_import_file_with_context(request, context.as_ref()).await,
Err(error) => Err(error),
};
let _ = state.detach_client_session_pool(&connection_id, database.as_deref(), &client_session_id).await;
result
}
pub async fn preview_table_import_file_core(file_path: &str) -> Result<TableImportPreview, String> {
preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: file_path.to_string(),
connection_id: None,
database: None,
source_ref: None,
source_format: None,
parse_options: TableImportParseOptions::default(),
@@ -6731,6 +7052,11 @@ where
format!("Import source is no longer available: {error}"),
));
}
let duckdb_context =
match duckdb_import_context_for_source(state, pool_key, db_type, &request.database, source_format).await {
Ok(context) => context,
Err(error) => return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error)),
};
let import_sql_hard_limit = mysql_import_sql_hard_limit(state, pool_key).await;
let prepared_source = validated_prepared_import_source(request, source_format);
let prepared_source_total_exact =
@@ -6831,11 +7157,12 @@ where
{
prepared
} else {
match parse_import_preview_file_with_options(
match parse_import_preview_file_with_context(
&request.file_path,
source_format,
&import_parse_options,
CREATE_TABLE_INFERENCE_ROWS,
duckdb_context.as_ref(),
)
.await
{
@@ -7769,6 +8096,7 @@ where
&import_parse_options,
text_source_columns,
effective_batch_size,
duckdb_context.clone(),
)
.await
{
@@ -8056,6 +8384,30 @@ mod tests {
.collect()
}
#[test]
fn parquet_extension_maps_to_the_duckdb_source_format() {
assert_eq!(source_format_for_path("sales.PARQUET").unwrap(), TableImportSourceFormat::Parquet);
assert_eq!(import_file_kind("sales.parquet").unwrap(), ImportFileKind::Parquet);
}
#[test]
fn parquet_scan_sql_escapes_paths_and_adds_offset_after_limit() {
let sql = duckdb_parquet_scan_sql(r"C:\data\customer's.parquet", 250, Some(500));
assert_eq!(sql, "SELECT * FROM read_parquet('C:\\data\\customer''s.parquet') LIMIT 250 OFFSET 500");
}
#[test]
fn parquet_source_size_is_not_rejected_by_the_materialized_file_limit() {
let path =
std::env::temp_dir().join(format!("dbx-table-import-parquet-limit-{}.parquet", uuid::Uuid::new_v4()));
let file = File::create(&path).unwrap();
file.set_len(MAX_NON_STREAMING_IMPORT_BYTES + 1).unwrap();
ensure_non_streaming_file_size(&path.to_string_lossy(), TableImportSourceFormat::Parquet).unwrap();
let _ = std::fs::remove_file(path);
}
#[test]
fn table_import_progress_and_summary_report_elapsed_ms() {
let started_at = std::time::Instant::now() - std::time::Duration::from_millis(25);
@@ -9248,6 +9600,8 @@ mod tests {
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: path.to_string_lossy().to_string(),
connection_id: None,
database: None,
source_ref: None,
source_format: Some(TableImportSourceFormat::Csv),
parse_options: TableImportParseOptions::default(),
@@ -12845,6 +13199,8 @@ mod tests {
let path = std::env::temp_dir().join(format!("dbx-missing-import-{}.csv", uuid::Uuid::new_v4()));
let error = preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: path.to_string_lossy().to_string(),
connection_id: None,
database: None,
source_ref: Some("missing".to_string()),
source_format: Some(TableImportSourceFormat::Csv),
parse_options: TableImportParseOptions::default(),