feat(table-import): add DuckDB Parquet imports
This commit is contained in:
1 parent
f61f64f727
commit
02f27dc294
26 files changed
+686
-46
No files matched your search
@@ -24,6 +24,7 @@ use sqlparser::dialect::{GenericDialect, MsSqlDialect, MySqlDialect, OracleDiale
|
||||
use sqlparser::parser::Parser;
|
||||
|
||||
use crate::connection::{task_client_session_id, AppState, PoolKind};
|
||||
use crate::db;
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::sql::SqlParsingOptions;
|
||||
use crate::sql_file_import::{SqlFileStreamDecoder, StreamingSqlFileSplitter};
|
||||
@@ -135,6 +136,7 @@ pub enum TableImportSourceFormat {
|
||||
Json,
|
||||
Excel,
|
||||
Sql,
|
||||
Parquet,
|
||||
}
|
||||
|
||||
impl TableImportSourceFormat {
|
||||
@@ -146,6 +148,7 @@ impl TableImportSourceFormat {
|
||||
TableImportSourceFormat::Json => "json",
|
||||
TableImportSourceFormat::Excel => "excel",
|
||||
TableImportSourceFormat::Sql => "sql",
|
||||
TableImportSourceFormat::Parquet => "parquet",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,6 +240,10 @@ impl Default for TableImportParseOptions {
|
||||
pub struct TableImportPreviewRequest {
|
||||
pub file_path: String,
|
||||
#[serde(default)]
|
||||
pub connection_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub database: Option<String>,
|
||||
#[serde(default)]
|
||||
pub source_ref: Option<String>,
|
||||
#[serde(default)]
|
||||
pub source_format: Option<TableImportSourceFormat>,
|
||||
@@ -447,6 +454,7 @@ pub enum ImportFileKind {
|
||||
Json,
|
||||
Xlsx,
|
||||
Sql,
|
||||
Parquet,
|
||||
}
|
||||
|
||||
impl ImportFileKind {
|
||||
@@ -458,6 +466,7 @@ impl ImportFileKind {
|
||||
ImportFileKind::Json => "json",
|
||||
ImportFileKind::Xlsx => "xlsx",
|
||||
ImportFileKind::Sql => "sql",
|
||||
ImportFileKind::Parquet => "parquet",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -476,6 +485,8 @@ pub fn import_file_kind(path: &str) -> Result<ImportFileKind, String> {
|
||||
Ok(ImportFileKind::Xlsx)
|
||||
} else if lower.ends_with(".sql") {
|
||||
Ok(ImportFileKind::Sql)
|
||||
} else if lower.ends_with(".parquet") {
|
||||
Ok(ImportFileKind::Parquet)
|
||||
} else {
|
||||
Err("Unsupported import file type".to_string())
|
||||
}
|
||||
@@ -489,6 +500,7 @@ pub fn source_format_for_path(path: &str) -> Result<TableImportSourceFormat, Str
|
||||
ImportFileKind::Json => TableImportSourceFormat::Json,
|
||||
ImportFileKind::Xlsx => TableImportSourceFormat::Excel,
|
||||
ImportFileKind::Sql => TableImportSourceFormat::Sql,
|
||||
ImportFileKind::Parquet => TableImportSourceFormat::Parquet,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -501,6 +513,117 @@ pub fn effective_source_format(
|
||||
.ok_or_else(|| "Unsupported import file type".to_string())
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
#[derive(Clone)]
|
||||
struct DuckDbImportContext {
|
||||
client: Arc<db::duckdb_worker_process::DuckDbWorkerClient>,
|
||||
database: Option<String>,
|
||||
query_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
#[derive(Clone)]
|
||||
struct DuckDbImportContext;
|
||||
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
impl DuckDbImportContext {
|
||||
async fn execute(&self, sql: String, max_rows: Option<usize>) -> Result<db::QueryResult, String> {
|
||||
self.client.execute(self.database.clone(), sql, max_rows, None, self.query_timeout).await
|
||||
}
|
||||
|
||||
async fn execute_preserving_insertion_order(
|
||||
&self,
|
||||
sql: String,
|
||||
max_rows: Option<usize>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
// Each Parquet page is an independent query. Ask the worker to scope
|
||||
// DuckDB's order-preservation setting to that request so OFFSET refers
|
||||
// to the same source order even when the connection configured it off.
|
||||
self.client
|
||||
.execute_preserving_insertion_order(self.database.clone(), sql, max_rows, None, self.query_timeout)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
async fn duckdb_import_context_for_pool(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
database: &str,
|
||||
) -> Result<DuckDbImportContext, String> {
|
||||
let pool = state.pool_handle(pool_key).await.ok_or_else(|| "Connection pool not found".to_string())?;
|
||||
let client = match pool {
|
||||
PoolKind::DuckDbWorker(client) => client,
|
||||
_ => return Err("Parquet import requires a DuckDB connection".to_string()),
|
||||
};
|
||||
let query_timeout = crate::query::operation_budget_for_pool_key(state, pool_key, None).await.query_timeout;
|
||||
Ok(DuckDbImportContext {
|
||||
client,
|
||||
database: (!database.trim().is_empty()).then_some(database.to_string()),
|
||||
query_timeout,
|
||||
})
|
||||
}
|
||||
|
||||
async fn duckdb_import_context_for_source(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
db_type: &DatabaseType,
|
||||
database: &str,
|
||||
source_format: TableImportSourceFormat,
|
||||
) -> Result<Option<DuckDbImportContext>, String> {
|
||||
if source_format != TableImportSourceFormat::Parquet {
|
||||
return Ok(None);
|
||||
}
|
||||
if *db_type != DatabaseType::DuckDb {
|
||||
return Err("Parquet import is only supported for DuckDB connections".to_string());
|
||||
}
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
{
|
||||
return duckdb_import_context_for_pool(state, pool_key, database).await.map(Some);
|
||||
}
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
{
|
||||
let _ = (state, pool_key, database);
|
||||
Err("DuckDB worker support is not compiled in this build".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn duckdb_sql_string_literal(value: &str) -> String {
|
||||
format!("'{}'", value.replace('\'', "''"))
|
||||
}
|
||||
|
||||
fn duckdb_parquet_scan_sql(path: &str, limit: usize, offset: Option<usize>) -> String {
|
||||
let mut sql = format!("SELECT * FROM read_parquet({}) LIMIT {}", duckdb_sql_string_literal(path), limit);
|
||||
if let Some(offset) = offset {
|
||||
sql.push_str(&format!(" OFFSET {offset}"));
|
||||
}
|
||||
sql
|
||||
}
|
||||
|
||||
fn duckdb_parquet_count_sql(path: &str) -> String {
|
||||
format!("SELECT COUNT(*) AS dbx_total_rows FROM read_parquet({})", duckdb_sql_string_literal(path))
|
||||
}
|
||||
|
||||
fn duckdb_count_result(result: &db::QueryResult) -> Result<usize, String> {
|
||||
let value = result
|
||||
.rows
|
||||
.first()
|
||||
.and_then(|row| row.first())
|
||||
.ok_or_else(|| "DuckDB Parquet row-count query returned no result".to_string())?;
|
||||
match value {
|
||||
serde_json::Value::Number(number) => number
|
||||
.as_u64()
|
||||
.and_then(|count| usize::try_from(count).ok())
|
||||
.ok_or_else(|| "DuckDB Parquet row count is outside the supported range".to_string()),
|
||||
serde_json::Value::String(value) => value
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.and_then(|count| usize::try_from(count).ok())
|
||||
.ok_or_else(|| "DuckDB Parquet row count is outside the supported range".to_string()),
|
||||
_ => Err("DuckDB Parquet row-count query returned an invalid value".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize_header(value: &str, index: usize) -> String {
|
||||
let trimmed = value.trim();
|
||||
if trimmed.is_empty() {
|
||||
@@ -2172,10 +2295,99 @@ fn stream_json_rows_to_channel(
|
||||
///
|
||||
/// 分隔文本与 Excel 仍然先把行解析进内存;`.sql` 脚本与 JSON 改用流式行来源
|
||||
/// 增量产出,内存不再随文件体积增长,因此二者都没有 100 MB 的体积上限。
|
||||
// Keep Parquet decoding in the DuckDB worker so compression and logical/nested type handling
|
||||
// stay identical to normal DuckDB queries; the import pipeline only owns mapping and writes.
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
struct DuckDbParquetRowStream {
|
||||
context: DuckDbImportContext,
|
||||
file_path: String,
|
||||
columns: Vec<String>,
|
||||
total_rows: usize,
|
||||
next_offset: usize,
|
||||
pending: Option<Vec<Vec<serde_json::Value>>>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
impl DuckDbParquetRowStream {
|
||||
async fn open(context: DuckDbImportContext, file_path: &str, batch_size: usize) -> Result<Self, String> {
|
||||
let total_rows = duckdb_count_result(&context.execute(duckdb_parquet_count_sql(file_path), Some(1)).await?)?;
|
||||
if total_rows == 0 {
|
||||
return Err("Parquet file has no rows".to_string());
|
||||
}
|
||||
let first_batch_size = batch_size.max(1);
|
||||
let first = context
|
||||
.execute_preserving_insertion_order(
|
||||
duckdb_parquet_scan_sql(file_path, first_batch_size, None),
|
||||
Some(first_batch_size),
|
||||
)
|
||||
.await?;
|
||||
if total_rows > 0 && first.rows.is_empty() {
|
||||
return Err("DuckDB Parquet scan returned no rows for a non-empty file".to_string());
|
||||
}
|
||||
Ok(Self {
|
||||
context,
|
||||
file_path: file_path.to_string(),
|
||||
columns: first.columns,
|
||||
total_rows,
|
||||
next_offset: first.rows.len(),
|
||||
pending: (!first.rows.is_empty()).then_some(first.rows),
|
||||
})
|
||||
}
|
||||
|
||||
fn columns(&self) -> Vec<String> {
|
||||
self.columns.clone()
|
||||
}
|
||||
|
||||
fn total_rows(&self) -> usize {
|
||||
self.total_rows
|
||||
}
|
||||
|
||||
async fn next_batch(&mut self, max_rows: usize) -> Result<Option<Vec<Vec<serde_json::Value>>>, String> {
|
||||
if let Some(rows) = self.pending.take() {
|
||||
return Ok(Some(rows));
|
||||
}
|
||||
if self.next_offset >= self.total_rows {
|
||||
return Ok(None);
|
||||
}
|
||||
let limit = max_rows.max(1);
|
||||
// The worker protocol exposes bounded query results but no cursor API. LIMIT/OFFSET keeps
|
||||
// each response bounded and lets the existing import writer retain its batch semantics.
|
||||
let result = self
|
||||
.context
|
||||
.execute_preserving_insertion_order(
|
||||
duckdb_parquet_scan_sql(&self.file_path, limit, Some(self.next_offset)),
|
||||
Some(limit),
|
||||
)
|
||||
.await?;
|
||||
if result.rows.is_empty() {
|
||||
return Err(format!(
|
||||
"DuckDB Parquet scan ended before the expected row count (read {}, expected {})",
|
||||
self.next_offset, self.total_rows
|
||||
));
|
||||
}
|
||||
self.next_offset = self.next_offset.saturating_add(result.rows.len());
|
||||
Ok(Some(result.rows))
|
||||
}
|
||||
}
|
||||
|
||||
enum ImportRowSource {
|
||||
Materialized { columns: Vec<String>, rows: std::vec::IntoIter<Vec<serde_json::Value>>, total_rows: usize },
|
||||
Sql { stream: Box<SqlImportRowStream>, bytes_read: Arc<AtomicU64>, pending: Option<Vec<Vec<serde_json::Value>>> },
|
||||
Json { stream: Box<JsonImportRowStream> },
|
||||
Materialized {
|
||||
columns: Vec<String>,
|
||||
rows: std::vec::IntoIter<Vec<serde_json::Value>>,
|
||||
total_rows: usize,
|
||||
},
|
||||
Sql {
|
||||
stream: Box<SqlImportRowStream>,
|
||||
bytes_read: Arc<AtomicU64>,
|
||||
pending: Option<Vec<Vec<serde_json::Value>>>,
|
||||
},
|
||||
Json {
|
||||
stream: Box<JsonImportRowStream>,
|
||||
},
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
DuckDb {
|
||||
stream: Box<DuckDbParquetRowStream>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ImportRowSource {
|
||||
@@ -2189,6 +2401,7 @@ impl ImportRowSource {
|
||||
parse_options: &TableImportParseOptions,
|
||||
text_source_columns: HashSet<String>,
|
||||
first_batch_rows: usize,
|
||||
duckdb_context: Option<DuckDbImportContext>,
|
||||
) -> Result<Self, String> {
|
||||
if source_format == TableImportSourceFormat::Sql {
|
||||
let bytes_read = Arc::new(AtomicU64::new(0));
|
||||
@@ -2200,6 +2413,12 @@ impl ImportRowSource {
|
||||
let stream = JsonImportRowStream::open(file_path, parse_options, first_batch_rows).await?;
|
||||
return Ok(Self::Json { stream: Box::new(stream) });
|
||||
}
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
if source_format == TableImportSourceFormat::Parquet {
|
||||
let context = duckdb_context.ok_or_else(|| "Parquet import requires a DuckDB connection".to_string())?;
|
||||
let stream = DuckDbParquetRowStream::open(context, file_path, first_batch_rows).await?;
|
||||
return Ok(Self::DuckDb { stream: Box::new(stream) });
|
||||
}
|
||||
let parsed = parse_import_file_with_options_and_text_columns(
|
||||
file_path,
|
||||
Some(source_format),
|
||||
@@ -2218,6 +2437,8 @@ impl ImportRowSource {
|
||||
stream.columns().ok_or_else(|| "No INSERT statements found in SQL file".to_string())
|
||||
}
|
||||
Self::Json { stream, .. } => Ok(stream.columns.clone()),
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
Self::DuckDb { stream } => Ok(stream.columns()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2226,6 +2447,8 @@ impl ImportRowSource {
|
||||
Self::Materialized { total_rows, .. } => *total_rows,
|
||||
Self::Sql { stream, .. } => stream.total_rows(),
|
||||
Self::Json { stream, .. } => stream.total_rows,
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
Self::DuckDb { stream } => stream.total_rows(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2243,6 +2466,8 @@ impl ImportRowSource {
|
||||
Self::Materialized { .. } => total_bytes,
|
||||
Self::Sql { bytes_read, .. } => bytes_read.load(Ordering::Relaxed).min(total_bytes),
|
||||
Self::Json { stream, .. } => stream.bytes_read.load(Ordering::Relaxed).min(total_bytes),
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
Self::DuckDb { .. } => total_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2259,6 +2484,8 @@ impl ImportRowSource {
|
||||
stream.next_batch(max_rows).await
|
||||
}
|
||||
Self::Json { stream, .. } => stream.receiver.recv().await.transpose(),
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
Self::DuckDb { stream } => stream.next_batch(max_rows).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4178,7 +4405,11 @@ pub fn parse_xlsx_file(path: &str, preview_limit: usize) -> Result<ParsedImportF
|
||||
|
||||
fn ensure_non_streaming_file_size(path: &str, format: TableImportSourceFormat) -> Result<(), String> {
|
||||
// 分隔文本、SQL 脚本与 JSON 都是流式解析,内存占用不随文件体积增长,没有体积上限。
|
||||
if format.is_delimited() || format == TableImportSourceFormat::Sql || format == TableImportSourceFormat::Json {
|
||||
if format.is_delimited()
|
||||
|| format == TableImportSourceFormat::Sql
|
||||
|| format == TableImportSourceFormat::Json
|
||||
|| format == TableImportSourceFormat::Parquet
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let metadata = std::fs::metadata(path).map_err(|e| e.to_string())?;
|
||||
@@ -4247,6 +4478,7 @@ async fn parse_import_file_with_options_and_text_columns(
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
TableImportSourceFormat::Parquet => Err("Parquet import requires a DuckDB connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4291,6 +4523,52 @@ async fn parse_import_preview_file_with_options(
|
||||
Ok((parsed, true, sheets))
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
async fn parse_duckdb_parquet_preview(
|
||||
context: &DuckDbImportContext,
|
||||
path: &str,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let total_rows = duckdb_count_result(&context.execute(duckdb_parquet_count_sql(path), Some(1)).await?)?;
|
||||
if total_rows == 0 {
|
||||
return Err("Parquet file has no rows".to_string());
|
||||
}
|
||||
let preview_limit = preview_limit.max(1);
|
||||
let result = context
|
||||
.execute_preserving_insertion_order(duckdb_parquet_scan_sql(path, preview_limit, None), Some(preview_limit))
|
||||
.await?;
|
||||
if result.columns.is_empty() {
|
||||
return Err("Parquet file has no columns".to_string());
|
||||
}
|
||||
if total_rows > 0 && result.rows.is_empty() {
|
||||
return Err("DuckDB Parquet scan returned no rows for a non-empty file".to_string());
|
||||
}
|
||||
Ok(ParsedImportFile { columns: result.columns, rows: result.rows, total_rows, effective_encoding: None })
|
||||
}
|
||||
|
||||
async fn parse_import_preview_file_with_context(
|
||||
path: &str,
|
||||
format: TableImportSourceFormat,
|
||||
options: &TableImportParseOptions,
|
||||
preview_limit: usize,
|
||||
duckdb_context: Option<&DuckDbImportContext>,
|
||||
) -> Result<(ParsedImportFile, bool, Vec<String>), String> {
|
||||
if format == TableImportSourceFormat::Parquet {
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
{
|
||||
let context = duckdb_context.ok_or_else(|| "Parquet import requires a DuckDB connection".to_string())?;
|
||||
let parsed = parse_duckdb_parquet_preview(context, path, preview_limit).await?;
|
||||
return Ok((parsed, true, Vec::new()));
|
||||
}
|
||||
#[cfg(not(feature = "duckdb-sidecar"))]
|
||||
{
|
||||
let _ = (path, options, preview_limit, duckdb_context);
|
||||
return Err("DuckDB worker support is not compiled in this build".to_string());
|
||||
}
|
||||
}
|
||||
parse_import_preview_file_with_options(path, format, options, preview_limit).await
|
||||
}
|
||||
|
||||
pub async fn parse_import_file(path: &str, preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
parse_import_file_with_options(path, None, &TableImportParseOptions::default(), preview_limit).await
|
||||
}
|
||||
@@ -6164,15 +6442,17 @@ fn validated_prepared_import_source(
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn preview_table_import_file_with_request(
|
||||
async fn preview_table_import_file_with_context(
|
||||
request: TableImportPreviewRequest,
|
||||
duckdb_context: Option<&DuckDbImportContext>,
|
||||
) -> Result<TableImportPreview, String> {
|
||||
let format = effective_source_format(&request.file_path, request.source_format)?;
|
||||
let (parsed, total_rows_exact, sheets) = parse_import_preview_file_with_options(
|
||||
let (parsed, total_rows_exact, sheets) = parse_import_preview_file_with_context(
|
||||
&request.file_path,
|
||||
format,
|
||||
&request.parse_options,
|
||||
request.preview_limit.unwrap_or(DEFAULT_PREVIEW_LIMIT),
|
||||
duckdb_context,
|
||||
)
|
||||
.await?;
|
||||
let metadata = tokio::fs::metadata(&request.file_path).await.map_err(|e| e.to_string())?;
|
||||
@@ -6198,9 +6478,50 @@ pub async fn preview_table_import_file_with_request(
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn preview_table_import_file_with_request(
|
||||
request: TableImportPreviewRequest,
|
||||
) -> Result<TableImportPreview, String> {
|
||||
preview_table_import_file_with_context(request, None).await
|
||||
}
|
||||
|
||||
pub async fn preview_table_import_file_with_state(
|
||||
state: &AppState,
|
||||
request: TableImportPreviewRequest,
|
||||
) -> Result<TableImportPreview, String> {
|
||||
let format = effective_source_format(&request.file_path, request.source_format)?;
|
||||
if format != TableImportSourceFormat::Parquet {
|
||||
return preview_table_import_file_with_request(request).await;
|
||||
}
|
||||
let connection_id = request
|
||||
.connection_id
|
||||
.as_deref()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.ok_or_else(|| "Parquet import preview requires a DuckDB connection".to_string())?
|
||||
.to_string();
|
||||
let db_type = crate::transfer::get_db_type(state, &connection_id).await?;
|
||||
if db_type != DatabaseType::DuckDb {
|
||||
return Err("Parquet import is only supported for DuckDB connections".to_string());
|
||||
}
|
||||
let database = request.database.clone().filter(|value| !value.trim().is_empty());
|
||||
let client_session_id = format!("table-import-preview-{}", uuid::Uuid::new_v4());
|
||||
let pool_key =
|
||||
state.get_or_create_pool_for_session(&connection_id, database.as_deref(), Some(&client_session_id)).await?;
|
||||
let context =
|
||||
duckdb_import_context_for_source(state, &pool_key, &db_type, database.as_deref().unwrap_or_default(), format)
|
||||
.await;
|
||||
let result = match context {
|
||||
Ok(context) => preview_table_import_file_with_context(request, context.as_ref()).await,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
let _ = state.detach_client_session_pool(&connection_id, database.as_deref(), &client_session_id).await;
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn preview_table_import_file_core(file_path: &str) -> Result<TableImportPreview, String> {
|
||||
preview_table_import_file_with_request(TableImportPreviewRequest {
|
||||
file_path: file_path.to_string(),
|
||||
connection_id: None,
|
||||
database: None,
|
||||
source_ref: None,
|
||||
source_format: None,
|
||||
parse_options: TableImportParseOptions::default(),
|
||||
@@ -6731,6 +7052,11 @@ where
|
||||
format!("Import source is no longer available: {error}"),
|
||||
));
|
||||
}
|
||||
let duckdb_context =
|
||||
match duckdb_import_context_for_source(state, pool_key, db_type, &request.database, source_format).await {
|
||||
Ok(context) => context,
|
||||
Err(error) => return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error)),
|
||||
};
|
||||
let import_sql_hard_limit = mysql_import_sql_hard_limit(state, pool_key).await;
|
||||
let prepared_source = validated_prepared_import_source(request, source_format);
|
||||
let prepared_source_total_exact =
|
||||
@@ -6831,11 +7157,12 @@ where
|
||||
{
|
||||
prepared
|
||||
} else {
|
||||
match parse_import_preview_file_with_options(
|
||||
match parse_import_preview_file_with_context(
|
||||
&request.file_path,
|
||||
source_format,
|
||||
&import_parse_options,
|
||||
CREATE_TABLE_INFERENCE_ROWS,
|
||||
duckdb_context.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -7769,6 +8096,7 @@ where
|
||||
&import_parse_options,
|
||||
text_source_columns,
|
||||
effective_batch_size,
|
||||
duckdb_context.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -8056,6 +8384,30 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parquet_extension_maps_to_the_duckdb_source_format() {
|
||||
assert_eq!(source_format_for_path("sales.PARQUET").unwrap(), TableImportSourceFormat::Parquet);
|
||||
assert_eq!(import_file_kind("sales.parquet").unwrap(), ImportFileKind::Parquet);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parquet_scan_sql_escapes_paths_and_adds_offset_after_limit() {
|
||||
let sql = duckdb_parquet_scan_sql(r"C:\data\customer's.parquet", 250, Some(500));
|
||||
|
||||
assert_eq!(sql, "SELECT * FROM read_parquet('C:\\data\\customer''s.parquet') LIMIT 250 OFFSET 500");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parquet_source_size_is_not_rejected_by_the_materialized_file_limit() {
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("dbx-table-import-parquet-limit-{}.parquet", uuid::Uuid::new_v4()));
|
||||
let file = File::create(&path).unwrap();
|
||||
file.set_len(MAX_NON_STREAMING_IMPORT_BYTES + 1).unwrap();
|
||||
|
||||
ensure_non_streaming_file_size(&path.to_string_lossy(), TableImportSourceFormat::Parquet).unwrap();
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_import_progress_and_summary_report_elapsed_ms() {
|
||||
let started_at = std::time::Instant::now() - std::time::Duration::from_millis(25);
|
||||
@@ -9248,6 +9600,8 @@ mod tests {
|
||||
|
||||
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
|
||||
file_path: path.to_string_lossy().to_string(),
|
||||
connection_id: None,
|
||||
database: None,
|
||||
source_ref: None,
|
||||
source_format: Some(TableImportSourceFormat::Csv),
|
||||
parse_options: TableImportParseOptions::default(),
|
||||
@@ -12845,6 +13199,8 @@ mod tests {
|
||||
let path = std::env::temp_dir().join(format!("dbx-missing-import-{}.csv", uuid::Uuid::new_v4()));
|
||||
let error = preview_table_import_file_with_request(TableImportPreviewRequest {
|
||||
file_path: path.to_string_lossy().to_string(),
|
||||
connection_id: None,
|
||||
database: None,
|
||||
source_ref: Some("missing".to_string()),
|
||||
source_format: Some(TableImportSourceFormat::Csv),
|
||||
parse_options: TableImportParseOptions::default(),
|
||||
|
||||
Reference in new issue
Block a user