feat(table-import): add DuckDB Parquet imports

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-09-28 21:38:32 +08:00
1 parent f61f64f727
commit 02f27dc294
26 files changed
+686 -46

No files matched your search

@@ -162,6 +162,7 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe
{ value: "json", icon: FileJson, labelKey: "tableImport.formatJson", descriptionKey: "tableImport.formatJsonDescription" },
{ value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" },
{ value: "sql", icon: FileCode, labelKey: "tableImport.formatSql", descriptionKey: "tableImport.formatSqlDescription" },
{ value: "parquet", icon: FileText, labelKey: "tableImport.formatParquet", descriptionKey: "tableImport.formatParquetDescription" },
];
const encodingOptions = TABLE_IMPORT_ENCODING_OPTIONS;
@@ -176,6 +177,9 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
const supportsDuckDbParquetImport = computed(() => structureDatabaseType.value === "duckdb");
const availableFormatOptions = computed(() => formatOptions.filter((format) => format.value !== "parquet" || supportsDuckDbParquetImport.value));
const importFileExtensions = computed(() => ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql", ...(supportsDuckDbParquetImport.value ? ["parquet"] : [])]);
// Mirrors the conflict SQL dispatch in transfer.rs. Other dialects must keep
// ordinary INSERT/error behavior instead of approximating an upsert.
const IMPORT_CONFLICT_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
@@ -426,6 +430,7 @@ function detectFormat(name: string): api.TableImportSourceFormat {
if (lower.endsWith(".json")) return "json";
if (lower.endsWith(".xls") || lower.endsWith(".xlsx") || lower.endsWith(".xlsm")) return "excel";
if (lower.endsWith(".sql")) return "sql";
if (lower.endsWith(".parquet")) return "parquet";
return "csv";
}
@@ -671,6 +676,8 @@ async function loadTargetColumns() {
async function previewSelectedImportFile(fileOrPath: string | File) {
const input = importPreviewInput(uploadedImportSourceFromPreview(preview.value), fileOrPath);
return api.previewTableImportFile(input.fileOrPath, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: sourceFormat.value,
parseOptions: parseOptions.value,
@@ -704,6 +711,13 @@ async function loadPreview(fileOrPath = selectedSource.value) {
}
function assignSelectedSource(source: string | File) {
const detectedFormat = detectFormat(typeof source === "string" ? source : source.name);
if (detectedFormat === "parquet" && !supportsDuckDbParquetImport.value) {
selectedSource.value = null;
preview.value = null;
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
return;
}
batchTasks.value = [];
selectedSource.value = source;
preview.value = null;
@@ -712,7 +726,7 @@ function assignSelectedSource(source: string | File) {
progress.value = null;
errorMessage.value = "";
const name = typeof source === "string" ? source : source.name;
sourceFormat.value = detectFormat(name);
sourceFormat.value = detectedFormat;
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
if (!newTableName.value.trim()) {
newTableName.value = suggestedTableName(name);
@@ -750,11 +764,18 @@ async function prepareBatchSources(sources: ImportSource[]) {
const tasks: BatchImportTask[] = [];
const usedNames = new Set<string>();
const formats = sources.map((source) => detectFormat(sourceName(source)));
if (formats.includes("parquet") && !supportsDuckDbParquetImport.value) {
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
loadingPreview.value = false;
return;
}
emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true;
try {
for (const [index, source] of sources.entries()) {
const format = formats[index]!;
const initialPreview = await api.previewTableImportFile(source, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceFormat: format,
parseOptions: taskParseOptions(format),
previewLimit: Math.max(1, Number(previewLimit.value) || 50),
@@ -765,6 +786,8 @@ async function prepareBatchSources(sources: ImportSource[]) {
const effectiveSheetName = sheetName && sheetName === initialPreview.sheets?.[0] ? "" : sheetName;
const taskPreview = effectiveSheetName
? await api.previewTableImportFile(input.fileOrPath, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: format,
parseOptions: taskParseOptions(format, effectiveSheetName),
@@ -809,11 +832,12 @@ async function selectFile() {
const selected = await open({
multiple: targetMode.value === "create",
filters: [
{ name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql"] },
{ name: "Data files", extensions: importFileExtensions.value },
{ name: "Text", extensions: ["csv", "tsv", "txt"] },
{ name: "JSON", extensions: ["json"] },
{ name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
{ name: "SQL", extensions: ["sql"] },
...(supportsDuckDbParquetImport.value ? [{ name: "Parquet", extensions: ["parquet"] }] : []),
],
});
if (!selected) return;
@@ -1156,6 +1180,8 @@ async function reloadBatchPreviewsForEncoding() {
if (!isDelimitedFormat(task.format) && task.format !== "sql") continue;
const input = importPreviewInput(uploadedImportSourceFromPreview(task.preview), task.source);
const nextPreview = await api.previewTableImportFile(input.fileOrPath, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: task.format,
parseOptions: taskParseOptions(task.format, task.sheetName),
@@ -1254,7 +1280,7 @@ watch(rawProgressPercent, (percent) => {
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1">
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
<input ref="fileInput" type="file" accept=".csv,.tsv,.txt,.json,.xlsx,.xlsm,.xls,.sql" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
<input ref="fileInput" type="file" :accept="importFileExtensions.map((extension) => `.${extension}`).join(',')" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
<span class="shrink-0 text-xs text-muted-foreground">{{ t("tableImport.target") }}</span>
<span class="min-w-0 truncate text-sm font-medium">
@@ -1329,8 +1355,15 @@ watch(rawProgressPercent, (percent) => {
{{ t("tableImport.selectFile") }}
</Button>
</div>
<div class="grid grid-cols-6 gap-2">
<button v-for="format in formatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
<div
data-testid="table-import-format-options"
class="grid grid-cols-2 gap-2 sm:grid-cols-3"
:class="{
'lg:grid-cols-6': availableFormatOptions.length === 6,
'lg:grid-cols-7': availableFormatOptions.length === 7,
}"
>
<button v-for="format in availableFormatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
<component :is="format.icon" class="mb-2 h-4 w-4 text-muted-foreground" />
<div class="text-xs font-medium">{{ t(format.labelKey) }}</div>
<div class="mt-1 text-[11px] leading-snug text-muted-foreground">{{ t(format.descriptionKey) }}</div>
@@ -1347,7 +1380,7 @@ watch(rawProgressPercent, (percent) => {
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem v-for="format in formatOptions" :key="format.value" :value="format.value">
<SelectItem v-for="format in availableFormatOptions" :key="format.value" :value="format.value">
{{ t(format.labelKey) }}
</SelectItem>
</SelectContent>
@@ -27,6 +27,7 @@ vi.mock("@/stores/connectionStore", () => ({
if (id === "connection-1") return { id, name: "SQLite", db_type: "sqlite" };
if (id === "postgres-1") return { id, name: "PostgreSQL", db_type: "postgres" };
if (id === "sqlserver-1") return { id, name: "SQL Server", db_type: "sqlserver" };
if (id === "duckdb-1") return { id, name: "DuckDB", db_type: "duckdb" };
return undefined;
},
ensureConnected: mocks.ensureConnected,
@@ -198,7 +199,7 @@ async function flushAsyncUpdates() {
}
}
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
async function mountSourceDialog(connectionId = "connection-1") {
const container = document.createElement("div");
document.body.append(container);
const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: connectionId, prefillDatabase: "main" }) }));
@@ -206,12 +207,20 @@ async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connec
app.use(i18n);
app.mount(container);
await flushAsyncUpdates();
}
async function selectFiles(files: File[]) {
const input = document.body.querySelector<HTMLInputElement>('input[type="file"]')!;
Object.defineProperty(input, "files", { configurable: true, value: files });
input.dispatchEvent(new Event("change", { bubbles: true }));
await flushAsyncUpdates();
}
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
await mountSourceDialog(connectionId);
await selectFiles(files);
}
function button(text: string) {
const result = [...document.body.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.trim() === text);
expect(result, `button ${text}`).toBeTruthy();
@@ -270,6 +279,26 @@ afterEach(() => {
});
describe("TableImportDialog batch selection", () => {
it("uses all desktop grid columns when Parquet is unavailable", async () => {
await mountSourceDialog();
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
expect(formats?.classList.contains("lg:grid-cols-6")).toBe(true);
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(false);
});
it("offers Parquet for DuckDB and forwards the connection context to preview", async () => {
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve({ ...delimitedPreview(), fileName: "rows.parquet", fileType: "parquet", sourceFingerprint: "rows-parquet" }));
await mountSourceDialog("duckdb-1");
const fileInput = document.body.querySelector<HTMLInputElement>('input[type="file"]');
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
expect(fileInput?.accept).toContain(".parquet");
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(true);
await selectFiles([new File(["parquet"], "rows.parquet")]);
expect(mocks.previewTableImportFile).toHaveBeenCalledWith(expect.any(File), expect.objectContaining({ connectionId: "duckdb-1", database: "main", sourceFormat: "parquet" }));
});
it("lets the import setup dialog be minimized and restored before import starts", async () => {
await mountDialog();
expect(button("Next").disabled).toBe(false);
+3
View File
@@ -6273,6 +6273,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL skriptlərində INSERT əmrləri",
formatParquet: "Parquet",
formatParquetDescription: "DuckDB üçün Parquet faylları",
parquetOnlyDuckdb: "Parquet idxalı yalnız DuckDB bağlantıları üçün dəstəklənir",
previewRows: "Sətirlərin önbaxışı",
delimiter: "Ayırıcı",
encoding: "Kodlaşdırma",
+3
View File
@@ -6881,6 +6881,9 @@ export default {
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "INSERT statements in SQL scripts",
formatParquet: "Parquet",
formatParquetDescription: "Parquet files for DuckDB",
parquetOnlyDuckdb: "Parquet import is only supported for DuckDB connections",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Encoding",
+3
View File
@@ -6311,6 +6311,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Sentencias INSERT en scripts SQL",
formatParquet: "Parquet",
formatParquetDescription: "Archivos Parquet para DuckDB",
parquetOnlyDuckdb: "La importación de Parquet solo es compatible con conexiones DuckDB",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Codificación",
+3
View File
@@ -6233,6 +6233,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Istruzioni INSERT negli script SQL",
formatParquet: "Parquet",
formatParquetDescription: "File Parquet per DuckDB",
parquetOnlyDuckdb: "L'importazione Parquet è supportata solo per le connessioni DuckDB",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Codifica",
+3
View File
@@ -6252,6 +6252,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL スクリプトの INSERT ステートメント",
formatParquet: "Parquet",
formatParquetDescription: "DuckDB 用の Parquet ファイル",
parquetOnlyDuckdb: "Parquet のインポートは DuckDB 接続でのみサポートされています",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "文字コード",
+3
View File
@@ -5963,6 +5963,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL 스크립트의 INSERT 문",
formatParquet: "Parquet",
formatParquetDescription: "DuckDB용 Parquet 파일",
parquetOnlyDuckdb: "Parquet 가져오기는 DuckDB 연결에서만 지원됩니다",
previewRows: "미리보기 행",
delimiter: "구분자",
encoding: "인코딩",
+3
View File
@@ -6235,6 +6235,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Instruções INSERT em scripts SQL",
formatParquet: "Parquet",
formatParquetDescription: "Arquivos Parquet para DuckDB",
parquetOnlyDuckdb: "A importação de Parquet é compatível apenas com conexões DuckDB",
previewRows: "Preview rows",
delimiter: "Delimiter",
encoding: "Codificação",
+3
View File
@@ -7400,6 +7400,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "Операторы INSERT в скриптах SQL",
formatParquet: "Parquet",
formatParquetDescription: "Файлы Parquet для DuckDB",
parquetOnlyDuckdb: "Импорт Parquet поддерживается только для подключений DuckDB",
previewRows: "Строк предпросмотра",
delimiter: "Разделитель",
encoding: "Кодировка",
+3
View File
@@ -6155,6 +6155,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS, XLSX, XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL betiklerindeki INSERT ifadeleri",
formatParquet: "Parquet",
formatParquetDescription: "DuckDB için Parquet dosyaları",
parquetOnlyDuckdb: "Parquet içe aktarma yalnızca DuckDB bağlantılarında desteklenir",
previewRows: "Satırları önizle",
delimiter: "Ayırıcı",
encoding: "Kodlama",
+3
View File
@@ -6848,6 +6848,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS、XLSX、XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL 脚本中的 INSERT 语句",
formatParquet: "Parquet",
formatParquetDescription: "DuckDB 的 Parquet 文件",
parquetOnlyDuckdb: "Parquet 导入仅支持 DuckDB 连接",
previewRows: "预览行数",
delimiter: "分隔符",
encoding: "文件编码",
+3
View File
@@ -5542,6 +5542,9 @@ export default withEnglishFallback({
formatExcelDescription: "XLS、XLSX、XLSM",
formatSql: "SQL",
formatSqlDescription: "SQL 指令檔中的 INSERT 語句",
formatParquet: "Parquet",
formatParquetDescription: "DuckDB 的 Parquet 檔案",
parquetOnlyDuckdb: "Parquet 匯入僅支援 DuckDB 連線",
previewRows: "預覽列數",
delimiter: "分隔符",
encoding: "檔案編碼",
+4
View File
@@ -2888,6 +2888,8 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
sourceRef: options.sourceRef,
connectionId: options.connectionId,
database: options.database,
sourceFormat: options.sourceFormat,
parseOptions: options.parseOptions,
previewLimit: options.previewLimit,
@@ -2898,6 +2900,8 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
}
const formData = new FormData();
formData.append("file", fileOrPath);
if (options.connectionId) formData.append("connectionId", options.connectionId);
if (options.database != null) formData.append("database", options.database);
if (options.sourceFormat) formData.append("sourceFormat", options.sourceFormat);
if (options.parseOptions) formData.append("parseOptions", JSON.stringify(options.parseOptions));
if (options.previewLimit != null) formData.append("previewLimit", String(options.previewLimit));
+3 -1
View File
@@ -5447,7 +5447,7 @@ export type TableImportMode = "append" | "truncate";
export type TableImportConflictPolicy = "error" | "skip" | "updateExisting";
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql";
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql" | "parquet";
export type TableImportJsonShape = "auto" | "objects" | "arrays";
export type TableImportTextEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
@@ -5474,6 +5474,8 @@ export interface TableImportParseOptions {
export interface TableImportPreviewRequest {
filePath: string;
connectionId?: string | null;
database?: string | null;
sourceRef?: string | null;
sourceFormat?: TableImportSourceFormat | null;
parseOptions?: TableImportParseOptions | null;