feat(table-import): add DuckDB Parquet imports

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-09-28 21:38:32 +08:00
1 parent f61f64f727
commit 02f27dc294
26 files changed
+686 -46

No files matched your search

@@ -162,6 +162,7 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe
{ value: "json", icon: FileJson, labelKey: "tableImport.formatJson", descriptionKey: "tableImport.formatJsonDescription" },
{ value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" },
{ value: "sql", icon: FileCode, labelKey: "tableImport.formatSql", descriptionKey: "tableImport.formatSqlDescription" },
{ value: "parquet", icon: FileText, labelKey: "tableImport.formatParquet", descriptionKey: "tableImport.formatParquetDescription" },
];
const encodingOptions = TABLE_IMPORT_ENCODING_OPTIONS;
@@ -176,6 +177,9 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
const supportsDuckDbParquetImport = computed(() => structureDatabaseType.value === "duckdb");
const availableFormatOptions = computed(() => formatOptions.filter((format) => format.value !== "parquet" || supportsDuckDbParquetImport.value));
const importFileExtensions = computed(() => ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql", ...(supportsDuckDbParquetImport.value ? ["parquet"] : [])]);
// Mirrors the conflict SQL dispatch in transfer.rs. Other dialects must keep
// ordinary INSERT/error behavior instead of approximating an upsert.
const IMPORT_CONFLICT_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
@@ -426,6 +430,7 @@ function detectFormat(name: string): api.TableImportSourceFormat {
if (lower.endsWith(".json")) return "json";
if (lower.endsWith(".xls") || lower.endsWith(".xlsx") || lower.endsWith(".xlsm")) return "excel";
if (lower.endsWith(".sql")) return "sql";
if (lower.endsWith(".parquet")) return "parquet";
return "csv";
}
@@ -671,6 +676,8 @@ async function loadTargetColumns() {
async function previewSelectedImportFile(fileOrPath: string | File) {
const input = importPreviewInput(uploadedImportSourceFromPreview(preview.value), fileOrPath);
return api.previewTableImportFile(input.fileOrPath, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: sourceFormat.value,
parseOptions: parseOptions.value,
@@ -704,6 +711,13 @@ async function loadPreview(fileOrPath = selectedSource.value) {
}
function assignSelectedSource(source: string | File) {
const detectedFormat = detectFormat(typeof source === "string" ? source : source.name);
if (detectedFormat === "parquet" && !supportsDuckDbParquetImport.value) {
selectedSource.value = null;
preview.value = null;
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
return;
}
batchTasks.value = [];
selectedSource.value = source;
preview.value = null;
@@ -712,7 +726,7 @@ function assignSelectedSource(source: string | File) {
progress.value = null;
errorMessage.value = "";
const name = typeof source === "string" ? source : source.name;
sourceFormat.value = detectFormat(name);
sourceFormat.value = detectedFormat;
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
if (!newTableName.value.trim()) {
newTableName.value = suggestedTableName(name);
@@ -750,11 +764,18 @@ async function prepareBatchSources(sources: ImportSource[]) {
const tasks: BatchImportTask[] = [];
const usedNames = new Set<string>();
const formats = sources.map((source) => detectFormat(sourceName(source)));
if (formats.includes("parquet") && !supportsDuckDbParquetImport.value) {
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
loadingPreview.value = false;
return;
}
emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true;
try {
for (const [index, source] of sources.entries()) {
const format = formats[index]!;
const initialPreview = await api.previewTableImportFile(source, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceFormat: format,
parseOptions: taskParseOptions(format),
previewLimit: Math.max(1, Number(previewLimit.value) || 50),
@@ -765,6 +786,8 @@ async function prepareBatchSources(sources: ImportSource[]) {
const effectiveSheetName = sheetName && sheetName === initialPreview.sheets?.[0] ? "" : sheetName;
const taskPreview = effectiveSheetName
? await api.previewTableImportFile(input.fileOrPath, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: format,
parseOptions: taskParseOptions(format, effectiveSheetName),
@@ -809,11 +832,12 @@ async function selectFile() {
const selected = await open({
multiple: targetMode.value === "create",
filters: [
{ name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql"] },
{ name: "Data files", extensions: importFileExtensions.value },
{ name: "Text", extensions: ["csv", "tsv", "txt"] },
{ name: "JSON", extensions: ["json"] },
{ name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
{ name: "SQL", extensions: ["sql"] },
...(supportsDuckDbParquetImport.value ? [{ name: "Parquet", extensions: ["parquet"] }] : []),
],
});
if (!selected) return;
@@ -1156,6 +1180,8 @@ async function reloadBatchPreviewsForEncoding() {
if (!isDelimitedFormat(task.format) && task.format !== "sql") continue;
const input = importPreviewInput(uploadedImportSourceFromPreview(task.preview), task.source);
const nextPreview = await api.previewTableImportFile(input.fileOrPath, {
connectionId: props.prefillConnectionId,
database: props.prefillDatabase || "",
sourceRef: input.sourceRef,
sourceFormat: task.format,
parseOptions: taskParseOptions(task.format, task.sheetName),
@@ -1254,7 +1280,7 @@ watch(rawProgressPercent, (percent) => {
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1">
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
<input ref="fileInput" type="file" accept=".csv,.tsv,.txt,.json,.xlsx,.xlsm,.xls,.sql" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
<input ref="fileInput" type="file" :accept="importFileExtensions.map((extension) => `.${extension}`).join(',')" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
<span class="shrink-0 text-xs text-muted-foreground">{{ t("tableImport.target") }}</span>
<span class="min-w-0 truncate text-sm font-medium">
@@ -1329,8 +1355,15 @@ watch(rawProgressPercent, (percent) => {
{{ t("tableImport.selectFile") }}
</Button>
</div>
<div class="grid grid-cols-6 gap-2">
<button v-for="format in formatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
<div
data-testid="table-import-format-options"
class="grid grid-cols-2 gap-2 sm:grid-cols-3"
:class="{
'lg:grid-cols-6': availableFormatOptions.length === 6,
'lg:grid-cols-7': availableFormatOptions.length === 7,
}"
>
<button v-for="format in availableFormatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
<component :is="format.icon" class="mb-2 h-4 w-4 text-muted-foreground" />
<div class="text-xs font-medium">{{ t(format.labelKey) }}</div>
<div class="mt-1 text-[11px] leading-snug text-muted-foreground">{{ t(format.descriptionKey) }}</div>
@@ -1347,7 +1380,7 @@ watch(rawProgressPercent, (percent) => {
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem v-for="format in formatOptions" :key="format.value" :value="format.value">
<SelectItem v-for="format in availableFormatOptions" :key="format.value" :value="format.value">
{{ t(format.labelKey) }}
</SelectItem>
</SelectContent>
@@ -27,6 +27,7 @@ vi.mock("@/stores/connectionStore", () => ({
if (id === "connection-1") return { id, name: "SQLite", db_type: "sqlite" };
if (id === "postgres-1") return { id, name: "PostgreSQL", db_type: "postgres" };
if (id === "sqlserver-1") return { id, name: "SQL Server", db_type: "sqlserver" };
if (id === "duckdb-1") return { id, name: "DuckDB", db_type: "duckdb" };
return undefined;
},
ensureConnected: mocks.ensureConnected,
@@ -198,7 +199,7 @@ async function flushAsyncUpdates() {
}
}
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
async function mountSourceDialog(connectionId = "connection-1") {
const container = document.createElement("div");
document.body.append(container);
const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: connectionId, prefillDatabase: "main" }) }));
@@ -206,12 +207,20 @@ async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connec
app.use(i18n);
app.mount(container);
await flushAsyncUpdates();
}
async function selectFiles(files: File[]) {
const input = document.body.querySelector<HTMLInputElement>('input[type="file"]')!;
Object.defineProperty(input, "files", { configurable: true, value: files });
input.dispatchEvent(new Event("change", { bubbles: true }));
await flushAsyncUpdates();
}
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
await mountSourceDialog(connectionId);
await selectFiles(files);
}
function button(text: string) {
const result = [...document.body.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.trim() === text);
expect(result, `button ${text}`).toBeTruthy();
@@ -270,6 +279,26 @@ afterEach(() => {
});
describe("TableImportDialog batch selection", () => {
it("uses all desktop grid columns when Parquet is unavailable", async () => {
await mountSourceDialog();
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
expect(formats?.classList.contains("lg:grid-cols-6")).toBe(true);
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(false);
});
it("offers Parquet for DuckDB and forwards the connection context to preview", async () => {
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve({ ...delimitedPreview(), fileName: "rows.parquet", fileType: "parquet", sourceFingerprint: "rows-parquet" }));
await mountSourceDialog("duckdb-1");
const fileInput = document.body.querySelector<HTMLInputElement>('input[type="file"]');
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
expect(fileInput?.accept).toContain(".parquet");
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(true);
await selectFiles([new File(["parquet"], "rows.parquet")]);
expect(mocks.previewTableImportFile).toHaveBeenCalledWith(expect.any(File), expect.objectContaining({ connectionId: "duckdb-1", database: "main", sourceFormat: "parquet" }));
});
it("lets the import setup dialog be minimized and restored before import starts", async () => {
await mountDialog();
expect(button("Next").disabled).toBe(false);