feat(table-import): add DuckDB Parquet imports
This commit is contained in:
1 parent
f61f64f727
commit
02f27dc294
26 files changed
+686
-46
No files matched your search
@@ -162,6 +162,7 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe
|
||||
{ value: "json", icon: FileJson, labelKey: "tableImport.formatJson", descriptionKey: "tableImport.formatJsonDescription" },
|
||||
{ value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" },
|
||||
{ value: "sql", icon: FileCode, labelKey: "tableImport.formatSql", descriptionKey: "tableImport.formatSqlDescription" },
|
||||
{ value: "parquet", icon: FileText, labelKey: "tableImport.formatParquet", descriptionKey: "tableImport.formatParquetDescription" },
|
||||
];
|
||||
|
||||
const encodingOptions = TABLE_IMPORT_ENCODING_OPTIONS;
|
||||
@@ -176,6 +177,9 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
|
||||
|
||||
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
|
||||
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
|
||||
const supportsDuckDbParquetImport = computed(() => structureDatabaseType.value === "duckdb");
|
||||
const availableFormatOptions = computed(() => formatOptions.filter((format) => format.value !== "parquet" || supportsDuckDbParquetImport.value));
|
||||
const importFileExtensions = computed(() => ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql", ...(supportsDuckDbParquetImport.value ? ["parquet"] : [])]);
|
||||
// Mirrors the conflict SQL dispatch in transfer.rs. Other dialects must keep
|
||||
// ordinary INSERT/error behavior instead of approximating an upsert.
|
||||
const IMPORT_CONFLICT_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
|
||||
@@ -426,6 +430,7 @@ function detectFormat(name: string): api.TableImportSourceFormat {
|
||||
if (lower.endsWith(".json")) return "json";
|
||||
if (lower.endsWith(".xls") || lower.endsWith(".xlsx") || lower.endsWith(".xlsm")) return "excel";
|
||||
if (lower.endsWith(".sql")) return "sql";
|
||||
if (lower.endsWith(".parquet")) return "parquet";
|
||||
return "csv";
|
||||
}
|
||||
|
||||
@@ -671,6 +676,8 @@ async function loadTargetColumns() {
|
||||
async function previewSelectedImportFile(fileOrPath: string | File) {
|
||||
const input = importPreviewInput(uploadedImportSourceFromPreview(preview.value), fileOrPath);
|
||||
return api.previewTableImportFile(input.fileOrPath, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceRef: input.sourceRef,
|
||||
sourceFormat: sourceFormat.value,
|
||||
parseOptions: parseOptions.value,
|
||||
@@ -704,6 +711,13 @@ async function loadPreview(fileOrPath = selectedSource.value) {
|
||||
}
|
||||
|
||||
function assignSelectedSource(source: string | File) {
|
||||
const detectedFormat = detectFormat(typeof source === "string" ? source : source.name);
|
||||
if (detectedFormat === "parquet" && !supportsDuckDbParquetImport.value) {
|
||||
selectedSource.value = null;
|
||||
preview.value = null;
|
||||
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
|
||||
return;
|
||||
}
|
||||
batchTasks.value = [];
|
||||
selectedSource.value = source;
|
||||
preview.value = null;
|
||||
@@ -712,7 +726,7 @@ function assignSelectedSource(source: string | File) {
|
||||
progress.value = null;
|
||||
errorMessage.value = "";
|
||||
const name = typeof source === "string" ? source : source.name;
|
||||
sourceFormat.value = detectFormat(name);
|
||||
sourceFormat.value = detectedFormat;
|
||||
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
|
||||
if (!newTableName.value.trim()) {
|
||||
newTableName.value = suggestedTableName(name);
|
||||
@@ -750,11 +764,18 @@ async function prepareBatchSources(sources: ImportSource[]) {
|
||||
const tasks: BatchImportTask[] = [];
|
||||
const usedNames = new Set<string>();
|
||||
const formats = sources.map((source) => detectFormat(sourceName(source)));
|
||||
if (formats.includes("parquet") && !supportsDuckDbParquetImport.value) {
|
||||
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
|
||||
loadingPreview.value = false;
|
||||
return;
|
||||
}
|
||||
emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true;
|
||||
try {
|
||||
for (const [index, source] of sources.entries()) {
|
||||
const format = formats[index]!;
|
||||
const initialPreview = await api.previewTableImportFile(source, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceFormat: format,
|
||||
parseOptions: taskParseOptions(format),
|
||||
previewLimit: Math.max(1, Number(previewLimit.value) || 50),
|
||||
@@ -765,6 +786,8 @@ async function prepareBatchSources(sources: ImportSource[]) {
|
||||
const effectiveSheetName = sheetName && sheetName === initialPreview.sheets?.[0] ? "" : sheetName;
|
||||
const taskPreview = effectiveSheetName
|
||||
? await api.previewTableImportFile(input.fileOrPath, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceRef: input.sourceRef,
|
||||
sourceFormat: format,
|
||||
parseOptions: taskParseOptions(format, effectiveSheetName),
|
||||
@@ -809,11 +832,12 @@ async function selectFile() {
|
||||
const selected = await open({
|
||||
multiple: targetMode.value === "create",
|
||||
filters: [
|
||||
{ name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql"] },
|
||||
{ name: "Data files", extensions: importFileExtensions.value },
|
||||
{ name: "Text", extensions: ["csv", "tsv", "txt"] },
|
||||
{ name: "JSON", extensions: ["json"] },
|
||||
{ name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
|
||||
{ name: "SQL", extensions: ["sql"] },
|
||||
...(supportsDuckDbParquetImport.value ? [{ name: "Parquet", extensions: ["parquet"] }] : []),
|
||||
],
|
||||
});
|
||||
if (!selected) return;
|
||||
@@ -1156,6 +1180,8 @@ async function reloadBatchPreviewsForEncoding() {
|
||||
if (!isDelimitedFormat(task.format) && task.format !== "sql") continue;
|
||||
const input = importPreviewInput(uploadedImportSourceFromPreview(task.preview), task.source);
|
||||
const nextPreview = await api.previewTableImportFile(input.fileOrPath, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceRef: input.sourceRef,
|
||||
sourceFormat: task.format,
|
||||
parseOptions: taskParseOptions(task.format, task.sheetName),
|
||||
@@ -1254,7 +1280,7 @@ watch(rawProgressPercent, (percent) => {
|
||||
|
||||
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1">
|
||||
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
|
||||
<input ref="fileInput" type="file" accept=".csv,.tsv,.txt,.json,.xlsx,.xlsm,.xls,.sql" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
|
||||
<input ref="fileInput" type="file" :accept="importFileExtensions.map((extension) => `.${extension}`).join(',')" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
|
||||
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
|
||||
<span class="shrink-0 text-xs text-muted-foreground">{{ t("tableImport.target") }}</span>
|
||||
<span class="min-w-0 truncate text-sm font-medium">
|
||||
@@ -1329,8 +1355,15 @@ watch(rawProgressPercent, (percent) => {
|
||||
{{ t("tableImport.selectFile") }}
|
||||
</Button>
|
||||
</div>
|
||||
<div class="grid grid-cols-6 gap-2">
|
||||
<button v-for="format in formatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
|
||||
<div
|
||||
data-testid="table-import-format-options"
|
||||
class="grid grid-cols-2 gap-2 sm:grid-cols-3"
|
||||
:class="{
|
||||
'lg:grid-cols-6': availableFormatOptions.length === 6,
|
||||
'lg:grid-cols-7': availableFormatOptions.length === 7,
|
||||
}"
|
||||
>
|
||||
<button v-for="format in availableFormatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
|
||||
<component :is="format.icon" class="mb-2 h-4 w-4 text-muted-foreground" />
|
||||
<div class="text-xs font-medium">{{ t(format.labelKey) }}</div>
|
||||
<div class="mt-1 text-[11px] leading-snug text-muted-foreground">{{ t(format.descriptionKey) }}</div>
|
||||
@@ -1347,7 +1380,7 @@ watch(rawProgressPercent, (percent) => {
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem v-for="format in formatOptions" :key="format.value" :value="format.value">
|
||||
<SelectItem v-for="format in availableFormatOptions" :key="format.value" :value="format.value">
|
||||
{{ t(format.labelKey) }}
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
|
||||
+30
-1
@@ -27,6 +27,7 @@ vi.mock("@/stores/connectionStore", () => ({
|
||||
if (id === "connection-1") return { id, name: "SQLite", db_type: "sqlite" };
|
||||
if (id === "postgres-1") return { id, name: "PostgreSQL", db_type: "postgres" };
|
||||
if (id === "sqlserver-1") return { id, name: "SQL Server", db_type: "sqlserver" };
|
||||
if (id === "duckdb-1") return { id, name: "DuckDB", db_type: "duckdb" };
|
||||
return undefined;
|
||||
},
|
||||
ensureConnected: mocks.ensureConnected,
|
||||
@@ -198,7 +199,7 @@ async function flushAsyncUpdates() {
|
||||
}
|
||||
}
|
||||
|
||||
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
|
||||
async function mountSourceDialog(connectionId = "connection-1") {
|
||||
const container = document.createElement("div");
|
||||
document.body.append(container);
|
||||
const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: connectionId, prefillDatabase: "main" }) }));
|
||||
@@ -206,12 +207,20 @@ async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connec
|
||||
app.use(i18n);
|
||||
app.mount(container);
|
||||
await flushAsyncUpdates();
|
||||
}
|
||||
|
||||
async function selectFiles(files: File[]) {
|
||||
const input = document.body.querySelector<HTMLInputElement>('input[type="file"]')!;
|
||||
Object.defineProperty(input, "files", { configurable: true, value: files });
|
||||
input.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
await flushAsyncUpdates();
|
||||
}
|
||||
|
||||
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
|
||||
await mountSourceDialog(connectionId);
|
||||
await selectFiles(files);
|
||||
}
|
||||
|
||||
function button(text: string) {
|
||||
const result = [...document.body.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.trim() === text);
|
||||
expect(result, `button ${text}`).toBeTruthy();
|
||||
@@ -270,6 +279,26 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("TableImportDialog batch selection", () => {
|
||||
it("uses all desktop grid columns when Parquet is unavailable", async () => {
|
||||
await mountSourceDialog();
|
||||
|
||||
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
|
||||
expect(formats?.classList.contains("lg:grid-cols-6")).toBe(true);
|
||||
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(false);
|
||||
});
|
||||
|
||||
it("offers Parquet for DuckDB and forwards the connection context to preview", async () => {
|
||||
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve({ ...delimitedPreview(), fileName: "rows.parquet", fileType: "parquet", sourceFingerprint: "rows-parquet" }));
|
||||
await mountSourceDialog("duckdb-1");
|
||||
|
||||
const fileInput = document.body.querySelector<HTMLInputElement>('input[type="file"]');
|
||||
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
|
||||
expect(fileInput?.accept).toContain(".parquet");
|
||||
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(true);
|
||||
await selectFiles([new File(["parquet"], "rows.parquet")]);
|
||||
expect(mocks.previewTableImportFile).toHaveBeenCalledWith(expect.any(File), expect.objectContaining({ connectionId: "duckdb-1", database: "main", sourceFormat: "parquet" }));
|
||||
});
|
||||
|
||||
it("lets the import setup dialog be minimized and restored before import starts", async () => {
|
||||
await mountDialog();
|
||||
expect(button("Next").disabled).toBe(false);
|
||||
|
||||
Reference in new issue
Block a user