feat(table-import): add DuckDB Parquet imports
This commit is contained in:
1 parent
f61f64f727
commit
02f27dc294
26 files changed
+686
-46
No files matched your search
@@ -162,6 +162,7 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe
|
||||
{ value: "json", icon: FileJson, labelKey: "tableImport.formatJson", descriptionKey: "tableImport.formatJsonDescription" },
|
||||
{ value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" },
|
||||
{ value: "sql", icon: FileCode, labelKey: "tableImport.formatSql", descriptionKey: "tableImport.formatSqlDescription" },
|
||||
{ value: "parquet", icon: FileText, labelKey: "tableImport.formatParquet", descriptionKey: "tableImport.formatParquetDescription" },
|
||||
];
|
||||
|
||||
const encodingOptions = TABLE_IMPORT_ENCODING_OPTIONS;
|
||||
@@ -176,6 +177,9 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
|
||||
|
||||
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
|
||||
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
|
||||
const supportsDuckDbParquetImport = computed(() => structureDatabaseType.value === "duckdb");
|
||||
const availableFormatOptions = computed(() => formatOptions.filter((format) => format.value !== "parquet" || supportsDuckDbParquetImport.value));
|
||||
const importFileExtensions = computed(() => ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql", ...(supportsDuckDbParquetImport.value ? ["parquet"] : [])]);
|
||||
// Mirrors the conflict SQL dispatch in transfer.rs. Other dialects must keep
|
||||
// ordinary INSERT/error behavior instead of approximating an upsert.
|
||||
const IMPORT_CONFLICT_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
|
||||
@@ -426,6 +430,7 @@ function detectFormat(name: string): api.TableImportSourceFormat {
|
||||
if (lower.endsWith(".json")) return "json";
|
||||
if (lower.endsWith(".xls") || lower.endsWith(".xlsx") || lower.endsWith(".xlsm")) return "excel";
|
||||
if (lower.endsWith(".sql")) return "sql";
|
||||
if (lower.endsWith(".parquet")) return "parquet";
|
||||
return "csv";
|
||||
}
|
||||
|
||||
@@ -671,6 +676,8 @@ async function loadTargetColumns() {
|
||||
async function previewSelectedImportFile(fileOrPath: string | File) {
|
||||
const input = importPreviewInput(uploadedImportSourceFromPreview(preview.value), fileOrPath);
|
||||
return api.previewTableImportFile(input.fileOrPath, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceRef: input.sourceRef,
|
||||
sourceFormat: sourceFormat.value,
|
||||
parseOptions: parseOptions.value,
|
||||
@@ -704,6 +711,13 @@ async function loadPreview(fileOrPath = selectedSource.value) {
|
||||
}
|
||||
|
||||
function assignSelectedSource(source: string | File) {
|
||||
const detectedFormat = detectFormat(typeof source === "string" ? source : source.name);
|
||||
if (detectedFormat === "parquet" && !supportsDuckDbParquetImport.value) {
|
||||
selectedSource.value = null;
|
||||
preview.value = null;
|
||||
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
|
||||
return;
|
||||
}
|
||||
batchTasks.value = [];
|
||||
selectedSource.value = source;
|
||||
preview.value = null;
|
||||
@@ -712,7 +726,7 @@ function assignSelectedSource(source: string | File) {
|
||||
progress.value = null;
|
||||
errorMessage.value = "";
|
||||
const name = typeof source === "string" ? source : source.name;
|
||||
sourceFormat.value = detectFormat(name);
|
||||
sourceFormat.value = detectedFormat;
|
||||
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
|
||||
if (!newTableName.value.trim()) {
|
||||
newTableName.value = suggestedTableName(name);
|
||||
@@ -750,11 +764,18 @@ async function prepareBatchSources(sources: ImportSource[]) {
|
||||
const tasks: BatchImportTask[] = [];
|
||||
const usedNames = new Set<string>();
|
||||
const formats = sources.map((source) => detectFormat(sourceName(source)));
|
||||
if (formats.includes("parquet") && !supportsDuckDbParquetImport.value) {
|
||||
errorMessage.value = t("tableImport.parquetOnlyDuckdb");
|
||||
loadingPreview.value = false;
|
||||
return;
|
||||
}
|
||||
emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true;
|
||||
try {
|
||||
for (const [index, source] of sources.entries()) {
|
||||
const format = formats[index]!;
|
||||
const initialPreview = await api.previewTableImportFile(source, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceFormat: format,
|
||||
parseOptions: taskParseOptions(format),
|
||||
previewLimit: Math.max(1, Number(previewLimit.value) || 50),
|
||||
@@ -765,6 +786,8 @@ async function prepareBatchSources(sources: ImportSource[]) {
|
||||
const effectiveSheetName = sheetName && sheetName === initialPreview.sheets?.[0] ? "" : sheetName;
|
||||
const taskPreview = effectiveSheetName
|
||||
? await api.previewTableImportFile(input.fileOrPath, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceRef: input.sourceRef,
|
||||
sourceFormat: format,
|
||||
parseOptions: taskParseOptions(format, effectiveSheetName),
|
||||
@@ -809,11 +832,12 @@ async function selectFile() {
|
||||
const selected = await open({
|
||||
multiple: targetMode.value === "create",
|
||||
filters: [
|
||||
{ name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql"] },
|
||||
{ name: "Data files", extensions: importFileExtensions.value },
|
||||
{ name: "Text", extensions: ["csv", "tsv", "txt"] },
|
||||
{ name: "JSON", extensions: ["json"] },
|
||||
{ name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
|
||||
{ name: "SQL", extensions: ["sql"] },
|
||||
...(supportsDuckDbParquetImport.value ? [{ name: "Parquet", extensions: ["parquet"] }] : []),
|
||||
],
|
||||
});
|
||||
if (!selected) return;
|
||||
@@ -1156,6 +1180,8 @@ async function reloadBatchPreviewsForEncoding() {
|
||||
if (!isDelimitedFormat(task.format) && task.format !== "sql") continue;
|
||||
const input = importPreviewInput(uploadedImportSourceFromPreview(task.preview), task.source);
|
||||
const nextPreview = await api.previewTableImportFile(input.fileOrPath, {
|
||||
connectionId: props.prefillConnectionId,
|
||||
database: props.prefillDatabase || "",
|
||||
sourceRef: input.sourceRef,
|
||||
sourceFormat: task.format,
|
||||
parseOptions: taskParseOptions(task.format, task.sheetName),
|
||||
@@ -1254,7 +1280,7 @@ watch(rawProgressPercent, (percent) => {
|
||||
|
||||
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1">
|
||||
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
|
||||
<input ref="fileInput" type="file" accept=".csv,.tsv,.txt,.json,.xlsx,.xlsm,.xls,.sql" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
|
||||
<input ref="fileInput" type="file" :accept="importFileExtensions.map((extension) => `.${extension}`).join(',')" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
|
||||
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
|
||||
<span class="shrink-0 text-xs text-muted-foreground">{{ t("tableImport.target") }}</span>
|
||||
<span class="min-w-0 truncate text-sm font-medium">
|
||||
@@ -1329,8 +1355,15 @@ watch(rawProgressPercent, (percent) => {
|
||||
{{ t("tableImport.selectFile") }}
|
||||
</Button>
|
||||
</div>
|
||||
<div class="grid grid-cols-6 gap-2">
|
||||
<button v-for="format in formatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
|
||||
<div
|
||||
data-testid="table-import-format-options"
|
||||
class="grid grid-cols-2 gap-2 sm:grid-cols-3"
|
||||
:class="{
|
||||
'lg:grid-cols-6': availableFormatOptions.length === 6,
|
||||
'lg:grid-cols-7': availableFormatOptions.length === 7,
|
||||
}"
|
||||
>
|
||||
<button v-for="format in availableFormatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
|
||||
<component :is="format.icon" class="mb-2 h-4 w-4 text-muted-foreground" />
|
||||
<div class="text-xs font-medium">{{ t(format.labelKey) }}</div>
|
||||
<div class="mt-1 text-[11px] leading-snug text-muted-foreground">{{ t(format.descriptionKey) }}</div>
|
||||
@@ -1347,7 +1380,7 @@ watch(rawProgressPercent, (percent) => {
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem v-for="format in formatOptions" :key="format.value" :value="format.value">
|
||||
<SelectItem v-for="format in availableFormatOptions" :key="format.value" :value="format.value">
|
||||
{{ t(format.labelKey) }}
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
|
||||
+30
-1
@@ -27,6 +27,7 @@ vi.mock("@/stores/connectionStore", () => ({
|
||||
if (id === "connection-1") return { id, name: "SQLite", db_type: "sqlite" };
|
||||
if (id === "postgres-1") return { id, name: "PostgreSQL", db_type: "postgres" };
|
||||
if (id === "sqlserver-1") return { id, name: "SQL Server", db_type: "sqlserver" };
|
||||
if (id === "duckdb-1") return { id, name: "DuckDB", db_type: "duckdb" };
|
||||
return undefined;
|
||||
},
|
||||
ensureConnected: mocks.ensureConnected,
|
||||
@@ -198,7 +199,7 @@ async function flushAsyncUpdates() {
|
||||
}
|
||||
}
|
||||
|
||||
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
|
||||
async function mountSourceDialog(connectionId = "connection-1") {
|
||||
const container = document.createElement("div");
|
||||
document.body.append(container);
|
||||
const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: connectionId, prefillDatabase: "main" }) }));
|
||||
@@ -206,12 +207,20 @@ async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connec
|
||||
app.use(i18n);
|
||||
app.mount(container);
|
||||
await flushAsyncUpdates();
|
||||
}
|
||||
|
||||
async function selectFiles(files: File[]) {
|
||||
const input = document.body.querySelector<HTMLInputElement>('input[type="file"]')!;
|
||||
Object.defineProperty(input, "files", { configurable: true, value: files });
|
||||
input.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
await flushAsyncUpdates();
|
||||
}
|
||||
|
||||
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
|
||||
await mountSourceDialog(connectionId);
|
||||
await selectFiles(files);
|
||||
}
|
||||
|
||||
function button(text: string) {
|
||||
const result = [...document.body.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.trim() === text);
|
||||
expect(result, `button ${text}`).toBeTruthy();
|
||||
@@ -270,6 +279,26 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("TableImportDialog batch selection", () => {
|
||||
it("uses all desktop grid columns when Parquet is unavailable", async () => {
|
||||
await mountSourceDialog();
|
||||
|
||||
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
|
||||
expect(formats?.classList.contains("lg:grid-cols-6")).toBe(true);
|
||||
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(false);
|
||||
});
|
||||
|
||||
it("offers Parquet for DuckDB and forwards the connection context to preview", async () => {
|
||||
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve({ ...delimitedPreview(), fileName: "rows.parquet", fileType: "parquet", sourceFingerprint: "rows-parquet" }));
|
||||
await mountSourceDialog("duckdb-1");
|
||||
|
||||
const fileInput = document.body.querySelector<HTMLInputElement>('input[type="file"]');
|
||||
const formats = document.body.querySelector<HTMLElement>('[data-testid="table-import-format-options"]');
|
||||
expect(fileInput?.accept).toContain(".parquet");
|
||||
expect(formats?.classList.contains("lg:grid-cols-7")).toBe(true);
|
||||
await selectFiles([new File(["parquet"], "rows.parquet")]);
|
||||
expect(mocks.previewTableImportFile).toHaveBeenCalledWith(expect.any(File), expect.objectContaining({ connectionId: "duckdb-1", database: "main", sourceFormat: "parquet" }));
|
||||
});
|
||||
|
||||
it("lets the import setup dialog be minimized and restored before import starts", async () => {
|
||||
await mountDialog();
|
||||
expect(button("Next").disabled).toBe(false);
|
||||
|
||||
@@ -6273,6 +6273,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL skriptlərində INSERT əmrləri",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "DuckDB üçün Parquet faylları",
|
||||
parquetOnlyDuckdb: "Parquet idxalı yalnız DuckDB bağlantıları üçün dəstəklənir",
|
||||
previewRows: "Sətirlərin önbaxışı",
|
||||
delimiter: "Ayırıcı",
|
||||
encoding: "Kodlaşdırma",
|
||||
|
||||
@@ -6881,6 +6881,9 @@ export default {
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "INSERT statements in SQL scripts",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "Parquet files for DuckDB",
|
||||
parquetOnlyDuckdb: "Parquet import is only supported for DuckDB connections",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Encoding",
|
||||
|
||||
@@ -6311,6 +6311,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "Sentencias INSERT en scripts SQL",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "Archivos Parquet para DuckDB",
|
||||
parquetOnlyDuckdb: "La importación de Parquet solo es compatible con conexiones DuckDB",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codificación",
|
||||
|
||||
@@ -6233,6 +6233,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "Istruzioni INSERT negli script SQL",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "File Parquet per DuckDB",
|
||||
parquetOnlyDuckdb: "L'importazione Parquet è supportata solo per le connessioni DuckDB",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codifica",
|
||||
|
||||
@@ -6252,6 +6252,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL スクリプトの INSERT ステートメント",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "DuckDB 用の Parquet ファイル",
|
||||
parquetOnlyDuckdb: "Parquet のインポートは DuckDB 接続でのみサポートされています",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "文字コード",
|
||||
|
||||
@@ -5963,6 +5963,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL 스크립트의 INSERT 문",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "DuckDB용 Parquet 파일",
|
||||
parquetOnlyDuckdb: "Parquet 가져오기는 DuckDB 연결에서만 지원됩니다",
|
||||
previewRows: "미리보기 행",
|
||||
delimiter: "구분자",
|
||||
encoding: "인코딩",
|
||||
|
||||
@@ -6235,6 +6235,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "Instruções INSERT em scripts SQL",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "Arquivos Parquet para DuckDB",
|
||||
parquetOnlyDuckdb: "A importação de Parquet é compatível apenas com conexões DuckDB",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codificação",
|
||||
|
||||
@@ -7400,6 +7400,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "Операторы INSERT в скриптах SQL",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "Файлы Parquet для DuckDB",
|
||||
parquetOnlyDuckdb: "Импорт Parquet поддерживается только для подключений DuckDB",
|
||||
previewRows: "Строк предпросмотра",
|
||||
delimiter: "Разделитель",
|
||||
encoding: "Кодировка",
|
||||
|
||||
@@ -6155,6 +6155,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL betiklerindeki INSERT ifadeleri",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "DuckDB için Parquet dosyaları",
|
||||
parquetOnlyDuckdb: "Parquet içe aktarma yalnızca DuckDB bağlantılarında desteklenir",
|
||||
previewRows: "Satırları önizle",
|
||||
delimiter: "Ayırıcı",
|
||||
encoding: "Kodlama",
|
||||
|
||||
@@ -6848,6 +6848,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS、XLSX、XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL 脚本中的 INSERT 语句",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "DuckDB 的 Parquet 文件",
|
||||
parquetOnlyDuckdb: "Parquet 导入仅支持 DuckDB 连接",
|
||||
previewRows: "预览行数",
|
||||
delimiter: "分隔符",
|
||||
encoding: "文件编码",
|
||||
|
||||
@@ -5542,6 +5542,9 @@ export default withEnglishFallback({
|
||||
formatExcelDescription: "XLS、XLSX、XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL 指令檔中的 INSERT 語句",
|
||||
formatParquet: "Parquet",
|
||||
formatParquetDescription: "DuckDB 的 Parquet 檔案",
|
||||
parquetOnlyDuckdb: "Parquet 匯入僅支援 DuckDB 連線",
|
||||
previewRows: "預覽列數",
|
||||
delimiter: "分隔符",
|
||||
encoding: "檔案編碼",
|
||||
|
||||
@@ -2888,6 +2888,8 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
sourceRef: options.sourceRef,
|
||||
connectionId: options.connectionId,
|
||||
database: options.database,
|
||||
sourceFormat: options.sourceFormat,
|
||||
parseOptions: options.parseOptions,
|
||||
previewLimit: options.previewLimit,
|
||||
@@ -2898,6 +2900,8 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
|
||||
}
|
||||
const formData = new FormData();
|
||||
formData.append("file", fileOrPath);
|
||||
if (options.connectionId) formData.append("connectionId", options.connectionId);
|
||||
if (options.database != null) formData.append("database", options.database);
|
||||
if (options.sourceFormat) formData.append("sourceFormat", options.sourceFormat);
|
||||
if (options.parseOptions) formData.append("parseOptions", JSON.stringify(options.parseOptions));
|
||||
if (options.previewLimit != null) formData.append("previewLimit", String(options.previewLimit));
|
||||
|
||||
@@ -5447,7 +5447,7 @@ export type TableImportMode = "append" | "truncate";
|
||||
export type TableImportConflictPolicy = "error" | "skip" | "updateExisting";
|
||||
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
|
||||
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
|
||||
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql";
|
||||
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql" | "parquet";
|
||||
export type TableImportJsonShape = "auto" | "objects" | "arrays";
|
||||
export type TableImportTextEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
|
||||
|
||||
@@ -5474,6 +5474,8 @@ export interface TableImportParseOptions {
|
||||
|
||||
export interface TableImportPreviewRequest {
|
||||
filePath: string;
|
||||
connectionId?: string | null;
|
||||
database?: string | null;
|
||||
sourceRef?: string | null;
|
||||
sourceFormat?: TableImportSourceFormat | null;
|
||||
parseOptions?: TableImportParseOptions | null;
|
||||
|
||||
Reference in new issue
Block a user