perf(import): optimize large file imports

This commit is contained in:
miracle authored and GitHub committed 2026-07-22 16:56:49 +08:00
1 parent 9602cc3329
commit 0e33c73035
29 files changed
+5032 -625

No files matched your search

+36 -1
View File
@@ -1,7 +1,42 @@
import { describe, expect, it } from "vitest";
import { autoMapImportColumns, buildTableImportParseOptions, nextTableImportWizardStep, previousTableImportWizardStep, requiredImportTargetColumns, suggestImportTargetDataTypes, validateImportMappings } from "@/lib/table/tableImport";
import {
autoMapImportColumns,
buildTableImportParseOptions,
formatTableImportElapsed,
nextTableImportWizardStep,
previousTableImportWizardStep,
requiredImportTargetColumns,
resolveTableImportElapsed,
suggestImportTargetDataTypes,
tableImportProgressPercent,
validateImportMappings,
} from "@/lib/table/tableImport";
describe("tableImport", () => {
it("formats import elapsed time for progress and terminal summaries", () => {
expect(formatTableImportElapsed(0)).toBe("0 ms");
expect(formatTableImportElapsed(999)).toBe("999 ms");
expect(formatTableImportElapsed(1_250)).toBe("1.3 s");
expect(formatTableImportElapsed(61_000)).toBe("1m 1s");
});
it("uses the backend elapsed time as the terminal source of truth", () => {
expect(resolveTableImportElapsed(1_500, 1_000, false)).toBe(1_500);
expect(resolveTableImportElapsed(1_500, 2_000, false)).toBe(2_000);
expect(resolveTableImportElapsed(2_500, 2_000, true)).toBe(2_000);
expect(resolveTableImportElapsed(2_500, undefined, true)).toBe(2_500);
});
it("uses byte progress for unknown totals and reserves 100 for done", () => {
expect(tableImportProgressPercent({ status: "running", rowsImported: 500, totalRows: 0, totalRowsExact: false, bytesRead: 50, totalBytes: 100 })).toBe(50);
expect(tableImportProgressPercent({ status: "running", rowsImported: 1000, totalRows: 0, totalRowsExact: false, bytesRead: 100, totalBytes: 100 })).toBe(99);
expect(tableImportProgressPercent({ status: "done", rowsImported: 1000, totalRows: 1000, totalRowsExact: true })).toBe(100);
expect(tableImportProgressPercent({ status: "running", phase: "detectingEncoding", rowsImported: 0, totalRows: 1000, totalRowsExact: true, bytesRead: 100, totalBytes: 100 })).toBe(10);
expect(tableImportProgressPercent({ status: "running", phase: "reading", rowsImported: 0, totalRows: 0, totalRowsExact: false, bytesRead: 0, totalBytes: 100 })).toBe(0);
expect(tableImportProgressPercent({ status: "running", phase: "writing", rowsImported: 500, totalRows: 1000, totalRowsExact: true, bytesRead: 50, totalBytes: 100 })).toBe(55);
expect(tableImportProgressPercent({ status: "running", phase: "finalizing", rowsImported: 1000, totalRows: 1000, totalRowsExact: true, bytesRead: 100, totalBytes: 100 })).toBe(99);
});
it("auto maps exact and normalized column names", () => {
expect(autoMapImportColumns(["id", "user name", "ignored"], ["id", "user_name"])).toEqual({
id: "id",
+2
View File
@@ -338,6 +338,7 @@ export const sortTablesByFkDependency = forward("sortTablesByFkDependency");
export const previewTableImportFile = forward("previewTableImportFile");
export const importTableFile = forward("importTableFile");
export const cancelTableImport = forward("cancelTableImport");
export const releaseTableImportSource = forward("releaseTableImportSource");
// Database Export
export const beginDatabaseBackupSnapshot = forward("beginDatabaseBackupSnapshot");
@@ -614,6 +615,7 @@ export type {
TableImportParseOptions,
TableImportPreviewRequest,
TableImportPreview,
TableImportPreparedSource,
TableImportRequest,
TableImportSummary,
TableImportProgress,
+13 -2
View File
@@ -1569,10 +1569,15 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
if (!options.sourceRef) {
throw new Error("previewTableImportFile in web mode requires a File object for new uploads");
}
const res = await fetch(apiUrl("/api/import/preview"), {
const res = await fetch(apiUrl("/api/import/preview-source"), {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ request: { ...options, filePath: fileOrPath } }),
body: JSON.stringify({
sourceRef: options.sourceRef,
sourceFormat: options.sourceFormat,
parseOptions: options.parseOptions,
previewLimit: options.previewLimit,
}),
});
if (!res.ok) throw new Error(await res.text());
return res.json();
@@ -1608,6 +1613,7 @@ export async function importTableFile(request: TableImportRequest, onProgress: (
importId: progress.importId,
rowsImported: progress.rowsImported,
totalRows: progress.totalRows,
elapsedMs: progress.elapsedMs,
};
es.close();
resolve(summary);
@@ -1627,6 +1633,11 @@ export async function cancelTableImport(importId: string): Promise<boolean> {
return post("/api/import/cancel", { importId });
}
export async function releaseTableImportSource(sourceRef: string): Promise<boolean> {
const result = await post<{ released: boolean }>("/api/import/source/release", { sourceRef });
return result.released;
}
// ---------------------------------------------------------------------------
// Database Export
// ---------------------------------------------------------------------------
+27 -1
View File
@@ -2287,6 +2287,7 @@ export async function sortTablesByFkDependency(options: SortTablesByFkOptions):
// --- Table File Import ---
export type TableImportMode = "append" | "truncate";
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel";
export type TableImportJsonShape = "auto" | "objects" | "arrays";
export type TableImportTextEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
@@ -2328,10 +2329,21 @@ export interface TableImportPreview {
columns: string[];
rows: unknown[][];
totalRows: number;
totalRowsExact?: boolean;
sourceFingerprint: string;
effectiveEncoding?: TableImportTextEncoding | null;
sheets?: string[];
}
export interface TableImportPreparedSource {
fingerprint: string;
columns: string[];
rows: unknown[][];
totalRows: number;
totalRowsExact?: boolean;
effectiveEncoding?: TableImportTextEncoding | null;
}
export interface TableImportRequest {
importId: string;
connectionId: string;
@@ -2347,19 +2359,27 @@ export interface TableImportRequest {
createTable?: boolean;
batchSize: number;
dateTimeFormat?: string;
preparedSource?: TableImportPreparedSource | null;
retainSource?: boolean;
}
export interface TableImportSummary {
importId: string;
rowsImported: number;
totalRows: number;
elapsedMs: number;
}
export interface TableImportProgress {
importId: string;
status: TableImportStatus;
phase?: TableImportPhase;
rowsImported: number;
totalRows: number;
totalRowsExact?: boolean;
bytesRead?: number;
totalBytes?: number;
elapsedMs: number;
error?: string | null;
}
@@ -2381,7 +2401,9 @@ export async function importTableFile(request: TableImportRequest, onProgress: (
}
});
try {
return await invoke("import_table_file", { request });
const summary = await invoke<TableImportSummary>("import_table_file", { request });
unlisten();
return summary;
} catch (e) {
unlisten();
throw e;
@@ -2392,6 +2414,10 @@ export async function cancelTableImport(importId: string): Promise<boolean> {
return invoke("cancel_table_import", { importId });
}
export async function releaseTableImportSource(_sourceRef: string): Promise<boolean> {
return false;
}
// --- Database Export ---
export interface DatabaseExportRequest {
exportId: string;
+48
View File
@@ -23,10 +23,58 @@ export interface ImportTargetColumnLike {
is_primary_key?: boolean;
}
export interface TableImportProgressLike {
status: "running" | "done" | "error" | "cancelled";
phase?: "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
rowsImported: number;
totalRows: number;
totalRowsExact?: boolean;
bytesRead?: number;
totalBytes?: number;
}
export type TableImportWizardStep = "source" | "options" | "mapping" | "review" | "execution";
export const TABLE_IMPORT_WIZARD_STEPS: TableImportWizardStep[] = ["source", "options", "mapping", "review", "execution"];
export function formatTableImportElapsed(ms: number): string {
const elapsedMs = Math.max(0, Number.isFinite(ms) ? Math.round(ms) : 0);
if (elapsedMs < 1000) return `${elapsedMs} ms`;
const seconds = elapsedMs / 1000;
if (seconds < 60) return `${seconds.toFixed(1)} s`;
const minutes = Math.floor(seconds / 60);
return `${minutes}m ${Math.round(seconds % 60)}s`;
}
export function resolveTableImportElapsed(liveElapsedMs: number, backendElapsedMs: number | undefined, terminal: boolean): number {
const live = Math.max(0, Number.isFinite(liveElapsedMs) ? liveElapsedMs : 0);
const backend = backendElapsedMs === undefined || !Number.isFinite(backendElapsedMs) ? undefined : Math.max(0, backendElapsedMs);
if (terminal && backend !== undefined) return backend;
return Math.max(live, backend ?? 0);
}
export function tableImportProgressPercent(progress: TableImportProgressLike | null | undefined): number {
if (!progress) return 0;
if (progress.status === "done") return 100;
let percent = 0;
const totalRows = progress.totalRows ?? 0;
const totalBytes = progress.totalBytes ?? 0;
if (progress.phase === "detectingEncoding") {
percent = totalBytes > 0 ? ((progress.bytesRead ?? 0) / totalBytes) * 10 : 0;
} else if (progress.phase === "writing" || progress.phase === "finalizing") {
if (progress.totalRowsExact !== false && totalRows > 0) {
percent = 10 + (progress.rowsImported / totalRows) * 89;
} else if (totalBytes > 0) {
percent = 10 + ((progress.bytesRead ?? 0) / totalBytes) * 89;
}
} else if (progress.totalRowsExact !== false && totalRows > 0) {
percent = (progress.rowsImported / totalRows) * 100;
} else if (totalBytes > 0) {
percent = ((progress.bytesRead ?? 0) / totalBytes) * 100;
}
return Math.min(99, Math.max(0, Math.round(percent)));
}
export interface TableImportParseSettings {
format: TableImportSourceFormat;
delimiter: string;