perf(import): optimize large file imports
This commit is contained in:
1 parent
9602cc3329
commit
0e33c73035
29 files changed
+5032
-625
No files matched your search
+36
-1
@@ -1,7 +1,42 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { autoMapImportColumns, buildTableImportParseOptions, nextTableImportWizardStep, previousTableImportWizardStep, requiredImportTargetColumns, suggestImportTargetDataTypes, validateImportMappings } from "@/lib/table/tableImport";
|
||||
import {
|
||||
autoMapImportColumns,
|
||||
buildTableImportParseOptions,
|
||||
formatTableImportElapsed,
|
||||
nextTableImportWizardStep,
|
||||
previousTableImportWizardStep,
|
||||
requiredImportTargetColumns,
|
||||
resolveTableImportElapsed,
|
||||
suggestImportTargetDataTypes,
|
||||
tableImportProgressPercent,
|
||||
validateImportMappings,
|
||||
} from "@/lib/table/tableImport";
|
||||
|
||||
describe("tableImport", () => {
|
||||
it("formats import elapsed time for progress and terminal summaries", () => {
|
||||
expect(formatTableImportElapsed(0)).toBe("0 ms");
|
||||
expect(formatTableImportElapsed(999)).toBe("999 ms");
|
||||
expect(formatTableImportElapsed(1_250)).toBe("1.3 s");
|
||||
expect(formatTableImportElapsed(61_000)).toBe("1m 1s");
|
||||
});
|
||||
|
||||
it("uses the backend elapsed time as the terminal source of truth", () => {
|
||||
expect(resolveTableImportElapsed(1_500, 1_000, false)).toBe(1_500);
|
||||
expect(resolveTableImportElapsed(1_500, 2_000, false)).toBe(2_000);
|
||||
expect(resolveTableImportElapsed(2_500, 2_000, true)).toBe(2_000);
|
||||
expect(resolveTableImportElapsed(2_500, undefined, true)).toBe(2_500);
|
||||
});
|
||||
|
||||
it("uses byte progress for unknown totals and reserves 100 for done", () => {
|
||||
expect(tableImportProgressPercent({ status: "running", rowsImported: 500, totalRows: 0, totalRowsExact: false, bytesRead: 50, totalBytes: 100 })).toBe(50);
|
||||
expect(tableImportProgressPercent({ status: "running", rowsImported: 1000, totalRows: 0, totalRowsExact: false, bytesRead: 100, totalBytes: 100 })).toBe(99);
|
||||
expect(tableImportProgressPercent({ status: "done", rowsImported: 1000, totalRows: 1000, totalRowsExact: true })).toBe(100);
|
||||
expect(tableImportProgressPercent({ status: "running", phase: "detectingEncoding", rowsImported: 0, totalRows: 1000, totalRowsExact: true, bytesRead: 100, totalBytes: 100 })).toBe(10);
|
||||
expect(tableImportProgressPercent({ status: "running", phase: "reading", rowsImported: 0, totalRows: 0, totalRowsExact: false, bytesRead: 0, totalBytes: 100 })).toBe(0);
|
||||
expect(tableImportProgressPercent({ status: "running", phase: "writing", rowsImported: 500, totalRows: 1000, totalRowsExact: true, bytesRead: 50, totalBytes: 100 })).toBe(55);
|
||||
expect(tableImportProgressPercent({ status: "running", phase: "finalizing", rowsImported: 1000, totalRows: 1000, totalRowsExact: true, bytesRead: 100, totalBytes: 100 })).toBe(99);
|
||||
});
|
||||
|
||||
it("auto maps exact and normalized column names", () => {
|
||||
expect(autoMapImportColumns(["id", "user name", "ignored"], ["id", "user_name"])).toEqual({
|
||||
id: "id",
|
||||
|
||||
@@ -338,6 +338,7 @@ export const sortTablesByFkDependency = forward("sortTablesByFkDependency");
|
||||
export const previewTableImportFile = forward("previewTableImportFile");
|
||||
export const importTableFile = forward("importTableFile");
|
||||
export const cancelTableImport = forward("cancelTableImport");
|
||||
export const releaseTableImportSource = forward("releaseTableImportSource");
|
||||
|
||||
// Database Export
|
||||
export const beginDatabaseBackupSnapshot = forward("beginDatabaseBackupSnapshot");
|
||||
@@ -614,6 +615,7 @@ export type {
|
||||
TableImportParseOptions,
|
||||
TableImportPreviewRequest,
|
||||
TableImportPreview,
|
||||
TableImportPreparedSource,
|
||||
TableImportRequest,
|
||||
TableImportSummary,
|
||||
TableImportProgress,
|
||||
|
||||
@@ -1569,10 +1569,15 @@ export async function previewTableImportFile(fileOrPath: string | File | TableIm
|
||||
if (!options.sourceRef) {
|
||||
throw new Error("previewTableImportFile in web mode requires a File object for new uploads");
|
||||
}
|
||||
const res = await fetch(apiUrl("/api/import/preview"), {
|
||||
const res = await fetch(apiUrl("/api/import/preview-source"), {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ request: { ...options, filePath: fileOrPath } }),
|
||||
body: JSON.stringify({
|
||||
sourceRef: options.sourceRef,
|
||||
sourceFormat: options.sourceFormat,
|
||||
parseOptions: options.parseOptions,
|
||||
previewLimit: options.previewLimit,
|
||||
}),
|
||||
});
|
||||
if (!res.ok) throw new Error(await res.text());
|
||||
return res.json();
|
||||
@@ -1608,6 +1613,7 @@ export async function importTableFile(request: TableImportRequest, onProgress: (
|
||||
importId: progress.importId,
|
||||
rowsImported: progress.rowsImported,
|
||||
totalRows: progress.totalRows,
|
||||
elapsedMs: progress.elapsedMs,
|
||||
};
|
||||
es.close();
|
||||
resolve(summary);
|
||||
@@ -1627,6 +1633,11 @@ export async function cancelTableImport(importId: string): Promise<boolean> {
|
||||
return post("/api/import/cancel", { importId });
|
||||
}
|
||||
|
||||
export async function releaseTableImportSource(sourceRef: string): Promise<boolean> {
|
||||
const result = await post<{ released: boolean }>("/api/import/source/release", { sourceRef });
|
||||
return result.released;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Database Export
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -2287,6 +2287,7 @@ export async function sortTablesByFkDependency(options: SortTablesByFkOptions):
|
||||
// --- Table File Import ---
|
||||
export type TableImportMode = "append" | "truncate";
|
||||
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
|
||||
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
|
||||
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel";
|
||||
export type TableImportJsonShape = "auto" | "objects" | "arrays";
|
||||
export type TableImportTextEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
|
||||
@@ -2328,10 +2329,21 @@ export interface TableImportPreview {
|
||||
columns: string[];
|
||||
rows: unknown[][];
|
||||
totalRows: number;
|
||||
totalRowsExact?: boolean;
|
||||
sourceFingerprint: string;
|
||||
effectiveEncoding?: TableImportTextEncoding | null;
|
||||
sheets?: string[];
|
||||
}
|
||||
|
||||
export interface TableImportPreparedSource {
|
||||
fingerprint: string;
|
||||
columns: string[];
|
||||
rows: unknown[][];
|
||||
totalRows: number;
|
||||
totalRowsExact?: boolean;
|
||||
effectiveEncoding?: TableImportTextEncoding | null;
|
||||
}
|
||||
|
||||
export interface TableImportRequest {
|
||||
importId: string;
|
||||
connectionId: string;
|
||||
@@ -2347,19 +2359,27 @@ export interface TableImportRequest {
|
||||
createTable?: boolean;
|
||||
batchSize: number;
|
||||
dateTimeFormat?: string;
|
||||
preparedSource?: TableImportPreparedSource | null;
|
||||
retainSource?: boolean;
|
||||
}
|
||||
|
||||
export interface TableImportSummary {
|
||||
importId: string;
|
||||
rowsImported: number;
|
||||
totalRows: number;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
export interface TableImportProgress {
|
||||
importId: string;
|
||||
status: TableImportStatus;
|
||||
phase?: TableImportPhase;
|
||||
rowsImported: number;
|
||||
totalRows: number;
|
||||
totalRowsExact?: boolean;
|
||||
bytesRead?: number;
|
||||
totalBytes?: number;
|
||||
elapsedMs: number;
|
||||
error?: string | null;
|
||||
}
|
||||
|
||||
@@ -2381,7 +2401,9 @@ export async function importTableFile(request: TableImportRequest, onProgress: (
|
||||
}
|
||||
});
|
||||
try {
|
||||
return await invoke("import_table_file", { request });
|
||||
const summary = await invoke<TableImportSummary>("import_table_file", { request });
|
||||
unlisten();
|
||||
return summary;
|
||||
} catch (e) {
|
||||
unlisten();
|
||||
throw e;
|
||||
@@ -2392,6 +2414,10 @@ export async function cancelTableImport(importId: string): Promise<boolean> {
|
||||
return invoke("cancel_table_import", { importId });
|
||||
}
|
||||
|
||||
export async function releaseTableImportSource(_sourceRef: string): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
|
||||
// --- Database Export ---
|
||||
export interface DatabaseExportRequest {
|
||||
exportId: string;
|
||||
|
||||
@@ -23,10 +23,58 @@ export interface ImportTargetColumnLike {
|
||||
is_primary_key?: boolean;
|
||||
}
|
||||
|
||||
export interface TableImportProgressLike {
|
||||
status: "running" | "done" | "error" | "cancelled";
|
||||
phase?: "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
|
||||
rowsImported: number;
|
||||
totalRows: number;
|
||||
totalRowsExact?: boolean;
|
||||
bytesRead?: number;
|
||||
totalBytes?: number;
|
||||
}
|
||||
|
||||
export type TableImportWizardStep = "source" | "options" | "mapping" | "review" | "execution";
|
||||
|
||||
export const TABLE_IMPORT_WIZARD_STEPS: TableImportWizardStep[] = ["source", "options", "mapping", "review", "execution"];
|
||||
|
||||
export function formatTableImportElapsed(ms: number): string {
|
||||
const elapsedMs = Math.max(0, Number.isFinite(ms) ? Math.round(ms) : 0);
|
||||
if (elapsedMs < 1000) return `${elapsedMs} ms`;
|
||||
const seconds = elapsedMs / 1000;
|
||||
if (seconds < 60) return `${seconds.toFixed(1)} s`;
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
return `${minutes}m ${Math.round(seconds % 60)}s`;
|
||||
}
|
||||
|
||||
export function resolveTableImportElapsed(liveElapsedMs: number, backendElapsedMs: number | undefined, terminal: boolean): number {
|
||||
const live = Math.max(0, Number.isFinite(liveElapsedMs) ? liveElapsedMs : 0);
|
||||
const backend = backendElapsedMs === undefined || !Number.isFinite(backendElapsedMs) ? undefined : Math.max(0, backendElapsedMs);
|
||||
if (terminal && backend !== undefined) return backend;
|
||||
return Math.max(live, backend ?? 0);
|
||||
}
|
||||
|
||||
export function tableImportProgressPercent(progress: TableImportProgressLike | null | undefined): number {
|
||||
if (!progress) return 0;
|
||||
if (progress.status === "done") return 100;
|
||||
let percent = 0;
|
||||
const totalRows = progress.totalRows ?? 0;
|
||||
const totalBytes = progress.totalBytes ?? 0;
|
||||
if (progress.phase === "detectingEncoding") {
|
||||
percent = totalBytes > 0 ? ((progress.bytesRead ?? 0) / totalBytes) * 10 : 0;
|
||||
} else if (progress.phase === "writing" || progress.phase === "finalizing") {
|
||||
if (progress.totalRowsExact !== false && totalRows > 0) {
|
||||
percent = 10 + (progress.rowsImported / totalRows) * 89;
|
||||
} else if (totalBytes > 0) {
|
||||
percent = 10 + ((progress.bytesRead ?? 0) / totalBytes) * 89;
|
||||
}
|
||||
} else if (progress.totalRowsExact !== false && totalRows > 0) {
|
||||
percent = (progress.rowsImported / totalRows) * 100;
|
||||
} else if (totalBytes > 0) {
|
||||
percent = ((progress.bytesRead ?? 0) / totalBytes) * 100;
|
||||
}
|
||||
return Math.min(99, Math.max(0, Math.round(percent)));
|
||||
}
|
||||
|
||||
export interface TableImportParseSettings {
|
||||
format: TableImportSourceFormat;
|
||||
delimiter: string;
|
||||
|
||||
Reference in new issue
Block a user