perf(import): optimize large file imports

This commit is contained in:
miracle authored and GitHub committed 2026-07-22 16:56:49 +08:00
1 parent 9602cc3329
commit 0e33c73035
29 files changed
+5032 -625

No files matched your search

+48
View File
@@ -23,10 +23,58 @@ export interface ImportTargetColumnLike {
is_primary_key?: boolean;
}
export interface TableImportProgressLike {
status: "running" | "done" | "error" | "cancelled";
phase?: "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
rowsImported: number;
totalRows: number;
totalRowsExact?: boolean;
bytesRead?: number;
totalBytes?: number;
}
export type TableImportWizardStep = "source" | "options" | "mapping" | "review" | "execution";
export const TABLE_IMPORT_WIZARD_STEPS: TableImportWizardStep[] = ["source", "options", "mapping", "review", "execution"];
export function formatTableImportElapsed(ms: number): string {
const elapsedMs = Math.max(0, Number.isFinite(ms) ? Math.round(ms) : 0);
if (elapsedMs < 1000) return `${elapsedMs} ms`;
const seconds = elapsedMs / 1000;
if (seconds < 60) return `${seconds.toFixed(1)} s`;
const minutes = Math.floor(seconds / 60);
return `${minutes}m ${Math.round(seconds % 60)}s`;
}
export function resolveTableImportElapsed(liveElapsedMs: number, backendElapsedMs: number | undefined, terminal: boolean): number {
const live = Math.max(0, Number.isFinite(liveElapsedMs) ? liveElapsedMs : 0);
const backend = backendElapsedMs === undefined || !Number.isFinite(backendElapsedMs) ? undefined : Math.max(0, backendElapsedMs);
if (terminal && backend !== undefined) return backend;
return Math.max(live, backend ?? 0);
}
export function tableImportProgressPercent(progress: TableImportProgressLike | null | undefined): number {
if (!progress) return 0;
if (progress.status === "done") return 100;
let percent = 0;
const totalRows = progress.totalRows ?? 0;
const totalBytes = progress.totalBytes ?? 0;
if (progress.phase === "detectingEncoding") {
percent = totalBytes > 0 ? ((progress.bytesRead ?? 0) / totalBytes) * 10 : 0;
} else if (progress.phase === "writing" || progress.phase === "finalizing") {
if (progress.totalRowsExact !== false && totalRows > 0) {
percent = 10 + (progress.rowsImported / totalRows) * 89;
} else if (totalBytes > 0) {
percent = 10 + ((progress.bytesRead ?? 0) / totalBytes) * 89;
}
} else if (progress.totalRowsExact !== false && totalRows > 0) {
percent = (progress.rowsImported / totalRows) * 100;
} else if (totalBytes > 0) {
percent = ((progress.bytesRead ?? 0) / totalBytes) * 100;
}
return Math.min(99, Math.max(0, Math.round(percent)));
}
export interface TableImportParseSettings {
format: TableImportSourceFormat;
delimiter: string;