perf(import): optimize large file imports
This commit is contained in:
1 parent
9602cc3329
commit
0e33c73035
29 files changed
+5032
-625
No files matched your search
@@ -23,10 +23,58 @@ export interface ImportTargetColumnLike {
|
||||
is_primary_key?: boolean;
|
||||
}
|
||||
|
||||
export interface TableImportProgressLike {
|
||||
status: "running" | "done" | "error" | "cancelled";
|
||||
phase?: "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
|
||||
rowsImported: number;
|
||||
totalRows: number;
|
||||
totalRowsExact?: boolean;
|
||||
bytesRead?: number;
|
||||
totalBytes?: number;
|
||||
}
|
||||
|
||||
export type TableImportWizardStep = "source" | "options" | "mapping" | "review" | "execution";
|
||||
|
||||
export const TABLE_IMPORT_WIZARD_STEPS: TableImportWizardStep[] = ["source", "options", "mapping", "review", "execution"];
|
||||
|
||||
export function formatTableImportElapsed(ms: number): string {
|
||||
const elapsedMs = Math.max(0, Number.isFinite(ms) ? Math.round(ms) : 0);
|
||||
if (elapsedMs < 1000) return `${elapsedMs} ms`;
|
||||
const seconds = elapsedMs / 1000;
|
||||
if (seconds < 60) return `${seconds.toFixed(1)} s`;
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
return `${minutes}m ${Math.round(seconds % 60)}s`;
|
||||
}
|
||||
|
||||
export function resolveTableImportElapsed(liveElapsedMs: number, backendElapsedMs: number | undefined, terminal: boolean): number {
|
||||
const live = Math.max(0, Number.isFinite(liveElapsedMs) ? liveElapsedMs : 0);
|
||||
const backend = backendElapsedMs === undefined || !Number.isFinite(backendElapsedMs) ? undefined : Math.max(0, backendElapsedMs);
|
||||
if (terminal && backend !== undefined) return backend;
|
||||
return Math.max(live, backend ?? 0);
|
||||
}
|
||||
|
||||
export function tableImportProgressPercent(progress: TableImportProgressLike | null | undefined): number {
|
||||
if (!progress) return 0;
|
||||
if (progress.status === "done") return 100;
|
||||
let percent = 0;
|
||||
const totalRows = progress.totalRows ?? 0;
|
||||
const totalBytes = progress.totalBytes ?? 0;
|
||||
if (progress.phase === "detectingEncoding") {
|
||||
percent = totalBytes > 0 ? ((progress.bytesRead ?? 0) / totalBytes) * 10 : 0;
|
||||
} else if (progress.phase === "writing" || progress.phase === "finalizing") {
|
||||
if (progress.totalRowsExact !== false && totalRows > 0) {
|
||||
percent = 10 + (progress.rowsImported / totalRows) * 89;
|
||||
} else if (totalBytes > 0) {
|
||||
percent = 10 + ((progress.bytesRead ?? 0) / totalBytes) * 89;
|
||||
}
|
||||
} else if (progress.totalRowsExact !== false && totalRows > 0) {
|
||||
percent = (progress.rowsImported / totalRows) * 100;
|
||||
} else if (totalBytes > 0) {
|
||||
percent = ((progress.bytesRead ?? 0) / totalBytes) * 100;
|
||||
}
|
||||
return Math.min(99, Math.max(0, Math.round(percent)));
|
||||
}
|
||||
|
||||
export interface TableImportParseSettings {
|
||||
format: TableImportSourceFormat;
|
||||
delimiter: string;
|
||||
|
||||
Reference in new issue
Block a user