fix(generate): stream bulk data generation with progress and configurable timeout

This commit is contained in:
gggaiitx authored and GitHub committed 2026-09-04 20:24:32 +08:00
1 parent 259071c09b
commit a3772fb442
14 files changed
+986 -99

No files matched your search

@@ -3,14 +3,28 @@ import { reactive, ref, computed, onMounted, watch, type ComponentPublicInstance
import { useI18n } from "vue-i18n";
import { useConnectionStore } from "@/stores/connectionStore";
import * as api from "@/lib/backend/api";
import type { GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
import { displayGeneratedValue, findGeneratorKey, formatGeneratedValue, generateTableData, defaultGeneratorParams, supportsGeneratedMultiRowValues, UniqueValueGenerationError } from "@/lib/dataGrid/dataGenerate";
import type { ColumnGenerateConfig, GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
import {
createTableGenerateState,
defaultGeneratorParams,
displayGeneratedValue,
findGeneratorKey,
formatGeneratedRowValues,
formatGeneratedValue,
generateInsertBatches,
generateTableData,
generateTableRowsChunk,
splitValueRowsByByteBudget,
supportsGeneratedMultiRowValues,
UniqueValueGenerationError,
} from "@/lib/dataGrid/dataGenerate";
import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "@/lib/dataGrid/generateInsertAccounting";
import { qualifiedTableName, quoteTableIdentifier } from "@/lib/table/tableSelectSql";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import { effectiveDatabaseTypeForConnection } from "@/lib/database/jdbcDialect";
import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard";
import GeneratorParamsPanel from "./params/GeneratorParamsPanel.vue";
import type { ColumnInfo, TableInfo } from "@/types/database";
import type { ColumnInfo, QueryResult, TableInfo } from "@/types/database";
import { Dialog, DialogHeader, DialogTitle, DialogScrollContent, DialogContent, DialogFooter } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
@@ -59,10 +73,36 @@ const currentStep = ref<"config" | "preview" | "result">("config");
interface GeneratedTableResult extends GenerateResult {
tableName: string;
schema: string;
/** Rows that will actually be inserted. `rows` only holds a preview sample. */
targetRowCount: number;
isSample: boolean;
/** Column config after auto-increment start values have been resolved. */
resolvedColumns: ColumnGenerateConfig[];
}
const generatedResults = ref<GeneratedTableResult[]>([]);
const generationError = ref("");
/**
* Preview only materializes a small sample. Generating the full row set up
* front froze the UI for hundreds of thousands of rows and exhausted memory
* at millions of rows.
*/
const PREVIEW_SAMPLE_ROWS = 50;
const MAX_ROW_COUNT = 100_000_000;
const DEFAULT_BATCH_ROWS = 1000;
/**
* Conservative per-statement budget. The backend clamps MySQL batches at 4MB
* and derives a `max_allowed_packet` margin, so staying near 1MB keeps the
* common 4MB / 16MB / 64MB server settings safe without extra round trips.
*/
const MAX_BATCH_BYTES = 1024 * 1024;
const LARGE_ROW_COUNT_HINT = 100_000;
function normalizeRowCount(value: number): number {
if (!Number.isFinite(value)) return 0;
return Math.max(0, Math.min(MAX_ROW_COUNT, Math.floor(value)));
}
const connectionName = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId)?.name : ""));
function tableKey(schema: string, table: string) {
@@ -366,7 +406,20 @@ function onPreviewColResizeStart(ci: number, event: MouseEvent) {
document.addEventListener("pointerup", onUp);
document.body.classList.add("select-none", "cursor-col-resize");
}
const currentPreview = computed<GeneratedTableResult>(() => generatedResults.value[previewTableIndex.value] ?? { tableName: "", schema: "", columns: [], rows: [], sql: "", statements: [] });
const currentPreview = computed<GeneratedTableResult>(
() =>
generatedResults.value[previewTableIndex.value] ?? {
tableName: "",
schema: "",
columns: [],
rows: [],
sql: "",
statements: [],
targetRowCount: 0,
isSample: false,
resolvedColumns: [],
},
);
function displayPreviewCell(cell: unknown): string {
return displayGeneratedValue(cell);
@@ -395,6 +448,19 @@ async function fetchMaxValues(cfg: TableGenerateConfig): Promise<Record<string,
return starts;
}
function buildSampleResult(cfg: TableGenerateConfig, columns: ColumnGenerateConfig[], targetRowCount: number): GeneratedTableResult {
const sampleCount = Math.min(targetRowCount, PREVIEW_SAMPLE_ROWS);
const result = generateTableData({ ...cfg, columns, rowCount: sampleCount }, dbType.value);
return {
tableName: cfg.tableName,
schema: cfg.schema,
targetRowCount,
isSample: targetRowCount > sampleCount,
resolvedColumns: columns,
...result,
};
}
async function doGenerate() {
if (!Object.values(checkedTables).some(Boolean)) return;
generationError.value = "";
@@ -416,8 +482,7 @@ async function doGenerate() {
return col;
});
}
const result = generateTableData({ ...cfg, columns }, dbType.value);
results.push({ tableName: cfg.tableName, schema: cfg.schema, ...result });
results.push(buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount)));
}
} catch (error) {
generationError.value = generationErrorMessage(error);
@@ -447,8 +512,7 @@ async function regenerate() {
}
generationError.value = "";
try {
const result = generateTableData({ ...cfg, columns }, dbType.value);
generatedResults.value[previewTableIndex.value] = { tableName: cfg.tableName, schema: cfg.schema, ...result };
generatedResults.value[previewTableIndex.value] = buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount));
} catch (error) {
generationError.value = generationErrorMessage(error);
}
@@ -461,7 +525,13 @@ function generationErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function copyAllSql() {
/**
* Copies the script behind the preview. The preview only materializes a
* `PREVIEW_SAMPLE_ROWS` sample, so this is the sample script rather than the
* full batch — the label says so, and the insert step generates the remaining
* rows incrementally instead of holding them in memory.
*/
function copySampleSql() {
const allSql = allSqlStatements().join("\n\n");
void navigator.clipboard.writeText(allSql);
}
@@ -474,6 +544,7 @@ interface TableResult {
ok: number;
err: number;
error?: string;
cancelled?: boolean;
}
const executeResults = ref<TableResult[]>([]);
@@ -482,6 +553,9 @@ const generateOptions = reactive({
truncate: false,
useTransaction: true,
extendedInsert: true,
/** 0 means "no timeout" — the backend maps 0 to an unbounded wait. */
timeoutSecs: 0,
batchRows: DEFAULT_BATCH_ROWS,
});
const supportsExtendedInsert = computed(() => supportsGeneratedMultiRowValues(dbType.value));
watch(
@@ -494,6 +568,35 @@ watch(
const optionsDialogOpen = ref(false);
interface InsertProgress {
tableName: string;
tableIndex: number;
tableCount: number;
insertedRows: number;
totalRows: number;
elapsedMs: number;
}
const insertProgress = ref<InsertProgress | null>(null);
const insertCancelled = ref(false);
let activeExecutionId: string | null = null;
const insertPercent = computed(() => {
const p = insertProgress.value;
if (!p || p.totalRows <= 0) return 0;
return Math.min(100, Math.round((p.insertedRows / p.totalRows) * 100));
});
function cancelInsert() {
if (!executing.value) return;
insertCancelled.value = true;
if (activeExecutionId) void api.cancelQuery(activeExecutionId);
}
function normalizeTimeoutSecs(value: number): number {
if (!Number.isFinite(value) || value < 0) return 0;
return Math.min(86_400, Math.floor(value));
}
function sqlStatementsForTable(r: GeneratedTableResult): string[] {
const stmts: string[] = [];
const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
@@ -516,71 +619,183 @@ function allSqlStatements(): string[] {
return generatedResults.value.flatMap((r) => sqlStatementsForTable(r));
}
/**
* Streams one table: generate a chunk, insert it, drop the references, repeat.
*
* Peak memory stays proportional to `batchRows` instead of the requested row
* count, and each awaited round trip gives the main thread a chance to paint
* the progress bar.
*/
async function streamInsertTable(cid: string, db: string, r: GeneratedTableResult, executionId: string, onRows: (insertedRows: number) => void): Promise<{ ok: number; attempted: number; error: string; cancelled: boolean }> {
const cfg = configs[tableKey(r.schema, r.tableName)];
const genCfg: TableGenerateConfig = {
tableName: r.tableName,
schema: r.schema,
database: props.prefillDatabase ?? cfg?.database ?? "",
tableType: cfg?.tableType,
rowCount: r.targetRowCount,
columns: r.resolvedColumns,
};
const state = createTableGenerateState(genCfg, dbType.value);
const schema = r.schema || props.prefillSchema;
const forceSingleRow = !generateOptions.extendedInsert;
const timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs);
const batchRows = Math.max(1, Math.floor(generateOptions.batchRows) || DEFAULT_BATCH_ROWS);
let ok = 0;
let attempted = 0;
let lastError = "";
let cancelled = false;
const run = (sql: string): Promise<QueryResult[]> =>
api.executeMultiWithProgress(cid, db, sql, () => {}, schema, {
timeoutSecs,
useTransaction: generateOptions.useTransaction,
continueOnError: generateOptions.continueOnError,
executionId,
});
/**
* Executes one statement batch and folds the outcome into the running totals.
* Returns false when the caller has to stop the table.
*
* The returned per-statement results are inspected instead of trusting the
* `await`: several backend paths report a failed statement inside the result
* array while still resolving, so counting rows on the await alone would
* report failed batches as inserted.
*/
const executeBatch = async (statements: string[], rowsPerStatement: number[]): Promise<boolean> => {
const expectedRows = rowsPerStatement.reduce((sum, rows) => sum + rows, 0);
attempted += expectedRows;
try {
const results = await run(statements.join("\n"));
// A row-carrying batch that comes back without per-statement results is
// unconfirmed; counting it would over-report inserts.
if (expectedRows > 0 && (!results || results.length === 0)) {
if (!lastError) lastError = "Backend returned no result for the batch";
return generateOptions.continueOnError;
}
const outcome = summarizeBatchResults(results, rowsPerStatement);
ok += outcome.insertedRows;
if (!outcome.failed) return true;
if (!lastError) lastError = outcome.error ?? "Statement failed";
console.error("[startInsert] SQL error:", lastError);
return generateOptions.continueOnError;
} catch (e: unknown) {
// Cancelling a transaction or a single-statement round trip surfaces as
// an error, so it has to be told apart from a real failure.
if (insertCancelled.value || isQueryCanceledError(e)) {
cancelled = true;
return false;
}
const msg = errorMessage(e);
console.error("[startInsert] SQL error:", msg);
if (!lastError) lastError = msg;
return generateOptions.continueOnError;
}
};
const finish = () => ({ ok, attempted, error: cancelled ? "" : lastError, cancelled });
if (generateOptions.truncate) {
const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
// TRUNCATE carries no generated rows, so it only gates the rest of the run.
const truncateOk = await executeBatch([`TRUNCATE TABLE ${targetTable};`], [0]);
if (!truncateOk) return finish();
}
while (state.nextIndex < r.targetRowCount && !insertCancelled.value) {
let valueRows: string[];
try {
const chunkRows = generateTableRowsChunk(genCfg, state, batchRows);
if (chunkRows.length === 0) break;
valueRows = chunkRows.map((row) => formatGeneratedRowValues(genCfg, dbType.value, state, row));
} catch (e: unknown) {
// Generation failures (unique space exhausted at row 500k, say) must not
// discard the rows that earlier chunks already inserted.
if (!lastError) lastError = generationErrorMessage(e);
return finish();
}
for (const group of splitValueRowsByByteBudget(state, valueRows, MAX_BATCH_BYTES)) {
const { statements, rowsPerStatement } = generateInsertBatches(dbType.value, state, group, forceSingleRow);
if (statements.length === 0) continue;
const keepGoing = await executeBatch(statements, rowsPerStatement);
onRows(ok);
if (!keepGoing) return finish();
}
onRows(ok);
// Yield so the progress bar repaints between chunks.
await new Promise((resolve) => setTimeout(resolve, 0));
}
if (insertCancelled.value) cancelled = true;
return finish();
}
async function startInsert() {
if (executing.value || generationError.value) return;
const cid = props.prefillConnectionId;
const db = props.prefillDatabase;
if (!cid || !db) return;
const sql = allSqlStatements().join("\n");
if (!sql.trim()) return;
const targets = generatedResults.value.filter((r) => r.targetRowCount > 0 && r.resolvedColumns.length > 0);
if (targets.length === 0) return;
const guardSql = allSqlStatements().join("\n") || `INSERT INTO ${targets[0].tableName}`;
try {
await executeWithProductionSqlGuard({
connection: store.getConfig(cid),
database: db,
sql,
sql: guardSql,
source: t("production.sourceDataGenerate"),
execute: async () => {
executing.value = true;
insertCancelled.value = false;
const executionId = crypto.randomUUID();
activeExecutionId = executionId;
const startedAt = performance.now();
const perTable: TableResult[] = [];
let stopAfterTable = false;
for (const r of generatedResults.value) {
const stmts = sqlStatementsForTable(r);
const rowCount = r.rows.length;
let ok = 0;
let lastError = "";
const executeAsStatementGroup = generateOptions.useTransaction && !supportsGeneratedMultiRowValues(dbType.value);
if (executeAsStatementGroup) {
let stopAll = false;
try {
for (let ti = 0; ti < targets.length && !stopAll; ti++) {
const r = targets[ti];
insertProgress.value = {
tableName: r.tableName,
tableIndex: ti + 1,
tableCount: targets.length,
insertedRows: 0,
totalRows: r.targetRowCount,
elapsedMs: 0,
};
let outcome: { ok: number; attempted: number; error: string; cancelled: boolean };
let insertedSoFar = 0;
try {
await api.executeInTransaction(cid, db, stmts, r.schema || props.prefillSchema);
ok = rowCount;
outcome = await streamInsertTable(cid, db, r, executionId, (insertedRows) => {
insertedSoFar = insertedRows;
if (insertProgress.value) {
insertProgress.value = { ...insertProgress.value, insertedRows, elapsedMs: Math.round(performance.now() - startedAt) };
}
});
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
console.error("[startInsert] SQL error:", msg);
lastError = msg;
stopAfterTable = !generateOptions.continueOnError;
// Safety net: even an unexpected throw must not hide the rows that
// earlier batches already inserted.
const wasCancelled = insertCancelled.value || isQueryCanceledError(e);
outcome = { ok: insertedSoFar, attempted: insertedSoFar, error: wasCancelled ? "" : errorMessage(e), cancelled: wasCancelled };
}
} else {
const usesMultiRowStatement = generateOptions.extendedInsert && supportsGeneratedMultiRowValues(dbType.value);
for (let si = 0; si < stmts.length; si++) {
try {
if (generateOptions.useTransaction) {
await api.executeInTransaction(cid, db, [stmts[si]], r.schema || props.prefillSchema);
} else {
await api.executeQuery(cid, db, stmts[si], r.schema || props.prefillSchema);
}
if (usesMultiRowStatement) {
ok = rowCount;
} else if (!(generateOptions.truncate && si === 0)) {
ok++;
}
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
console.error("[startInsert] SQL error:", msg);
if (!lastError) lastError = msg;
if (usesMultiRowStatement) ok = 0;
if (!generateOptions.continueOnError) {
stopAfterTable = true;
break;
}
}
const failedRows = Math.max(0, Math.max(outcome.attempted, outcome.ok) - outcome.ok);
perTable.push({
table: r.tableName,
total: r.targetRowCount,
ok: outcome.ok,
err: failedRows,
error: outcome.error || undefined,
cancelled: outcome.cancelled || undefined,
});
if (outcome.ok > 0) {
store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
}
if ((outcome.error && !generateOptions.continueOnError) || outcome.cancelled) stopAll = true;
}
perTable.push({ table: r.tableName, total: rowCount, ok, err: rowCount - ok, error: lastError || undefined });
if (ok > 0) {
store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
}
if (stopAfterTable) break;
} finally {
activeExecutionId = null;
}
executeResults.value = perTable;
currentStep.value = "result";
@@ -589,6 +804,7 @@ async function startInsert() {
});
} finally {
executing.value = false;
insertProgress.value = null;
}
}
@@ -919,8 +1135,11 @@ async function onFileSelected(event: Event) {
</div>
<div class="flex items-center gap-3 rounded-md bg-muted/20 px-3 py-2">
<Label class="text-xs shrink-0">{{ t("dataGenerate.rowCount") }}:</Label>
<Input v-model.number="activeCfg.rowCount" class="h-7 w-24 text-xs" />
<Input v-model.number="activeCfg.rowCount" type="number" min="0" :max="MAX_ROW_COUNT" class="h-7 w-28 text-xs" @blur="activeCfg.rowCount = normalizeRowCount(activeCfg.rowCount)" />
</div>
<p v-if="activeCfg.rowCount > LARGE_ROW_COUNT_HINT" class="px-1 text-[11px] leading-relaxed text-muted-foreground">
{{ t("dataGenerate.largeRowCountHint", { count: activeCfg.rowCount.toLocaleString() }) }}
</p>
</div>
</template>
</div>
@@ -951,9 +1170,24 @@ async function onFileSelected(event: Event) {
<option v-for="(r, i) in generatedResults" :key="i" :value="i">{{ r.tableName }}</option>
</select>
<span v-else class="text-sm font-medium">{{ generatedResults[0].tableName }}</span>
<span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.rows.length }) }}</span>
<span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.targetRowCount }) }}</span>
</div>
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
</div>
<div v-if="executing && insertProgress" class="space-y-1.5 border-b bg-muted/10 px-3 py-2">
<div class="flex items-center justify-between gap-2 text-xs">
<span class="truncate text-muted-foreground">
{{ t("dataGenerate.insertingTable", { table: insertProgress.tableName, index: insertProgress.tableIndex, count: insertProgress.tableCount }) }}
</span>
<span class="shrink-0 font-medium tabular-nums">{{ insertProgress.insertedRows.toLocaleString() }} / {{ insertProgress.totalRows.toLocaleString() }}</span>
</div>
<div class="h-1.5 w-full overflow-hidden rounded-full bg-muted">
<div class="h-full bg-primary transition-[width] duration-150" :style="{ width: insertPercent + '%' }" />
</div>
<div class="flex items-center justify-between text-[11px] text-muted-foreground">
<span>{{ insertPercent }}%</span>
<span>{{ t("dataGenerate.elapsed", { seconds: Math.round(insertProgress.elapsedMs / 1000) }) }}</span>
</div>
<Button variant="outline" size="sm" class="h-7 text-xs" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
</div>
<div class="flex flex-col h-[380px]">
<div class="flex-1 overflow-auto overscroll-none bg-background">
@@ -982,7 +1216,9 @@ async function onFileSelected(event: Event) {
</tr>
</tbody>
</table>
<div v-if="currentPreview.rows.length > 50" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">{{ t("dataGenerate.first50Rows", { count: currentPreview.rows.length }) }}</div>
<div v-if="currentPreview.isSample" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">
{{ t("dataGenerate.samplePreviewNotice", { shown: currentPreview.rows.length, count: currentPreview.targetRowCount }) }}
</div>
</div>
</div>
</template>
@@ -1003,6 +1239,7 @@ async function onFileSelected(event: Event) {
</span>
</div>
<div v-if="r.error" class="mt-1 text-destructive/80 break-all leading-relaxed">{{ r.error }}</div>
<div v-else-if="r.cancelled" class="mt-1 text-muted-foreground">{{ t("dataGenerate.cancelledLabel") }}</div>
</div>
</div>
<div v-if="executeResults.length === 0" class="flex h-24 items-center justify-center text-xs text-muted-foreground">{{ t("dataGenerate.noResult") }}</div>
@@ -1030,7 +1267,7 @@ async function onFileSelected(event: Event) {
</Button>
</template>
<template v-else-if="currentStep === 'preview' && generatedResults.length > 0">
<Button variant="outline" size="sm" class="h-7 text-xs" @click="copyAllSql">{{ t("dataGenerate.copyAllSql") }}</Button>
<Button variant="outline" size="sm" class="h-7 text-xs" @click="copySampleSql">{{ t("dataGenerate.copySampleSql") }}</Button>
</template>
</div>
<div class="flex items-center gap-2">
@@ -1044,7 +1281,8 @@ async function onFileSelected(event: Event) {
</Button>
</template>
<template v-else-if="currentStep === 'preview'">
<Button variant="outline" size="sm" class="h-7 text-xs" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button>
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="!executing" @click="cancelInsert">{{ t("dataGenerate.cancelInsert") }}</Button>
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button>
<Button variant="default" size="sm" class="h-7 text-xs" :disabled="executing || !!generationError" @click="startInsert">
<Loader2 v-if="executing" class="mr-1 h-3 w-3 animate-spin" />
{{ t("dataGenerate.startInsert") }}
@@ -1081,6 +1319,16 @@ async function onFileSelected(event: Event) {
<input type="checkbox" v-model="generateOptions.extendedInsert" class="h-4 w-4 accent-primary" :disabled="!supportsExtendedInsert" />
<span class="text-xs">{{ t("dataGenerate.extendedInsert") }}</span>
</label>
<div class="flex items-center gap-3">
<Label class="text-xs shrink-0">{{ t("dataGenerate.timeoutSecs") }}:</Label>
<Input v-model.number="generateOptions.timeoutSecs" type="number" min="0" max="86400" class="h-7 w-24 text-xs" @blur="generateOptions.timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs)" />
</div>
<p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.timeoutHint") }}</p>
<div class="flex items-center gap-3">
<Label class="text-xs shrink-0">{{ t("dataGenerate.batchRows") }}:</Label>
<Input v-model.number="generateOptions.batchRows" type="number" min="1" max="100000" class="h-7 w-24 text-xs" />
</div>
<p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.batchRowsHint") }}</p>
</div>
<DialogFooter>
<Button size="sm" class="h-7 text-xs" @click="optionsDialogOpen = false">{{ t("dataGenerate.ok") }}</Button>