diff --git a/apps/desktop/src/components/generate/DataGenerateDialog.vue b/apps/desktop/src/components/generate/DataGenerateDialog.vue index ca9ee9a92b..3979aa77b7 100644 --- a/apps/desktop/src/components/generate/DataGenerateDialog.vue +++ b/apps/desktop/src/components/generate/DataGenerateDialog.vue @@ -3,14 +3,28 @@ import { reactive, ref, computed, onMounted, watch, type ComponentPublicInstance import { useI18n } from "vue-i18n"; import { useConnectionStore } from "@/stores/connectionStore"; import * as api from "@/lib/backend/api"; -import type { GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate"; -import { displayGeneratedValue, findGeneratorKey, formatGeneratedValue, generateTableData, defaultGeneratorParams, supportsGeneratedMultiRowValues, UniqueValueGenerationError } from "@/lib/dataGrid/dataGenerate"; +import type { ColumnGenerateConfig, GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate"; +import { + createTableGenerateState, + defaultGeneratorParams, + displayGeneratedValue, + findGeneratorKey, + formatGeneratedRowValues, + formatGeneratedValue, + generateInsertBatches, + generateTableData, + generateTableRowsChunk, + splitValueRowsByByteBudget, + supportsGeneratedMultiRowValues, + UniqueValueGenerationError, +} from "@/lib/dataGrid/dataGenerate"; +import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "@/lib/dataGrid/generateInsertAccounting"; import { qualifiedTableName, quoteTableIdentifier } from "@/lib/table/tableSelectSql"; import { isTauriRuntime } from "@/lib/backend/tauriRuntime"; import { effectiveDatabaseTypeForConnection } from "@/lib/database/jdbcDialect"; import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard"; import GeneratorParamsPanel from "./params/GeneratorParamsPanel.vue"; -import type { ColumnInfo, TableInfo } from "@/types/database"; +import type { ColumnInfo, QueryResult, TableInfo } from "@/types/database"; import { Dialog, DialogHeader, DialogTitle, DialogScrollContent, DialogContent, DialogFooter } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; @@ -59,10 +73,36 @@ const currentStep = ref<"config" | "preview" | "result">("config"); interface GeneratedTableResult extends GenerateResult { tableName: string; schema: string; + /** Rows that will actually be inserted. `rows` only holds a preview sample. */ + targetRowCount: number; + isSample: boolean; + /** Column config after auto-increment start values have been resolved. */ + resolvedColumns: ColumnGenerateConfig[]; } const generatedResults = ref([]); const generationError = ref(""); +/** + * Preview only materializes a small sample. Generating the full row set up + * front froze the UI for hundreds of thousands of rows and exhausted memory + * at millions of rows. + */ +const PREVIEW_SAMPLE_ROWS = 50; +const MAX_ROW_COUNT = 100_000_000; +const DEFAULT_BATCH_ROWS = 1000; +/** + * Conservative per-statement budget. The backend clamps MySQL batches at 4MB + * and derives a `max_allowed_packet` margin, so staying near 1MB keeps the + * common 4MB / 16MB / 64MB server settings safe without extra round trips. + */ +const MAX_BATCH_BYTES = 1024 * 1024; +const LARGE_ROW_COUNT_HINT = 100_000; + +function normalizeRowCount(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.min(MAX_ROW_COUNT, Math.floor(value))); +} + const connectionName = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId)?.name : "")); function tableKey(schema: string, table: string) { @@ -366,7 +406,20 @@ function onPreviewColResizeStart(ci: number, event: MouseEvent) { document.addEventListener("pointerup", onUp); document.body.classList.add("select-none", "cursor-col-resize"); } -const currentPreview = computed(() => generatedResults.value[previewTableIndex.value] ?? { tableName: "", schema: "", columns: [], rows: [], sql: "", statements: [] }); +const currentPreview = computed( + () => + generatedResults.value[previewTableIndex.value] ?? { + tableName: "", + schema: "", + columns: [], + rows: [], + sql: "", + statements: [], + targetRowCount: 0, + isSample: false, + resolvedColumns: [], + }, +); function displayPreviewCell(cell: unknown): string { return displayGeneratedValue(cell); @@ -395,6 +448,19 @@ async function fetchMaxValues(cfg: TableGenerateConfig): Promise sampleCount, + resolvedColumns: columns, + ...result, + }; +} + async function doGenerate() { if (!Object.values(checkedTables).some(Boolean)) return; generationError.value = ""; @@ -416,8 +482,7 @@ async function doGenerate() { return col; }); } - const result = generateTableData({ ...cfg, columns }, dbType.value); - results.push({ tableName: cfg.tableName, schema: cfg.schema, ...result }); + results.push(buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount))); } } catch (error) { generationError.value = generationErrorMessage(error); @@ -447,8 +512,7 @@ async function regenerate() { } generationError.value = ""; try { - const result = generateTableData({ ...cfg, columns }, dbType.value); - generatedResults.value[previewTableIndex.value] = { tableName: cfg.tableName, schema: cfg.schema, ...result }; + generatedResults.value[previewTableIndex.value] = buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount)); } catch (error) { generationError.value = generationErrorMessage(error); } @@ -461,7 +525,13 @@ function generationErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } -function copyAllSql() { +/** + * Copies the script behind the preview. The preview only materializes a + * `PREVIEW_SAMPLE_ROWS` sample, so this is the sample script rather than the + * full batch — the label says so, and the insert step generates the remaining + * rows incrementally instead of holding them in memory. + */ +function copySampleSql() { const allSql = allSqlStatements().join("\n\n"); void navigator.clipboard.writeText(allSql); } @@ -474,6 +544,7 @@ interface TableResult { ok: number; err: number; error?: string; + cancelled?: boolean; } const executeResults = ref([]); @@ -482,6 +553,9 @@ const generateOptions = reactive({ truncate: false, useTransaction: true, extendedInsert: true, + /** 0 means "no timeout" — the backend maps 0 to an unbounded wait. */ + timeoutSecs: 0, + batchRows: DEFAULT_BATCH_ROWS, }); const supportsExtendedInsert = computed(() => supportsGeneratedMultiRowValues(dbType.value)); watch( @@ -494,6 +568,35 @@ watch( const optionsDialogOpen = ref(false); +interface InsertProgress { + tableName: string; + tableIndex: number; + tableCount: number; + insertedRows: number; + totalRows: number; + elapsedMs: number; +} +const insertProgress = ref(null); +const insertCancelled = ref(false); +let activeExecutionId: string | null = null; + +const insertPercent = computed(() => { + const p = insertProgress.value; + if (!p || p.totalRows <= 0) return 0; + return Math.min(100, Math.round((p.insertedRows / p.totalRows) * 100)); +}); + +function cancelInsert() { + if (!executing.value) return; + insertCancelled.value = true; + if (activeExecutionId) void api.cancelQuery(activeExecutionId); +} + +function normalizeTimeoutSecs(value: number): number { + if (!Number.isFinite(value) || value < 0) return 0; + return Math.min(86_400, Math.floor(value)); +} + function sqlStatementsForTable(r: GeneratedTableResult): string[] { const stmts: string[] = []; const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase }); @@ -516,71 +619,183 @@ function allSqlStatements(): string[] { return generatedResults.value.flatMap((r) => sqlStatementsForTable(r)); } +/** + * Streams one table: generate a chunk, insert it, drop the references, repeat. + * + * Peak memory stays proportional to `batchRows` instead of the requested row + * count, and each awaited round trip gives the main thread a chance to paint + * the progress bar. + */ +async function streamInsertTable(cid: string, db: string, r: GeneratedTableResult, executionId: string, onRows: (insertedRows: number) => void): Promise<{ ok: number; attempted: number; error: string; cancelled: boolean }> { + const cfg = configs[tableKey(r.schema, r.tableName)]; + const genCfg: TableGenerateConfig = { + tableName: r.tableName, + schema: r.schema, + database: props.prefillDatabase ?? cfg?.database ?? "", + tableType: cfg?.tableType, + rowCount: r.targetRowCount, + columns: r.resolvedColumns, + }; + const state = createTableGenerateState(genCfg, dbType.value); + const schema = r.schema || props.prefillSchema; + const forceSingleRow = !generateOptions.extendedInsert; + const timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs); + const batchRows = Math.max(1, Math.floor(generateOptions.batchRows) || DEFAULT_BATCH_ROWS); + let ok = 0; + let attempted = 0; + let lastError = ""; + let cancelled = false; + + const run = (sql: string): Promise => + api.executeMultiWithProgress(cid, db, sql, () => {}, schema, { + timeoutSecs, + useTransaction: generateOptions.useTransaction, + continueOnError: generateOptions.continueOnError, + executionId, + }); + + /** + * Executes one statement batch and folds the outcome into the running totals. + * Returns false when the caller has to stop the table. + * + * The returned per-statement results are inspected instead of trusting the + * `await`: several backend paths report a failed statement inside the result + * array while still resolving, so counting rows on the await alone would + * report failed batches as inserted. + */ + const executeBatch = async (statements: string[], rowsPerStatement: number[]): Promise => { + const expectedRows = rowsPerStatement.reduce((sum, rows) => sum + rows, 0); + attempted += expectedRows; + try { + const results = await run(statements.join("\n")); + // A row-carrying batch that comes back without per-statement results is + // unconfirmed; counting it would over-report inserts. + if (expectedRows > 0 && (!results || results.length === 0)) { + if (!lastError) lastError = "Backend returned no result for the batch"; + return generateOptions.continueOnError; + } + const outcome = summarizeBatchResults(results, rowsPerStatement); + ok += outcome.insertedRows; + if (!outcome.failed) return true; + if (!lastError) lastError = outcome.error ?? "Statement failed"; + console.error("[startInsert] SQL error:", lastError); + return generateOptions.continueOnError; + } catch (e: unknown) { + // Cancelling a transaction or a single-statement round trip surfaces as + // an error, so it has to be told apart from a real failure. + if (insertCancelled.value || isQueryCanceledError(e)) { + cancelled = true; + return false; + } + const msg = errorMessage(e); + console.error("[startInsert] SQL error:", msg); + if (!lastError) lastError = msg; + return generateOptions.continueOnError; + } + }; + + const finish = () => ({ ok, attempted, error: cancelled ? "" : lastError, cancelled }); + + if (generateOptions.truncate) { + const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase }); + // TRUNCATE carries no generated rows, so it only gates the rest of the run. + const truncateOk = await executeBatch([`TRUNCATE TABLE ${targetTable};`], [0]); + if (!truncateOk) return finish(); + } + + while (state.nextIndex < r.targetRowCount && !insertCancelled.value) { + let valueRows: string[]; + try { + const chunkRows = generateTableRowsChunk(genCfg, state, batchRows); + if (chunkRows.length === 0) break; + valueRows = chunkRows.map((row) => formatGeneratedRowValues(genCfg, dbType.value, state, row)); + } catch (e: unknown) { + // Generation failures (unique space exhausted at row 500k, say) must not + // discard the rows that earlier chunks already inserted. + if (!lastError) lastError = generationErrorMessage(e); + return finish(); + } + + for (const group of splitValueRowsByByteBudget(state, valueRows, MAX_BATCH_BYTES)) { + const { statements, rowsPerStatement } = generateInsertBatches(dbType.value, state, group, forceSingleRow); + if (statements.length === 0) continue; + const keepGoing = await executeBatch(statements, rowsPerStatement); + onRows(ok); + if (!keepGoing) return finish(); + } + onRows(ok); + // Yield so the progress bar repaints between chunks. + await new Promise((resolve) => setTimeout(resolve, 0)); + } + + if (insertCancelled.value) cancelled = true; + return finish(); +} + async function startInsert() { if (executing.value || generationError.value) return; const cid = props.prefillConnectionId; const db = props.prefillDatabase; if (!cid || !db) return; - const sql = allSqlStatements().join("\n"); - if (!sql.trim()) return; + const targets = generatedResults.value.filter((r) => r.targetRowCount > 0 && r.resolvedColumns.length > 0); + if (targets.length === 0) return; + const guardSql = allSqlStatements().join("\n") || `INSERT INTO ${targets[0].tableName}`; try { await executeWithProductionSqlGuard({ connection: store.getConfig(cid), database: db, - sql, + sql: guardSql, source: t("production.sourceDataGenerate"), execute: async () => { executing.value = true; + insertCancelled.value = false; + const executionId = crypto.randomUUID(); + activeExecutionId = executionId; + const startedAt = performance.now(); const perTable: TableResult[] = []; - let stopAfterTable = false; - for (const r of generatedResults.value) { - const stmts = sqlStatementsForTable(r); - const rowCount = r.rows.length; - let ok = 0; - let lastError = ""; - - const executeAsStatementGroup = generateOptions.useTransaction && !supportsGeneratedMultiRowValues(dbType.value); - if (executeAsStatementGroup) { + let stopAll = false; + try { + for (let ti = 0; ti < targets.length && !stopAll; ti++) { + const r = targets[ti]; + insertProgress.value = { + tableName: r.tableName, + tableIndex: ti + 1, + tableCount: targets.length, + insertedRows: 0, + totalRows: r.targetRowCount, + elapsedMs: 0, + }; + let outcome: { ok: number; attempted: number; error: string; cancelled: boolean }; + let insertedSoFar = 0; try { - await api.executeInTransaction(cid, db, stmts, r.schema || props.prefillSchema); - ok = rowCount; + outcome = await streamInsertTable(cid, db, r, executionId, (insertedRows) => { + insertedSoFar = insertedRows; + if (insertProgress.value) { + insertProgress.value = { ...insertProgress.value, insertedRows, elapsedMs: Math.round(performance.now() - startedAt) }; + } + }); } catch (e: unknown) { - const msg = e instanceof Error ? e.message : String(e); - console.error("[startInsert] SQL error:", msg); - lastError = msg; - stopAfterTable = !generateOptions.continueOnError; + // Safety net: even an unexpected throw must not hide the rows that + // earlier batches already inserted. + const wasCancelled = insertCancelled.value || isQueryCanceledError(e); + outcome = { ok: insertedSoFar, attempted: insertedSoFar, error: wasCancelled ? "" : errorMessage(e), cancelled: wasCancelled }; } - } else { - const usesMultiRowStatement = generateOptions.extendedInsert && supportsGeneratedMultiRowValues(dbType.value); - for (let si = 0; si < stmts.length; si++) { - try { - if (generateOptions.useTransaction) { - await api.executeInTransaction(cid, db, [stmts[si]], r.schema || props.prefillSchema); - } else { - await api.executeQuery(cid, db, stmts[si], r.schema || props.prefillSchema); - } - if (usesMultiRowStatement) { - ok = rowCount; - } else if (!(generateOptions.truncate && si === 0)) { - ok++; - } - } catch (e: unknown) { - const msg = e instanceof Error ? e.message : String(e); - console.error("[startInsert] SQL error:", msg); - if (!lastError) lastError = msg; - if (usesMultiRowStatement) ok = 0; - if (!generateOptions.continueOnError) { - stopAfterTable = true; - break; - } - } + const failedRows = Math.max(0, Math.max(outcome.attempted, outcome.ok) - outcome.ok); + perTable.push({ + table: r.tableName, + total: r.targetRowCount, + ok: outcome.ok, + err: failedRows, + error: outcome.error || undefined, + cancelled: outcome.cancelled || undefined, + }); + if (outcome.ok > 0) { + store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName); } + if ((outcome.error && !generateOptions.continueOnError) || outcome.cancelled) stopAll = true; } - perTable.push({ table: r.tableName, total: rowCount, ok, err: rowCount - ok, error: lastError || undefined }); - if (ok > 0) { - store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName); - } - if (stopAfterTable) break; + } finally { + activeExecutionId = null; } executeResults.value = perTable; currentStep.value = "result"; @@ -589,6 +804,7 @@ async function startInsert() { }); } finally { executing.value = false; + insertProgress.value = null; } } @@ -919,8 +1135,11 @@ async function onFileSelected(event: Event) {
- +
+

+ {{ t("dataGenerate.largeRowCountHint", { count: activeCfg.rowCount.toLocaleString() }) }} +

@@ -951,9 +1170,24 @@ async function onFileSelected(event: Event) { {{ generatedResults[0].tableName }} - {{ t("dataGenerate.previewRowCount", { count: currentPreview.rows.length }) }} + {{ t("dataGenerate.previewRowCount", { count: currentPreview.targetRowCount }) }} + + + +
+
+ + {{ t("dataGenerate.insertingTable", { table: insertProgress.tableName, index: insertProgress.tableIndex, count: insertProgress.tableCount }) }} + + {{ insertProgress.insertedRows.toLocaleString() }} / {{ insertProgress.totalRows.toLocaleString() }} +
+
+
+
+
+ {{ insertPercent }}% + {{ t("dataGenerate.elapsed", { seconds: Math.round(insertProgress.elapsedMs / 1000) }) }}
-
@@ -982,7 +1216,9 @@ async function onFileSelected(event: Event) { -
{{ t("dataGenerate.first50Rows", { count: currentPreview.rows.length }) }}
+
+ {{ t("dataGenerate.samplePreviewNotice", { shown: currentPreview.rows.length, count: currentPreview.targetRowCount }) }} +
@@ -1003,6 +1239,7 @@ async function onFileSelected(event: Event) {
{{ r.error }}
+
{{ t("dataGenerate.cancelledLabel") }}
{{ t("dataGenerate.noResult") }}
@@ -1030,7 +1267,7 @@ async function onFileSelected(event: Event) {
@@ -1044,7 +1281,8 @@ async function onFileSelected(event: Event) {