fix(generate): stream bulk data generation with progress and configurable timeout

This commit is contained in:
gggaiitx authored and GitHub committed 2026-09-04 20:24:32 +08:00
1 parent 259071c09b
commit a3772fb442
14 files changed
+986 -99

No files matched your search

@@ -3,14 +3,28 @@ import { reactive, ref, computed, onMounted, watch, type ComponentPublicInstance
import { useI18n } from "vue-i18n"; import { useI18n } from "vue-i18n";
import { useConnectionStore } from "@/stores/connectionStore"; import { useConnectionStore } from "@/stores/connectionStore";
import * as api from "@/lib/backend/api"; import * as api from "@/lib/backend/api";
import type { GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate"; import type { ColumnGenerateConfig, GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
import { displayGeneratedValue, findGeneratorKey, formatGeneratedValue, generateTableData, defaultGeneratorParams, supportsGeneratedMultiRowValues, UniqueValueGenerationError } from "@/lib/dataGrid/dataGenerate"; import {
createTableGenerateState,
defaultGeneratorParams,
displayGeneratedValue,
findGeneratorKey,
formatGeneratedRowValues,
formatGeneratedValue,
generateInsertBatches,
generateTableData,
generateTableRowsChunk,
splitValueRowsByByteBudget,
supportsGeneratedMultiRowValues,
UniqueValueGenerationError,
} from "@/lib/dataGrid/dataGenerate";
import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "@/lib/dataGrid/generateInsertAccounting";
import { qualifiedTableName, quoteTableIdentifier } from "@/lib/table/tableSelectSql"; import { qualifiedTableName, quoteTableIdentifier } from "@/lib/table/tableSelectSql";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime"; import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import { effectiveDatabaseTypeForConnection } from "@/lib/database/jdbcDialect"; import { effectiveDatabaseTypeForConnection } from "@/lib/database/jdbcDialect";
import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard"; import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard";
import GeneratorParamsPanel from "./params/GeneratorParamsPanel.vue"; import GeneratorParamsPanel from "./params/GeneratorParamsPanel.vue";
import type { ColumnInfo, TableInfo } from "@/types/database"; import type { ColumnInfo, QueryResult, TableInfo } from "@/types/database";
import { Dialog, DialogHeader, DialogTitle, DialogScrollContent, DialogContent, DialogFooter } from "@/components/ui/dialog"; import { Dialog, DialogHeader, DialogTitle, DialogScrollContent, DialogContent, DialogFooter } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
@@ -59,10 +73,36 @@ const currentStep = ref<"config" | "preview" | "result">("config");
interface GeneratedTableResult extends GenerateResult { interface GeneratedTableResult extends GenerateResult {
tableName: string; tableName: string;
schema: string; schema: string;
/** Rows that will actually be inserted. `rows` only holds a preview sample. */
targetRowCount: number;
isSample: boolean;
/** Column config after auto-increment start values have been resolved. */
resolvedColumns: ColumnGenerateConfig[];
} }
const generatedResults = ref<GeneratedTableResult[]>([]); const generatedResults = ref<GeneratedTableResult[]>([]);
const generationError = ref(""); const generationError = ref("");
/**
* Preview only materializes a small sample. Generating the full row set up
* front froze the UI for hundreds of thousands of rows and exhausted memory
* at millions of rows.
*/
const PREVIEW_SAMPLE_ROWS = 50;
const MAX_ROW_COUNT = 100_000_000;
const DEFAULT_BATCH_ROWS = 1000;
/**
* Conservative per-statement budget. The backend clamps MySQL batches at 4MB
* and derives a `max_allowed_packet` margin, so staying near 1MB keeps the
* common 4MB / 16MB / 64MB server settings safe without extra round trips.
*/
const MAX_BATCH_BYTES = 1024 * 1024;
const LARGE_ROW_COUNT_HINT = 100_000;
function normalizeRowCount(value: number): number {
if (!Number.isFinite(value)) return 0;
return Math.max(0, Math.min(MAX_ROW_COUNT, Math.floor(value)));
}
const connectionName = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId)?.name : "")); const connectionName = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId)?.name : ""));
function tableKey(schema: string, table: string) { function tableKey(schema: string, table: string) {
@@ -366,7 +406,20 @@ function onPreviewColResizeStart(ci: number, event: MouseEvent) {
document.addEventListener("pointerup", onUp); document.addEventListener("pointerup", onUp);
document.body.classList.add("select-none", "cursor-col-resize"); document.body.classList.add("select-none", "cursor-col-resize");
} }
const currentPreview = computed<GeneratedTableResult>(() => generatedResults.value[previewTableIndex.value] ?? { tableName: "", schema: "", columns: [], rows: [], sql: "", statements: [] }); const currentPreview = computed<GeneratedTableResult>(
() =>
generatedResults.value[previewTableIndex.value] ?? {
tableName: "",
schema: "",
columns: [],
rows: [],
sql: "",
statements: [],
targetRowCount: 0,
isSample: false,
resolvedColumns: [],
},
);
function displayPreviewCell(cell: unknown): string { function displayPreviewCell(cell: unknown): string {
return displayGeneratedValue(cell); return displayGeneratedValue(cell);
@@ -395,6 +448,19 @@ async function fetchMaxValues(cfg: TableGenerateConfig): Promise<Record<string,
return starts; return starts;
} }
function buildSampleResult(cfg: TableGenerateConfig, columns: ColumnGenerateConfig[], targetRowCount: number): GeneratedTableResult {
const sampleCount = Math.min(targetRowCount, PREVIEW_SAMPLE_ROWS);
const result = generateTableData({ ...cfg, columns, rowCount: sampleCount }, dbType.value);
return {
tableName: cfg.tableName,
schema: cfg.schema,
targetRowCount,
isSample: targetRowCount > sampleCount,
resolvedColumns: columns,
...result,
};
}
async function doGenerate() { async function doGenerate() {
if (!Object.values(checkedTables).some(Boolean)) return; if (!Object.values(checkedTables).some(Boolean)) return;
generationError.value = ""; generationError.value = "";
@@ -416,8 +482,7 @@ async function doGenerate() {
return col; return col;
}); });
} }
const result = generateTableData({ ...cfg, columns }, dbType.value); results.push(buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount)));
results.push({ tableName: cfg.tableName, schema: cfg.schema, ...result });
} }
} catch (error) { } catch (error) {
generationError.value = generationErrorMessage(error); generationError.value = generationErrorMessage(error);
@@ -447,8 +512,7 @@ async function regenerate() {
} }
generationError.value = ""; generationError.value = "";
try { try {
const result = generateTableData({ ...cfg, columns }, dbType.value); generatedResults.value[previewTableIndex.value] = buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount));
generatedResults.value[previewTableIndex.value] = { tableName: cfg.tableName, schema: cfg.schema, ...result };
} catch (error) { } catch (error) {
generationError.value = generationErrorMessage(error); generationError.value = generationErrorMessage(error);
} }
@@ -461,7 +525,13 @@ function generationErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error); return error instanceof Error ? error.message : String(error);
} }
function copyAllSql() { /**
* Copies the script behind the preview. The preview only materializes a
* `PREVIEW_SAMPLE_ROWS` sample, so this is the sample script rather than the
* full batch — the label says so, and the insert step generates the remaining
* rows incrementally instead of holding them in memory.
*/
function copySampleSql() {
const allSql = allSqlStatements().join("\n\n"); const allSql = allSqlStatements().join("\n\n");
void navigator.clipboard.writeText(allSql); void navigator.clipboard.writeText(allSql);
} }
@@ -474,6 +544,7 @@ interface TableResult {
ok: number; ok: number;
err: number; err: number;
error?: string; error?: string;
cancelled?: boolean;
} }
const executeResults = ref<TableResult[]>([]); const executeResults = ref<TableResult[]>([]);
@@ -482,6 +553,9 @@ const generateOptions = reactive({
truncate: false, truncate: false,
useTransaction: true, useTransaction: true,
extendedInsert: true, extendedInsert: true,
/** 0 means "no timeout" — the backend maps 0 to an unbounded wait. */
timeoutSecs: 0,
batchRows: DEFAULT_BATCH_ROWS,
}); });
const supportsExtendedInsert = computed(() => supportsGeneratedMultiRowValues(dbType.value)); const supportsExtendedInsert = computed(() => supportsGeneratedMultiRowValues(dbType.value));
watch( watch(
@@ -494,6 +568,35 @@ watch(
const optionsDialogOpen = ref(false); const optionsDialogOpen = ref(false);
interface InsertProgress {
tableName: string;
tableIndex: number;
tableCount: number;
insertedRows: number;
totalRows: number;
elapsedMs: number;
}
const insertProgress = ref<InsertProgress | null>(null);
const insertCancelled = ref(false);
let activeExecutionId: string | null = null;
const insertPercent = computed(() => {
const p = insertProgress.value;
if (!p || p.totalRows <= 0) return 0;
return Math.min(100, Math.round((p.insertedRows / p.totalRows) * 100));
});
function cancelInsert() {
if (!executing.value) return;
insertCancelled.value = true;
if (activeExecutionId) void api.cancelQuery(activeExecutionId);
}
function normalizeTimeoutSecs(value: number): number {
if (!Number.isFinite(value) || value < 0) return 0;
return Math.min(86_400, Math.floor(value));
}
function sqlStatementsForTable(r: GeneratedTableResult): string[] { function sqlStatementsForTable(r: GeneratedTableResult): string[] {
const stmts: string[] = []; const stmts: string[] = [];
const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase }); const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
@@ -516,71 +619,183 @@ function allSqlStatements(): string[] {
return generatedResults.value.flatMap((r) => sqlStatementsForTable(r)); return generatedResults.value.flatMap((r) => sqlStatementsForTable(r));
} }
/**
* Streams one table: generate a chunk, insert it, drop the references, repeat.
*
* Peak memory stays proportional to `batchRows` instead of the requested row
* count, and each awaited round trip gives the main thread a chance to paint
* the progress bar.
*/
async function streamInsertTable(cid: string, db: string, r: GeneratedTableResult, executionId: string, onRows: (insertedRows: number) => void): Promise<{ ok: number; attempted: number; error: string; cancelled: boolean }> {
const cfg = configs[tableKey(r.schema, r.tableName)];
const genCfg: TableGenerateConfig = {
tableName: r.tableName,
schema: r.schema,
database: props.prefillDatabase ?? cfg?.database ?? "",
tableType: cfg?.tableType,
rowCount: r.targetRowCount,
columns: r.resolvedColumns,
};
const state = createTableGenerateState(genCfg, dbType.value);
const schema = r.schema || props.prefillSchema;
const forceSingleRow = !generateOptions.extendedInsert;
const timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs);
const batchRows = Math.max(1, Math.floor(generateOptions.batchRows) || DEFAULT_BATCH_ROWS);
let ok = 0;
let attempted = 0;
let lastError = "";
let cancelled = false;
const run = (sql: string): Promise<QueryResult[]> =>
api.executeMultiWithProgress(cid, db, sql, () => {}, schema, {
timeoutSecs,
useTransaction: generateOptions.useTransaction,
continueOnError: generateOptions.continueOnError,
executionId,
});
/**
* Executes one statement batch and folds the outcome into the running totals.
* Returns false when the caller has to stop the table.
*
* The returned per-statement results are inspected instead of trusting the
* `await`: several backend paths report a failed statement inside the result
* array while still resolving, so counting rows on the await alone would
* report failed batches as inserted.
*/
const executeBatch = async (statements: string[], rowsPerStatement: number[]): Promise<boolean> => {
const expectedRows = rowsPerStatement.reduce((sum, rows) => sum + rows, 0);
attempted += expectedRows;
try {
const results = await run(statements.join("\n"));
// A row-carrying batch that comes back without per-statement results is
// unconfirmed; counting it would over-report inserts.
if (expectedRows > 0 && (!results || results.length === 0)) {
if (!lastError) lastError = "Backend returned no result for the batch";
return generateOptions.continueOnError;
}
const outcome = summarizeBatchResults(results, rowsPerStatement);
ok += outcome.insertedRows;
if (!outcome.failed) return true;
if (!lastError) lastError = outcome.error ?? "Statement failed";
console.error("[startInsert] SQL error:", lastError);
return generateOptions.continueOnError;
} catch (e: unknown) {
// Cancelling a transaction or a single-statement round trip surfaces as
// an error, so it has to be told apart from a real failure.
if (insertCancelled.value || isQueryCanceledError(e)) {
cancelled = true;
return false;
}
const msg = errorMessage(e);
console.error("[startInsert] SQL error:", msg);
if (!lastError) lastError = msg;
return generateOptions.continueOnError;
}
};
const finish = () => ({ ok, attempted, error: cancelled ? "" : lastError, cancelled });
if (generateOptions.truncate) {
const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
// TRUNCATE carries no generated rows, so it only gates the rest of the run.
const truncateOk = await executeBatch([`TRUNCATE TABLE ${targetTable};`], [0]);
if (!truncateOk) return finish();
}
while (state.nextIndex < r.targetRowCount && !insertCancelled.value) {
let valueRows: string[];
try {
const chunkRows = generateTableRowsChunk(genCfg, state, batchRows);
if (chunkRows.length === 0) break;
valueRows = chunkRows.map((row) => formatGeneratedRowValues(genCfg, dbType.value, state, row));
} catch (e: unknown) {
// Generation failures (unique space exhausted at row 500k, say) must not
// discard the rows that earlier chunks already inserted.
if (!lastError) lastError = generationErrorMessage(e);
return finish();
}
for (const group of splitValueRowsByByteBudget(state, valueRows, MAX_BATCH_BYTES)) {
const { statements, rowsPerStatement } = generateInsertBatches(dbType.value, state, group, forceSingleRow);
if (statements.length === 0) continue;
const keepGoing = await executeBatch(statements, rowsPerStatement);
onRows(ok);
if (!keepGoing) return finish();
}
onRows(ok);
// Yield so the progress bar repaints between chunks.
await new Promise((resolve) => setTimeout(resolve, 0));
}
if (insertCancelled.value) cancelled = true;
return finish();
}
async function startInsert() { async function startInsert() {
if (executing.value || generationError.value) return; if (executing.value || generationError.value) return;
const cid = props.prefillConnectionId; const cid = props.prefillConnectionId;
const db = props.prefillDatabase; const db = props.prefillDatabase;
if (!cid || !db) return; if (!cid || !db) return;
const sql = allSqlStatements().join("\n"); const targets = generatedResults.value.filter((r) => r.targetRowCount > 0 && r.resolvedColumns.length > 0);
if (!sql.trim()) return; if (targets.length === 0) return;
const guardSql = allSqlStatements().join("\n") || `INSERT INTO ${targets[0].tableName}`;
try { try {
await executeWithProductionSqlGuard({ await executeWithProductionSqlGuard({
connection: store.getConfig(cid), connection: store.getConfig(cid),
database: db, database: db,
sql, sql: guardSql,
source: t("production.sourceDataGenerate"), source: t("production.sourceDataGenerate"),
execute: async () => { execute: async () => {
executing.value = true; executing.value = true;
insertCancelled.value = false;
const executionId = crypto.randomUUID();
activeExecutionId = executionId;
const startedAt = performance.now();
const perTable: TableResult[] = []; const perTable: TableResult[] = [];
let stopAfterTable = false; let stopAll = false;
for (const r of generatedResults.value) { try {
const stmts = sqlStatementsForTable(r); for (let ti = 0; ti < targets.length && !stopAll; ti++) {
const rowCount = r.rows.length; const r = targets[ti];
let ok = 0; insertProgress.value = {
let lastError = ""; tableName: r.tableName,
tableIndex: ti + 1,
const executeAsStatementGroup = generateOptions.useTransaction && !supportsGeneratedMultiRowValues(dbType.value); tableCount: targets.length,
if (executeAsStatementGroup) { insertedRows: 0,
totalRows: r.targetRowCount,
elapsedMs: 0,
};
let outcome: { ok: number; attempted: number; error: string; cancelled: boolean };
let insertedSoFar = 0;
try { try {
await api.executeInTransaction(cid, db, stmts, r.schema || props.prefillSchema); outcome = await streamInsertTable(cid, db, r, executionId, (insertedRows) => {
ok = rowCount; insertedSoFar = insertedRows;
if (insertProgress.value) {
insertProgress.value = { ...insertProgress.value, insertedRows, elapsedMs: Math.round(performance.now() - startedAt) };
}
});
} catch (e: unknown) { } catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e); // Safety net: even an unexpected throw must not hide the rows that
console.error("[startInsert] SQL error:", msg); // earlier batches already inserted.
lastError = msg; const wasCancelled = insertCancelled.value || isQueryCanceledError(e);
stopAfterTable = !generateOptions.continueOnError; outcome = { ok: insertedSoFar, attempted: insertedSoFar, error: wasCancelled ? "" : errorMessage(e), cancelled: wasCancelled };
} }
} else { const failedRows = Math.max(0, Math.max(outcome.attempted, outcome.ok) - outcome.ok);
const usesMultiRowStatement = generateOptions.extendedInsert && supportsGeneratedMultiRowValues(dbType.value); perTable.push({
for (let si = 0; si < stmts.length; si++) { table: r.tableName,
try { total: r.targetRowCount,
if (generateOptions.useTransaction) { ok: outcome.ok,
await api.executeInTransaction(cid, db, [stmts[si]], r.schema || props.prefillSchema); err: failedRows,
} else { error: outcome.error || undefined,
await api.executeQuery(cid, db, stmts[si], r.schema || props.prefillSchema); cancelled: outcome.cancelled || undefined,
} });
if (usesMultiRowStatement) { if (outcome.ok > 0) {
ok = rowCount; store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
} else if (!(generateOptions.truncate && si === 0)) {
ok++;
}
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
console.error("[startInsert] SQL error:", msg);
if (!lastError) lastError = msg;
if (usesMultiRowStatement) ok = 0;
if (!generateOptions.continueOnError) {
stopAfterTable = true;
break;
}
}
} }
if ((outcome.error && !generateOptions.continueOnError) || outcome.cancelled) stopAll = true;
} }
perTable.push({ table: r.tableName, total: rowCount, ok, err: rowCount - ok, error: lastError || undefined }); } finally {
if (ok > 0) { activeExecutionId = null;
store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
}
if (stopAfterTable) break;
} }
executeResults.value = perTable; executeResults.value = perTable;
currentStep.value = "result"; currentStep.value = "result";
@@ -589,6 +804,7 @@ async function startInsert() {
}); });
} finally { } finally {
executing.value = false; executing.value = false;
insertProgress.value = null;
} }
} }
@@ -919,8 +1135,11 @@ async function onFileSelected(event: Event) {
</div> </div>
<div class="flex items-center gap-3 rounded-md bg-muted/20 px-3 py-2"> <div class="flex items-center gap-3 rounded-md bg-muted/20 px-3 py-2">
<Label class="text-xs shrink-0">{{ t("dataGenerate.rowCount") }}:</Label> <Label class="text-xs shrink-0">{{ t("dataGenerate.rowCount") }}:</Label>
<Input v-model.number="activeCfg.rowCount" class="h-7 w-24 text-xs" /> <Input v-model.number="activeCfg.rowCount" type="number" min="0" :max="MAX_ROW_COUNT" class="h-7 w-28 text-xs" @blur="activeCfg.rowCount = normalizeRowCount(activeCfg.rowCount)" />
</div> </div>
<p v-if="activeCfg.rowCount > LARGE_ROW_COUNT_HINT" class="px-1 text-[11px] leading-relaxed text-muted-foreground">
{{ t("dataGenerate.largeRowCountHint", { count: activeCfg.rowCount.toLocaleString() }) }}
</p>
</div> </div>
</template> </template>
</div> </div>
@@ -951,9 +1170,24 @@ async function onFileSelected(event: Event) {
<option v-for="(r, i) in generatedResults" :key="i" :value="i">{{ r.tableName }}</option> <option v-for="(r, i) in generatedResults" :key="i" :value="i">{{ r.tableName }}</option>
</select> </select>
<span v-else class="text-sm font-medium">{{ generatedResults[0].tableName }}</span> <span v-else class="text-sm font-medium">{{ generatedResults[0].tableName }}</span>
<span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.rows.length }) }}</span> <span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.targetRowCount }) }}</span>
</div>
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
</div>
<div v-if="executing && insertProgress" class="space-y-1.5 border-b bg-muted/10 px-3 py-2">
<div class="flex items-center justify-between gap-2 text-xs">
<span class="truncate text-muted-foreground">
{{ t("dataGenerate.insertingTable", { table: insertProgress.tableName, index: insertProgress.tableIndex, count: insertProgress.tableCount }) }}
</span>
<span class="shrink-0 font-medium tabular-nums">{{ insertProgress.insertedRows.toLocaleString() }} / {{ insertProgress.totalRows.toLocaleString() }}</span>
</div>
<div class="h-1.5 w-full overflow-hidden rounded-full bg-muted">
<div class="h-full bg-primary transition-[width] duration-150" :style="{ width: insertPercent + '%' }" />
</div>
<div class="flex items-center justify-between text-[11px] text-muted-foreground">
<span>{{ insertPercent }}%</span>
<span>{{ t("dataGenerate.elapsed", { seconds: Math.round(insertProgress.elapsedMs / 1000) }) }}</span>
</div> </div>
<Button variant="outline" size="sm" class="h-7 text-xs" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
</div> </div>
<div class="flex flex-col h-[380px]"> <div class="flex flex-col h-[380px]">
<div class="flex-1 overflow-auto overscroll-none bg-background"> <div class="flex-1 overflow-auto overscroll-none bg-background">
@@ -982,7 +1216,9 @@ async function onFileSelected(event: Event) {
</tr> </tr>
</tbody> </tbody>
</table> </table>
<div v-if="currentPreview.rows.length > 50" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">{{ t("dataGenerate.first50Rows", { count: currentPreview.rows.length }) }}</div> <div v-if="currentPreview.isSample" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">
{{ t("dataGenerate.samplePreviewNotice", { shown: currentPreview.rows.length, count: currentPreview.targetRowCount }) }}
</div>
</div> </div>
</div> </div>
</template> </template>
@@ -1003,6 +1239,7 @@ async function onFileSelected(event: Event) {
</span> </span>
</div> </div>
<div v-if="r.error" class="mt-1 text-destructive/80 break-all leading-relaxed">{{ r.error }}</div> <div v-if="r.error" class="mt-1 text-destructive/80 break-all leading-relaxed">{{ r.error }}</div>
<div v-else-if="r.cancelled" class="mt-1 text-muted-foreground">{{ t("dataGenerate.cancelledLabel") }}</div>
</div> </div>
</div> </div>
<div v-if="executeResults.length === 0" class="flex h-24 items-center justify-center text-xs text-muted-foreground">{{ t("dataGenerate.noResult") }}</div> <div v-if="executeResults.length === 0" class="flex h-24 items-center justify-center text-xs text-muted-foreground">{{ t("dataGenerate.noResult") }}</div>
@@ -1030,7 +1267,7 @@ async function onFileSelected(event: Event) {
</Button> </Button>
</template> </template>
<template v-else-if="currentStep === 'preview' && generatedResults.length > 0"> <template v-else-if="currentStep === 'preview' && generatedResults.length > 0">
<Button variant="outline" size="sm" class="h-7 text-xs" @click="copyAllSql">{{ t("dataGenerate.copyAllSql") }}</Button> <Button variant="outline" size="sm" class="h-7 text-xs" @click="copySampleSql">{{ t("dataGenerate.copySampleSql") }}</Button>
</template> </template>
</div> </div>
<div class="flex items-center gap-2"> <div class="flex items-center gap-2">
@@ -1044,7 +1281,8 @@ async function onFileSelected(event: Event) {
</Button> </Button>
</template> </template>
<template v-else-if="currentStep === 'preview'"> <template v-else-if="currentStep === 'preview'">
<Button variant="outline" size="sm" class="h-7 text-xs" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button> <Button variant="outline" size="sm" class="h-7 text-xs" :disabled="!executing" @click="cancelInsert">{{ t("dataGenerate.cancelInsert") }}</Button>
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button>
<Button variant="default" size="sm" class="h-7 text-xs" :disabled="executing || !!generationError" @click="startInsert"> <Button variant="default" size="sm" class="h-7 text-xs" :disabled="executing || !!generationError" @click="startInsert">
<Loader2 v-if="executing" class="mr-1 h-3 w-3 animate-spin" /> <Loader2 v-if="executing" class="mr-1 h-3 w-3 animate-spin" />
{{ t("dataGenerate.startInsert") }} {{ t("dataGenerate.startInsert") }}
@@ -1081,6 +1319,16 @@ async function onFileSelected(event: Event) {
<input type="checkbox" v-model="generateOptions.extendedInsert" class="h-4 w-4 accent-primary" :disabled="!supportsExtendedInsert" /> <input type="checkbox" v-model="generateOptions.extendedInsert" class="h-4 w-4 accent-primary" :disabled="!supportsExtendedInsert" />
<span class="text-xs">{{ t("dataGenerate.extendedInsert") }}</span> <span class="text-xs">{{ t("dataGenerate.extendedInsert") }}</span>
</label> </label>
<div class="flex items-center gap-3">
<Label class="text-xs shrink-0">{{ t("dataGenerate.timeoutSecs") }}:</Label>
<Input v-model.number="generateOptions.timeoutSecs" type="number" min="0" max="86400" class="h-7 w-24 text-xs" @blur="generateOptions.timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs)" />
</div>
<p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.timeoutHint") }}</p>
<div class="flex items-center gap-3">
<Label class="text-xs shrink-0">{{ t("dataGenerate.batchRows") }}:</Label>
<Input v-model.number="generateOptions.batchRows" type="number" min="1" max="100000" class="h-7 w-24 text-xs" />
</div>
<p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.batchRowsHint") }}</p>
</div> </div>
<DialogFooter> <DialogFooter>
<Button size="sm" class="h-7 text-xs" @click="optionsDialogOpen = false">{{ t("dataGenerate.ok") }}</Button> <Button size="sm" class="h-7 text-xs" @click="optionsDialogOpen = false">{{ t("dataGenerate.ok") }}</Button>
+11 -1
View File
@@ -5643,7 +5643,7 @@ export default {
successLabel: "{count} succeeded", successLabel: "{count} succeeded",
failLabel: "{count} failed", failLabel: "{count} failed",
totalLabel: "/ {count} entries", totalLabel: "/ {count} entries",
copyAllSql: "Copy All SQL", copySampleSql: "Copy sample SQL",
prevStep: "Previous", prevStep: "Previous",
startInsert: "Start Insert", startInsert: "Start Insert",
confirm: "Confirm", confirm: "Confirm",
@@ -5869,6 +5869,16 @@ export default {
drivers_license: "Driver's License", drivers_license: "Driver's License",
custom: "Custom", custom: "Custom",
}, },
insertingTable: "Writing to {table} (table {index}/{count})",
elapsed: "{seconds}s elapsed",
samplePreviewNotice: "Showing the first {shown} sample rows; all {count} rows are generated while inserting",
largeRowCountHint: "Large row count ({count}); rows are generated and inserted in batches and you can cancel at any time",
timeoutSecs: "Timeout (seconds)",
timeoutHint: "0 means no timeout; keep 0 or raise it for large data volumes",
batchRows: "Rows per batch",
batchRowsHint: "Rows generated and inserted per round trip; oversized batches are split automatically",
cancelInsert: "Cancel insert",
cancelledLabel: "Cancelled; rows already written remain in the table",
}, },
tableToolbox: { tableToolbox: {
title: "Data Tools", title: "Data Tools",
+11 -1
View File
@@ -5363,7 +5363,7 @@ export default withEnglishFallback({
successLabel: "{count} exitosas", successLabel: "{count} exitosas",
failLabel: "{count} fallidas", failLabel: "{count} fallidas",
totalLabel: "/ {count} filas", totalLabel: "/ {count} filas",
copyAllSql: "Copy All SQL", copySampleSql: "Copiar SQL de ejemplo",
prevStep: "Anterior", prevStep: "Anterior",
startInsert: "Start Insert", startInsert: "Start Insert",
confirm: "Confirmar", confirm: "Confirmar",
@@ -5589,6 +5589,16 @@ export default withEnglishFallback({
drivers_license: "Carnet de conducir", drivers_license: "Carnet de conducir",
custom: "Personalizado", custom: "Personalizado",
}, },
insertingTable: "Escribiendo en {table} (tabla {index}/{count})",
elapsed: "{seconds} s transcurridos",
samplePreviewNotice: "Se muestran las primeras {shown} filas de ejemplo; las {count} filas se generan al insertar",
largeRowCountHint: "Número de filas elevado ({count}); se generan e insertan por lotes y puede cancelar en cualquier momento",
timeoutSecs: "Tiempo de espera (segundos)",
timeoutHint: "0 significa sin límite; mantenga 0 o auméntelo para grandes volúmenes",
batchRows: "Filas por lote",
batchRowsHint: "Filas generadas e insertadas en cada envío; los lotes grandes se dividen automáticamente",
cancelInsert: "Cancelar inserción",
cancelledLabel: "Cancelado; las filas ya insertadas permanecen en la tabla",
}, },
tableToolbox: { tableToolbox: {
title: "Herramientas de datos", title: "Herramientas de datos",
+11 -1
View File
@@ -5361,7 +5361,7 @@ export default withEnglishFallback({
successLabel: "{count} riuscite", successLabel: "{count} riuscite",
failLabel: "{count} non riuscite", failLabel: "{count} non riuscite",
totalLabel: "/ {count} righe", totalLabel: "/ {count} righe",
copyAllSql: "Copia tutto SQL", copySampleSql: "Copia SQL di esempio",
prevStep: "Precedente", prevStep: "Precedente",
startInsert: "Avvia inserimento", startInsert: "Avvia inserimento",
confirm: "Conferma", confirm: "Conferma",
@@ -5587,6 +5587,16 @@ export default withEnglishFallback({
drivers_license: "Patente di guida", drivers_license: "Patente di guida",
custom: "Personalizzato", custom: "Personalizzato",
}, },
insertingTable: "Scrittura in {table} (tabella {index}/{count})",
elapsed: "{seconds}s trascorsi",
samplePreviewNotice: "Prime {shown} righe di esempio; le {count} righe vengono generate durante l'inserimento",
largeRowCountHint: "Numero elevato di righe ({count}); le righe vengono generate e inserite in batch ed è possibile annullare in qualsiasi momento",
timeoutSecs: "Timeout (secondi)",
timeoutHint: "0 significa nessun timeout; mantieni 0 o aumentalo per grandi volumi",
batchRows: "Righe per batch",
batchRowsHint: "Righe generate e inserite per invio; i batch troppo grandi vengono divisi automaticamente",
cancelInsert: "Annulla inserimento",
cancelledLabel: "Annullato; le righe già scritte restano nella tabella",
}, },
tableToolbox: { tableToolbox: {
title: "Strumenti dati", title: "Strumenti dati",
+11 -1
View File
@@ -5399,7 +5399,7 @@ export default withEnglishFallback({
successLabel: "{count}件成功", successLabel: "{count}件成功",
failLabel: "{count}件失敗", failLabel: "{count}件失敗",
totalLabel: "/ {count}エントリ", totalLabel: "/ {count}エントリ",
copyAllSql: "すべてのSQLをコピー", copySampleSql: "サンプル SQL をコピー",
prevStep: "戻る", prevStep: "戻る",
startInsert: "挿入開始", startInsert: "挿入開始",
confirm: "確認", confirm: "確認",
@@ -5625,6 +5625,16 @@ export default withEnglishFallback({
drivers_license: "運転免許証", drivers_license: "運転免許証",
custom: "カスタム", custom: "カスタム",
}, },
insertingTable: "{table} に書き込み中({index}/{count} テーブル)",
elapsed: "経過時間 {seconds} 秒",
samplePreviewNotice: "先頭 {shown} 行はサンプルです。{count} 行は挿入時に順次生成されます",
largeRowCountHint: "行数が多いため({count} 行)、生成と挿入をバッチで行います。いつでもキャンセルできます",
timeoutSecs: "タイムアウト(秒)",
timeoutHint: "0 はタイムアウトなしです。大量データでは 0 のままにするか、より長い時間を設定してください",
batchRows: "1 バッチの行数",
batchRowsHint: "1 回で生成・挿入する行数。上限を超える場合は自動的に分割されます",
cancelInsert: "挿入をキャンセル",
cancelledLabel: "キャンセルしました。書き込み済みの行はテーブルに残ります",
}, },
tableToolbox: { tableToolbox: {
title: "データツール", title: "データツール",
+11 -1
View File
@@ -5060,7 +5060,7 @@ export default withEnglishFallback({
successLabel: "{count}개 성공", successLabel: "{count}개 성공",
failLabel: "{count}개 실패", failLabel: "{count}개 실패",
totalLabel: "/ {count}개 항목", totalLabel: "/ {count}개 항목",
copyAllSql: "모든 SQL 복사", copySampleSql: "샘플 SQL 복사",
prevStep: "이전", prevStep: "이전",
startInsert: "삽입 시작", startInsert: "삽입 시작",
confirm: "확인", confirm: "확인",
@@ -5286,6 +5286,16 @@ export default withEnglishFallback({
drivers_license: "운전면허증", drivers_license: "운전면허증",
custom: "사용자 지정", custom: "사용자 지정",
}, },
insertingTable: "{table}에 쓰는 중 (표 {index}/{count})",
elapsed: "{seconds}초 경과",
samplePreviewNotice: "앞의 {shown}행은 예시입니다. 전체 {count}행은 삽입할 때 생성됩니다",
largeRowCountHint: "행 수가 많습니다({count}행). 배치로 생성과 삽입을 진행하며 언제든 취소할 수 있습니다",
timeoutSecs: "제한 시간(초)",
timeoutHint: "0이면 제한 시간 없음입니다. 대량 데이터에서는 0을 유지하거나 더 길게 설정하세요",
batchRows: "배치당 행 수",
batchRowsHint: "한 번에 생성하고 삽입할 행 수입니다. 서버 한도를 넘으면 자동으로 분할됩니다",
cancelInsert: "삽입 취소",
cancelledLabel: "취소되었습니다. 이미 삽입된 행은 테이블에 남습니다",
}, },
tableToolbox: { tableToolbox: {
title: "데이터 도구", title: "데이터 도구",
+11 -1
View File
@@ -5363,7 +5363,7 @@ export default withEnglishFallback({
successLabel: "{count} bem-sucedidas", successLabel: "{count} bem-sucedidas",
failLabel: "{count} falhas", failLabel: "{count} falhas",
totalLabel: "/ {count} linhas", totalLabel: "/ {count} linhas",
copyAllSql: "Copy All SQL", copySampleSql: "Copiar SQL de exemplo",
prevStep: "Anterior", prevStep: "Anterior",
startInsert: "Start Insert", startInsert: "Start Insert",
confirm: "Confirmar", confirm: "Confirmar",
@@ -5589,6 +5589,16 @@ export default withEnglishFallback({
drivers_license: "Carteira de motorista", drivers_license: "Carteira de motorista",
custom: "Personalizado", custom: "Personalizado",
}, },
insertingTable: "Gravando em {table} (tabela {index}/{count})",
elapsed: "{seconds}s decorridos",
samplePreviewNotice: "Exibindo as primeiras {shown} linhas de exemplo; as {count} linhas são geradas durante a inserção",
largeRowCountHint: "Grande número de linhas ({count}); as linhas são geradas e inseridas em lotes e você pode cancelar a qualquer momento",
timeoutSecs: "Tempo limite (segundos)",
timeoutHint: "0 significa sem limite; mantenha 0 ou aumente para grandes volumes",
batchRows: "Linhas por lote",
batchRowsHint: "Linhas geradas e inseridas por envio; lotes grandes são divididos automaticamente",
cancelInsert: "Cancelar inserção",
cancelledLabel: "Cancelado; as linhas já gravadas permanecem na tabela",
}, },
tableToolbox: { tableToolbox: {
title: "Ferramentas de dados", title: "Ferramentas de dados",
+11 -1
View File
@@ -5633,7 +5633,7 @@ export default withEnglishFallback({
successLabel: "{count} başarılı", successLabel: "{count} başarılı",
failLabel: "{count} başarısız", failLabel: "{count} başarısız",
totalLabel: "/ {count} girdi", totalLabel: "/ {count} girdi",
copyAllSql: "Tüm SQL'i Kopyala", copySampleSql: "Örnek SQL'i kopyala",
prevStep: "Geri", prevStep: "Geri",
startInsert: "Eklemeyi Başlat", startInsert: "Eklemeyi Başlat",
confirm: "Onayla", confirm: "Onayla",
@@ -5859,6 +5859,16 @@ export default withEnglishFallback({
drivers_license: "Sürücü Belgesi", drivers_license: "Sürücü Belgesi",
custom: "Özel", custom: "Özel",
}, },
insertingTable: "{table} yazılıyor (tablo {index}/{count})",
elapsed: "{seconds}s geçti",
samplePreviewNotice: "İlk {shown} örnek satır gösteriliyor; {count} satır ekleme sırasında oluşturulur",
largeRowCountHint: "Satır sayısı yüksek ({count}); satırlar toplu olarak oluşturulup eklenir ve istediğiniz zaman iptal edebilirsiniz",
timeoutSecs: "Zaman aşımı (saniye)",
timeoutHint: "0 sınırsız anlamına gelir; büyük verilerde 0 olarak bırakın veya daha uzun bir süre ayarlayın",
batchRows: "Toplu iş başına satır",
batchRowsHint: "Her gönderimde oluşturulup eklenen satır sayısı; büyük toplu işler otomatik olarak bölünür",
cancelInsert: "Eklemeyi iptal et",
cancelledLabel: "İptal edildi; yazılan satırlar tabloda kalır",
}, },
tableToolbox: { tableToolbox: {
title: "Veri Araçları", title: "Veri Araçları",
+11 -1
View File
@@ -5627,7 +5627,7 @@ export default withEnglishFallback({
successLabel: "{count} 成功", successLabel: "{count} 成功",
failLabel: "{count} 失败", failLabel: "{count} 失败",
totalLabel: "/ {count} 条", totalLabel: "/ {count} 条",
copyAllSql: "复制全部 SQL", copySampleSql: "复制样例 SQL",
prevStep: "上一步", prevStep: "上一步",
startInsert: "开始插入", startInsert: "开始插入",
confirm: "确定", confirm: "确定",
@@ -5853,6 +5853,16 @@ export default withEnglishFallback({
drivers_license: "驾驶证号", drivers_license: "驾驶证号",
custom: "自定义", custom: "自定义",
}, },
insertingTable: "正在写入 {table}(第 {index}/{count} 张表)",
elapsed: "已用时 {seconds} 秒",
samplePreviewNotice: "以上为前 {shown} 行样例数据,共 {count} 行将在插入时流式生成",
largeRowCountHint: "行数较大({count} 行),插入时将采用边生成边写入的批量模式,可随时取消",
timeoutSecs: "超时时间(秒)",
timeoutHint: "0 表示不限制超时;数据量较大时建议保持 0 或设置更长时间",
batchRows: "每批行数",
batchRowsHint: "每次生成并写入的行数;超过服务端报文上限时会自动再拆分",
cancelInsert: "取消插入",
cancelledLabel: "已取消,已写入的行保留在表中",
}, },
tableToolbox: { tableToolbox: {
title: "数据工具", title: "数据工具",
+11 -1
View File
@@ -4683,7 +4683,7 @@ export default withEnglishFallback({
successLabel: "{count} 成功", successLabel: "{count} 成功",
failLabel: "{count} 失敗", failLabel: "{count} 失敗",
totalLabel: "/ {count} 條", totalLabel: "/ {count} 條",
copyAllSql: "複製全部 SQL", copySampleSql: "複製範例 SQL",
prevStep: "上一步", prevStep: "上一步",
startInsert: "開始插入", startInsert: "開始插入",
confirm: "確定", confirm: "確定",
@@ -4909,6 +4909,16 @@ export default withEnglishFallback({
drivers_license: "駕駛證號", drivers_license: "駕駛證號",
custom: "自訂", custom: "自訂",
}, },
insertingTable: "正在寫入 {table}(第 {index}/{count} 張表)",
elapsed: "已用時 {seconds} 秒",
samplePreviewNotice: "以上為前 {shown} 列範例資料,共 {count} 列將在插入時串流產生",
largeRowCountHint: "列數較大({count} 列),插入時將採用邊產生邊寫入的批次模式,可隨時取消",
timeoutSecs: "逾時時間(秒)",
timeoutHint: "0 表示不限制逾時;資料量較大時建議保持 0 或設定更長時間",
batchRows: "每批列數",
batchRowsHint: "每次產生並寫入的列數;超過伺服端封包上限時會自動再拆分",
cancelInsert: "取消插入",
cancelledLabel: "已取消,已寫入的列保留在表中",
}, },
tableToolbox: { tableToolbox: {
title: "數據工具", title: "數據工具",
+135 -28
View File
@@ -1827,24 +1827,62 @@ function generateTdengineChildTableName(): string {
return `dbx_gen_${Date.now().toString(36)}_${randomSuffix}`; return `dbx_gen_${Date.now().toString(36)}_${randomSuffix}`;
} }
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult { /**
* Cross-chunk generation state.
*
* Bulk generation is streamed one chunk at a time so the UI never holds the
* full row set in memory. Anything that must stay coherent across chunks
* (uniqueness sets, TDengine tag values, the global row index, the TDengine
* child-table name) therefore has to live outside the chunk function.
*/
export interface TableGenerateChunkState {
readonly isTdengineStable: boolean;
readonly shouldAddTbname: boolean;
readonly tbname: string | null;
readonly colNames: string[];
readonly targetTable: string;
readonly columnList: string;
readonly insertPrefix: string;
readonly uniqueValues: (Set<string> | null)[];
readonly tagValues: Map<string, unknown>;
nextIndex: number;
}
export function createTableGenerateState(config: TableGenerateConfig, databaseType?: DatabaseType): TableGenerateChunkState {
const isTdengineStable = isTdengineStableGenerate(config, databaseType); const isTdengineStable = isTdengineStableGenerate(config, databaseType);
const hasTbnameColumn = config.columns.some((column) => column.columnName.toLowerCase() === "tbname"); const hasTbnameColumn = config.columns.some((column) => column.columnName.toLowerCase() === "tbname");
const shouldAddTbname = isTdengineStable && !hasTbnameColumn; const shouldAddTbname = isTdengineStable && !hasTbnameColumn;
const tdengineChildTableName = shouldAddTbname ? generateTdengineChildTableName() : null;
const tagValues = new Map<string, unknown>();
const uniqueValues = config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null));
const colNames = shouldAddTbname ? ["tbname", ...config.columns.map((c) => c.columnName)] : config.columns.map((c) => c.columnName); const colNames = shouldAddTbname ? ["tbname", ...config.columns.map((c) => c.columnName)] : config.columns.map((c) => c.columnName);
const rows: unknown[][] = []; const columnList = colNames.map((column) => quoteTableIdentifier(databaseType, column)).join(", ");
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
return {
isTdengineStable,
shouldAddTbname,
tbname: shouldAddTbname ? generateTdengineChildTableName() : null,
colNames,
targetTable,
columnList,
insertPrefix: `INSERT INTO ${targetTable} (${columnList}) VALUES`,
uniqueValues: config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null)),
tagValues: new Map<string, unknown>(),
nextIndex: 0,
};
}
for (let i = 0; i < config.rowCount; i++) { /** Generates the next `chunkSize` rows (or whatever remains) and advances `state.nextIndex`. */
export function generateTableRowsChunk(config: TableGenerateConfig, state: TableGenerateChunkState, chunkSize: number): unknown[][] {
const rows: unknown[][] = [];
if (chunkSize <= 0) return rows;
const total = Math.max(0, config.rowCount);
const end = Math.min(total, state.nextIndex + chunkSize);
for (let i = state.nextIndex; i < end; i++) {
const row = config.columns.map((col, columnIndex) => { const row = config.columns.map((col, columnIndex) => {
if (isTdengineStable && col.isTag && tagValues.has(col.columnName)) { if (state.isTdengineStable && col.isTag && state.tagValues.has(col.columnName)) {
return tagValues.get(col.columnName); return state.tagValues.get(col.columnName);
} }
let value: unknown; let value: unknown;
if (col.generatorParams?.unique) { if (col.generatorParams?.unique) {
const seen = uniqueValues[columnIndex]!; const seen = state.uniqueValues[columnIndex]!;
let found = false; let found = false;
for (let attempt = 0; attempt < MAX_UNIQUE_GENERATION_ATTEMPTS; attempt++) { for (let attempt = 0; attempt < MAX_UNIQUE_GENERATION_ATTEMPTS; attempt++) {
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault); value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
@@ -1861,29 +1899,98 @@ export function generateTableData(config: TableGenerateConfig, databaseType?: Da
} else { } else {
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault); value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
} }
if (isTdengineStable && col.isTag) { if (state.isTdengineStable && col.isTag) {
tagValues.set(col.columnName, value); state.tagValues.set(col.columnName, value);
} }
return value; return value;
}); });
rows.push(shouldAddTbname ? [tdengineChildTableName, ...row] : row); rows.push(state.tbname !== null ? [state.tbname, ...row] : row);
} }
state.nextIndex = end;
return rows;
}
const quotedCols = colNames.map((column) => quoteTableIdentifier(databaseType, column)); export function formatGeneratedRowValues(config: TableGenerateConfig, databaseType: DatabaseType | undefined, state: TableGenerateChunkState, row: unknown[]): string {
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database }); const values = row
const columnList = quotedCols.join(", "); .map((value, index) => {
const insertPrefix = `INSERT INTO ${targetTable} (${columnList}) VALUES`; const configIndex = state.shouldAddTbname ? index - 1 : index;
const valueRows = rows.map( return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
(row) => })
`(${row .join(", ");
.map((value, index) => { return `(${values})`;
const configIndex = shouldAddTbname ? index - 1 : index; }
return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
})
.join(", ")})`,
);
const statements = databaseType === "oracle" ? buildOracleInsertStatements(targetTable, columnList, valueRows) : supportsGeneratedMultiRowValues(databaseType) ? [`${insertPrefix}\n${valueRows.join(",\n")};`] : valueRows.map((values) => `${insertPrefix} ${values};`);
const sql = statements.join("\n");
return { columns: colNames, rows, sql, statements }; /**
* Builds the INSERT statements for a batch together with the number of value
* rows each statement carries.
*
* `rowsPerStatement` is aligned index-by-index with `statements`, so a caller
* that executes the statements can map each returned per-statement result back
* to the rows it inserted. This is what makes failed-batch accounting possible
* on backends that report failures inside the result array instead of throwing.
*/
export function generateInsertBatches(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): { statements: string[]; rowsPerStatement: number[] } {
if (databaseType === "oracle") {
const statements = buildOracleInsertStatements(state.targetTable, state.columnList, valueRows);
return { statements, rowsPerStatement: oracleRowsPerStatement(valueRows, statements) };
}
if (!supportsGeneratedMultiRowValues(databaseType) || forceSingleRow) {
return { statements: valueRows.map((values) => `${state.insertPrefix} ${values};`), rowsPerStatement: valueRows.map(() => 1) };
}
if (valueRows.length === 0) return { statements: [], rowsPerStatement: [] };
return { statements: [`${state.insertPrefix}\n${valueRows.join(",\n")};`], rowsPerStatement: [valueRows.length] };
}
/**
* Oracle batches INSERT ALL statements `ORACLE_INSERT_ALL_BATCH_SIZE` rows at a
* time, so the per-statement row counts have to mirror that chunking.
*/
function oracleRowsPerStatement(valueRows: string[], statements: string[]): number[] {
if (statements.length === 0) return [];
// A single value row is emitted as a plain VALUES statement, not INSERT ALL.
if (valueRows.length === 1) return [1];
const counts: number[] = [];
for (let remaining = valueRows.length; remaining > 0; remaining -= ORACLE_INSERT_ALL_BATCH_SIZE) {
counts.push(Math.min(ORACLE_INSERT_ALL_BATCH_SIZE, remaining));
}
return counts;
}
export function buildGenerateInsertStatements(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): string[] {
return generateInsertBatches(databaseType, state, valueRows, forceSingleRow).statements;
}
/**
* Splits formatted value rows into statement groups that stay within a byte budget.
*
* Servers reject statements larger than `max_allowed_packet`, so an oversized
* multi-row INSERT has to be cut into several statements before it is sent.
* Splitting happens after generation (never by rewinding the generator state)
* so uniqueness bookkeeping stays intact.
*/
export function splitValueRowsByByteBudget(state: TableGenerateChunkState, valueRows: string[], maxBytes: number): string[][] {
if (maxBytes <= 0) return valueRows.length > 0 ? [valueRows] : [];
const groups: string[][] = [];
let current: string[] = [];
let currentBytes = 0;
for (const values of valueRows) {
const bytes = values.length + state.insertPrefix.length + 4;
if (current.length > 0 && currentBytes + bytes > maxBytes) {
groups.push(current);
current = [];
currentBytes = 0;
}
current.push(values);
currentBytes += bytes;
}
if (current.length > 0) groups.push(current);
return groups;
}
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
const state = createTableGenerateState(config, databaseType);
const rows = generateTableRowsChunk(config, state, config.rowCount);
const valueRows = rows.map((row) => formatGeneratedRowValues(config, databaseType, state, row));
const statements = buildGenerateInsertStatements(databaseType, state, valueRows);
return { columns: state.colNames, rows, sql: statements.join("\n"), statements };
} }
@@ -0,0 +1,95 @@
import { BackendErrorException } from "@/lib/backend/errorUtils";
import type { QueryResult } from "@/types/database";
export interface BatchExecutionOutcome {
/** Rows the backend confirmed, counted only from statements that succeeded. */
readonly insertedRows: number;
/** True when at least one statement in the batch failed. */
readonly failed: boolean;
/** Message of the first failed statement, when one is available. */
readonly error?: string;
/** Index of the first failed statement, when one is available. */
readonly failedStatement?: number;
}
const EMPTY_OUTCOME: BatchExecutionOutcome = { insertedRows: 0, failed: false };
/**
* Maps the per-statement results of one executed batch back to the rows that
* were really inserted.
*
* Not every backend throws when a statement fails: the sequential executor
* (`query.rs`) records the failure in the result array and still returns `Ok`,
* and the MySQL batch route returns `Ok` with an embedded error result even on
* connection checkout failure. Counting rows on the `await` alone therefore
* over-reports inserts and can leave `continueOnError` silently dropping rows.
*
* `rowsPerStatement` comes from `generateInsertBatches()` and is aligned with
* the statements that were sent; a statement counts as successful only when it
* carries neither `execution_error` nor an `error`.
*/
export function summarizeBatchResults(results: QueryResult[] | null | undefined, rowsPerStatement: number[]): BatchExecutionOutcome {
if (!results || results.length === 0 || rowsPerStatement.length === 0) return EMPTY_OUTCOME;
let insertedRows = 0;
let error: string | undefined;
let failedStatement: number | undefined;
for (let index = 0; index < results.length && index < rowsPerStatement.length; index++) {
const result = results[index];
const statementError = statementErrorMessage(result);
if (statementError === undefined) {
insertedRows += Math.max(0, rowsPerStatement[index]);
continue;
}
if (failedStatement === undefined) {
failedStatement = index;
error = statementError;
}
}
const failed = failedStatement !== undefined;
return failed ? { insertedRows, failed, error, failedStatement } : { insertedRows, failed };
}
function statementErrorMessage(result: QueryResult): string | undefined {
if (!result) return "Unknown statement failure";
if (result.execution_error === true) {
return result.error?.detail ?? result.error?.messageKey ?? errorMessageFromRows(result);
}
if (result.error) {
return result.error.detail ?? result.error.messageKey;
}
return undefined;
}
/** Error-shaped results carry the driver message in a single "Error" row. */
function errorMessageFromRows(result: QueryResult): string | undefined {
const firstRow = result.rows?.[0];
const firstCell = Array.isArray(firstRow) ? firstRow[0] : undefined;
return typeof firstCell === "string" && firstCell.trim().length > 0 ? firstCell : undefined;
}
const CANCELLATION_PATTERN = /\bcancel(?:ed|led|ing|ling|lation)?\b/i;
/**
* Recognizes a failure produced by cancelling a query.
*
* Cancelling a transaction or a single-statement round trip surfaces as an
* error instead of a quiet stop, so a user-initiated cancel has to be told
* apart from a real SQL failure. The backend reports cancellation as
* "Query canceled" (`QUERY_CANCELED`); structured backend errors are matched on
* their code/message key as well.
*/
export function isQueryCanceledError(error: unknown): boolean {
if (error instanceof BackendErrorException) {
const { code, messageKey, detail } = error.backendError;
if (CANCELLATION_PATTERN.test(code) || CANCELLATION_PATTERN.test(messageKey)) return true;
if (detail !== undefined && CANCELLATION_PATTERN.test(detail)) return true;
}
return CANCELLATION_PATTERN.test(errorMessage(error));
}
export function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
+209
View File
@@ -0,0 +1,209 @@
import { strict as assert } from "node:assert";
import { test } from "vitest";
import {
buildGenerateInsertStatements,
createTableGenerateState,
generateTableData,
generateTableRowsChunk,
splitValueRowsByByteBudget,
type TableGenerateConfig,
} from "../../apps/desktop/src/lib/dataGrid/dataGenerate.ts";
function makeConfig(overrides: Partial<TableGenerateConfig> = {}): TableGenerateConfig {
return {
tableName: "users",
schema: "public",
database: "app",
rowCount: 10,
columns: [
{ columnName: "id", dataType: "bigint", rowCount: 10, generatorKey: "sequence", generatorParams: { startValue: 1, increment: 1 } },
{ columnName: "name", dataType: "varchar(32)", rowCount: 10, generatorKey: "text" },
],
...overrides,
};
}
function deterministicConfig(rowCount: number): TableGenerateConfig {
return {
tableName: "users",
schema: "public",
database: "app",
rowCount,
columns: [
{ columnName: "id", dataType: "bigint", rowCount, generatorKey: "sequence", generatorParams: { startValue: 100, increment: 5 } },
{ columnName: "copy_of_id", dataType: "bigint", rowCount, generatorKey: "sequence", generatorParams: { startValue: 0, increment: -1 } },
],
};
}
test("chunked generation produces exactly the same rows as one-shot generation", () => {
const config = deterministicConfig(11);
const expected = generateTableData(config, "mysql");
const state = createTableGenerateState(config, "mysql");
const chunked: unknown[][] = [];
for (;;) {
const rows = generateTableRowsChunk(config, state, 3);
if (rows.length === 0) break;
chunked.push(...rows);
}
assert.deepEqual(chunked, expected.rows);
});
test("unique values stay distinct across chunks", () => {
const config = makeConfig({
rowCount: 24,
columns: [{ columnName: "code", dataType: "varchar(8)", rowCount: 24, generatorKey: "text", generatorParams: { unique: true } }],
});
const state = createTableGenerateState(config, "mysql");
const all: unknown[] = [];
for (;;) {
const rows = generateTableRowsChunk(config, state, 5);
if (rows.length === 0) break;
all.push(...rows.map((row) => row[0]));
}
assert.equal(all.length, 24);
assert.equal(new Set(all.map((value) => String(value))).size, 24);
});
test("sequence values continue monotonically across chunk boundaries", () => {
const config = makeConfig({ rowCount: 7 });
const state = createTableGenerateState(config, "mysql");
const ids: number[] = [];
for (;;) {
const rows = generateTableRowsChunk(config, state, 2);
if (rows.length === 0) break;
ids.push(...rows.map((row) => row[0] as number));
}
assert.deepEqual(ids, [1, 2, 3, 4, 5, 6, 7]);
});
test("chunking respects exact boundaries and terminates cleanly", () => {
const config = makeConfig({ rowCount: 6 });
const state = createTableGenerateState(config, "mysql");
assert.equal(generateTableRowsChunk(config, state, 0).length, 0);
assert.equal(generateTableRowsChunk(config, state, -1).length, 0);
assert.equal(generateTableRowsChunk(config, state, 6).length, 6);
assert.equal(generateTableRowsChunk(config, state, 6).length, 0);
assert.equal(generateTableRowsChunk(config, state, 100).length, 0);
});
test("a chunk larger than the remaining row count returns only the remainder", () => {
const config = makeConfig({ rowCount: 4 });
const state = createTableGenerateState(config, "mysql");
assert.equal(generateTableRowsChunk(config, state, 3).length, 3);
assert.equal(generateTableRowsChunk(config, state, 10).length, 1);
});
test("tdengine stable table keeps tag values and tbname coherent across chunks", () => {
const config = makeConfig({
tableName: "metrics",
schema: "",
database: "db",
rowCount: 9,
tableType: "STABLE",
columns: [
{ columnName: "ts", dataType: "timestamp", rowCount: 9, generatorKey: "time" },
{ columnName: "region", dataType: "varchar(16)", rowCount: 9, generatorKey: "text", isTag: true },
],
});
const state = createTableGenerateState(config, "tdengine");
assert.equal(state.shouldAddTbname, true);
assert.ok(state.tbname);
const tagValues = new Set<unknown>();
const tbnames = new Set<unknown>();
let total = 0;
for (;;) {
const rows = generateTableRowsChunk(config, state, 4);
if (rows.length === 0) break;
for (const row of rows) {
assert.equal(row.length, 3); // tbname + ts + region
tbnames.add(row[0]);
tagValues.add(row[2]);
total++;
}
}
assert.equal(total, 9);
assert.equal(tbnames.size, 1);
assert.equal(tagValues.size, 1); // tag is generated once and reused
assert.equal(state.colNames[0], "tbname");
});
test("buildGenerateInsertStatements merges mysql rows into one multi-row INSERT", () => {
const config = makeConfig({ rowCount: 3 });
const state = createTableGenerateState(config, "mysql");
const statements = buildGenerateInsertStatements("mysql", state, ["(1, 'a')", "(2, 'b')", "(3, 'c')"]);
assert.equal(statements.length, 1);
assert.match(statements[0], /^INSERT INTO /);
assert.match(statements[0], /VALUES\n\(1, 'a'\),\n\(2, 'b'\),\n\(3, 'c'\);$/);
});
test("buildGenerateInsertStatements falls back to one statement per row without multi-row support", () => {
const config = makeConfig({ rowCount: 2 });
const state = createTableGenerateState(config, "iris");
const statements = buildGenerateInsertStatements("iris", state, ["(1, 'a')", "(2, 'b')"]);
assert.equal(statements.length, 2);
assert.match(statements[0], / VALUES \(1, 'a'\);$/);
assert.match(statements[1], / VALUES \(2, 'b'\);$/);
});
test("buildGenerateInsertStatements honours forceSingleRow and empty input", () => {
const config = makeConfig();
const state = createTableGenerateState(config, "mysql");
assert.deepEqual(buildGenerateInsertStatements("mysql", state, [], false), []);
assert.deepEqual(buildGenerateInsertStatements("mysql", state, [], true), []);
const forced = buildGenerateInsertStatements("mysql", state, ["(1, 'a')", "(2, 'b')"], true);
assert.equal(forced.length, 2);
});
test("splitValueRowsByByteBudget keeps every group within the budget", () => {
const config = makeConfig();
const state = createTableGenerateState(config, "mysql");
const valueRows = ["(1, 'a')", "(2, 'b')", "(3, 'c')", "(4, 'd')"];
// Budget that only fits the prefix plus one row forces per-row grouping.
const prefixBytes = state.insertPrefix.length + 4;
const groups = splitValueRowsByByteBudget(state, valueRows, prefixBytes + 12);
assert.ok(groups.length > 1);
for (const group of groups) {
const bytes = state.insertPrefix.length + 4 + group.reduce((sum, values) => sum + values.length + 4, 0);
assert.ok(bytes <= prefixBytes + 12, `group exceeds budget: ${bytes}`);
}
assert.deepEqual(groups.flat(), valueRows);
});
test("splitValueRowsByByteBudget keeps an oversized single row in its own group", () => {
const config = makeConfig();
const state = createTableGenerateState(config, "mysql");
const huge = `(${"x".repeat(500)})`;
const groups = splitValueRowsByByteBudget(state, [huge, "(2, 'b')"], 64);
assert.equal(groups.length, 2);
assert.equal(groups[0][0], huge);
assert.equal(groups[1][0], "(2, 'b')");
});
test("splitValueRowsByByteBudget handles degenerate inputs", () => {
const config = makeConfig();
const state = createTableGenerateState(config, "mysql");
assert.deepEqual(splitValueRowsByByteBudget(state, [], 1024), []);
// maxBytes <= 0 disables splitting and returns a single group.
assert.deepEqual(splitValueRowsByByteBudget(state, ["(1, 'a')", "(2, 'b')"], 0), [["(1, 'a')", "(2, 'b')"]]);
assert.deepEqual(splitValueRowsByByteBudget(state, ["(1, 'a')"], -5), [["(1, 'a')"]]);
});
+138
View File
@@ -0,0 +1,138 @@
import { strict as assert } from "node:assert";
import { test } from "vitest";
import { BackendErrorException } from "../../apps/desktop/src/lib/backend/errorUtils.ts";
import { createTableGenerateState, generateInsertBatches, type TableGenerateConfig } from "../../apps/desktop/src/lib/dataGrid/dataGenerate.ts";
import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "../../apps/desktop/src/lib/dataGrid/generateInsertAccounting.ts";
import type { QueryResult } from "../../apps/desktop/src/types/database.ts";
function makeConfig(rowCount: number): TableGenerateConfig {
return {
tableName: "users",
schema: "public",
database: "app",
rowCount,
columns: [{ columnName: "id", dataType: "bigint", rowCount, generatorKey: "sequence", generatorParams: { startValue: 1, increment: 1 } }],
};
}
function state(databaseType: string, rowCount: number) {
return createTableGenerateState(makeConfig(rowCount), databaseType as never);
}
function result(options: { executionError?: boolean; error?: { detail?: string; messageKey?: string }; rows?: unknown[][]; affected?: number } = {}): QueryResult {
return {
columns: ["affected"],
rows: options.rows ?? [],
affected_rows: options.affected ?? 0,
execution_time_ms: 1,
...(options.executionError ? { execution_error: true as const } : {}),
...(options.error ? { error: { version: 1, code: "DBX-TEST-0001", messageKey: options.error.messageKey ?? "backendErrors.test", messageParams: {}, source: "test", operationOutcome: "unknown", ...(options.error.detail ? { detail: options.error.detail } : {}) } } : {}),
} as QueryResult;
}
function valueRows(count: number): string[] {
return Array.from({ length: count }, (_unused, index) => `(${index + 1})`);
}
test("generateInsertBatches maps one multi-row statement to all of its rows", () => {
const batches = generateInsertBatches("mysql", state("mysql", 3), valueRows(3));
assert.equal(batches.statements.length, 1);
assert.deepEqual(batches.rowsPerStatement, [3]);
});
test("generateInsertBatches maps single-row statements one row each", () => {
const batches = generateInsertBatches("mysql", state("mysql", 3), valueRows(3), true);
assert.equal(batches.statements.length, 3);
assert.deepEqual(batches.rowsPerStatement, [1, 1, 1]);
});
test("generateInsertBatches keeps counts aligned for databases without multi-row support", () => {
const batches = generateInsertBatches("iris", state("iris", 2), valueRows(2));
assert.equal(batches.statements.length, 2);
assert.deepEqual(batches.rowsPerStatement, [1, 1]);
});
test("generateInsertBatches mirrors Oracle INSERT ALL chunking", () => {
const batches = generateInsertBatches("oracle", state("oracle", 101), valueRows(101));
assert.equal(batches.statements.length, 2);
assert.deepEqual(batches.rowsPerStatement, [100, 1]);
const single = generateInsertBatches("oracle", state("oracle", 1), valueRows(1));
assert.equal(single.statements.length, 1);
assert.deepEqual(single.rowsPerStatement, [1]);
});
test("generateInsertBatches returns nothing for an empty batch", () => {
const batches = generateInsertBatches("mysql", state("mysql", 0), []);
assert.deepEqual(batches.statements, []);
assert.deepEqual(batches.rowsPerStatement, []);
});
test("summarizeBatchResults counts every row when all statements succeed", () => {
const outcome = summarizeBatchResults([result({ affected: 3 })], [3]);
assert.equal(outcome.insertedRows, 3);
assert.equal(outcome.failed, false);
assert.equal(outcome.error, undefined);
});
test("summarizeBatchResults ignores failed statements that do not throw", () => {
// Sequential executor and the MySQL batch route resolve with the failure
// recorded in the result array instead of rejecting.
const outcome = summarizeBatchResults(
[result({ affected: 2 }), result({ executionError: true, error: { detail: "Duplicate entry" } }), result({ affected: 1 })],
[2, 5, 1],
);
assert.equal(outcome.insertedRows, 3);
assert.equal(outcome.failed, true);
assert.equal(outcome.error, "Duplicate entry");
assert.equal(outcome.failedStatement, 1);
});
test("summarizeBatchResults reads driver messages from error-shaped rows", () => {
const outcome = summarizeBatchResults([result({ executionError: true, rows: [["relation missing_table does not exist"]] })], [4]);
assert.equal(outcome.insertedRows, 0);
assert.equal(outcome.failed, true);
assert.equal(outcome.error, "relation missing_table does not exist");
});
test("summarizeBatchResults flags results that carry a structured error", () => {
const outcome = summarizeBatchResults([result({ error: { messageKey: "backendErrors.poolCheckoutFailed" } })], [7]);
assert.equal(outcome.failed, true);
assert.equal(outcome.insertedRows, 0);
assert.equal(outcome.error, "backendErrors.poolCheckoutFailed");
});
test("summarizeBatchResults ignores results beyond the statements that were sent", () => {
const outcome = summarizeBatchResults([result({ affected: 1 }), result({ affected: 1 }), result({ executionError: true })], [1, 1]);
assert.equal(outcome.insertedRows, 2);
assert.equal(outcome.failed, false);
});
test("summarizeBatchResults treats a missing result array as zero confirmed rows", () => {
const outcome = summarizeBatchResults(null, [3]);
assert.equal(outcome.insertedRows, 0);
assert.equal(outcome.failed, false);
});
test("summarizeBatchResults handles an empty batch", () => {
const outcome = summarizeBatchResults([], []);
assert.equal(outcome.insertedRows, 0);
assert.equal(outcome.failed, false);
});
test("isQueryCanceledError recognizes the backend cancellation message", () => {
assert.equal(isQueryCanceledError(new Error("Query canceled")), true);
assert.equal(isQueryCanceledError("Query canceled"), true);
assert.equal(isQueryCanceledError(new BackendErrorException({ message: "Query canceled" })), true);
});
test("isQueryCanceledError does not treat SQL failures as cancellations", () => {
assert.equal(isQueryCanceledError(new Error("relation \"users\" does not exist")), false);
assert.equal(isQueryCanceledError(new Error("Deadlock found when trying to get lock")), false);
assert.equal(isQueryCanceledError(new BackendErrorException({ message: "Syntax error" })), false);
});
test("errorMessage unwraps Error instances and plain values", () => {
assert.equal(errorMessage(new Error("boom")), "boom");
assert.equal(errorMessage("plain failure"), "plain failure");
});