fix(generate): stream bulk data generation with progress and configurable timeout
This commit is contained in:
1 parent
259071c09b
commit
a3772fb442
14 files changed
+986
-99
No files matched your search
@@ -3,14 +3,28 @@ import { reactive, ref, computed, onMounted, watch, type ComponentPublicInstance
|
||||
import { useI18n } from "vue-i18n";
|
||||
import { useConnectionStore } from "@/stores/connectionStore";
|
||||
import * as api from "@/lib/backend/api";
|
||||
import type { GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
|
||||
import { displayGeneratedValue, findGeneratorKey, formatGeneratedValue, generateTableData, defaultGeneratorParams, supportsGeneratedMultiRowValues, UniqueValueGenerationError } from "@/lib/dataGrid/dataGenerate";
|
||||
import type { ColumnGenerateConfig, GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
|
||||
import {
|
||||
createTableGenerateState,
|
||||
defaultGeneratorParams,
|
||||
displayGeneratedValue,
|
||||
findGeneratorKey,
|
||||
formatGeneratedRowValues,
|
||||
formatGeneratedValue,
|
||||
generateInsertBatches,
|
||||
generateTableData,
|
||||
generateTableRowsChunk,
|
||||
splitValueRowsByByteBudget,
|
||||
supportsGeneratedMultiRowValues,
|
||||
UniqueValueGenerationError,
|
||||
} from "@/lib/dataGrid/dataGenerate";
|
||||
import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "@/lib/dataGrid/generateInsertAccounting";
|
||||
import { qualifiedTableName, quoteTableIdentifier } from "@/lib/table/tableSelectSql";
|
||||
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
|
||||
import { effectiveDatabaseTypeForConnection } from "@/lib/database/jdbcDialect";
|
||||
import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard";
|
||||
import GeneratorParamsPanel from "./params/GeneratorParamsPanel.vue";
|
||||
import type { ColumnInfo, TableInfo } from "@/types/database";
|
||||
import type { ColumnInfo, QueryResult, TableInfo } from "@/types/database";
|
||||
|
||||
import { Dialog, DialogHeader, DialogTitle, DialogScrollContent, DialogContent, DialogFooter } from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -59,10 +73,36 @@ const currentStep = ref<"config" | "preview" | "result">("config");
|
||||
interface GeneratedTableResult extends GenerateResult {
|
||||
tableName: string;
|
||||
schema: string;
|
||||
/** Rows that will actually be inserted. `rows` only holds a preview sample. */
|
||||
targetRowCount: number;
|
||||
isSample: boolean;
|
||||
/** Column config after auto-increment start values have been resolved. */
|
||||
resolvedColumns: ColumnGenerateConfig[];
|
||||
}
|
||||
const generatedResults = ref<GeneratedTableResult[]>([]);
|
||||
const generationError = ref("");
|
||||
|
||||
/**
|
||||
* Preview only materializes a small sample. Generating the full row set up
|
||||
* front froze the UI for hundreds of thousands of rows and exhausted memory
|
||||
* at millions of rows.
|
||||
*/
|
||||
const PREVIEW_SAMPLE_ROWS = 50;
|
||||
const MAX_ROW_COUNT = 100_000_000;
|
||||
const DEFAULT_BATCH_ROWS = 1000;
|
||||
/**
|
||||
* Conservative per-statement budget. The backend clamps MySQL batches at 4MB
|
||||
* and derives a `max_allowed_packet` margin, so staying near 1MB keeps the
|
||||
* common 4MB / 16MB / 64MB server settings safe without extra round trips.
|
||||
*/
|
||||
const MAX_BATCH_BYTES = 1024 * 1024;
|
||||
const LARGE_ROW_COUNT_HINT = 100_000;
|
||||
|
||||
function normalizeRowCount(value: number): number {
|
||||
if (!Number.isFinite(value)) return 0;
|
||||
return Math.max(0, Math.min(MAX_ROW_COUNT, Math.floor(value)));
|
||||
}
|
||||
|
||||
const connectionName = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId)?.name : ""));
|
||||
|
||||
function tableKey(schema: string, table: string) {
|
||||
@@ -366,7 +406,20 @@ function onPreviewColResizeStart(ci: number, event: MouseEvent) {
|
||||
document.addEventListener("pointerup", onUp);
|
||||
document.body.classList.add("select-none", "cursor-col-resize");
|
||||
}
|
||||
const currentPreview = computed<GeneratedTableResult>(() => generatedResults.value[previewTableIndex.value] ?? { tableName: "", schema: "", columns: [], rows: [], sql: "", statements: [] });
|
||||
const currentPreview = computed<GeneratedTableResult>(
|
||||
() =>
|
||||
generatedResults.value[previewTableIndex.value] ?? {
|
||||
tableName: "",
|
||||
schema: "",
|
||||
columns: [],
|
||||
rows: [],
|
||||
sql: "",
|
||||
statements: [],
|
||||
targetRowCount: 0,
|
||||
isSample: false,
|
||||
resolvedColumns: [],
|
||||
},
|
||||
);
|
||||
|
||||
function displayPreviewCell(cell: unknown): string {
|
||||
return displayGeneratedValue(cell);
|
||||
@@ -395,6 +448,19 @@ async function fetchMaxValues(cfg: TableGenerateConfig): Promise<Record<string,
|
||||
return starts;
|
||||
}
|
||||
|
||||
function buildSampleResult(cfg: TableGenerateConfig, columns: ColumnGenerateConfig[], targetRowCount: number): GeneratedTableResult {
|
||||
const sampleCount = Math.min(targetRowCount, PREVIEW_SAMPLE_ROWS);
|
||||
const result = generateTableData({ ...cfg, columns, rowCount: sampleCount }, dbType.value);
|
||||
return {
|
||||
tableName: cfg.tableName,
|
||||
schema: cfg.schema,
|
||||
targetRowCount,
|
||||
isSample: targetRowCount > sampleCount,
|
||||
resolvedColumns: columns,
|
||||
...result,
|
||||
};
|
||||
}
|
||||
|
||||
async function doGenerate() {
|
||||
if (!Object.values(checkedTables).some(Boolean)) return;
|
||||
generationError.value = "";
|
||||
@@ -416,8 +482,7 @@ async function doGenerate() {
|
||||
return col;
|
||||
});
|
||||
}
|
||||
const result = generateTableData({ ...cfg, columns }, dbType.value);
|
||||
results.push({ tableName: cfg.tableName, schema: cfg.schema, ...result });
|
||||
results.push(buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount)));
|
||||
}
|
||||
} catch (error) {
|
||||
generationError.value = generationErrorMessage(error);
|
||||
@@ -447,8 +512,7 @@ async function regenerate() {
|
||||
}
|
||||
generationError.value = "";
|
||||
try {
|
||||
const result = generateTableData({ ...cfg, columns }, dbType.value);
|
||||
generatedResults.value[previewTableIndex.value] = { tableName: cfg.tableName, schema: cfg.schema, ...result };
|
||||
generatedResults.value[previewTableIndex.value] = buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount));
|
||||
} catch (error) {
|
||||
generationError.value = generationErrorMessage(error);
|
||||
}
|
||||
@@ -461,7 +525,13 @@ function generationErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function copyAllSql() {
|
||||
/**
|
||||
* Copies the script behind the preview. The preview only materializes a
|
||||
* `PREVIEW_SAMPLE_ROWS` sample, so this is the sample script rather than the
|
||||
* full batch — the label says so, and the insert step generates the remaining
|
||||
* rows incrementally instead of holding them in memory.
|
||||
*/
|
||||
function copySampleSql() {
|
||||
const allSql = allSqlStatements().join("\n\n");
|
||||
void navigator.clipboard.writeText(allSql);
|
||||
}
|
||||
@@ -474,6 +544,7 @@ interface TableResult {
|
||||
ok: number;
|
||||
err: number;
|
||||
error?: string;
|
||||
cancelled?: boolean;
|
||||
}
|
||||
const executeResults = ref<TableResult[]>([]);
|
||||
|
||||
@@ -482,6 +553,9 @@ const generateOptions = reactive({
|
||||
truncate: false,
|
||||
useTransaction: true,
|
||||
extendedInsert: true,
|
||||
/** 0 means "no timeout" — the backend maps 0 to an unbounded wait. */
|
||||
timeoutSecs: 0,
|
||||
batchRows: DEFAULT_BATCH_ROWS,
|
||||
});
|
||||
const supportsExtendedInsert = computed(() => supportsGeneratedMultiRowValues(dbType.value));
|
||||
watch(
|
||||
@@ -494,6 +568,35 @@ watch(
|
||||
|
||||
const optionsDialogOpen = ref(false);
|
||||
|
||||
interface InsertProgress {
|
||||
tableName: string;
|
||||
tableIndex: number;
|
||||
tableCount: number;
|
||||
insertedRows: number;
|
||||
totalRows: number;
|
||||
elapsedMs: number;
|
||||
}
|
||||
const insertProgress = ref<InsertProgress | null>(null);
|
||||
const insertCancelled = ref(false);
|
||||
let activeExecutionId: string | null = null;
|
||||
|
||||
const insertPercent = computed(() => {
|
||||
const p = insertProgress.value;
|
||||
if (!p || p.totalRows <= 0) return 0;
|
||||
return Math.min(100, Math.round((p.insertedRows / p.totalRows) * 100));
|
||||
});
|
||||
|
||||
function cancelInsert() {
|
||||
if (!executing.value) return;
|
||||
insertCancelled.value = true;
|
||||
if (activeExecutionId) void api.cancelQuery(activeExecutionId);
|
||||
}
|
||||
|
||||
function normalizeTimeoutSecs(value: number): number {
|
||||
if (!Number.isFinite(value) || value < 0) return 0;
|
||||
return Math.min(86_400, Math.floor(value));
|
||||
}
|
||||
|
||||
function sqlStatementsForTable(r: GeneratedTableResult): string[] {
|
||||
const stmts: string[] = [];
|
||||
const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
|
||||
@@ -516,71 +619,183 @@ function allSqlStatements(): string[] {
|
||||
return generatedResults.value.flatMap((r) => sqlStatementsForTable(r));
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams one table: generate a chunk, insert it, drop the references, repeat.
|
||||
*
|
||||
* Peak memory stays proportional to `batchRows` instead of the requested row
|
||||
* count, and each awaited round trip gives the main thread a chance to paint
|
||||
* the progress bar.
|
||||
*/
|
||||
async function streamInsertTable(cid: string, db: string, r: GeneratedTableResult, executionId: string, onRows: (insertedRows: number) => void): Promise<{ ok: number; attempted: number; error: string; cancelled: boolean }> {
|
||||
const cfg = configs[tableKey(r.schema, r.tableName)];
|
||||
const genCfg: TableGenerateConfig = {
|
||||
tableName: r.tableName,
|
||||
schema: r.schema,
|
||||
database: props.prefillDatabase ?? cfg?.database ?? "",
|
||||
tableType: cfg?.tableType,
|
||||
rowCount: r.targetRowCount,
|
||||
columns: r.resolvedColumns,
|
||||
};
|
||||
const state = createTableGenerateState(genCfg, dbType.value);
|
||||
const schema = r.schema || props.prefillSchema;
|
||||
const forceSingleRow = !generateOptions.extendedInsert;
|
||||
const timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs);
|
||||
const batchRows = Math.max(1, Math.floor(generateOptions.batchRows) || DEFAULT_BATCH_ROWS);
|
||||
let ok = 0;
|
||||
let attempted = 0;
|
||||
let lastError = "";
|
||||
let cancelled = false;
|
||||
|
||||
const run = (sql: string): Promise<QueryResult[]> =>
|
||||
api.executeMultiWithProgress(cid, db, sql, () => {}, schema, {
|
||||
timeoutSecs,
|
||||
useTransaction: generateOptions.useTransaction,
|
||||
continueOnError: generateOptions.continueOnError,
|
||||
executionId,
|
||||
});
|
||||
|
||||
/**
|
||||
* Executes one statement batch and folds the outcome into the running totals.
|
||||
* Returns false when the caller has to stop the table.
|
||||
*
|
||||
* The returned per-statement results are inspected instead of trusting the
|
||||
* `await`: several backend paths report a failed statement inside the result
|
||||
* array while still resolving, so counting rows on the await alone would
|
||||
* report failed batches as inserted.
|
||||
*/
|
||||
const executeBatch = async (statements: string[], rowsPerStatement: number[]): Promise<boolean> => {
|
||||
const expectedRows = rowsPerStatement.reduce((sum, rows) => sum + rows, 0);
|
||||
attempted += expectedRows;
|
||||
try {
|
||||
const results = await run(statements.join("\n"));
|
||||
// A row-carrying batch that comes back without per-statement results is
|
||||
// unconfirmed; counting it would over-report inserts.
|
||||
if (expectedRows > 0 && (!results || results.length === 0)) {
|
||||
if (!lastError) lastError = "Backend returned no result for the batch";
|
||||
return generateOptions.continueOnError;
|
||||
}
|
||||
const outcome = summarizeBatchResults(results, rowsPerStatement);
|
||||
ok += outcome.insertedRows;
|
||||
if (!outcome.failed) return true;
|
||||
if (!lastError) lastError = outcome.error ?? "Statement failed";
|
||||
console.error("[startInsert] SQL error:", lastError);
|
||||
return generateOptions.continueOnError;
|
||||
} catch (e: unknown) {
|
||||
// Cancelling a transaction or a single-statement round trip surfaces as
|
||||
// an error, so it has to be told apart from a real failure.
|
||||
if (insertCancelled.value || isQueryCanceledError(e)) {
|
||||
cancelled = true;
|
||||
return false;
|
||||
}
|
||||
const msg = errorMessage(e);
|
||||
console.error("[startInsert] SQL error:", msg);
|
||||
if (!lastError) lastError = msg;
|
||||
return generateOptions.continueOnError;
|
||||
}
|
||||
};
|
||||
|
||||
const finish = () => ({ ok, attempted, error: cancelled ? "" : lastError, cancelled });
|
||||
|
||||
if (generateOptions.truncate) {
|
||||
const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
|
||||
// TRUNCATE carries no generated rows, so it only gates the rest of the run.
|
||||
const truncateOk = await executeBatch([`TRUNCATE TABLE ${targetTable};`], [0]);
|
||||
if (!truncateOk) return finish();
|
||||
}
|
||||
|
||||
while (state.nextIndex < r.targetRowCount && !insertCancelled.value) {
|
||||
let valueRows: string[];
|
||||
try {
|
||||
const chunkRows = generateTableRowsChunk(genCfg, state, batchRows);
|
||||
if (chunkRows.length === 0) break;
|
||||
valueRows = chunkRows.map((row) => formatGeneratedRowValues(genCfg, dbType.value, state, row));
|
||||
} catch (e: unknown) {
|
||||
// Generation failures (unique space exhausted at row 500k, say) must not
|
||||
// discard the rows that earlier chunks already inserted.
|
||||
if (!lastError) lastError = generationErrorMessage(e);
|
||||
return finish();
|
||||
}
|
||||
|
||||
for (const group of splitValueRowsByByteBudget(state, valueRows, MAX_BATCH_BYTES)) {
|
||||
const { statements, rowsPerStatement } = generateInsertBatches(dbType.value, state, group, forceSingleRow);
|
||||
if (statements.length === 0) continue;
|
||||
const keepGoing = await executeBatch(statements, rowsPerStatement);
|
||||
onRows(ok);
|
||||
if (!keepGoing) return finish();
|
||||
}
|
||||
onRows(ok);
|
||||
// Yield so the progress bar repaints between chunks.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
if (insertCancelled.value) cancelled = true;
|
||||
return finish();
|
||||
}
|
||||
|
||||
async function startInsert() {
|
||||
if (executing.value || generationError.value) return;
|
||||
const cid = props.prefillConnectionId;
|
||||
const db = props.prefillDatabase;
|
||||
if (!cid || !db) return;
|
||||
const sql = allSqlStatements().join("\n");
|
||||
if (!sql.trim()) return;
|
||||
const targets = generatedResults.value.filter((r) => r.targetRowCount > 0 && r.resolvedColumns.length > 0);
|
||||
if (targets.length === 0) return;
|
||||
const guardSql = allSqlStatements().join("\n") || `INSERT INTO ${targets[0].tableName}`;
|
||||
try {
|
||||
await executeWithProductionSqlGuard({
|
||||
connection: store.getConfig(cid),
|
||||
database: db,
|
||||
sql,
|
||||
sql: guardSql,
|
||||
source: t("production.sourceDataGenerate"),
|
||||
execute: async () => {
|
||||
executing.value = true;
|
||||
insertCancelled.value = false;
|
||||
const executionId = crypto.randomUUID();
|
||||
activeExecutionId = executionId;
|
||||
const startedAt = performance.now();
|
||||
const perTable: TableResult[] = [];
|
||||
let stopAfterTable = false;
|
||||
for (const r of generatedResults.value) {
|
||||
const stmts = sqlStatementsForTable(r);
|
||||
const rowCount = r.rows.length;
|
||||
let ok = 0;
|
||||
let lastError = "";
|
||||
|
||||
const executeAsStatementGroup = generateOptions.useTransaction && !supportsGeneratedMultiRowValues(dbType.value);
|
||||
if (executeAsStatementGroup) {
|
||||
let stopAll = false;
|
||||
try {
|
||||
for (let ti = 0; ti < targets.length && !stopAll; ti++) {
|
||||
const r = targets[ti];
|
||||
insertProgress.value = {
|
||||
tableName: r.tableName,
|
||||
tableIndex: ti + 1,
|
||||
tableCount: targets.length,
|
||||
insertedRows: 0,
|
||||
totalRows: r.targetRowCount,
|
||||
elapsedMs: 0,
|
||||
};
|
||||
let outcome: { ok: number; attempted: number; error: string; cancelled: boolean };
|
||||
let insertedSoFar = 0;
|
||||
try {
|
||||
await api.executeInTransaction(cid, db, stmts, r.schema || props.prefillSchema);
|
||||
ok = rowCount;
|
||||
outcome = await streamInsertTable(cid, db, r, executionId, (insertedRows) => {
|
||||
insertedSoFar = insertedRows;
|
||||
if (insertProgress.value) {
|
||||
insertProgress.value = { ...insertProgress.value, insertedRows, elapsedMs: Math.round(performance.now() - startedAt) };
|
||||
}
|
||||
});
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
console.error("[startInsert] SQL error:", msg);
|
||||
lastError = msg;
|
||||
stopAfterTable = !generateOptions.continueOnError;
|
||||
// Safety net: even an unexpected throw must not hide the rows that
|
||||
// earlier batches already inserted.
|
||||
const wasCancelled = insertCancelled.value || isQueryCanceledError(e);
|
||||
outcome = { ok: insertedSoFar, attempted: insertedSoFar, error: wasCancelled ? "" : errorMessage(e), cancelled: wasCancelled };
|
||||
}
|
||||
} else {
|
||||
const usesMultiRowStatement = generateOptions.extendedInsert && supportsGeneratedMultiRowValues(dbType.value);
|
||||
for (let si = 0; si < stmts.length; si++) {
|
||||
try {
|
||||
if (generateOptions.useTransaction) {
|
||||
await api.executeInTransaction(cid, db, [stmts[si]], r.schema || props.prefillSchema);
|
||||
} else {
|
||||
await api.executeQuery(cid, db, stmts[si], r.schema || props.prefillSchema);
|
||||
}
|
||||
if (usesMultiRowStatement) {
|
||||
ok = rowCount;
|
||||
} else if (!(generateOptions.truncate && si === 0)) {
|
||||
ok++;
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
console.error("[startInsert] SQL error:", msg);
|
||||
if (!lastError) lastError = msg;
|
||||
if (usesMultiRowStatement) ok = 0;
|
||||
if (!generateOptions.continueOnError) {
|
||||
stopAfterTable = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const failedRows = Math.max(0, Math.max(outcome.attempted, outcome.ok) - outcome.ok);
|
||||
perTable.push({
|
||||
table: r.tableName,
|
||||
total: r.targetRowCount,
|
||||
ok: outcome.ok,
|
||||
err: failedRows,
|
||||
error: outcome.error || undefined,
|
||||
cancelled: outcome.cancelled || undefined,
|
||||
});
|
||||
if (outcome.ok > 0) {
|
||||
store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
|
||||
}
|
||||
if ((outcome.error && !generateOptions.continueOnError) || outcome.cancelled) stopAll = true;
|
||||
}
|
||||
perTable.push({ table: r.tableName, total: rowCount, ok, err: rowCount - ok, error: lastError || undefined });
|
||||
if (ok > 0) {
|
||||
store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
|
||||
}
|
||||
if (stopAfterTable) break;
|
||||
} finally {
|
||||
activeExecutionId = null;
|
||||
}
|
||||
executeResults.value = perTable;
|
||||
currentStep.value = "result";
|
||||
@@ -589,6 +804,7 @@ async function startInsert() {
|
||||
});
|
||||
} finally {
|
||||
executing.value = false;
|
||||
insertProgress.value = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -919,8 +1135,11 @@ async function onFileSelected(event: Event) {
|
||||
</div>
|
||||
<div class="flex items-center gap-3 rounded-md bg-muted/20 px-3 py-2">
|
||||
<Label class="text-xs shrink-0">{{ t("dataGenerate.rowCount") }}:</Label>
|
||||
<Input v-model.number="activeCfg.rowCount" class="h-7 w-24 text-xs" />
|
||||
<Input v-model.number="activeCfg.rowCount" type="number" min="0" :max="MAX_ROW_COUNT" class="h-7 w-28 text-xs" @blur="activeCfg.rowCount = normalizeRowCount(activeCfg.rowCount)" />
|
||||
</div>
|
||||
<p v-if="activeCfg.rowCount > LARGE_ROW_COUNT_HINT" class="px-1 text-[11px] leading-relaxed text-muted-foreground">
|
||||
{{ t("dataGenerate.largeRowCountHint", { count: activeCfg.rowCount.toLocaleString() }) }}
|
||||
</p>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
@@ -951,9 +1170,24 @@ async function onFileSelected(event: Event) {
|
||||
<option v-for="(r, i) in generatedResults" :key="i" :value="i">{{ r.tableName }}</option>
|
||||
</select>
|
||||
<span v-else class="text-sm font-medium">{{ generatedResults[0].tableName }}</span>
|
||||
<span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.rows.length }) }}</span>
|
||||
<span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.targetRowCount }) }}</span>
|
||||
</div>
|
||||
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
|
||||
</div>
|
||||
<div v-if="executing && insertProgress" class="space-y-1.5 border-b bg-muted/10 px-3 py-2">
|
||||
<div class="flex items-center justify-between gap-2 text-xs">
|
||||
<span class="truncate text-muted-foreground">
|
||||
{{ t("dataGenerate.insertingTable", { table: insertProgress.tableName, index: insertProgress.tableIndex, count: insertProgress.tableCount }) }}
|
||||
</span>
|
||||
<span class="shrink-0 font-medium tabular-nums">{{ insertProgress.insertedRows.toLocaleString() }} / {{ insertProgress.totalRows.toLocaleString() }}</span>
|
||||
</div>
|
||||
<div class="h-1.5 w-full overflow-hidden rounded-full bg-muted">
|
||||
<div class="h-full bg-primary transition-[width] duration-150" :style="{ width: insertPercent + '%' }" />
|
||||
</div>
|
||||
<div class="flex items-center justify-between text-[11px] text-muted-foreground">
|
||||
<span>{{ insertPercent }}%</span>
|
||||
<span>{{ t("dataGenerate.elapsed", { seconds: Math.round(insertProgress.elapsedMs / 1000) }) }}</span>
|
||||
</div>
|
||||
<Button variant="outline" size="sm" class="h-7 text-xs" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
|
||||
</div>
|
||||
<div class="flex flex-col h-[380px]">
|
||||
<div class="flex-1 overflow-auto overscroll-none bg-background">
|
||||
@@ -982,7 +1216,9 @@ async function onFileSelected(event: Event) {
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<div v-if="currentPreview.rows.length > 50" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">{{ t("dataGenerate.first50Rows", { count: currentPreview.rows.length }) }}</div>
|
||||
<div v-if="currentPreview.isSample" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">
|
||||
{{ t("dataGenerate.samplePreviewNotice", { shown: currentPreview.rows.length, count: currentPreview.targetRowCount }) }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1003,6 +1239,7 @@ async function onFileSelected(event: Event) {
|
||||
</span>
|
||||
</div>
|
||||
<div v-if="r.error" class="mt-1 text-destructive/80 break-all leading-relaxed">{{ r.error }}</div>
|
||||
<div v-else-if="r.cancelled" class="mt-1 text-muted-foreground">{{ t("dataGenerate.cancelledLabel") }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="executeResults.length === 0" class="flex h-24 items-center justify-center text-xs text-muted-foreground">{{ t("dataGenerate.noResult") }}</div>
|
||||
@@ -1030,7 +1267,7 @@ async function onFileSelected(event: Event) {
|
||||
</Button>
|
||||
</template>
|
||||
<template v-else-if="currentStep === 'preview' && generatedResults.length > 0">
|
||||
<Button variant="outline" size="sm" class="h-7 text-xs" @click="copyAllSql">{{ t("dataGenerate.copyAllSql") }}</Button>
|
||||
<Button variant="outline" size="sm" class="h-7 text-xs" @click="copySampleSql">{{ t("dataGenerate.copySampleSql") }}</Button>
|
||||
</template>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
@@ -1044,7 +1281,8 @@ async function onFileSelected(event: Event) {
|
||||
</Button>
|
||||
</template>
|
||||
<template v-else-if="currentStep === 'preview'">
|
||||
<Button variant="outline" size="sm" class="h-7 text-xs" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button>
|
||||
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="!executing" @click="cancelInsert">{{ t("dataGenerate.cancelInsert") }}</Button>
|
||||
<Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button>
|
||||
<Button variant="default" size="sm" class="h-7 text-xs" :disabled="executing || !!generationError" @click="startInsert">
|
||||
<Loader2 v-if="executing" class="mr-1 h-3 w-3 animate-spin" />
|
||||
{{ t("dataGenerate.startInsert") }}
|
||||
@@ -1081,6 +1319,16 @@ async function onFileSelected(event: Event) {
|
||||
<input type="checkbox" v-model="generateOptions.extendedInsert" class="h-4 w-4 accent-primary" :disabled="!supportsExtendedInsert" />
|
||||
<span class="text-xs">{{ t("dataGenerate.extendedInsert") }}</span>
|
||||
</label>
|
||||
<div class="flex items-center gap-3">
|
||||
<Label class="text-xs shrink-0">{{ t("dataGenerate.timeoutSecs") }}:</Label>
|
||||
<Input v-model.number="generateOptions.timeoutSecs" type="number" min="0" max="86400" class="h-7 w-24 text-xs" @blur="generateOptions.timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs)" />
|
||||
</div>
|
||||
<p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.timeoutHint") }}</p>
|
||||
<div class="flex items-center gap-3">
|
||||
<Label class="text-xs shrink-0">{{ t("dataGenerate.batchRows") }}:</Label>
|
||||
<Input v-model.number="generateOptions.batchRows" type="number" min="1" max="100000" class="h-7 w-24 text-xs" />
|
||||
</div>
|
||||
<p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.batchRowsHint") }}</p>
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<Button size="sm" class="h-7 text-xs" @click="optionsDialogOpen = false">{{ t("dataGenerate.ok") }}</Button>
|
||||
|
||||
@@ -5643,7 +5643,7 @@ export default {
|
||||
successLabel: "{count} succeeded",
|
||||
failLabel: "{count} failed",
|
||||
totalLabel: "/ {count} entries",
|
||||
copyAllSql: "Copy All SQL",
|
||||
copySampleSql: "Copy sample SQL",
|
||||
prevStep: "Previous",
|
||||
startInsert: "Start Insert",
|
||||
confirm: "Confirm",
|
||||
@@ -5869,6 +5869,16 @@ export default {
|
||||
drivers_license: "Driver's License",
|
||||
custom: "Custom",
|
||||
},
|
||||
insertingTable: "Writing to {table} (table {index}/{count})",
|
||||
elapsed: "{seconds}s elapsed",
|
||||
samplePreviewNotice: "Showing the first {shown} sample rows; all {count} rows are generated while inserting",
|
||||
largeRowCountHint: "Large row count ({count}); rows are generated and inserted in batches and you can cancel at any time",
|
||||
timeoutSecs: "Timeout (seconds)",
|
||||
timeoutHint: "0 means no timeout; keep 0 or raise it for large data volumes",
|
||||
batchRows: "Rows per batch",
|
||||
batchRowsHint: "Rows generated and inserted per round trip; oversized batches are split automatically",
|
||||
cancelInsert: "Cancel insert",
|
||||
cancelledLabel: "Cancelled; rows already written remain in the table",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "Data Tools",
|
||||
|
||||
@@ -5363,7 +5363,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count} exitosas",
|
||||
failLabel: "{count} fallidas",
|
||||
totalLabel: "/ {count} filas",
|
||||
copyAllSql: "Copy All SQL",
|
||||
copySampleSql: "Copiar SQL de ejemplo",
|
||||
prevStep: "Anterior",
|
||||
startInsert: "Start Insert",
|
||||
confirm: "Confirmar",
|
||||
@@ -5589,6 +5589,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "Carnet de conducir",
|
||||
custom: "Personalizado",
|
||||
},
|
||||
insertingTable: "Escribiendo en {table} (tabla {index}/{count})",
|
||||
elapsed: "{seconds} s transcurridos",
|
||||
samplePreviewNotice: "Se muestran las primeras {shown} filas de ejemplo; las {count} filas se generan al insertar",
|
||||
largeRowCountHint: "Número de filas elevado ({count}); se generan e insertan por lotes y puede cancelar en cualquier momento",
|
||||
timeoutSecs: "Tiempo de espera (segundos)",
|
||||
timeoutHint: "0 significa sin límite; mantenga 0 o auméntelo para grandes volúmenes",
|
||||
batchRows: "Filas por lote",
|
||||
batchRowsHint: "Filas generadas e insertadas en cada envío; los lotes grandes se dividen automáticamente",
|
||||
cancelInsert: "Cancelar inserción",
|
||||
cancelledLabel: "Cancelado; las filas ya insertadas permanecen en la tabla",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "Herramientas de datos",
|
||||
|
||||
@@ -5361,7 +5361,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count} riuscite",
|
||||
failLabel: "{count} non riuscite",
|
||||
totalLabel: "/ {count} righe",
|
||||
copyAllSql: "Copia tutto SQL",
|
||||
copySampleSql: "Copia SQL di esempio",
|
||||
prevStep: "Precedente",
|
||||
startInsert: "Avvia inserimento",
|
||||
confirm: "Conferma",
|
||||
@@ -5587,6 +5587,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "Patente di guida",
|
||||
custom: "Personalizzato",
|
||||
},
|
||||
insertingTable: "Scrittura in {table} (tabella {index}/{count})",
|
||||
elapsed: "{seconds}s trascorsi",
|
||||
samplePreviewNotice: "Prime {shown} righe di esempio; le {count} righe vengono generate durante l'inserimento",
|
||||
largeRowCountHint: "Numero elevato di righe ({count}); le righe vengono generate e inserite in batch ed è possibile annullare in qualsiasi momento",
|
||||
timeoutSecs: "Timeout (secondi)",
|
||||
timeoutHint: "0 significa nessun timeout; mantieni 0 o aumentalo per grandi volumi",
|
||||
batchRows: "Righe per batch",
|
||||
batchRowsHint: "Righe generate e inserite per invio; i batch troppo grandi vengono divisi automaticamente",
|
||||
cancelInsert: "Annulla inserimento",
|
||||
cancelledLabel: "Annullato; le righe già scritte restano nella tabella",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "Strumenti dati",
|
||||
|
||||
@@ -5399,7 +5399,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count}件成功",
|
||||
failLabel: "{count}件失敗",
|
||||
totalLabel: "/ {count}エントリ",
|
||||
copyAllSql: "すべてのSQLをコピー",
|
||||
copySampleSql: "サンプル SQL をコピー",
|
||||
prevStep: "戻る",
|
||||
startInsert: "挿入開始",
|
||||
confirm: "確認",
|
||||
@@ -5625,6 +5625,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "運転免許証",
|
||||
custom: "カスタム",
|
||||
},
|
||||
insertingTable: "{table} に書き込み中({index}/{count} テーブル)",
|
||||
elapsed: "経過時間 {seconds} 秒",
|
||||
samplePreviewNotice: "先頭 {shown} 行はサンプルです。{count} 行は挿入時に順次生成されます",
|
||||
largeRowCountHint: "行数が多いため({count} 行)、生成と挿入をバッチで行います。いつでもキャンセルできます",
|
||||
timeoutSecs: "タイムアウト(秒)",
|
||||
timeoutHint: "0 はタイムアウトなしです。大量データでは 0 のままにするか、より長い時間を設定してください",
|
||||
batchRows: "1 バッチの行数",
|
||||
batchRowsHint: "1 回で生成・挿入する行数。上限を超える場合は自動的に分割されます",
|
||||
cancelInsert: "挿入をキャンセル",
|
||||
cancelledLabel: "キャンセルしました。書き込み済みの行はテーブルに残ります",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "データツール",
|
||||
|
||||
@@ -5060,7 +5060,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count}개 성공",
|
||||
failLabel: "{count}개 실패",
|
||||
totalLabel: "/ {count}개 항목",
|
||||
copyAllSql: "모든 SQL 복사",
|
||||
copySampleSql: "샘플 SQL 복사",
|
||||
prevStep: "이전",
|
||||
startInsert: "삽입 시작",
|
||||
confirm: "확인",
|
||||
@@ -5286,6 +5286,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "운전면허증",
|
||||
custom: "사용자 지정",
|
||||
},
|
||||
insertingTable: "{table}에 쓰는 중 (표 {index}/{count})",
|
||||
elapsed: "{seconds}초 경과",
|
||||
samplePreviewNotice: "앞의 {shown}행은 예시입니다. 전체 {count}행은 삽입할 때 생성됩니다",
|
||||
largeRowCountHint: "행 수가 많습니다({count}행). 배치로 생성과 삽입을 진행하며 언제든 취소할 수 있습니다",
|
||||
timeoutSecs: "제한 시간(초)",
|
||||
timeoutHint: "0이면 제한 시간 없음입니다. 대량 데이터에서는 0을 유지하거나 더 길게 설정하세요",
|
||||
batchRows: "배치당 행 수",
|
||||
batchRowsHint: "한 번에 생성하고 삽입할 행 수입니다. 서버 한도를 넘으면 자동으로 분할됩니다",
|
||||
cancelInsert: "삽입 취소",
|
||||
cancelledLabel: "취소되었습니다. 이미 삽입된 행은 테이블에 남습니다",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "데이터 도구",
|
||||
|
||||
@@ -5363,7 +5363,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count} bem-sucedidas",
|
||||
failLabel: "{count} falhas",
|
||||
totalLabel: "/ {count} linhas",
|
||||
copyAllSql: "Copy All SQL",
|
||||
copySampleSql: "Copiar SQL de exemplo",
|
||||
prevStep: "Anterior",
|
||||
startInsert: "Start Insert",
|
||||
confirm: "Confirmar",
|
||||
@@ -5589,6 +5589,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "Carteira de motorista",
|
||||
custom: "Personalizado",
|
||||
},
|
||||
insertingTable: "Gravando em {table} (tabela {index}/{count})",
|
||||
elapsed: "{seconds}s decorridos",
|
||||
samplePreviewNotice: "Exibindo as primeiras {shown} linhas de exemplo; as {count} linhas são geradas durante a inserção",
|
||||
largeRowCountHint: "Grande número de linhas ({count}); as linhas são geradas e inseridas em lotes e você pode cancelar a qualquer momento",
|
||||
timeoutSecs: "Tempo limite (segundos)",
|
||||
timeoutHint: "0 significa sem limite; mantenha 0 ou aumente para grandes volumes",
|
||||
batchRows: "Linhas por lote",
|
||||
batchRowsHint: "Linhas geradas e inseridas por envio; lotes grandes são divididos automaticamente",
|
||||
cancelInsert: "Cancelar inserção",
|
||||
cancelledLabel: "Cancelado; as linhas já gravadas permanecem na tabela",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "Ferramentas de dados",
|
||||
|
||||
@@ -5633,7 +5633,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count} başarılı",
|
||||
failLabel: "{count} başarısız",
|
||||
totalLabel: "/ {count} girdi",
|
||||
copyAllSql: "Tüm SQL'i Kopyala",
|
||||
copySampleSql: "Örnek SQL'i kopyala",
|
||||
prevStep: "Geri",
|
||||
startInsert: "Eklemeyi Başlat",
|
||||
confirm: "Onayla",
|
||||
@@ -5859,6 +5859,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "Sürücü Belgesi",
|
||||
custom: "Özel",
|
||||
},
|
||||
insertingTable: "{table} yazılıyor (tablo {index}/{count})",
|
||||
elapsed: "{seconds}s geçti",
|
||||
samplePreviewNotice: "İlk {shown} örnek satır gösteriliyor; {count} satır ekleme sırasında oluşturulur",
|
||||
largeRowCountHint: "Satır sayısı yüksek ({count}); satırlar toplu olarak oluşturulup eklenir ve istediğiniz zaman iptal edebilirsiniz",
|
||||
timeoutSecs: "Zaman aşımı (saniye)",
|
||||
timeoutHint: "0 sınırsız anlamına gelir; büyük verilerde 0 olarak bırakın veya daha uzun bir süre ayarlayın",
|
||||
batchRows: "Toplu iş başına satır",
|
||||
batchRowsHint: "Her gönderimde oluşturulup eklenen satır sayısı; büyük toplu işler otomatik olarak bölünür",
|
||||
cancelInsert: "Eklemeyi iptal et",
|
||||
cancelledLabel: "İptal edildi; yazılan satırlar tabloda kalır",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "Veri Araçları",
|
||||
|
||||
@@ -5627,7 +5627,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count} 成功",
|
||||
failLabel: "{count} 失败",
|
||||
totalLabel: "/ {count} 条",
|
||||
copyAllSql: "复制全部 SQL",
|
||||
copySampleSql: "复制样例 SQL",
|
||||
prevStep: "上一步",
|
||||
startInsert: "开始插入",
|
||||
confirm: "确定",
|
||||
@@ -5853,6 +5853,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "驾驶证号",
|
||||
custom: "自定义",
|
||||
},
|
||||
insertingTable: "正在写入 {table}(第 {index}/{count} 张表)",
|
||||
elapsed: "已用时 {seconds} 秒",
|
||||
samplePreviewNotice: "以上为前 {shown} 行样例数据,共 {count} 行将在插入时流式生成",
|
||||
largeRowCountHint: "行数较大({count} 行),插入时将采用边生成边写入的批量模式,可随时取消",
|
||||
timeoutSecs: "超时时间(秒)",
|
||||
timeoutHint: "0 表示不限制超时;数据量较大时建议保持 0 或设置更长时间",
|
||||
batchRows: "每批行数",
|
||||
batchRowsHint: "每次生成并写入的行数;超过服务端报文上限时会自动再拆分",
|
||||
cancelInsert: "取消插入",
|
||||
cancelledLabel: "已取消,已写入的行保留在表中",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "数据工具",
|
||||
|
||||
@@ -4683,7 +4683,7 @@ export default withEnglishFallback({
|
||||
successLabel: "{count} 成功",
|
||||
failLabel: "{count} 失敗",
|
||||
totalLabel: "/ {count} 條",
|
||||
copyAllSql: "複製全部 SQL",
|
||||
copySampleSql: "複製範例 SQL",
|
||||
prevStep: "上一步",
|
||||
startInsert: "開始插入",
|
||||
confirm: "確定",
|
||||
@@ -4909,6 +4909,16 @@ export default withEnglishFallback({
|
||||
drivers_license: "駕駛證號",
|
||||
custom: "自訂",
|
||||
},
|
||||
insertingTable: "正在寫入 {table}(第 {index}/{count} 張表)",
|
||||
elapsed: "已用時 {seconds} 秒",
|
||||
samplePreviewNotice: "以上為前 {shown} 列範例資料,共 {count} 列將在插入時串流產生",
|
||||
largeRowCountHint: "列數較大({count} 列),插入時將採用邊產生邊寫入的批次模式,可隨時取消",
|
||||
timeoutSecs: "逾時時間(秒)",
|
||||
timeoutHint: "0 表示不限制逾時;資料量較大時建議保持 0 或設定更長時間",
|
||||
batchRows: "每批列數",
|
||||
batchRowsHint: "每次產生並寫入的列數;超過伺服端封包上限時會自動再拆分",
|
||||
cancelInsert: "取消插入",
|
||||
cancelledLabel: "已取消,已寫入的列保留在表中",
|
||||
},
|
||||
tableToolbox: {
|
||||
title: "數據工具",
|
||||
|
||||
@@ -1827,24 +1827,62 @@ function generateTdengineChildTableName(): string {
|
||||
return `dbx_gen_${Date.now().toString(36)}_${randomSuffix}`;
|
||||
}
|
||||
|
||||
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
|
||||
/**
|
||||
* Cross-chunk generation state.
|
||||
*
|
||||
* Bulk generation is streamed one chunk at a time so the UI never holds the
|
||||
* full row set in memory. Anything that must stay coherent across chunks
|
||||
* (uniqueness sets, TDengine tag values, the global row index, the TDengine
|
||||
* child-table name) therefore has to live outside the chunk function.
|
||||
*/
|
||||
export interface TableGenerateChunkState {
|
||||
readonly isTdengineStable: boolean;
|
||||
readonly shouldAddTbname: boolean;
|
||||
readonly tbname: string | null;
|
||||
readonly colNames: string[];
|
||||
readonly targetTable: string;
|
||||
readonly columnList: string;
|
||||
readonly insertPrefix: string;
|
||||
readonly uniqueValues: (Set<string> | null)[];
|
||||
readonly tagValues: Map<string, unknown>;
|
||||
nextIndex: number;
|
||||
}
|
||||
|
||||
export function createTableGenerateState(config: TableGenerateConfig, databaseType?: DatabaseType): TableGenerateChunkState {
|
||||
const isTdengineStable = isTdengineStableGenerate(config, databaseType);
|
||||
const hasTbnameColumn = config.columns.some((column) => column.columnName.toLowerCase() === "tbname");
|
||||
const shouldAddTbname = isTdengineStable && !hasTbnameColumn;
|
||||
const tdengineChildTableName = shouldAddTbname ? generateTdengineChildTableName() : null;
|
||||
const tagValues = new Map<string, unknown>();
|
||||
const uniqueValues = config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null));
|
||||
const colNames = shouldAddTbname ? ["tbname", ...config.columns.map((c) => c.columnName)] : config.columns.map((c) => c.columnName);
|
||||
const rows: unknown[][] = [];
|
||||
const columnList = colNames.map((column) => quoteTableIdentifier(databaseType, column)).join(", ");
|
||||
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
|
||||
return {
|
||||
isTdengineStable,
|
||||
shouldAddTbname,
|
||||
tbname: shouldAddTbname ? generateTdengineChildTableName() : null,
|
||||
colNames,
|
||||
targetTable,
|
||||
columnList,
|
||||
insertPrefix: `INSERT INTO ${targetTable} (${columnList}) VALUES`,
|
||||
uniqueValues: config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null)),
|
||||
tagValues: new Map<string, unknown>(),
|
||||
nextIndex: 0,
|
||||
};
|
||||
}
|
||||
|
||||
for (let i = 0; i < config.rowCount; i++) {
|
||||
/** Generates the next `chunkSize` rows (or whatever remains) and advances `state.nextIndex`. */
|
||||
export function generateTableRowsChunk(config: TableGenerateConfig, state: TableGenerateChunkState, chunkSize: number): unknown[][] {
|
||||
const rows: unknown[][] = [];
|
||||
if (chunkSize <= 0) return rows;
|
||||
const total = Math.max(0, config.rowCount);
|
||||
const end = Math.min(total, state.nextIndex + chunkSize);
|
||||
for (let i = state.nextIndex; i < end; i++) {
|
||||
const row = config.columns.map((col, columnIndex) => {
|
||||
if (isTdengineStable && col.isTag && tagValues.has(col.columnName)) {
|
||||
return tagValues.get(col.columnName);
|
||||
if (state.isTdengineStable && col.isTag && state.tagValues.has(col.columnName)) {
|
||||
return state.tagValues.get(col.columnName);
|
||||
}
|
||||
let value: unknown;
|
||||
if (col.generatorParams?.unique) {
|
||||
const seen = uniqueValues[columnIndex]!;
|
||||
const seen = state.uniqueValues[columnIndex]!;
|
||||
let found = false;
|
||||
for (let attempt = 0; attempt < MAX_UNIQUE_GENERATION_ATTEMPTS; attempt++) {
|
||||
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
|
||||
@@ -1861,29 +1899,98 @@ export function generateTableData(config: TableGenerateConfig, databaseType?: Da
|
||||
} else {
|
||||
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
|
||||
}
|
||||
if (isTdengineStable && col.isTag) {
|
||||
tagValues.set(col.columnName, value);
|
||||
if (state.isTdengineStable && col.isTag) {
|
||||
state.tagValues.set(col.columnName, value);
|
||||
}
|
||||
return value;
|
||||
});
|
||||
rows.push(shouldAddTbname ? [tdengineChildTableName, ...row] : row);
|
||||
rows.push(state.tbname !== null ? [state.tbname, ...row] : row);
|
||||
}
|
||||
state.nextIndex = end;
|
||||
return rows;
|
||||
}
|
||||
|
||||
const quotedCols = colNames.map((column) => quoteTableIdentifier(databaseType, column));
|
||||
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
|
||||
const columnList = quotedCols.join(", ");
|
||||
const insertPrefix = `INSERT INTO ${targetTable} (${columnList}) VALUES`;
|
||||
const valueRows = rows.map(
|
||||
(row) =>
|
||||
`(${row
|
||||
.map((value, index) => {
|
||||
const configIndex = shouldAddTbname ? index - 1 : index;
|
||||
return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
|
||||
})
|
||||
.join(", ")})`,
|
||||
);
|
||||
const statements = databaseType === "oracle" ? buildOracleInsertStatements(targetTable, columnList, valueRows) : supportsGeneratedMultiRowValues(databaseType) ? [`${insertPrefix}\n${valueRows.join(",\n")};`] : valueRows.map((values) => `${insertPrefix} ${values};`);
|
||||
const sql = statements.join("\n");
|
||||
export function formatGeneratedRowValues(config: TableGenerateConfig, databaseType: DatabaseType | undefined, state: TableGenerateChunkState, row: unknown[]): string {
|
||||
const values = row
|
||||
.map((value, index) => {
|
||||
const configIndex = state.shouldAddTbname ? index - 1 : index;
|
||||
return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
|
||||
})
|
||||
.join(", ");
|
||||
return `(${values})`;
|
||||
}
|
||||
|
||||
return { columns: colNames, rows, sql, statements };
|
||||
/**
|
||||
* Builds the INSERT statements for a batch together with the number of value
|
||||
* rows each statement carries.
|
||||
*
|
||||
* `rowsPerStatement` is aligned index-by-index with `statements`, so a caller
|
||||
* that executes the statements can map each returned per-statement result back
|
||||
* to the rows it inserted. This is what makes failed-batch accounting possible
|
||||
* on backends that report failures inside the result array instead of throwing.
|
||||
*/
|
||||
export function generateInsertBatches(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): { statements: string[]; rowsPerStatement: number[] } {
|
||||
if (databaseType === "oracle") {
|
||||
const statements = buildOracleInsertStatements(state.targetTable, state.columnList, valueRows);
|
||||
return { statements, rowsPerStatement: oracleRowsPerStatement(valueRows, statements) };
|
||||
}
|
||||
if (!supportsGeneratedMultiRowValues(databaseType) || forceSingleRow) {
|
||||
return { statements: valueRows.map((values) => `${state.insertPrefix} ${values};`), rowsPerStatement: valueRows.map(() => 1) };
|
||||
}
|
||||
if (valueRows.length === 0) return { statements: [], rowsPerStatement: [] };
|
||||
return { statements: [`${state.insertPrefix}\n${valueRows.join(",\n")};`], rowsPerStatement: [valueRows.length] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Oracle batches INSERT ALL statements `ORACLE_INSERT_ALL_BATCH_SIZE` rows at a
|
||||
* time, so the per-statement row counts have to mirror that chunking.
|
||||
*/
|
||||
function oracleRowsPerStatement(valueRows: string[], statements: string[]): number[] {
|
||||
if (statements.length === 0) return [];
|
||||
// A single value row is emitted as a plain VALUES statement, not INSERT ALL.
|
||||
if (valueRows.length === 1) return [1];
|
||||
const counts: number[] = [];
|
||||
for (let remaining = valueRows.length; remaining > 0; remaining -= ORACLE_INSERT_ALL_BATCH_SIZE) {
|
||||
counts.push(Math.min(ORACLE_INSERT_ALL_BATCH_SIZE, remaining));
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
export function buildGenerateInsertStatements(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): string[] {
|
||||
return generateInsertBatches(databaseType, state, valueRows, forceSingleRow).statements;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits formatted value rows into statement groups that stay within a byte budget.
|
||||
*
|
||||
* Servers reject statements larger than `max_allowed_packet`, so an oversized
|
||||
* multi-row INSERT has to be cut into several statements before it is sent.
|
||||
* Splitting happens after generation (never by rewinding the generator state)
|
||||
* so uniqueness bookkeeping stays intact.
|
||||
*/
|
||||
export function splitValueRowsByByteBudget(state: TableGenerateChunkState, valueRows: string[], maxBytes: number): string[][] {
|
||||
if (maxBytes <= 0) return valueRows.length > 0 ? [valueRows] : [];
|
||||
const groups: string[][] = [];
|
||||
let current: string[] = [];
|
||||
let currentBytes = 0;
|
||||
for (const values of valueRows) {
|
||||
const bytes = values.length + state.insertPrefix.length + 4;
|
||||
if (current.length > 0 && currentBytes + bytes > maxBytes) {
|
||||
groups.push(current);
|
||||
current = [];
|
||||
currentBytes = 0;
|
||||
}
|
||||
current.push(values);
|
||||
currentBytes += bytes;
|
||||
}
|
||||
if (current.length > 0) groups.push(current);
|
||||
return groups;
|
||||
}
|
||||
|
||||
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
|
||||
const state = createTableGenerateState(config, databaseType);
|
||||
const rows = generateTableRowsChunk(config, state, config.rowCount);
|
||||
const valueRows = rows.map((row) => formatGeneratedRowValues(config, databaseType, state, row));
|
||||
const statements = buildGenerateInsertStatements(databaseType, state, valueRows);
|
||||
return { columns: state.colNames, rows, sql: statements.join("\n"), statements };
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { BackendErrorException } from "@/lib/backend/errorUtils";
|
||||
import type { QueryResult } from "@/types/database";
|
||||
|
||||
export interface BatchExecutionOutcome {
|
||||
/** Rows the backend confirmed, counted only from statements that succeeded. */
|
||||
readonly insertedRows: number;
|
||||
/** True when at least one statement in the batch failed. */
|
||||
readonly failed: boolean;
|
||||
/** Message of the first failed statement, when one is available. */
|
||||
readonly error?: string;
|
||||
/** Index of the first failed statement, when one is available. */
|
||||
readonly failedStatement?: number;
|
||||
}
|
||||
|
||||
const EMPTY_OUTCOME: BatchExecutionOutcome = { insertedRows: 0, failed: false };
|
||||
|
||||
/**
|
||||
* Maps the per-statement results of one executed batch back to the rows that
|
||||
* were really inserted.
|
||||
*
|
||||
* Not every backend throws when a statement fails: the sequential executor
|
||||
* (`query.rs`) records the failure in the result array and still returns `Ok`,
|
||||
* and the MySQL batch route returns `Ok` with an embedded error result even on
|
||||
* connection checkout failure. Counting rows on the `await` alone therefore
|
||||
* over-reports inserts and can leave `continueOnError` silently dropping rows.
|
||||
*
|
||||
* `rowsPerStatement` comes from `generateInsertBatches()` and is aligned with
|
||||
* the statements that were sent; a statement counts as successful only when it
|
||||
* carries neither `execution_error` nor an `error`.
|
||||
*/
|
||||
export function summarizeBatchResults(results: QueryResult[] | null | undefined, rowsPerStatement: number[]): BatchExecutionOutcome {
|
||||
if (!results || results.length === 0 || rowsPerStatement.length === 0) return EMPTY_OUTCOME;
|
||||
|
||||
let insertedRows = 0;
|
||||
let error: string | undefined;
|
||||
let failedStatement: number | undefined;
|
||||
|
||||
for (let index = 0; index < results.length && index < rowsPerStatement.length; index++) {
|
||||
const result = results[index];
|
||||
const statementError = statementErrorMessage(result);
|
||||
if (statementError === undefined) {
|
||||
insertedRows += Math.max(0, rowsPerStatement[index]);
|
||||
continue;
|
||||
}
|
||||
if (failedStatement === undefined) {
|
||||
failedStatement = index;
|
||||
error = statementError;
|
||||
}
|
||||
}
|
||||
|
||||
const failed = failedStatement !== undefined;
|
||||
return failed ? { insertedRows, failed, error, failedStatement } : { insertedRows, failed };
|
||||
}
|
||||
|
||||
function statementErrorMessage(result: QueryResult): string | undefined {
|
||||
if (!result) return "Unknown statement failure";
|
||||
if (result.execution_error === true) {
|
||||
return result.error?.detail ?? result.error?.messageKey ?? errorMessageFromRows(result);
|
||||
}
|
||||
if (result.error) {
|
||||
return result.error.detail ?? result.error.messageKey;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Error-shaped results carry the driver message in a single "Error" row. */
|
||||
function errorMessageFromRows(result: QueryResult): string | undefined {
|
||||
const firstRow = result.rows?.[0];
|
||||
const firstCell = Array.isArray(firstRow) ? firstRow[0] : undefined;
|
||||
return typeof firstCell === "string" && firstCell.trim().length > 0 ? firstCell : undefined;
|
||||
}
|
||||
|
||||
const CANCELLATION_PATTERN = /\bcancel(?:ed|led|ing|ling|lation)?\b/i;
|
||||
|
||||
/**
|
||||
* Recognizes a failure produced by cancelling a query.
|
||||
*
|
||||
* Cancelling a transaction or a single-statement round trip surfaces as an
|
||||
* error instead of a quiet stop, so a user-initiated cancel has to be told
|
||||
* apart from a real SQL failure. The backend reports cancellation as
|
||||
* "Query canceled" (`QUERY_CANCELED`); structured backend errors are matched on
|
||||
* their code/message key as well.
|
||||
*/
|
||||
export function isQueryCanceledError(error: unknown): boolean {
|
||||
if (error instanceof BackendErrorException) {
|
||||
const { code, messageKey, detail } = error.backendError;
|
||||
if (CANCELLATION_PATTERN.test(code) || CANCELLATION_PATTERN.test(messageKey)) return true;
|
||||
if (detail !== undefined && CANCELLATION_PATTERN.test(detail)) return true;
|
||||
}
|
||||
return CANCELLATION_PATTERN.test(errorMessage(error));
|
||||
}
|
||||
|
||||
export function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
Reference in new issue
Block a user