fix(generate): stream bulk data generation with progress and configurable timeout

This commit is contained in:
gggaiitx authored and GitHub committed 2026-09-04 20:24:32 +08:00
1 parent 259071c09b
commit a3772fb442
14 files changed
+986 -99

No files matched your search

+135 -28
View File
@@ -1827,24 +1827,62 @@ function generateTdengineChildTableName(): string {
return `dbx_gen_${Date.now().toString(36)}_${randomSuffix}`;
}
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
/**
* Cross-chunk generation state.
*
* Bulk generation is streamed one chunk at a time so the UI never holds the
* full row set in memory. Anything that must stay coherent across chunks
* (uniqueness sets, TDengine tag values, the global row index, the TDengine
* child-table name) therefore has to live outside the chunk function.
*/
export interface TableGenerateChunkState {
readonly isTdengineStable: boolean;
readonly shouldAddTbname: boolean;
readonly tbname: string | null;
readonly colNames: string[];
readonly targetTable: string;
readonly columnList: string;
readonly insertPrefix: string;
readonly uniqueValues: (Set<string> | null)[];
readonly tagValues: Map<string, unknown>;
nextIndex: number;
}
export function createTableGenerateState(config: TableGenerateConfig, databaseType?: DatabaseType): TableGenerateChunkState {
const isTdengineStable = isTdengineStableGenerate(config, databaseType);
const hasTbnameColumn = config.columns.some((column) => column.columnName.toLowerCase() === "tbname");
const shouldAddTbname = isTdengineStable && !hasTbnameColumn;
const tdengineChildTableName = shouldAddTbname ? generateTdengineChildTableName() : null;
const tagValues = new Map<string, unknown>();
const uniqueValues = config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null));
const colNames = shouldAddTbname ? ["tbname", ...config.columns.map((c) => c.columnName)] : config.columns.map((c) => c.columnName);
const rows: unknown[][] = [];
const columnList = colNames.map((column) => quoteTableIdentifier(databaseType, column)).join(", ");
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
return {
isTdengineStable,
shouldAddTbname,
tbname: shouldAddTbname ? generateTdengineChildTableName() : null,
colNames,
targetTable,
columnList,
insertPrefix: `INSERT INTO ${targetTable} (${columnList}) VALUES`,
uniqueValues: config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null)),
tagValues: new Map<string, unknown>(),
nextIndex: 0,
};
}
for (let i = 0; i < config.rowCount; i++) {
/** Generates the next `chunkSize` rows (or whatever remains) and advances `state.nextIndex`. */
export function generateTableRowsChunk(config: TableGenerateConfig, state: TableGenerateChunkState, chunkSize: number): unknown[][] {
const rows: unknown[][] = [];
if (chunkSize <= 0) return rows;
const total = Math.max(0, config.rowCount);
const end = Math.min(total, state.nextIndex + chunkSize);
for (let i = state.nextIndex; i < end; i++) {
const row = config.columns.map((col, columnIndex) => {
if (isTdengineStable && col.isTag && tagValues.has(col.columnName)) {
return tagValues.get(col.columnName);
if (state.isTdengineStable && col.isTag && state.tagValues.has(col.columnName)) {
return state.tagValues.get(col.columnName);
}
let value: unknown;
if (col.generatorParams?.unique) {
const seen = uniqueValues[columnIndex]!;
const seen = state.uniqueValues[columnIndex]!;
let found = false;
for (let attempt = 0; attempt < MAX_UNIQUE_GENERATION_ATTEMPTS; attempt++) {
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
@@ -1861,29 +1899,98 @@ export function generateTableData(config: TableGenerateConfig, databaseType?: Da
} else {
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
}
if (isTdengineStable && col.isTag) {
tagValues.set(col.columnName, value);
if (state.isTdengineStable && col.isTag) {
state.tagValues.set(col.columnName, value);
}
return value;
});
rows.push(shouldAddTbname ? [tdengineChildTableName, ...row] : row);
rows.push(state.tbname !== null ? [state.tbname, ...row] : row);
}
state.nextIndex = end;
return rows;
}
const quotedCols = colNames.map((column) => quoteTableIdentifier(databaseType, column));
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
const columnList = quotedCols.join(", ");
const insertPrefix = `INSERT INTO ${targetTable} (${columnList}) VALUES`;
const valueRows = rows.map(
(row) =>
`(${row
.map((value, index) => {
const configIndex = shouldAddTbname ? index - 1 : index;
return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
})
.join(", ")})`,
);
const statements = databaseType === "oracle" ? buildOracleInsertStatements(targetTable, columnList, valueRows) : supportsGeneratedMultiRowValues(databaseType) ? [`${insertPrefix}\n${valueRows.join(",\n")};`] : valueRows.map((values) => `${insertPrefix} ${values};`);
const sql = statements.join("\n");
export function formatGeneratedRowValues(config: TableGenerateConfig, databaseType: DatabaseType | undefined, state: TableGenerateChunkState, row: unknown[]): string {
const values = row
.map((value, index) => {
const configIndex = state.shouldAddTbname ? index - 1 : index;
return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
})
.join(", ");
return `(${values})`;
}
return { columns: colNames, rows, sql, statements };
/**
* Builds the INSERT statements for a batch together with the number of value
* rows each statement carries.
*
* `rowsPerStatement` is aligned index-by-index with `statements`, so a caller
* that executes the statements can map each returned per-statement result back
* to the rows it inserted. This is what makes failed-batch accounting possible
* on backends that report failures inside the result array instead of throwing.
*/
export function generateInsertBatches(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): { statements: string[]; rowsPerStatement: number[] } {
if (databaseType === "oracle") {
const statements = buildOracleInsertStatements(state.targetTable, state.columnList, valueRows);
return { statements, rowsPerStatement: oracleRowsPerStatement(valueRows, statements) };
}
if (!supportsGeneratedMultiRowValues(databaseType) || forceSingleRow) {
return { statements: valueRows.map((values) => `${state.insertPrefix} ${values};`), rowsPerStatement: valueRows.map(() => 1) };
}
if (valueRows.length === 0) return { statements: [], rowsPerStatement: [] };
return { statements: [`${state.insertPrefix}\n${valueRows.join(",\n")};`], rowsPerStatement: [valueRows.length] };
}
/**
* Oracle batches INSERT ALL statements `ORACLE_INSERT_ALL_BATCH_SIZE` rows at a
* time, so the per-statement row counts have to mirror that chunking.
*/
function oracleRowsPerStatement(valueRows: string[], statements: string[]): number[] {
if (statements.length === 0) return [];
// A single value row is emitted as a plain VALUES statement, not INSERT ALL.
if (valueRows.length === 1) return [1];
const counts: number[] = [];
for (let remaining = valueRows.length; remaining > 0; remaining -= ORACLE_INSERT_ALL_BATCH_SIZE) {
counts.push(Math.min(ORACLE_INSERT_ALL_BATCH_SIZE, remaining));
}
return counts;
}
export function buildGenerateInsertStatements(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): string[] {
return generateInsertBatches(databaseType, state, valueRows, forceSingleRow).statements;
}
/**
* Splits formatted value rows into statement groups that stay within a byte budget.
*
* Servers reject statements larger than `max_allowed_packet`, so an oversized
* multi-row INSERT has to be cut into several statements before it is sent.
* Splitting happens after generation (never by rewinding the generator state)
* so uniqueness bookkeeping stays intact.
*/
export function splitValueRowsByByteBudget(state: TableGenerateChunkState, valueRows: string[], maxBytes: number): string[][] {
if (maxBytes <= 0) return valueRows.length > 0 ? [valueRows] : [];
const groups: string[][] = [];
let current: string[] = [];
let currentBytes = 0;
for (const values of valueRows) {
const bytes = values.length + state.insertPrefix.length + 4;
if (current.length > 0 && currentBytes + bytes > maxBytes) {
groups.push(current);
current = [];
currentBytes = 0;
}
current.push(values);
currentBytes += bytes;
}
if (current.length > 0) groups.push(current);
return groups;
}
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
const state = createTableGenerateState(config, databaseType);
const rows = generateTableRowsChunk(config, state, config.rowCount);
const valueRows = rows.map((row) => formatGeneratedRowValues(config, databaseType, state, row));
const statements = buildGenerateInsertStatements(databaseType, state, valueRows);
return { columns: state.colNames, rows, sql: statements.join("\n"), statements };
}
@@ -0,0 +1,95 @@
import { BackendErrorException } from "@/lib/backend/errorUtils";
import type { QueryResult } from "@/types/database";
export interface BatchExecutionOutcome {
/** Rows the backend confirmed, counted only from statements that succeeded. */
readonly insertedRows: number;
/** True when at least one statement in the batch failed. */
readonly failed: boolean;
/** Message of the first failed statement, when one is available. */
readonly error?: string;
/** Index of the first failed statement, when one is available. */
readonly failedStatement?: number;
}
const EMPTY_OUTCOME: BatchExecutionOutcome = { insertedRows: 0, failed: false };
/**
* Maps the per-statement results of one executed batch back to the rows that
* were really inserted.
*
* Not every backend throws when a statement fails: the sequential executor
* (`query.rs`) records the failure in the result array and still returns `Ok`,
* and the MySQL batch route returns `Ok` with an embedded error result even on
* connection checkout failure. Counting rows on the `await` alone therefore
* over-reports inserts and can leave `continueOnError` silently dropping rows.
*
* `rowsPerStatement` comes from `generateInsertBatches()` and is aligned with
* the statements that were sent; a statement counts as successful only when it
* carries neither `execution_error` nor an `error`.
*/
export function summarizeBatchResults(results: QueryResult[] | null | undefined, rowsPerStatement: number[]): BatchExecutionOutcome {
if (!results || results.length === 0 || rowsPerStatement.length === 0) return EMPTY_OUTCOME;
let insertedRows = 0;
let error: string | undefined;
let failedStatement: number | undefined;
for (let index = 0; index < results.length && index < rowsPerStatement.length; index++) {
const result = results[index];
const statementError = statementErrorMessage(result);
if (statementError === undefined) {
insertedRows += Math.max(0, rowsPerStatement[index]);
continue;
}
if (failedStatement === undefined) {
failedStatement = index;
error = statementError;
}
}
const failed = failedStatement !== undefined;
return failed ? { insertedRows, failed, error, failedStatement } : { insertedRows, failed };
}
function statementErrorMessage(result: QueryResult): string | undefined {
if (!result) return "Unknown statement failure";
if (result.execution_error === true) {
return result.error?.detail ?? result.error?.messageKey ?? errorMessageFromRows(result);
}
if (result.error) {
return result.error.detail ?? result.error.messageKey;
}
return undefined;
}
/** Error-shaped results carry the driver message in a single "Error" row. */
function errorMessageFromRows(result: QueryResult): string | undefined {
const firstRow = result.rows?.[0];
const firstCell = Array.isArray(firstRow) ? firstRow[0] : undefined;
return typeof firstCell === "string" && firstCell.trim().length > 0 ? firstCell : undefined;
}
const CANCELLATION_PATTERN = /\bcancel(?:ed|led|ing|ling|lation)?\b/i;
/**
* Recognizes a failure produced by cancelling a query.
*
* Cancelling a transaction or a single-statement round trip surfaces as an
* error instead of a quiet stop, so a user-initiated cancel has to be told
* apart from a real SQL failure. The backend reports cancellation as
* "Query canceled" (`QUERY_CANCELED`); structured backend errors are matched on
* their code/message key as well.
*/
export function isQueryCanceledError(error: unknown): boolean {
if (error instanceof BackendErrorException) {
const { code, messageKey, detail } = error.backendError;
if (CANCELLATION_PATTERN.test(code) || CANCELLATION_PATTERN.test(messageKey)) return true;
if (detail !== undefined && CANCELLATION_PATTERN.test(detail)) return true;
}
return CANCELLATION_PATTERN.test(errorMessage(error));
}
export function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}