fix(generate): stream bulk data generation with progress and configurable timeout
This commit is contained in:
1 parent
259071c09b
commit
a3772fb442
14 files changed
+986
-99
No files matched your search
@@ -1827,24 +1827,62 @@ function generateTdengineChildTableName(): string {
|
||||
return `dbx_gen_${Date.now().toString(36)}_${randomSuffix}`;
|
||||
}
|
||||
|
||||
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
|
||||
/**
|
||||
* Cross-chunk generation state.
|
||||
*
|
||||
* Bulk generation is streamed one chunk at a time so the UI never holds the
|
||||
* full row set in memory. Anything that must stay coherent across chunks
|
||||
* (uniqueness sets, TDengine tag values, the global row index, the TDengine
|
||||
* child-table name) therefore has to live outside the chunk function.
|
||||
*/
|
||||
export interface TableGenerateChunkState {
|
||||
readonly isTdengineStable: boolean;
|
||||
readonly shouldAddTbname: boolean;
|
||||
readonly tbname: string | null;
|
||||
readonly colNames: string[];
|
||||
readonly targetTable: string;
|
||||
readonly columnList: string;
|
||||
readonly insertPrefix: string;
|
||||
readonly uniqueValues: (Set<string> | null)[];
|
||||
readonly tagValues: Map<string, unknown>;
|
||||
nextIndex: number;
|
||||
}
|
||||
|
||||
export function createTableGenerateState(config: TableGenerateConfig, databaseType?: DatabaseType): TableGenerateChunkState {
|
||||
const isTdengineStable = isTdengineStableGenerate(config, databaseType);
|
||||
const hasTbnameColumn = config.columns.some((column) => column.columnName.toLowerCase() === "tbname");
|
||||
const shouldAddTbname = isTdengineStable && !hasTbnameColumn;
|
||||
const tdengineChildTableName = shouldAddTbname ? generateTdengineChildTableName() : null;
|
||||
const tagValues = new Map<string, unknown>();
|
||||
const uniqueValues = config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null));
|
||||
const colNames = shouldAddTbname ? ["tbname", ...config.columns.map((c) => c.columnName)] : config.columns.map((c) => c.columnName);
|
||||
const rows: unknown[][] = [];
|
||||
const columnList = colNames.map((column) => quoteTableIdentifier(databaseType, column)).join(", ");
|
||||
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
|
||||
return {
|
||||
isTdengineStable,
|
||||
shouldAddTbname,
|
||||
tbname: shouldAddTbname ? generateTdengineChildTableName() : null,
|
||||
colNames,
|
||||
targetTable,
|
||||
columnList,
|
||||
insertPrefix: `INSERT INTO ${targetTable} (${columnList}) VALUES`,
|
||||
uniqueValues: config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null)),
|
||||
tagValues: new Map<string, unknown>(),
|
||||
nextIndex: 0,
|
||||
};
|
||||
}
|
||||
|
||||
for (let i = 0; i < config.rowCount; i++) {
|
||||
/** Generates the next `chunkSize` rows (or whatever remains) and advances `state.nextIndex`. */
|
||||
export function generateTableRowsChunk(config: TableGenerateConfig, state: TableGenerateChunkState, chunkSize: number): unknown[][] {
|
||||
const rows: unknown[][] = [];
|
||||
if (chunkSize <= 0) return rows;
|
||||
const total = Math.max(0, config.rowCount);
|
||||
const end = Math.min(total, state.nextIndex + chunkSize);
|
||||
for (let i = state.nextIndex; i < end; i++) {
|
||||
const row = config.columns.map((col, columnIndex) => {
|
||||
if (isTdengineStable && col.isTag && tagValues.has(col.columnName)) {
|
||||
return tagValues.get(col.columnName);
|
||||
if (state.isTdengineStable && col.isTag && state.tagValues.has(col.columnName)) {
|
||||
return state.tagValues.get(col.columnName);
|
||||
}
|
||||
let value: unknown;
|
||||
if (col.generatorParams?.unique) {
|
||||
const seen = uniqueValues[columnIndex]!;
|
||||
const seen = state.uniqueValues[columnIndex]!;
|
||||
let found = false;
|
||||
for (let attempt = 0; attempt < MAX_UNIQUE_GENERATION_ATTEMPTS; attempt++) {
|
||||
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
|
||||
@@ -1861,29 +1899,98 @@ export function generateTableData(config: TableGenerateConfig, databaseType?: Da
|
||||
} else {
|
||||
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
|
||||
}
|
||||
if (isTdengineStable && col.isTag) {
|
||||
tagValues.set(col.columnName, value);
|
||||
if (state.isTdengineStable && col.isTag) {
|
||||
state.tagValues.set(col.columnName, value);
|
||||
}
|
||||
return value;
|
||||
});
|
||||
rows.push(shouldAddTbname ? [tdengineChildTableName, ...row] : row);
|
||||
rows.push(state.tbname !== null ? [state.tbname, ...row] : row);
|
||||
}
|
||||
state.nextIndex = end;
|
||||
return rows;
|
||||
}
|
||||
|
||||
const quotedCols = colNames.map((column) => quoteTableIdentifier(databaseType, column));
|
||||
const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
|
||||
const columnList = quotedCols.join(", ");
|
||||
const insertPrefix = `INSERT INTO ${targetTable} (${columnList}) VALUES`;
|
||||
const valueRows = rows.map(
|
||||
(row) =>
|
||||
`(${row
|
||||
.map((value, index) => {
|
||||
const configIndex = shouldAddTbname ? index - 1 : index;
|
||||
return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
|
||||
})
|
||||
.join(", ")})`,
|
||||
);
|
||||
const statements = databaseType === "oracle" ? buildOracleInsertStatements(targetTable, columnList, valueRows) : supportsGeneratedMultiRowValues(databaseType) ? [`${insertPrefix}\n${valueRows.join(",\n")};`] : valueRows.map((values) => `${insertPrefix} ${values};`);
|
||||
const sql = statements.join("\n");
|
||||
export function formatGeneratedRowValues(config: TableGenerateConfig, databaseType: DatabaseType | undefined, state: TableGenerateChunkState, row: unknown[]): string {
|
||||
const values = row
|
||||
.map((value, index) => {
|
||||
const configIndex = state.shouldAddTbname ? index - 1 : index;
|
||||
return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
|
||||
})
|
||||
.join(", ");
|
||||
return `(${values})`;
|
||||
}
|
||||
|
||||
return { columns: colNames, rows, sql, statements };
|
||||
/**
|
||||
* Builds the INSERT statements for a batch together with the number of value
|
||||
* rows each statement carries.
|
||||
*
|
||||
* `rowsPerStatement` is aligned index-by-index with `statements`, so a caller
|
||||
* that executes the statements can map each returned per-statement result back
|
||||
* to the rows it inserted. This is what makes failed-batch accounting possible
|
||||
* on backends that report failures inside the result array instead of throwing.
|
||||
*/
|
||||
export function generateInsertBatches(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): { statements: string[]; rowsPerStatement: number[] } {
|
||||
if (databaseType === "oracle") {
|
||||
const statements = buildOracleInsertStatements(state.targetTable, state.columnList, valueRows);
|
||||
return { statements, rowsPerStatement: oracleRowsPerStatement(valueRows, statements) };
|
||||
}
|
||||
if (!supportsGeneratedMultiRowValues(databaseType) || forceSingleRow) {
|
||||
return { statements: valueRows.map((values) => `${state.insertPrefix} ${values};`), rowsPerStatement: valueRows.map(() => 1) };
|
||||
}
|
||||
if (valueRows.length === 0) return { statements: [], rowsPerStatement: [] };
|
||||
return { statements: [`${state.insertPrefix}\n${valueRows.join(",\n")};`], rowsPerStatement: [valueRows.length] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Oracle batches INSERT ALL statements `ORACLE_INSERT_ALL_BATCH_SIZE` rows at a
|
||||
* time, so the per-statement row counts have to mirror that chunking.
|
||||
*/
|
||||
function oracleRowsPerStatement(valueRows: string[], statements: string[]): number[] {
|
||||
if (statements.length === 0) return [];
|
||||
// A single value row is emitted as a plain VALUES statement, not INSERT ALL.
|
||||
if (valueRows.length === 1) return [1];
|
||||
const counts: number[] = [];
|
||||
for (let remaining = valueRows.length; remaining > 0; remaining -= ORACLE_INSERT_ALL_BATCH_SIZE) {
|
||||
counts.push(Math.min(ORACLE_INSERT_ALL_BATCH_SIZE, remaining));
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
export function buildGenerateInsertStatements(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): string[] {
|
||||
return generateInsertBatches(databaseType, state, valueRows, forceSingleRow).statements;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits formatted value rows into statement groups that stay within a byte budget.
|
||||
*
|
||||
* Servers reject statements larger than `max_allowed_packet`, so an oversized
|
||||
* multi-row INSERT has to be cut into several statements before it is sent.
|
||||
* Splitting happens after generation (never by rewinding the generator state)
|
||||
* so uniqueness bookkeeping stays intact.
|
||||
*/
|
||||
export function splitValueRowsByByteBudget(state: TableGenerateChunkState, valueRows: string[], maxBytes: number): string[][] {
|
||||
if (maxBytes <= 0) return valueRows.length > 0 ? [valueRows] : [];
|
||||
const groups: string[][] = [];
|
||||
let current: string[] = [];
|
||||
let currentBytes = 0;
|
||||
for (const values of valueRows) {
|
||||
const bytes = values.length + state.insertPrefix.length + 4;
|
||||
if (current.length > 0 && currentBytes + bytes > maxBytes) {
|
||||
groups.push(current);
|
||||
current = [];
|
||||
currentBytes = 0;
|
||||
}
|
||||
current.push(values);
|
||||
currentBytes += bytes;
|
||||
}
|
||||
if (current.length > 0) groups.push(current);
|
||||
return groups;
|
||||
}
|
||||
|
||||
export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
|
||||
const state = createTableGenerateState(config, databaseType);
|
||||
const rows = generateTableRowsChunk(config, state, config.rowCount);
|
||||
const valueRows = rows.map((row) => formatGeneratedRowValues(config, databaseType, state, row));
|
||||
const statements = buildGenerateInsertStatements(databaseType, state, valueRows);
|
||||
return { columns: state.colNames, rows, sql: statements.join("\n"), statements };
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { BackendErrorException } from "@/lib/backend/errorUtils";
|
||||
import type { QueryResult } from "@/types/database";
|
||||
|
||||
export interface BatchExecutionOutcome {
|
||||
/** Rows the backend confirmed, counted only from statements that succeeded. */
|
||||
readonly insertedRows: number;
|
||||
/** True when at least one statement in the batch failed. */
|
||||
readonly failed: boolean;
|
||||
/** Message of the first failed statement, when one is available. */
|
||||
readonly error?: string;
|
||||
/** Index of the first failed statement, when one is available. */
|
||||
readonly failedStatement?: number;
|
||||
}
|
||||
|
||||
const EMPTY_OUTCOME: BatchExecutionOutcome = { insertedRows: 0, failed: false };
|
||||
|
||||
/**
|
||||
* Maps the per-statement results of one executed batch back to the rows that
|
||||
* were really inserted.
|
||||
*
|
||||
* Not every backend throws when a statement fails: the sequential executor
|
||||
* (`query.rs`) records the failure in the result array and still returns `Ok`,
|
||||
* and the MySQL batch route returns `Ok` with an embedded error result even on
|
||||
* connection checkout failure. Counting rows on the `await` alone therefore
|
||||
* over-reports inserts and can leave `continueOnError` silently dropping rows.
|
||||
*
|
||||
* `rowsPerStatement` comes from `generateInsertBatches()` and is aligned with
|
||||
* the statements that were sent; a statement counts as successful only when it
|
||||
* carries neither `execution_error` nor an `error`.
|
||||
*/
|
||||
export function summarizeBatchResults(results: QueryResult[] | null | undefined, rowsPerStatement: number[]): BatchExecutionOutcome {
|
||||
if (!results || results.length === 0 || rowsPerStatement.length === 0) return EMPTY_OUTCOME;
|
||||
|
||||
let insertedRows = 0;
|
||||
let error: string | undefined;
|
||||
let failedStatement: number | undefined;
|
||||
|
||||
for (let index = 0; index < results.length && index < rowsPerStatement.length; index++) {
|
||||
const result = results[index];
|
||||
const statementError = statementErrorMessage(result);
|
||||
if (statementError === undefined) {
|
||||
insertedRows += Math.max(0, rowsPerStatement[index]);
|
||||
continue;
|
||||
}
|
||||
if (failedStatement === undefined) {
|
||||
failedStatement = index;
|
||||
error = statementError;
|
||||
}
|
||||
}
|
||||
|
||||
const failed = failedStatement !== undefined;
|
||||
return failed ? { insertedRows, failed, error, failedStatement } : { insertedRows, failed };
|
||||
}
|
||||
|
||||
function statementErrorMessage(result: QueryResult): string | undefined {
|
||||
if (!result) return "Unknown statement failure";
|
||||
if (result.execution_error === true) {
|
||||
return result.error?.detail ?? result.error?.messageKey ?? errorMessageFromRows(result);
|
||||
}
|
||||
if (result.error) {
|
||||
return result.error.detail ?? result.error.messageKey;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Error-shaped results carry the driver message in a single "Error" row. */
|
||||
function errorMessageFromRows(result: QueryResult): string | undefined {
|
||||
const firstRow = result.rows?.[0];
|
||||
const firstCell = Array.isArray(firstRow) ? firstRow[0] : undefined;
|
||||
return typeof firstCell === "string" && firstCell.trim().length > 0 ? firstCell : undefined;
|
||||
}
|
||||
|
||||
const CANCELLATION_PATTERN = /\bcancel(?:ed|led|ing|ling|lation)?\b/i;
|
||||
|
||||
/**
|
||||
* Recognizes a failure produced by cancelling a query.
|
||||
*
|
||||
* Cancelling a transaction or a single-statement round trip surfaces as an
|
||||
* error instead of a quiet stop, so a user-initiated cancel has to be told
|
||||
* apart from a real SQL failure. The backend reports cancellation as
|
||||
* "Query canceled" (`QUERY_CANCELED`); structured backend errors are matched on
|
||||
* their code/message key as well.
|
||||
*/
|
||||
export function isQueryCanceledError(error: unknown): boolean {
|
||||
if (error instanceof BackendErrorException) {
|
||||
const { code, messageKey, detail } = error.backendError;
|
||||
if (CANCELLATION_PATTERN.test(code) || CANCELLATION_PATTERN.test(messageKey)) return true;
|
||||
if (detail !== undefined && CANCELLATION_PATTERN.test(detail)) return true;
|
||||
}
|
||||
return CANCELLATION_PATTERN.test(errorMessage(error));
|
||||
}
|
||||
|
||||
export function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
Reference in new issue
Block a user