feat(data): run generation and transfer tasks in background
Closes #10390
This commit is contained in:
1 parent
1d07cb05e0
commit
78dcb85713
23 files changed
+1014
-247
No files matched your search
@@ -0,0 +1,177 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { QueryResult } from "@/types/database";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
cancelQuery: vi.fn(),
|
||||
executeMultiWithProgress: vi.fn(),
|
||||
openDataGenerateSession: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/backend/api", () => ({
|
||||
cancelDatabaseExport: vi.fn(),
|
||||
cancelQuery: mocks.cancelQuery,
|
||||
cancelSqlFileExecution: vi.fn(),
|
||||
cancelTableExport: vi.fn(),
|
||||
cancelTransfer: vi.fn(),
|
||||
executeMultiWithProgress: mocks.executeMultiWithProgress,
|
||||
startTransfer: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/composables/useDialogSources", () => ({
|
||||
openDataGenerateSession: mocks.openDataGenerateSession,
|
||||
}));
|
||||
|
||||
import { getDataGenerateSession, startDataGenerateSession } from "@/composables/useDataGenerateSession";
|
||||
import { useExportTracker } from "@/composables/useExportTracker";
|
||||
|
||||
function successfulResult(): QueryResult[] {
|
||||
return [
|
||||
{
|
||||
columns: [],
|
||||
rows: [],
|
||||
affected_rows: 2,
|
||||
execution_time_ms: 1,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function startSession() {
|
||||
return startDataGenerateSession(
|
||||
{
|
||||
connectionId: "connection-1",
|
||||
database: "app",
|
||||
prefillSchema: "public",
|
||||
databaseType: "postgres",
|
||||
label: "app.users",
|
||||
targets: [
|
||||
{
|
||||
tableName: "users",
|
||||
schema: "public",
|
||||
database: "app",
|
||||
targetRowCount: 2,
|
||||
isSample: false,
|
||||
columns: ["id"],
|
||||
rows: [[1], [2]],
|
||||
sql: "INSERT INTO users VALUES (1), (2);",
|
||||
statements: ["INSERT INTO users VALUES (1), (2);"],
|
||||
resolvedColumns: [
|
||||
{
|
||||
columnName: "id",
|
||||
dataType: "integer",
|
||||
rowCount: 2,
|
||||
generatorKey: "sequence",
|
||||
generatorParams: { startValue: 1, increment: 1 },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
options: {
|
||||
continueOnError: false,
|
||||
truncate: false,
|
||||
useTransaction: true,
|
||||
extendedInsert: true,
|
||||
timeoutSecs: 0,
|
||||
batchRows: 2,
|
||||
},
|
||||
},
|
||||
{ invalidateMetadataCache: vi.fn() },
|
||||
);
|
||||
}
|
||||
|
||||
function resetTracker() {
|
||||
const tracker = useExportTracker();
|
||||
for (const task of tracker.tasks.value) tracker.removeTask(task.exportId);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
resetTracker();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetTracker();
|
||||
});
|
||||
|
||||
describe("data generation background sessions", () => {
|
||||
it("continues outside the dialog, reopens from the task entry, and cleans up after completion", async () => {
|
||||
let finishBatch!: (result: QueryResult[]) => void;
|
||||
mocks.executeMultiWithProgress.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<QueryResult[]>((resolve) => {
|
||||
finishBatch = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const tracker = useExportTracker();
|
||||
const session = startSession();
|
||||
const task = tracker.tasks.value.find((candidate) => candidate.exportId === session.id)!;
|
||||
|
||||
await vi.waitFor(() => expect(mocks.executeMultiWithProgress).toHaveBeenCalledOnce());
|
||||
expect(session.status).toBe("running");
|
||||
expect(task.status).toBe("Running");
|
||||
|
||||
task.onOpen?.();
|
||||
expect(mocks.openDataGenerateSession).toHaveBeenCalledWith(session.id, {
|
||||
connectionId: "connection-1",
|
||||
database: "app",
|
||||
schema: "public",
|
||||
tableName: "users",
|
||||
});
|
||||
|
||||
finishBatch(successfulResult());
|
||||
await vi.waitFor(() => expect(session.status).toBe("completed"));
|
||||
expect(task.status).toBe("Done");
|
||||
expect(task.rowsExported).toBe(2);
|
||||
expect(session.results).toEqual([{ table: "users", total: 2, ok: 2, err: 0, error: undefined, cancelled: undefined }]);
|
||||
|
||||
tracker.removeTask(session.id);
|
||||
expect(getDataGenerateSession(session.id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps cancel separate from minimize and publishes the cancelled terminal state", async () => {
|
||||
let failBatch!: (error: Error) => void;
|
||||
mocks.executeMultiWithProgress.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<QueryResult[]>((_resolve, reject) => {
|
||||
failBatch = reject;
|
||||
}),
|
||||
);
|
||||
mocks.cancelQuery.mockResolvedValueOnce(true);
|
||||
|
||||
const tracker = useExportTracker();
|
||||
const session = startSession();
|
||||
const task = tracker.tasks.value.find((candidate) => candidate.exportId === session.id)!;
|
||||
await vi.waitFor(() => expect(mocks.executeMultiWithProgress).toHaveBeenCalledOnce());
|
||||
|
||||
await tracker.cancelTask(session.id);
|
||||
expect(mocks.cancelQuery).toHaveBeenCalledWith(session.id);
|
||||
expect(session.status).toBe("cancelling");
|
||||
expect(task.status).toBe("Cancelling");
|
||||
|
||||
failBatch(new Error("Query canceled"));
|
||||
await vi.waitFor(() => expect(session.status).toBe("cancelled"));
|
||||
expect(task.status).toBe("Cancelled");
|
||||
expect(session.results[0]?.cancelled).toBe(true);
|
||||
});
|
||||
|
||||
it("retains per-table failures in the resumable result", async () => {
|
||||
mocks.executeMultiWithProgress.mockResolvedValueOnce([
|
||||
{
|
||||
columns: ["Error"],
|
||||
rows: [["duplicate key"]],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 1,
|
||||
execution_error: true,
|
||||
} as QueryResult,
|
||||
]);
|
||||
|
||||
const tracker = useExportTracker();
|
||||
const session = startSession();
|
||||
const task = tracker.tasks.value.find((candidate) => candidate.exportId === session.id)!;
|
||||
|
||||
await vi.waitFor(() => expect(session.status).toBe("completed"));
|
||||
expect(task.status).toBe("Error");
|
||||
expect(task.errorMessage).toBe("duplicate key");
|
||||
expect(session.results[0]).toMatchObject({ table: "users", ok: 0, err: 2, error: "duplicate key" });
|
||||
});
|
||||
});
|
||||
@@ -169,14 +169,17 @@ describe("data transfer task duration", () => {
|
||||
});
|
||||
const tracker = useExportTracker();
|
||||
const onStarted = vi.fn();
|
||||
const onOpen = vi.fn();
|
||||
const request = { ...transferRequest(`start-${_flow}`), ...overrides };
|
||||
|
||||
const task = tracker.startDataTransferTask(request, _flow, { onStarted });
|
||||
const task = tracker.startDataTransferTask(request, _flow, { onStarted, onOpen });
|
||||
|
||||
expect(task.status).toBe("Running");
|
||||
expect(api.startTransfer).toHaveBeenCalledTimes(1);
|
||||
expect(vi.mocked(api.startTransfer).mock.calls[0]?.[0]).toMatchObject(overrides);
|
||||
expect(onStarted).toHaveBeenCalledTimes(1);
|
||||
task.onOpen?.();
|
||||
expect(onOpen).toHaveBeenCalledTimes(1);
|
||||
|
||||
finishTransfer();
|
||||
await Promise.resolve();
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
import { reactive, shallowReactive } from "vue";
|
||||
import { openDataGenerateSession } from "@/composables/useDialogSources";
|
||||
import { useExportTracker } from "@/composables/useExportTracker";
|
||||
import * as api from "@/lib/backend/api";
|
||||
import { createTableGenerateState, formatGeneratedRowValues, generateInsertBatches, generateTableRowsChunk, splitValueRowsByByteBudget, type ColumnGenerateConfig, type GenerateResult, type TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
|
||||
import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "@/lib/dataGrid/generateInsertAccounting";
|
||||
import { qualifiedTableName } from "@/lib/table/tableSelectSql";
|
||||
import { uuid } from "@/lib/common/utils";
|
||||
import type { DatabaseType, QueryResult } from "@/types/database";
|
||||
|
||||
const DEFAULT_BATCH_ROWS = 1000;
|
||||
const MAX_BATCH_BYTES = 1024 * 1024;
|
||||
|
||||
export interface DataGenerateTarget extends GenerateResult {
|
||||
tableName: string;
|
||||
schema: string;
|
||||
database: string;
|
||||
tableType?: string;
|
||||
targetRowCount: number;
|
||||
isSample: boolean;
|
||||
resolvedColumns: ColumnGenerateConfig[];
|
||||
}
|
||||
|
||||
export interface DataGenerateRunOptions {
|
||||
continueOnError: boolean;
|
||||
truncate: boolean;
|
||||
useTransaction: boolean;
|
||||
extendedInsert: boolean;
|
||||
timeoutSecs: number;
|
||||
batchRows: number;
|
||||
}
|
||||
|
||||
export interface DataGenerateTableResult {
|
||||
table: string;
|
||||
total: number;
|
||||
ok: number;
|
||||
err: number;
|
||||
error?: string;
|
||||
cancelled?: boolean;
|
||||
}
|
||||
|
||||
export interface DataGenerateSessionProgress {
|
||||
tableName: string;
|
||||
tableIndex: number;
|
||||
tableCount: number;
|
||||
insertedRows: number;
|
||||
totalRows: number;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
export type DataGenerateSessionStatus = "running" | "cancelling" | "completed" | "failed" | "cancelled";
|
||||
|
||||
export interface DataGenerateSessionConfig {
|
||||
connectionId: string;
|
||||
database: string;
|
||||
prefillSchema?: string;
|
||||
databaseType?: DatabaseType;
|
||||
label: string;
|
||||
targets: DataGenerateTarget[];
|
||||
options: DataGenerateRunOptions;
|
||||
}
|
||||
|
||||
export interface DataGenerateSession {
|
||||
id: string;
|
||||
version: number;
|
||||
status: DataGenerateSessionStatus;
|
||||
config: DataGenerateSessionConfig;
|
||||
progress: DataGenerateSessionProgress | null;
|
||||
results: DataGenerateTableResult[];
|
||||
error: string | null;
|
||||
startedAt: number;
|
||||
finishedAt?: number;
|
||||
}
|
||||
|
||||
interface MutableDataGenerateSession extends DataGenerateSession {
|
||||
cancelRequested: boolean;
|
||||
activeExecutionId: string | null;
|
||||
}
|
||||
|
||||
export interface DataGenerateSessionDependencies {
|
||||
invalidateMetadataCache(connectionId: string, database: string, schema?: string, table?: string): void;
|
||||
formatGenerationError?: (error: unknown) => string;
|
||||
}
|
||||
|
||||
interface TableInsertOutcome {
|
||||
ok: number;
|
||||
attempted: number;
|
||||
error: string;
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
const sessions = reactive(new Map<string, MutableDataGenerateSession>());
|
||||
|
||||
function normalizeTimeoutSecs(value: number): number {
|
||||
if (!Number.isFinite(value) || value < 0) return 0;
|
||||
return Math.min(86_400, Math.floor(value));
|
||||
}
|
||||
|
||||
function cloneColumn(column: ColumnGenerateConfig): ColumnGenerateConfig {
|
||||
return {
|
||||
...column,
|
||||
generatorParams: column.generatorParams
|
||||
? {
|
||||
...column.generatorParams,
|
||||
languages: column.generatorParams.languages ? [...column.generatorParams.languages] : undefined,
|
||||
weekdays: column.generatorParams.weekdays ? [...column.generatorParams.weekdays] : undefined,
|
||||
phoneRegions: column.generatorParams.phoneRegions ? [...column.generatorParams.phoneRegions] : undefined,
|
||||
cardTypes: column.generatorParams.cardTypes ? [...column.generatorParams.cardTypes] : undefined,
|
||||
regions: column.generatorParams.regions ? [...column.generatorParams.regions] : undefined,
|
||||
idTypes: column.generatorParams.idTypes ? [...column.generatorParams.idTypes] : undefined,
|
||||
barcodeTypes: column.generatorParams.barcodeTypes ? [...column.generatorParams.barcodeTypes] : undefined,
|
||||
pathTypes: column.generatorParams.pathTypes ? [...column.generatorParams.pathTypes] : undefined,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function cloneTarget(target: DataGenerateTarget): DataGenerateTarget {
|
||||
return {
|
||||
...target,
|
||||
columns: [...target.columns],
|
||||
rows: target.rows.map((row) => [...row]),
|
||||
statements: [...target.statements],
|
||||
resolvedColumns: target.resolvedColumns.map(cloneColumn),
|
||||
};
|
||||
}
|
||||
|
||||
function cloneConfig(config: DataGenerateSessionConfig): DataGenerateSessionConfig {
|
||||
return {
|
||||
...config,
|
||||
targets: config.targets.map(cloneTarget),
|
||||
options: { ...config.options },
|
||||
};
|
||||
}
|
||||
|
||||
function generationError(dependencies: DataGenerateSessionDependencies, error: unknown): string {
|
||||
return dependencies.formatGenerationError?.(error) ?? errorMessage(error);
|
||||
}
|
||||
|
||||
function totalTargetRows(session: DataGenerateSession): number {
|
||||
return session.config.targets.reduce((sum, target) => sum + target.targetRowCount, 0);
|
||||
}
|
||||
|
||||
function insertedRows(session: DataGenerateSession, currentRows = 0): number {
|
||||
return session.results.reduce((sum, result) => sum + result.ok, 0) + currentRows;
|
||||
}
|
||||
|
||||
function publishProgress(session: MutableDataGenerateSession, progress: DataGenerateSessionProgress, currentRows = progress.insertedRows): void {
|
||||
session.version += 1;
|
||||
session.progress = progress;
|
||||
useExportTracker().updateDataGenerationTask(session.id, {
|
||||
status: session.status === "cancelling" ? "Cancelling" : "Running",
|
||||
tableIndex: progress.tableIndex,
|
||||
totalTables: progress.tableCount,
|
||||
currentTable: progress.tableName,
|
||||
rowsGenerated: insertedRows(session, currentRows),
|
||||
totalRows: totalTargetRows(session),
|
||||
});
|
||||
}
|
||||
|
||||
async function streamInsertTable(session: MutableDataGenerateSession, target: DataGenerateTarget, dependencies: DataGenerateSessionDependencies, onRows: (insertedRows: number) => void): Promise<TableInsertOutcome> {
|
||||
const { connectionId, database, databaseType, options, prefillSchema } = session.config;
|
||||
const config: TableGenerateConfig = {
|
||||
tableName: target.tableName,
|
||||
schema: target.schema,
|
||||
database: target.database || database,
|
||||
tableType: target.tableType,
|
||||
rowCount: target.targetRowCount,
|
||||
columns: target.resolvedColumns,
|
||||
};
|
||||
const state = createTableGenerateState(config, databaseType);
|
||||
const schema = target.schema || prefillSchema;
|
||||
const forceSingleRow = !options.extendedInsert;
|
||||
const timeoutSecs = normalizeTimeoutSecs(options.timeoutSecs);
|
||||
const batchRows = Math.max(1, Math.floor(options.batchRows) || DEFAULT_BATCH_ROWS);
|
||||
let ok = 0;
|
||||
let attempted = 0;
|
||||
let lastError = "";
|
||||
let cancelled = false;
|
||||
|
||||
const run = (sql: string): Promise<QueryResult[]> =>
|
||||
api.executeMultiWithProgress(connectionId, database, sql, () => {}, schema, {
|
||||
timeoutSecs,
|
||||
useTransaction: options.useTransaction,
|
||||
continueOnError: options.continueOnError,
|
||||
executionId: session.id,
|
||||
});
|
||||
|
||||
const executeBatch = async (statements: string[], rowsPerStatement: number[]): Promise<boolean> => {
|
||||
const expectedRows = rowsPerStatement.reduce((sum, rows) => sum + rows, 0);
|
||||
attempted += expectedRows;
|
||||
try {
|
||||
const results = await run(statements.join("\n"));
|
||||
if (expectedRows > 0 && (!results || results.length === 0)) {
|
||||
if (!lastError) lastError = "Backend returned no result for the batch";
|
||||
return options.continueOnError;
|
||||
}
|
||||
const outcome = summarizeBatchResults(results, rowsPerStatement);
|
||||
ok += outcome.insertedRows;
|
||||
if (!outcome.failed) return true;
|
||||
if (!lastError) lastError = outcome.error ?? "Statement failed";
|
||||
return options.continueOnError;
|
||||
} catch (error: unknown) {
|
||||
if (session.cancelRequested || isQueryCanceledError(error)) {
|
||||
cancelled = true;
|
||||
return false;
|
||||
}
|
||||
if (!lastError) lastError = errorMessage(error);
|
||||
return options.continueOnError;
|
||||
}
|
||||
};
|
||||
|
||||
const finish = (): TableInsertOutcome => ({ ok, attempted, error: cancelled ? "" : lastError, cancelled });
|
||||
|
||||
if (options.truncate) {
|
||||
const table = qualifiedTableName({ databaseType, schema: target.schema, tableName: target.tableName, database });
|
||||
if (!(await executeBatch([`TRUNCATE TABLE ${table};`], [0]))) return finish();
|
||||
}
|
||||
|
||||
while (state.nextIndex < target.targetRowCount && !session.cancelRequested) {
|
||||
let valueRows: string[];
|
||||
try {
|
||||
const rows = generateTableRowsChunk(config, state, batchRows);
|
||||
if (rows.length === 0) break;
|
||||
valueRows = rows.map((row) => formatGeneratedRowValues(config, databaseType, state, row));
|
||||
} catch (error: unknown) {
|
||||
if (!lastError) lastError = generationError(dependencies, error);
|
||||
return finish();
|
||||
}
|
||||
|
||||
for (const group of splitValueRowsByByteBudget(state, valueRows, MAX_BATCH_BYTES)) {
|
||||
if (session.cancelRequested) break;
|
||||
const { statements, rowsPerStatement } = generateInsertBatches(databaseType, state, group, forceSingleRow);
|
||||
if (statements.length === 0) continue;
|
||||
const keepGoing = await executeBatch(statements, rowsPerStatement);
|
||||
onRows(ok);
|
||||
if (!keepGoing) return finish();
|
||||
}
|
||||
onRows(ok);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
if (session.cancelRequested) cancelled = true;
|
||||
return finish();
|
||||
}
|
||||
|
||||
function finishSession(session: MutableDataGenerateSession, status: "completed" | "failed" | "cancelled", error: string | null): void {
|
||||
session.status = status;
|
||||
session.error = error;
|
||||
session.progress = null;
|
||||
session.finishedAt ??= Date.now();
|
||||
session.version += 1;
|
||||
const lastResult = session.results[session.results.length - 1];
|
||||
useExportTracker().updateDataGenerationTask(session.id, {
|
||||
status: status === "cancelled" ? "Cancelled" : status === "failed" || session.results.some((result) => result.err > 0 || !!result.error) ? "Error" : "Done",
|
||||
tableIndex: session.results.length,
|
||||
totalTables: session.config.targets.length,
|
||||
currentTable: lastResult?.table ?? "",
|
||||
rowsGenerated: insertedRows(session),
|
||||
totalRows: totalTargetRows(session),
|
||||
errorMessage: error ?? session.results.find((result) => result.error)?.error ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
async function runDataGenerateSession(session: MutableDataGenerateSession, dependencies: DataGenerateSessionDependencies): Promise<void> {
|
||||
const { connectionId, database, prefillSchema, targets, options } = session.config;
|
||||
try {
|
||||
let stopAll = false;
|
||||
for (const [index, target] of targets.entries()) {
|
||||
if (stopAll || session.cancelRequested) break;
|
||||
const initialProgress: DataGenerateSessionProgress = {
|
||||
tableName: target.tableName,
|
||||
tableIndex: index + 1,
|
||||
tableCount: targets.length,
|
||||
insertedRows: 0,
|
||||
totalRows: target.targetRowCount,
|
||||
elapsedMs: Date.now() - session.startedAt,
|
||||
};
|
||||
publishProgress(session, initialProgress, 0);
|
||||
|
||||
let insertedSoFar = 0;
|
||||
let outcome: TableInsertOutcome;
|
||||
try {
|
||||
outcome = await streamInsertTable(session, target, dependencies, (rows) => {
|
||||
insertedSoFar = rows;
|
||||
publishProgress(
|
||||
session,
|
||||
{
|
||||
...initialProgress,
|
||||
insertedRows: rows,
|
||||
elapsedMs: Date.now() - session.startedAt,
|
||||
},
|
||||
rows,
|
||||
);
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
const wasCancelled = session.cancelRequested || isQueryCanceledError(error);
|
||||
outcome = { ok: insertedSoFar, attempted: insertedSoFar, error: wasCancelled ? "" : errorMessage(error), cancelled: wasCancelled };
|
||||
}
|
||||
|
||||
const failedRows = Math.max(0, Math.max(outcome.attempted, outcome.ok) - outcome.ok);
|
||||
session.results = [
|
||||
...session.results,
|
||||
{
|
||||
table: target.tableName,
|
||||
total: target.targetRowCount,
|
||||
ok: outcome.ok,
|
||||
err: failedRows,
|
||||
error: outcome.error || undefined,
|
||||
cancelled: outcome.cancelled || undefined,
|
||||
},
|
||||
];
|
||||
session.version += 1;
|
||||
if (outcome.ok > 0) dependencies.invalidateMetadataCache(connectionId, database, target.schema || prefillSchema || undefined, target.tableName);
|
||||
if ((outcome.error && !options.continueOnError) || outcome.cancelled) stopAll = true;
|
||||
}
|
||||
|
||||
if (session.cancelRequested || session.results.some((result) => result.cancelled)) {
|
||||
finishSession(session, "cancelled", null);
|
||||
} else {
|
||||
finishSession(session, "completed", null);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
finishSession(session, "failed", generationError(dependencies, error));
|
||||
} finally {
|
||||
session.activeExecutionId = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function startDataGenerateSession(config: DataGenerateSessionConfig, dependencies: DataGenerateSessionDependencies): DataGenerateSession {
|
||||
const id = uuid();
|
||||
const session = shallowReactive<MutableDataGenerateSession>({
|
||||
id,
|
||||
version: 0,
|
||||
status: "running",
|
||||
config: cloneConfig(config),
|
||||
progress: null,
|
||||
results: [],
|
||||
error: null,
|
||||
startedAt: Date.now(),
|
||||
cancelRequested: false,
|
||||
activeExecutionId: id,
|
||||
});
|
||||
sessions.set(id, session);
|
||||
|
||||
const firstTarget = session.config.targets[0];
|
||||
const tracker = useExportTracker();
|
||||
tracker.addDataGenerationTask(
|
||||
id,
|
||||
config.label,
|
||||
totalTargetRows(session),
|
||||
session.config.targets.length,
|
||||
() =>
|
||||
openDataGenerateSession(id, {
|
||||
connectionId: config.connectionId,
|
||||
database: config.database,
|
||||
schema: firstTarget?.schema ?? config.prefillSchema ?? "",
|
||||
tableName: firstTarget?.tableName ?? "",
|
||||
}),
|
||||
() => removeDataGenerateSession(id),
|
||||
);
|
||||
tracker.registerTaskCancelHandler(id, async () => {
|
||||
await cancelDataGenerateSession(id);
|
||||
});
|
||||
void runDataGenerateSession(session, dependencies);
|
||||
return session;
|
||||
}
|
||||
|
||||
export function getDataGenerateSession(id: string | null | undefined): DataGenerateSession | undefined {
|
||||
return id ? sessions.get(id) : undefined;
|
||||
}
|
||||
|
||||
export async function cancelDataGenerateSession(id: string): Promise<boolean> {
|
||||
const session = sessions.get(id);
|
||||
if (!session || (session.status !== "running" && session.status !== "cancelling")) return false;
|
||||
session.cancelRequested = true;
|
||||
session.status = "cancelling";
|
||||
session.version += 1;
|
||||
const progress = session.progress;
|
||||
useExportTracker().updateDataGenerationTask(session.id, {
|
||||
status: "Cancelling",
|
||||
tableIndex: progress?.tableIndex ?? session.results.length,
|
||||
totalTables: session.config.targets.length,
|
||||
currentTable: progress?.tableName ?? "",
|
||||
rowsGenerated: insertedRows(session, progress?.insertedRows ?? 0),
|
||||
totalRows: totalTargetRows(session),
|
||||
});
|
||||
if (session.activeExecutionId) await api.cancelQuery(session.activeExecutionId).catch(() => undefined);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function removeDataGenerateSession(id: string): boolean {
|
||||
const session = sessions.get(id);
|
||||
if (session && (session.status === "running" || session.status === "cancelling")) return false;
|
||||
return sessions.delete(id);
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import type { ConnectionConfigBundle } from "@/lib/connection/connectionConfigTr
|
||||
import type { ConnectionConfig, SidebarLayout } from "@/types/database";
|
||||
|
||||
const showTransferDialog = ref(false);
|
||||
const transferTaskId = ref<string | null>(null);
|
||||
const showSchemaDiffDialog = ref(false);
|
||||
const showDataCompareDialog = ref(false);
|
||||
const showSqlFileDialog = ref(false);
|
||||
@@ -22,6 +23,7 @@ const mongoDatabaseDumpPrefillConnectionId = ref("");
|
||||
const mongoDatabaseDumpPrefillDatabase = ref("");
|
||||
const mongoDatabaseDumpMode = ref<"dump" | "restore">("dump");
|
||||
const showTableDataGenerateDialog = ref(false);
|
||||
const tableDataGenerateSessionId = ref<string | null>(null);
|
||||
const showFieldLineageDialog = ref(false);
|
||||
const showDatabaseSearchDialog = ref(false);
|
||||
const showDatabaseExportDialog = ref(false);
|
||||
@@ -125,6 +127,20 @@ export function openDataCompareSession(sessionId: string): void {
|
||||
showDataCompareDialog.value = true;
|
||||
}
|
||||
|
||||
export function openDataTransferTask(taskId: string): void {
|
||||
transferTaskId.value = taskId;
|
||||
showTransferDialog.value = true;
|
||||
}
|
||||
|
||||
export function openDataGenerateSession(sessionId: string, target: { connectionId: string; database: string; schema?: string; tableName?: string }): void {
|
||||
tableDataGenerateSessionId.value = sessionId;
|
||||
tableDataGeneratePrefillConnectionId.value = target.connectionId;
|
||||
tableDataGeneratePrefillDatabase.value = target.database;
|
||||
tableDataGeneratePrefillSchema.value = target.schema ?? "";
|
||||
tableDataGeneratePrefillTable.value = target.tableName ?? "";
|
||||
showTableDataGenerateDialog.value = true;
|
||||
}
|
||||
|
||||
export function useDialogSources() {
|
||||
const { t } = useI18n();
|
||||
const connectionStore = useConnectionStore();
|
||||
@@ -138,6 +154,7 @@ export function useDialogSources() {
|
||||
() => connectionStore.transferSource,
|
||||
(v) => {
|
||||
if (v) {
|
||||
transferTaskId.value = null;
|
||||
transferPrefillConnectionId.value = v.connectionId;
|
||||
transferPrefillDatabase.value = v.database;
|
||||
transferPrefillCatalog.value = v.catalog ?? "";
|
||||
@@ -153,7 +170,10 @@ export function useDialogSources() {
|
||||
);
|
||||
|
||||
watch(showTransferDialog, (open) => {
|
||||
if (!open) clearTransferPrefill();
|
||||
if (!open) {
|
||||
transferTaskId.value = null;
|
||||
clearTransferPrefill();
|
||||
}
|
||||
});
|
||||
|
||||
watch(
|
||||
@@ -308,6 +328,7 @@ export function useDialogSources() {
|
||||
() => connectionStore.tableDataGenerateSource,
|
||||
(v) => {
|
||||
if (v) {
|
||||
tableDataGenerateSessionId.value = null;
|
||||
tableDataGeneratePrefillConnectionId.value = v.connectionId;
|
||||
tableDataGeneratePrefillDatabase.value = v.database;
|
||||
tableDataGeneratePrefillSchema.value = v.schema ?? "";
|
||||
@@ -318,6 +339,10 @@ export function useDialogSources() {
|
||||
},
|
||||
);
|
||||
|
||||
watch(showTableDataGenerateDialog, (open) => {
|
||||
if (!open) tableDataGenerateSessionId.value = null;
|
||||
});
|
||||
|
||||
watch(
|
||||
() => connectionStore.fieldLineageSource,
|
||||
(v) => {
|
||||
@@ -553,6 +578,7 @@ export function useDialogSources() {
|
||||
|
||||
return {
|
||||
showTransferDialog,
|
||||
transferTaskId,
|
||||
showSchemaDiffDialog,
|
||||
showDataCompareDialog,
|
||||
showSqlFileDialog,
|
||||
@@ -566,6 +592,7 @@ export function useDialogSources() {
|
||||
mongoDatabaseDumpPrefillDatabase,
|
||||
mongoDatabaseDumpMode,
|
||||
showTableDataGenerateDialog,
|
||||
tableDataGenerateSessionId,
|
||||
showFieldLineageDialog,
|
||||
showDatabaseSearchDialog,
|
||||
showDatabaseExportDialog,
|
||||
|
||||
@@ -4,7 +4,7 @@ import { isTerminalTransferProgress } from "@/lib/backend/transferProgress";
|
||||
import { uuid } from "@/lib/common/utils";
|
||||
import { formatQueryDuration } from "@/lib/format/duration";
|
||||
|
||||
export type BackgroundTaskKind = "table-export" | "database-export" | "data-dictionary" | "sql-file" | "data-transfer" | "multi-db-execution" | "schema-diff" | "data-compare";
|
||||
export type BackgroundTaskKind = "table-export" | "database-export" | "data-dictionary" | "sql-file" | "data-transfer" | "data-generation" | "multi-db-execution" | "schema-diff" | "data-compare";
|
||||
export type BackgroundTaskStatus = "Running" | "Writing" | "Cancelling" | "Done" | "Error" | "Cancelled";
|
||||
export type DatabaseExportSource = "manual" | "scheduled";
|
||||
|
||||
@@ -129,6 +129,16 @@ export interface MultiDbExecutionTaskProgress {
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
export interface DataGenerationTaskProgress {
|
||||
status: BackgroundTaskStatus;
|
||||
tableIndex: number;
|
||||
totalTables: number;
|
||||
currentTable: string;
|
||||
rowsGenerated: number;
|
||||
totalRows: number;
|
||||
errorMessage?: string | null;
|
||||
}
|
||||
|
||||
export const MAX_TRANSFER_FAILURE_DETAILS = 100;
|
||||
export const MAX_TRANSFER_FAILURE_DETAIL_BYTES = 128 * 1024;
|
||||
export const MAX_TRANSFER_FAILURE_ERROR_BYTES = 8 * 1024;
|
||||
@@ -505,6 +515,29 @@ export function useExportTracker() {
|
||||
return task;
|
||||
}
|
||||
|
||||
function addDataGenerationTask(executionId: string, label: string, totalRows: number, totalTables: number, onOpen?: () => void, onRemove?: () => void): ExportTask {
|
||||
const task = reactive<ExportTask>({
|
||||
exportId: executionId,
|
||||
kind: "data-generation",
|
||||
tableName: label,
|
||||
format: "generate",
|
||||
filePath: "",
|
||||
rowsExported: 0,
|
||||
totalRows,
|
||||
status: "Running",
|
||||
errorMessage: null,
|
||||
tableIndex: 0,
|
||||
totalTables,
|
||||
currentTable: "",
|
||||
canCancel: true,
|
||||
onOpen,
|
||||
onRemove,
|
||||
startedAt: Date.now(),
|
||||
});
|
||||
taskMap.set(executionId, task);
|
||||
return task;
|
||||
}
|
||||
|
||||
function addCompareTask(kind: "schema-diff" | "data-compare", sessionId: string, label: string, onOpen?: () => void, onRemove?: () => void): ExportTask {
|
||||
const task = reactive<ExportTask>({
|
||||
exportId: sessionId,
|
||||
@@ -584,11 +617,13 @@ export function useExportTracker() {
|
||||
options: {
|
||||
onStarted?: () => void;
|
||||
onDone?: () => void | Promise<void>;
|
||||
onOpen?: () => void;
|
||||
formatOverlapError?: (tables: string[]) => string;
|
||||
} = {},
|
||||
): ExportTask {
|
||||
const existingTask = taskMap.get(request.transferId);
|
||||
const task = existingTask ?? addDataTransferTask(request.transferId, label, request.tables.length);
|
||||
task.onOpen = options.onOpen;
|
||||
task.startedAt ??= Date.now();
|
||||
task.targetConnectionId = request.targetConnectionId;
|
||||
task.targetCatalog = request.targetCatalog;
|
||||
@@ -734,6 +769,19 @@ export function useExportTracker() {
|
||||
task.totalRows = progress.totalRows ?? task.totalRows;
|
||||
}
|
||||
|
||||
function updateDataGenerationTask(executionId: string, progress: DataGenerationTaskProgress): void {
|
||||
const task = taskMap.get(executionId);
|
||||
if (!task || task.kind !== "data-generation") return;
|
||||
task.status = progress.status;
|
||||
task.tableIndex = progress.tableIndex;
|
||||
task.totalTables = progress.totalTables;
|
||||
task.currentTable = progress.currentTable;
|
||||
task.rowsExported = progress.rowsGenerated;
|
||||
task.totalRows = progress.totalRows;
|
||||
task.errorMessage = progress.errorMessage ?? null;
|
||||
if (task.status === "Done" || task.status === "Error" || task.status === "Cancelled") finishExportTask(task);
|
||||
}
|
||||
|
||||
function updateCompareTask(sessionId: string, progress: CompareTaskProgress): void {
|
||||
const task = taskMap.get(sessionId);
|
||||
if (!task || (task.kind !== "schema-diff" && task.kind !== "data-compare")) return;
|
||||
@@ -821,6 +869,7 @@ export function useExportTracker() {
|
||||
updateDataDictionaryTask,
|
||||
addSqlFileTask,
|
||||
addDataTransferTask,
|
||||
addDataGenerationTask,
|
||||
addSchemaDiffTask,
|
||||
addDataCompareTask,
|
||||
addMultiDbExecutionTask,
|
||||
@@ -832,6 +881,7 @@ export function useExportTracker() {
|
||||
restoreDatabaseExportTaskRunning,
|
||||
updateSqlFileTask,
|
||||
updateDataTransferTask,
|
||||
updateDataGenerationTask,
|
||||
updateCompareTask,
|
||||
registerTaskCancelHandler,
|
||||
unregisterTaskCancelHandler,
|
||||
|
||||
Reference in new issue
Block a user