feat(sql-file): show all execution failures

This commit is contained in:
Guoyu Su authored and GitHub committed 2026-09-03 01:12:25 +08:00
1 parent 460b413da8
commit a13a7302fa
6 files changed
+414 -20

No files matched your search

@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { TransferProgress, TransferRequest } from "@/lib/backend/api";
import type { SqlFileProgress, TransferProgress, TransferRequest } from "@/lib/backend/api";
vi.mock("@/lib/backend/api", () => ({
startTransfer: vi.fn(),
@@ -10,7 +10,17 @@ vi.mock("@/lib/backend/api", () => ({
}));
import * as api from "@/lib/backend/api";
import { formatDataTransferDuration, MAX_TRANSFER_FAILURE_DETAIL_BYTES, MAX_TRANSFER_FAILURE_DETAILS, MAX_TRANSFER_FAILURE_ERROR_BYTES, useExportTracker } from "@/composables/useExportTracker";
import {
formatDataTransferDuration,
MAX_SQL_FILE_FAILURE_DETAIL_BYTES,
MAX_SQL_FILE_FAILURE_DETAILS,
MAX_SQL_FILE_FAILURE_ERROR_BYTES,
MAX_SQL_FILE_FAILURE_SUMMARY_BYTES,
MAX_TRANSFER_FAILURE_DETAIL_BYTES,
MAX_TRANSFER_FAILURE_DETAILS,
MAX_TRANSFER_FAILURE_ERROR_BYTES,
useExportTracker,
} from "@/composables/useExportTracker";
let now = 0;
@@ -46,6 +56,21 @@ function transferProgress(transferId: string, status: TransferProgress["status"]
};
}
function sqlFileProgress(executionId: string, statementIndex: number, overrides: Partial<SqlFileProgress> = {}): SqlFileProgress {
return {
executionId,
status: "statementFailed",
statementIndex,
successCount: 0,
failureCount: statementIndex,
affectedRows: 0,
elapsedMs: statementIndex,
statementSummary: `SELECT missing_${statementIndex}`,
error: `failure ${statementIndex}`,
...overrides,
};
}
function resetTracker() {
const tracker = useExportTracker();
for (const task of tracker.tasks.value) tracker.removeTask(task.exportId);
@@ -330,6 +355,85 @@ describe("data transfer failure details", () => {
});
});
describe("SQL-file failure details", () => {
it("preserves failures in event order through terminal completion and updates duplicate statements", () => {
const tracker = useExportTracker();
const task = tracker.addSqlFileTask("sql-failures", "migration.sql", "/tmp/migration.sql");
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, 3));
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, 7));
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, 3, { error: "updated failure 3" }));
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, 8, { status: "done", successCount: 5, failureCount: 2, error: null }));
expect(task.status).toBe("Done");
expect(task.sqlFileFailures).toEqual([
{ statementIndex: 3, statementSummary: "SELECT missing_3", error: "updated failure 3" },
{ statementIndex: 7, statementSummary: "SELECT missing_7", error: "failure 7" },
]);
expect(task.failureCount).toBe(2);
});
it("retains reliable multi-file context for equal per-file statement indexes", () => {
const tracker = useExportTracker();
const task = tracker.addSqlFileTask("multi-file-failures", "first.sql (+1)", "/tmp/first.sql; /tmp/second.sql");
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, 1), { fileIndex: 0, fileName: "first.sql" });
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, 1), { fileIndex: 1, fileName: "second.sql" });
expect(task.sqlFileFailures).toEqual([
{ statementIndex: 1, statementSummary: "SELECT missing_1", error: "failure 1", fileIndex: 0, fileName: "first.sql" },
{ statementIndex: 1, statementSummary: "SELECT missing_1", error: "failure 1", fileIndex: 1, fileName: "second.sql" },
]);
});
it("bounds retained count and reports omitted failures without double-counting replays", () => {
const tracker = useExportTracker();
const task = tracker.addSqlFileTask("many-sql-failures", "migration.sql", "/tmp/migration.sql");
for (let index = 1; index <= MAX_SQL_FILE_FAILURE_DETAILS + 2; index += 1) {
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, index));
}
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, MAX_SQL_FILE_FAILURE_DETAILS + 1));
expect(task.sqlFileFailures).toHaveLength(MAX_SQL_FILE_FAILURE_DETAILS);
expect(task.sqlFileFailuresOmitted).toBe(2);
});
it("bounds UTF-8 summaries, errors, and total retained bytes", () => {
const tracker = useExportTracker();
const task = tracker.addSqlFileTask("long-sql-failures", "migration.sql", "/tmp/migration.sql");
const longText = "错误🙂".repeat(4_000);
for (let index = 1; index <= 300; index += 1) {
tracker.updateSqlFileTask(task.exportId, sqlFileProgress(task.exportId, index, { statementSummary: longText, error: longText }));
}
const encoder = new TextEncoder();
const retainedBytes = task.sqlFileFailures!.reduce((total, failure) => {
expect(encoder.encode(failure.statementSummary).length).toBeLessThanOrEqual(MAX_SQL_FILE_FAILURE_SUMMARY_BYTES);
expect(encoder.encode(failure.error).length).toBeLessThanOrEqual(MAX_SQL_FILE_FAILURE_ERROR_BYTES);
expect(failure.statementSummary.endsWith("\ud83d")).toBe(false);
expect(failure.error.endsWith("\ud83d")).toBe(false);
expect(failure.truncated).toBe(true);
return total + encoder.encode(failure.statementSummary).length + encoder.encode(failure.error).length + encoder.encode(failure.fileName ?? "").length;
}, 0);
expect(retainedBytes).toBeLessThanOrEqual(MAX_SQL_FILE_FAILURE_DETAIL_BYTES);
expect(task.sqlFileFailuresOmitted).toBeGreaterThan(0);
});
it("starts a replacement execution with an empty failure list", () => {
const tracker = useExportTracker();
const first = tracker.addSqlFileTask("reused-id", "first.sql", "/tmp/first.sql");
tracker.updateSqlFileTask(first.exportId, sqlFileProgress(first.exportId, 1));
const replacement = tracker.addSqlFileTask("reused-id", "second.sql", "/tmp/second.sql");
expect(replacement.sqlFileFailures).toEqual([]);
expect(replacement.sqlFileFailuresOmitted).toBe(0);
});
});
describe("formatDataTransferDuration", () => {
it("formats millisecond, second, minute, and hour boundaries", () => {
expect(formatDataTransferDuration(Number.NaN)).toBe("0 ms");
@@ -12,6 +12,20 @@ export interface DataTransferFailure {
truncated?: boolean;
}
export interface SqlFileFailure {
statementIndex: number;
statementSummary: string;
error: string;
fileIndex?: number;
fileName?: string;
truncated?: boolean;
}
export interface SqlFileFailureContext {
fileIndex?: number;
fileName?: string;
}
export interface ExportTask {
exportId: string;
kind: BackgroundTaskKind;
@@ -46,6 +60,8 @@ export interface ExportTask {
targetTables?: string[];
transferFailures?: DataTransferFailure[];
transferFailuresOmitted?: number;
sqlFileFailures?: SqlFileFailure[];
sqlFileFailuresOmitted?: number;
multiDbSourceTabId?: string;
multiDbTotal?: number;
multiDbCompleted?: number;
@@ -76,6 +92,11 @@ export interface MultiDbExecutionTaskProgress {
export const MAX_TRANSFER_FAILURE_DETAILS = 100;
export const MAX_TRANSFER_FAILURE_DETAIL_BYTES = 128 * 1024;
export const MAX_TRANSFER_FAILURE_ERROR_BYTES = 8 * 1024;
export const MAX_SQL_FILE_FAILURE_DETAILS = 500;
export const MAX_SQL_FILE_FAILURE_DETAIL_BYTES = 2 * 1024 * 1024;
export const MAX_SQL_FILE_FAILURE_ERROR_BYTES = 8 * 1024;
export const MAX_SQL_FILE_FAILURE_SUMMARY_BYTES = 2 * 1024;
const MAX_SQL_FILE_FAILURE_NAME_BYTES = 1024;
const MAX_TRACKED_OMITTED_FAILURES = 4096;
interface TransferFailureState {
@@ -86,10 +107,18 @@ interface TransferFailureState {
replayOmittedCount: number;
}
interface SqlFileFailureState {
indexes: Map<string, number>;
retainedBytes: number;
omittedKeys: Set<string>;
omittedCount: number;
}
const taskMap = reactive<Map<string, ExportTask>>(new Map());
const activeTransferRuns = new Set<string>();
const taskCancelHandlers = new Map<string, () => void | Promise<void>>();
const transferFailureStates = new Map<string, TransferFailureState>();
const sqlFileFailureStates = new Map<string, SqlFileFailureState>();
const textEncoder = new TextEncoder();
function utf8ByteLength(value: string): number {
@@ -180,6 +209,82 @@ function recordTransferFailure(task: ExportTask, table: string, error: string) {
task.transferFailures.push(failure);
}
function sqlFileFailureKey(statementIndex: number, fileIndex?: number): string {
return `${fileIndex ?? -1}:${statementIndex}`;
}
function sqlFileFailureBytes(failure: SqlFileFailure): number {
return utf8ByteLength(failure.statementSummary) + utf8ByteLength(failure.error) + utf8ByteLength(failure.fileName ?? "");
}
function getSqlFileFailureState(task: ExportTask): SqlFileFailureState {
let state = sqlFileFailureStates.get(task.exportId);
if (state) return state;
const failures = task.sqlFileFailures ?? [];
state = {
indexes: new Map(failures.map((failure, index) => [sqlFileFailureKey(failure.statementIndex, failure.fileIndex), index])),
retainedBytes: failures.reduce((total, failure) => total + sqlFileFailureBytes(failure), 0),
omittedKeys: new Set(),
omittedCount: task.sqlFileFailuresOmitted ?? 0,
};
sqlFileFailureStates.set(task.exportId, state);
return state;
}
function recordOmittedSqlFileFailure(task: ExportTask, state: SqlFileFailureState, key: string) {
if (state.omittedKeys.has(key) || state.omittedKeys.size >= MAX_TRACKED_OMITTED_FAILURES) return;
state.omittedKeys.add(key);
state.omittedCount += 1;
task.sqlFileFailuresOmitted = state.omittedCount;
}
function buildSqlFileFailure(progress: api.SqlFileProgress, context?: SqlFileFailureContext, availableBytes = MAX_SQL_FILE_FAILURE_DETAIL_BYTES): SqlFileFailure | null {
const retainedName = truncateUtf8(context?.fileName ?? progress.fileName ?? "", MAX_SQL_FILE_FAILURE_NAME_BYTES);
const retainedSummary = truncateUtf8(progress.statementSummary, MAX_SQL_FILE_FAILURE_SUMMARY_BYTES);
const fixedBytes = retainedName.bytes + retainedSummary.bytes;
if (fixedBytes >= availableBytes) return null;
const retainedError = truncateUtf8(progress.error ?? "", Math.min(MAX_SQL_FILE_FAILURE_ERROR_BYTES, availableBytes - fixedBytes));
const failure: SqlFileFailure = {
statementIndex: progress.statementIndex,
statementSummary: retainedSummary.value,
error: retainedError.value,
};
const fileIndex = context?.fileIndex ?? progress.fileIndex;
if (fileIndex !== undefined) failure.fileIndex = fileIndex;
if (retainedName.value) failure.fileName = retainedName.value;
if (retainedName.truncated || retainedSummary.truncated || retainedError.truncated) failure.truncated = true;
return failure;
}
function recordSqlFileFailure(task: ExportTask, progress: api.SqlFileProgress, context?: SqlFileFailureContext) {
if (progress.status !== "statementFailed" || !progress.error) return;
task.sqlFileFailures ??= [];
const state = getSqlFileFailureState(task);
const fileIndex = context?.fileIndex ?? progress.fileIndex;
const key = sqlFileFailureKey(progress.statementIndex, fileIndex);
const existingIndex = state.indexes.get(key);
const previousBytes = existingIndex === undefined ? 0 : sqlFileFailureBytes(task.sqlFileFailures[existingIndex]!);
const availableBytes = MAX_SQL_FILE_FAILURE_DETAIL_BYTES - state.retainedBytes + previousBytes;
const failure = buildSqlFileFailure(progress, context, availableBytes);
if (!failure || (existingIndex === undefined && task.sqlFileFailures.length >= MAX_SQL_FILE_FAILURE_DETAILS)) {
recordOmittedSqlFileFailure(task, state, key);
return;
}
if (existingIndex !== undefined) {
task.sqlFileFailures[existingIndex] = failure;
state.retainedBytes += sqlFileFailureBytes(failure) - previousBytes;
return;
}
state.indexes.set(key, task.sqlFileFailures.length);
state.retainedBytes += sqlFileFailureBytes(failure);
task.sqlFileFailures.push(failure);
}
function normalizeExportStatus(status: string): BackgroundTaskStatus {
if (status === "Writing" || status === "Done" || status === "Error" || status === "Cancelled") return status;
return "Running";
@@ -306,6 +411,7 @@ export function useExportTracker() {
}
function addSqlFileTask(executionId: string, fileName: string, filePath: string): ExportTask {
sqlFileFailureStates.delete(executionId);
const task = reactive<ExportTask>({
exportId: executionId,
kind: "sql-file",
@@ -322,6 +428,8 @@ export function useExportTracker() {
affectedRows: 0,
elapsedMs: 0,
statementSummary: "",
sqlFileFailures: [],
sqlFileFailuresOmitted: 0,
});
taskMap.set(executionId, task);
return task;
@@ -493,9 +601,10 @@ export function useExportTracker() {
if (task?.kind === "database-export" && task.status === "Cancelling") task.status = "Running";
}
function updateSqlFileTask(executionId: string, progress: api.SqlFileProgress) {
function updateSqlFileTask(executionId: string, progress: api.SqlFileProgress, context?: SqlFileFailureContext) {
const task = taskMap.get(executionId);
if (!task) return;
recordSqlFileFailure(task, progress, context);
task.status = normalizeSqlFileStatus(progress.status);
task.errorMessage = progress.error || null;
task.statementIndex = progress.statementIndex;
@@ -537,6 +646,7 @@ export function useExportTracker() {
taskMap.delete(exportId);
taskCancelHandlers.delete(exportId);
transferFailureStates.delete(exportId);
sqlFileFailureStates.delete(exportId);
}
function clearFinished() {
@@ -545,6 +655,7 @@ export function useExportTracker() {
taskMap.delete(id);
taskCancelHandlers.delete(id);
transferFailureStates.delete(id);
sqlFileFailureStates.delete(id);
}
}
}