feat(transfer): show per-table failure details
This commit is contained in:
20 files changed
+622
-65
No files matched your search
+104
-1
@@ -10,7 +10,7 @@ vi.mock("@/lib/backend/api", () => ({
|
||||
}));
|
||||
|
||||
import * as api from "@/lib/backend/api";
|
||||
import { formatDataTransferDuration, useExportTracker } from "@/composables/useExportTracker";
|
||||
import { formatDataTransferDuration, MAX_TRANSFER_FAILURE_DETAIL_BYTES, MAX_TRANSFER_FAILURE_DETAILS, MAX_TRANSFER_FAILURE_ERROR_BYTES, useExportTracker } from "@/composables/useExportTracker";
|
||||
|
||||
let now = 0;
|
||||
|
||||
@@ -161,6 +161,109 @@ describe("data transfer task duration", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("data transfer failure details", () => {
|
||||
it("preserves per-table failures after the terminal summary and deduplicates table updates", () => {
|
||||
const tracker = useExportTracker();
|
||||
const task = tracker.addDataTransferTask("failure-details", "users and orders", 2);
|
||||
|
||||
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error", false), table: "users", error: "permission denied" });
|
||||
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error", false), table: "orders", error: "table missing" });
|
||||
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error", false), table: "users", error: "permission denied by policy" });
|
||||
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error"), table: "", error: "2 table(s) failed: users, orders" });
|
||||
|
||||
expect(task.errorMessage).toBe("2 table(s) failed: users, orders");
|
||||
expect(task.transferFailures).toEqual([
|
||||
{ table: "users", error: "permission denied by policy" },
|
||||
{ table: "orders", error: "table missing" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("bounds many distinct failures and deduplicates retained and omitted table updates", () => {
|
||||
const tracker = useExportTracker();
|
||||
const task = tracker.addDataTransferTask("many-failures", "many tables", 250);
|
||||
|
||||
for (let index = 0; index < 250; index += 1) {
|
||||
tracker.updateDataTransferTask(task.exportId, {
|
||||
...transferProgress(task.exportId, "error", false),
|
||||
table: `table_${index}`,
|
||||
error: `failure ${index}`,
|
||||
});
|
||||
}
|
||||
|
||||
expect(task.transferFailures).toHaveLength(MAX_TRANSFER_FAILURE_DETAILS);
|
||||
expect(task.transferFailuresOmitted).toBe(150);
|
||||
|
||||
tracker.updateDataTransferTask(task.exportId, {
|
||||
...transferProgress(task.exportId, "error", false),
|
||||
table: "table_150",
|
||||
error: "updated omitted failure",
|
||||
});
|
||||
tracker.updateDataTransferTask(task.exportId, {
|
||||
...transferProgress(task.exportId, "error", false),
|
||||
table: "table_0",
|
||||
error: "updated retained failure",
|
||||
});
|
||||
|
||||
expect(task.transferFailuresOmitted).toBe(150);
|
||||
expect(task.transferFailures?.[0]).toEqual({ table: "table_0", error: "updated retained failure" });
|
||||
|
||||
tracker.updateDataTransferTask(task.exportId, {
|
||||
...transferProgress(task.exportId, "done"),
|
||||
transferFailuresOmitted: 12,
|
||||
});
|
||||
|
||||
expect(task.transferFailuresOmitted).toBe(162);
|
||||
});
|
||||
|
||||
it("limits individual and total UTF-8 error detail bytes without splitting characters", () => {
|
||||
const tracker = useExportTracker();
|
||||
const task = tracker.addDataTransferTask("long-failures", "long errors", 40);
|
||||
const longError = "错误🙂".repeat(4_000);
|
||||
|
||||
for (let index = 0; index < 40; index += 1) {
|
||||
tracker.updateDataTransferTask(task.exportId, {
|
||||
...transferProgress(task.exportId, "error", false),
|
||||
table: `long_table_${index}`,
|
||||
error: longError,
|
||||
});
|
||||
}
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const retainedBytes = task.transferFailures!.reduce((total, failure) => {
|
||||
expect(encoder.encode(failure.error).length).toBeLessThanOrEqual(MAX_TRANSFER_FAILURE_ERROR_BYTES);
|
||||
expect(failure.error.endsWith("\ud83d")).toBe(false);
|
||||
expect(failure.truncated).toBe(true);
|
||||
return total + encoder.encode(failure.table).length + encoder.encode(failure.error).length;
|
||||
}, 0);
|
||||
|
||||
expect(retainedBytes).toBeLessThanOrEqual(MAX_TRANSFER_FAILURE_DETAIL_BYTES);
|
||||
expect(task.transferFailuresOmitted).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("saturates omitted failure counting after the deduplication cap", () => {
|
||||
const tracker = useExportTracker();
|
||||
const task = tracker.addDataTransferTask("omitted-cap", "many tables", 4_197);
|
||||
|
||||
for (let index = 0; index < MAX_TRANSFER_FAILURE_DETAILS + 4_097; index += 1) {
|
||||
tracker.updateDataTransferTask(task.exportId, {
|
||||
...transferProgress(task.exportId, "error", false),
|
||||
table: `table_${index}`,
|
||||
error: `failure ${index}`,
|
||||
});
|
||||
}
|
||||
|
||||
expect(task.transferFailuresOmitted).toBe(4_096);
|
||||
|
||||
tracker.updateDataTransferTask(task.exportId, {
|
||||
...transferProgress(task.exportId, "error", false),
|
||||
table: "table_4_196",
|
||||
error: "repeated omitted failure",
|
||||
});
|
||||
|
||||
expect(task.transferFailuresOmitted).toBe(4_096);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatDataTransferDuration", () => {
|
||||
it("formats millisecond, second, minute, and hour boundaries", () => {
|
||||
expect(formatDataTransferDuration(Number.NaN)).toBe("0 ms");
|
||||
|
||||
@@ -5,6 +5,12 @@ import { isTerminalTransferProgress } from "@/lib/backend/transferProgress";
|
||||
export type BackgroundTaskKind = "table-export" | "database-export" | "sql-file" | "data-transfer";
|
||||
export type BackgroundTaskStatus = "Running" | "Writing" | "Done" | "Error" | "Cancelled";
|
||||
|
||||
export interface DataTransferFailure {
|
||||
table: string;
|
||||
error: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
export interface ExportTask {
|
||||
exportId: string;
|
||||
kind: BackgroundTaskKind;
|
||||
@@ -32,11 +38,116 @@ export interface ExportTask {
|
||||
targetDatabase?: string;
|
||||
targetSchema?: string;
|
||||
targetTables?: string[];
|
||||
transferFailures?: DataTransferFailure[];
|
||||
transferFailuresOmitted?: number;
|
||||
}
|
||||
|
||||
export const MAX_TRANSFER_FAILURE_DETAILS = 100;
|
||||
export const MAX_TRANSFER_FAILURE_DETAIL_BYTES = 128 * 1024;
|
||||
export const MAX_TRANSFER_FAILURE_ERROR_BYTES = 8 * 1024;
|
||||
const MAX_TRACKED_OMITTED_FAILURES = 4096;
|
||||
|
||||
interface TransferFailureState {
|
||||
indexes: Map<string, number>;
|
||||
retainedBytes: number;
|
||||
omittedHashes: Set<number>;
|
||||
localOmittedCount: number;
|
||||
replayOmittedCount: number;
|
||||
}
|
||||
|
||||
const taskMap = reactive<Map<string, ExportTask>>(new Map());
|
||||
const activeTransferRuns = new Set<string>();
|
||||
const taskCancelHandlers = new Map<string, () => void | Promise<void>>();
|
||||
const transferFailureStates = new Map<string, TransferFailureState>();
|
||||
const textEncoder = new TextEncoder();
|
||||
|
||||
function utf8ByteLength(value: string): number {
|
||||
return textEncoder.encode(value).length;
|
||||
}
|
||||
|
||||
function truncateUtf8(value: string, maxBytes: number): { value: string; bytes: number; truncated: boolean } {
|
||||
const encodedBytes = utf8ByteLength(value);
|
||||
if (encodedBytes <= maxBytes) return { value, bytes: encodedBytes, truncated: false };
|
||||
|
||||
let result = "";
|
||||
let bytes = 0;
|
||||
for (const character of value) {
|
||||
const characterBytes = utf8ByteLength(character);
|
||||
if (bytes + characterBytes > maxBytes) break;
|
||||
result += character;
|
||||
bytes += characterBytes;
|
||||
}
|
||||
return { value: result, bytes, truncated: true };
|
||||
}
|
||||
|
||||
function failureTableHash(value: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
hash ^= value.charCodeAt(index);
|
||||
hash = Math.imul(hash, 0x01000193);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
function getTransferFailureState(task: ExportTask): TransferFailureState {
|
||||
let state = transferFailureStates.get(task.exportId);
|
||||
if (state) return state;
|
||||
|
||||
const failures = task.transferFailures ?? [];
|
||||
state = {
|
||||
indexes: new Map(failures.map((failure, index) => [failure.table, index])),
|
||||
retainedBytes: failures.reduce((total, failure) => total + utf8ByteLength(failure.table) + utf8ByteLength(failure.error), 0),
|
||||
omittedHashes: new Set(),
|
||||
localOmittedCount: task.transferFailuresOmitted ?? 0,
|
||||
replayOmittedCount: 0,
|
||||
};
|
||||
transferFailureStates.set(task.exportId, state);
|
||||
return state;
|
||||
}
|
||||
|
||||
function syncTransferFailureOmittedCount(task: ExportTask, state: TransferFailureState) {
|
||||
task.transferFailuresOmitted = state.localOmittedCount + state.replayOmittedCount;
|
||||
}
|
||||
|
||||
function recordOmittedTransferFailure(task: ExportTask, state: TransferFailureState, table: string) {
|
||||
const hash = failureTableHash(table);
|
||||
if (state.omittedHashes.has(hash)) return;
|
||||
if (state.omittedHashes.size >= MAX_TRACKED_OMITTED_FAILURES) return;
|
||||
state.omittedHashes.add(hash);
|
||||
state.localOmittedCount += 1;
|
||||
syncTransferFailureOmittedCount(task, state);
|
||||
}
|
||||
|
||||
function recordTransferFailure(task: ExportTask, table: string, error: string) {
|
||||
task.transferFailures ??= [];
|
||||
const state = getTransferFailureState(task);
|
||||
const existingIndex = state.indexes.get(table);
|
||||
if (existingIndex !== undefined) {
|
||||
const existing = task.transferFailures[existingIndex];
|
||||
const previousErrorBytes = utf8ByteLength(existing.error);
|
||||
const availableBytes = Math.min(MAX_TRANSFER_FAILURE_ERROR_BYTES, MAX_TRANSFER_FAILURE_DETAIL_BYTES - (state.retainedBytes - previousErrorBytes));
|
||||
const nextError = truncateUtf8(error, Math.max(0, availableBytes));
|
||||
existing.error = nextError.value;
|
||||
if (nextError.truncated) existing.truncated = true;
|
||||
else delete existing.truncated;
|
||||
state.retainedBytes += nextError.bytes - previousErrorBytes;
|
||||
return;
|
||||
}
|
||||
|
||||
const tableBytes = utf8ByteLength(table);
|
||||
const availableBytes = Math.min(MAX_TRANSFER_FAILURE_ERROR_BYTES, MAX_TRANSFER_FAILURE_DETAIL_BYTES - state.retainedBytes - tableBytes);
|
||||
if (task.transferFailures.length >= MAX_TRANSFER_FAILURE_DETAILS || availableBytes <= 0) {
|
||||
recordOmittedTransferFailure(task, state, table);
|
||||
return;
|
||||
}
|
||||
|
||||
const retainedError = truncateUtf8(error, availableBytes);
|
||||
const failure: DataTransferFailure = { table, error: retainedError.value };
|
||||
if (retainedError.truncated) failure.truncated = true;
|
||||
state.indexes.set(table, task.transferFailures.length);
|
||||
state.retainedBytes += tableBytes + retainedError.bytes;
|
||||
task.transferFailures.push(failure);
|
||||
}
|
||||
|
||||
function normalizeExportStatus(status: string): BackgroundTaskStatus {
|
||||
if (status === "Writing" || status === "Done" || status === "Error" || status === "Cancelled") return status;
|
||||
@@ -176,6 +287,7 @@ export function useExportTracker() {
|
||||
}
|
||||
|
||||
function addDataTransferTask(transferId: string, label: string, totalTables: number): ExportTask {
|
||||
transferFailureStates.delete(transferId);
|
||||
const task = reactive<ExportTask>({
|
||||
exportId: transferId,
|
||||
kind: "data-transfer",
|
||||
@@ -190,6 +302,8 @@ export function useExportTracker() {
|
||||
totalTables,
|
||||
currentTable: "",
|
||||
startedAt: Date.now(),
|
||||
transferFailures: [],
|
||||
transferFailuresOmitted: 0,
|
||||
});
|
||||
taskMap.set(transferId, task);
|
||||
return task;
|
||||
@@ -294,6 +408,14 @@ export function useExportTracker() {
|
||||
function updateDataTransferTask(transferId: string, progress: api.TransferProgress) {
|
||||
const task = taskMap.get(transferId);
|
||||
if (!task) return;
|
||||
if (progress.transferFailuresOmitted !== undefined) {
|
||||
const state = getTransferFailureState(task);
|
||||
state.replayOmittedCount = Math.max(state.replayOmittedCount, progress.transferFailuresOmitted);
|
||||
syncTransferFailureOmittedCount(task, state);
|
||||
}
|
||||
if (progress.status === "error" && !progress.terminal && progress.table && progress.error) {
|
||||
recordTransferFailure(task, progress.table, progress.error);
|
||||
}
|
||||
const nextStatus = normalizeTransferStatus(progress.status, progress.terminal);
|
||||
const hadError = task.status === "Error";
|
||||
task.status = hadError && nextStatus === "Done" ? "Error" : nextStatus;
|
||||
@@ -311,6 +433,7 @@ export function useExportTracker() {
|
||||
function removeTask(exportId: string) {
|
||||
taskMap.delete(exportId);
|
||||
taskCancelHandlers.delete(exportId);
|
||||
transferFailureStates.delete(exportId);
|
||||
}
|
||||
|
||||
function clearFinished() {
|
||||
@@ -318,6 +441,7 @@ export function useExportTracker() {
|
||||
if (task.status === "Done" || task.status === "Error" || task.status === "Cancelled") {
|
||||
taskMap.delete(id);
|
||||
taskCancelHandlers.delete(id);
|
||||
transferFailureStates.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user