feat(transfer): show per-table failure details

This commit is contained in:
zipg authored and GitHub committed 2026-07-28 11:47:58 +08:00
1 parent 44cb4acbb7
commit 1e72d28ed3
20 files changed
+622 -65

No files matched your search

@@ -3,13 +3,14 @@ import { computed, ref } from "vue";
import { useI18n } from "vue-i18n";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Button } from "@/components/ui/button";
import { Loader2, CheckCircle2, XCircle, AlertCircle, X, FileDown, DatabaseBackup, FileCode2, ArrowRightLeft } from "@lucide/vue";
import { Loader2, CheckCircle2, XCircle, AlertCircle, X, FileDown, DatabaseBackup, FileCode2, ArrowRightLeft, ChevronRight } from "@lucide/vue";
import { formatDataTransferDuration, useExportTracker, type ExportTask } from "@/composables/useExportTracker";
const { t } = useI18n();
const { tasks, activeCount, hasActive, clearFinished, cancelTask, removeTask } = useExportTracker();
const open = ref(false);
const showAll = ref(false);
const expandedFailureTaskIds = ref<string[]>([]);
const MAX_VISIBLE = 5;
const reversedTasks = computed(() => {
@@ -132,6 +133,18 @@ const statusColor = (status: string) => {
function toggleShowAll() {
showAll.value = !showAll.value;
}
function failureDetailsExpanded(exportId: string) {
return expandedFailureTaskIds.value.includes(exportId);
}
function toggleFailureDetails(exportId: string) {
expandedFailureTaskIds.value = failureDetailsExpanded(exportId) ? expandedFailureTaskIds.value.filter((id) => id !== exportId) : [...expandedFailureTaskIds.value, exportId];
}
function failureDetailCount(task: ExportTask) {
return (task.transferFailures?.length ?? 0) + (task.transferFailuresOmitted ?? 0);
}
</script>
<template>
@@ -175,6 +188,20 @@ function toggleShowAll() {
<span v-if="task.status === 'Error' && task.errorMessage" class="mt-1 block whitespace-normal break-words text-destructive" :title="task.errorMessage">
{{ task.errorMessage }}
</span>
<template v-if="task.kind === 'data-transfer' && failureDetailCount(task) > 0">
<button class="mt-1.5 flex items-center gap-1 text-xs font-medium text-foreground hover:text-primary" :aria-expanded="failureDetailsExpanded(task.exportId)" @click="toggleFailureDetails(task.exportId)">
<ChevronRight class="h-3.5 w-3.5 shrink-0 transition-transform" :class="{ 'rotate-90': failureDetailsExpanded(task.exportId) }" />
{{ failureDetailsExpanded(task.exportId) ? t("exportProgress.hideFailureDetails") : t("exportProgress.showFailureDetails", { count: failureDetailCount(task) }) }}
</button>
<div v-if="failureDetailsExpanded(task.exportId)" class="mt-1.5 max-h-44 overflow-y-auto rounded border border-destructive/20 bg-destructive/5">
<div v-for="failure in task.transferFailures" :key="failure.table" class="border-b border-destructive/15 px-2.5 py-2 last:border-b-0">
<div class="break-all font-mono font-medium text-foreground">{{ failure.table }}</div>
<div class="mt-0.5 select-text whitespace-pre-wrap break-words text-destructive">{{ failure.error }}</div>
<div v-if="failure.truncated" class="mt-0.5 text-muted-foreground">{{ t("exportProgress.failureDetailTruncated") }}</div>
</div>
<div v-if="task.transferFailuresOmitted" class="px-2.5 py-2 text-muted-foreground">{{ t("exportProgress.failureDetailsOmitted", { count: task.transferFailuresOmitted }) }}</div>
</div>
</template>
</div>
</div>
@@ -91,4 +91,72 @@ describe("ExportProgressPopover data transfer duration", () => {
expect(document.body.textContent).toContain("Elapsed: 1m 5s");
expect(document.body.textContent?.match(/Elapsed:/g)).toHaveLength(1);
});
it("expands complete per-table transfer failure details", async () => {
const tracker = useExportTracker();
const task = tracker.addDataTransferTask("failed-transfer", "source to target", 2);
tracker.updateDataTransferTask(task.exportId, {
transferId: task.exportId,
table: "users",
tableIndex: 0,
totalTables: 2,
rowsTransferred: 0,
totalRows: null,
status: "error",
error: "permission denied for relation users",
terminal: false,
});
tracker.updateDataTransferTask(task.exportId, {
transferId: task.exportId,
table: "",
tableIndex: 2,
totalTables: 2,
rowsTransferred: 0,
totalRows: null,
status: "error",
error: "1 table(s) failed: users",
terminal: true,
});
await mountPopover();
const detailsButton = document.body.querySelector<HTMLButtonElement>('button[aria-expanded="false"]');
expect(detailsButton?.textContent).toContain("Failure details (1)");
expect(document.body.textContent).not.toContain("permission denied for relation users");
detailsButton?.click();
await nextTick();
expect(detailsButton?.getAttribute("aria-expanded")).toBe("true");
expect(document.body.textContent).toContain("users");
expect(document.body.textContent).toContain("permission denied for relation users");
});
it("shows the total failure count and omitted-detail notice when the bounded list is full", async () => {
const tracker = useExportTracker();
const task = tracker.addDataTransferTask("bounded-failures", "source to target", 101);
for (let index = 0; index < 101; index += 1) {
tracker.updateDataTransferTask(task.exportId, {
transferId: task.exportId,
table: `table_${index}`,
tableIndex: index,
totalTables: 101,
rowsTransferred: 0,
totalRows: null,
status: "error",
error: `failure ${index}`,
terminal: false,
});
}
await mountPopover();
const detailsButton = document.body.querySelector<HTMLButtonElement>('button[aria-expanded="false"]');
expect(detailsButton?.textContent).toContain("Failure details (101)");
detailsButton?.click();
await nextTick();
expect(document.body.textContent).toContain("1 additional failure detail(s) not shown");
expect(document.body.textContent).not.toContain("failure 100");
});
});
@@ -10,7 +10,7 @@ vi.mock("@/lib/backend/api", () => ({
}));
import * as api from "@/lib/backend/api";
import { formatDataTransferDuration, useExportTracker } from "@/composables/useExportTracker";
import { formatDataTransferDuration, MAX_TRANSFER_FAILURE_DETAIL_BYTES, MAX_TRANSFER_FAILURE_DETAILS, MAX_TRANSFER_FAILURE_ERROR_BYTES, useExportTracker } from "@/composables/useExportTracker";
let now = 0;
@@ -161,6 +161,109 @@ describe("data transfer task duration", () => {
});
});
describe("data transfer failure details", () => {
it("preserves per-table failures after the terminal summary and deduplicates table updates", () => {
const tracker = useExportTracker();
const task = tracker.addDataTransferTask("failure-details", "users and orders", 2);
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error", false), table: "users", error: "permission denied" });
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error", false), table: "orders", error: "table missing" });
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error", false), table: "users", error: "permission denied by policy" });
tracker.updateDataTransferTask(task.exportId, { ...transferProgress(task.exportId, "error"), table: "", error: "2 table(s) failed: users, orders" });
expect(task.errorMessage).toBe("2 table(s) failed: users, orders");
expect(task.transferFailures).toEqual([
{ table: "users", error: "permission denied by policy" },
{ table: "orders", error: "table missing" },
]);
});
it("bounds many distinct failures and deduplicates retained and omitted table updates", () => {
const tracker = useExportTracker();
const task = tracker.addDataTransferTask("many-failures", "many tables", 250);
for (let index = 0; index < 250; index += 1) {
tracker.updateDataTransferTask(task.exportId, {
...transferProgress(task.exportId, "error", false),
table: `table_${index}`,
error: `failure ${index}`,
});
}
expect(task.transferFailures).toHaveLength(MAX_TRANSFER_FAILURE_DETAILS);
expect(task.transferFailuresOmitted).toBe(150);
tracker.updateDataTransferTask(task.exportId, {
...transferProgress(task.exportId, "error", false),
table: "table_150",
error: "updated omitted failure",
});
tracker.updateDataTransferTask(task.exportId, {
...transferProgress(task.exportId, "error", false),
table: "table_0",
error: "updated retained failure",
});
expect(task.transferFailuresOmitted).toBe(150);
expect(task.transferFailures?.[0]).toEqual({ table: "table_0", error: "updated retained failure" });
tracker.updateDataTransferTask(task.exportId, {
...transferProgress(task.exportId, "done"),
transferFailuresOmitted: 12,
});
expect(task.transferFailuresOmitted).toBe(162);
});
it("limits individual and total UTF-8 error detail bytes without splitting characters", () => {
const tracker = useExportTracker();
const task = tracker.addDataTransferTask("long-failures", "long errors", 40);
const longError = "错误🙂".repeat(4_000);
for (let index = 0; index < 40; index += 1) {
tracker.updateDataTransferTask(task.exportId, {
...transferProgress(task.exportId, "error", false),
table: `long_table_${index}`,
error: longError,
});
}
const encoder = new TextEncoder();
const retainedBytes = task.transferFailures!.reduce((total, failure) => {
expect(encoder.encode(failure.error).length).toBeLessThanOrEqual(MAX_TRANSFER_FAILURE_ERROR_BYTES);
expect(failure.error.endsWith("\ud83d")).toBe(false);
expect(failure.truncated).toBe(true);
return total + encoder.encode(failure.table).length + encoder.encode(failure.error).length;
}, 0);
expect(retainedBytes).toBeLessThanOrEqual(MAX_TRANSFER_FAILURE_DETAIL_BYTES);
expect(task.transferFailuresOmitted).toBeGreaterThan(0);
});
it("saturates omitted failure counting after the deduplication cap", () => {
const tracker = useExportTracker();
const task = tracker.addDataTransferTask("omitted-cap", "many tables", 4_197);
for (let index = 0; index < MAX_TRANSFER_FAILURE_DETAILS + 4_097; index += 1) {
tracker.updateDataTransferTask(task.exportId, {
...transferProgress(task.exportId, "error", false),
table: `table_${index}`,
error: `failure ${index}`,
});
}
expect(task.transferFailuresOmitted).toBe(4_096);
tracker.updateDataTransferTask(task.exportId, {
...transferProgress(task.exportId, "error", false),
table: "table_4_196",
error: "repeated omitted failure",
});
expect(task.transferFailuresOmitted).toBe(4_096);
});
});
describe("formatDataTransferDuration", () => {
it("formats millisecond, second, minute, and hour boundaries", () => {
expect(formatDataTransferDuration(Number.NaN)).toBe("0 ms");
@@ -5,6 +5,12 @@ import { isTerminalTransferProgress } from "@/lib/backend/transferProgress";
export type BackgroundTaskKind = "table-export" | "database-export" | "sql-file" | "data-transfer";
export type BackgroundTaskStatus = "Running" | "Writing" | "Done" | "Error" | "Cancelled";
export interface DataTransferFailure {
table: string;
error: string;
truncated?: boolean;
}
export interface ExportTask {
exportId: string;
kind: BackgroundTaskKind;
@@ -32,11 +38,116 @@ export interface ExportTask {
targetDatabase?: string;
targetSchema?: string;
targetTables?: string[];
transferFailures?: DataTransferFailure[];
transferFailuresOmitted?: number;
}
export const MAX_TRANSFER_FAILURE_DETAILS = 100;
export const MAX_TRANSFER_FAILURE_DETAIL_BYTES = 128 * 1024;
export const MAX_TRANSFER_FAILURE_ERROR_BYTES = 8 * 1024;
const MAX_TRACKED_OMITTED_FAILURES = 4096;
interface TransferFailureState {
indexes: Map<string, number>;
retainedBytes: number;
omittedHashes: Set<number>;
localOmittedCount: number;
replayOmittedCount: number;
}
const taskMap = reactive<Map<string, ExportTask>>(new Map());
const activeTransferRuns = new Set<string>();
const taskCancelHandlers = new Map<string, () => void | Promise<void>>();
const transferFailureStates = new Map<string, TransferFailureState>();
const textEncoder = new TextEncoder();
function utf8ByteLength(value: string): number {
return textEncoder.encode(value).length;
}
function truncateUtf8(value: string, maxBytes: number): { value: string; bytes: number; truncated: boolean } {
const encodedBytes = utf8ByteLength(value);
if (encodedBytes <= maxBytes) return { value, bytes: encodedBytes, truncated: false };
let result = "";
let bytes = 0;
for (const character of value) {
const characterBytes = utf8ByteLength(character);
if (bytes + characterBytes > maxBytes) break;
result += character;
bytes += characterBytes;
}
return { value: result, bytes, truncated: true };
}
function failureTableHash(value: string): number {
let hash = 0x811c9dc5;
for (let index = 0; index < value.length; index += 1) {
hash ^= value.charCodeAt(index);
hash = Math.imul(hash, 0x01000193);
}
return hash >>> 0;
}
function getTransferFailureState(task: ExportTask): TransferFailureState {
let state = transferFailureStates.get(task.exportId);
if (state) return state;
const failures = task.transferFailures ?? [];
state = {
indexes: new Map(failures.map((failure, index) => [failure.table, index])),
retainedBytes: failures.reduce((total, failure) => total + utf8ByteLength(failure.table) + utf8ByteLength(failure.error), 0),
omittedHashes: new Set(),
localOmittedCount: task.transferFailuresOmitted ?? 0,
replayOmittedCount: 0,
};
transferFailureStates.set(task.exportId, state);
return state;
}
function syncTransferFailureOmittedCount(task: ExportTask, state: TransferFailureState) {
task.transferFailuresOmitted = state.localOmittedCount + state.replayOmittedCount;
}
function recordOmittedTransferFailure(task: ExportTask, state: TransferFailureState, table: string) {
const hash = failureTableHash(table);
if (state.omittedHashes.has(hash)) return;
if (state.omittedHashes.size >= MAX_TRACKED_OMITTED_FAILURES) return;
state.omittedHashes.add(hash);
state.localOmittedCount += 1;
syncTransferFailureOmittedCount(task, state);
}
function recordTransferFailure(task: ExportTask, table: string, error: string) {
task.transferFailures ??= [];
const state = getTransferFailureState(task);
const existingIndex = state.indexes.get(table);
if (existingIndex !== undefined) {
const existing = task.transferFailures[existingIndex];
const previousErrorBytes = utf8ByteLength(existing.error);
const availableBytes = Math.min(MAX_TRANSFER_FAILURE_ERROR_BYTES, MAX_TRANSFER_FAILURE_DETAIL_BYTES - (state.retainedBytes - previousErrorBytes));
const nextError = truncateUtf8(error, Math.max(0, availableBytes));
existing.error = nextError.value;
if (nextError.truncated) existing.truncated = true;
else delete existing.truncated;
state.retainedBytes += nextError.bytes - previousErrorBytes;
return;
}
const tableBytes = utf8ByteLength(table);
const availableBytes = Math.min(MAX_TRANSFER_FAILURE_ERROR_BYTES, MAX_TRANSFER_FAILURE_DETAIL_BYTES - state.retainedBytes - tableBytes);
if (task.transferFailures.length >= MAX_TRANSFER_FAILURE_DETAILS || availableBytes <= 0) {
recordOmittedTransferFailure(task, state, table);
return;
}
const retainedError = truncateUtf8(error, availableBytes);
const failure: DataTransferFailure = { table, error: retainedError.value };
if (retainedError.truncated) failure.truncated = true;
state.indexes.set(table, task.transferFailures.length);
state.retainedBytes += tableBytes + retainedError.bytes;
task.transferFailures.push(failure);
}
function normalizeExportStatus(status: string): BackgroundTaskStatus {
if (status === "Writing" || status === "Done" || status === "Error" || status === "Cancelled") return status;
@@ -176,6 +287,7 @@ export function useExportTracker() {
}
function addDataTransferTask(transferId: string, label: string, totalTables: number): ExportTask {
transferFailureStates.delete(transferId);
const task = reactive<ExportTask>({
exportId: transferId,
kind: "data-transfer",
@@ -190,6 +302,8 @@ export function useExportTracker() {
totalTables,
currentTable: "",
startedAt: Date.now(),
transferFailures: [],
transferFailuresOmitted: 0,
});
taskMap.set(transferId, task);
return task;
@@ -294,6 +408,14 @@ export function useExportTracker() {
function updateDataTransferTask(transferId: string, progress: api.TransferProgress) {
const task = taskMap.get(transferId);
if (!task) return;
if (progress.transferFailuresOmitted !== undefined) {
const state = getTransferFailureState(task);
state.replayOmittedCount = Math.max(state.replayOmittedCount, progress.transferFailuresOmitted);
syncTransferFailureOmittedCount(task, state);
}
if (progress.status === "error" && !progress.terminal && progress.table && progress.error) {
recordTransferFailure(task, progress.table, progress.error);
}
const nextStatus = normalizeTransferStatus(progress.status, progress.terminal);
const hadError = task.status === "Error";
task.status = hadError && nextStatus === "Done" ? "Error" : nextStatus;
@@ -311,6 +433,7 @@ export function useExportTracker() {
function removeTask(exportId: string) {
taskMap.delete(exportId);
taskCancelHandlers.delete(exportId);
transferFailureStates.delete(exportId);
}
function clearFinished() {
@@ -318,6 +441,7 @@ export function useExportTracker() {
if (task.status === "Done" || task.status === "Error" || task.status === "Cancelled") {
taskMap.delete(id);
taskCancelHandlers.delete(id);
transferFailureStates.delete(id);
}
}
}
+4
View File
@@ -1381,6 +1381,10 @@ export default {
tablesCount: "{current} / {total} tables",
statementsCount: "{done} succeeded, {failed} failed",
elapsed: "Elapsed: {duration}",
showFailureDetails: "Failure details ({count})",
hideFailureDetails: "Hide failure details",
failureDetailTruncated: "Error details were truncated",
failureDetailsOmitted: "{count} additional failure detail(s) not shown",
},
welcome: {
title: "Database Workspace",
+4
View File
@@ -1324,6 +1324,10 @@ export default withEnglishFallback({
statementsCount: "{done} exitosas, {failed} fallidas",
elapsed: "Transcurrido: {duration}",
delete: "Eliminar",
showFailureDetails: "Detalles de fallos ({count})",
hideFailureDetails: "Ocultar detalles de fallos",
failureDetailTruncated: "Los detalles del error se han truncado",
failureDetailsOmitted: "No se muestran {count} detalles de fallos adicionales",
},
welcome: {
title: "Espacio de trabajo de base de datos",
+4
View File
@@ -1322,6 +1322,10 @@ export default withEnglishFallback({
tablesCount: "{current} / {total} tabelle",
statementsCount: "{done} riuscite, {failed} non riuscite",
elapsed: "Tempo trascorso: {duration}",
showFailureDetails: "Dettagli errori ({count})",
hideFailureDetails: "Nascondi dettagli errori",
failureDetailTruncated: "I dettagli dell'errore sono stati troncati",
failureDetailsOmitted: "Altri {count} dettagli di errore non sono visualizzati",
},
welcome: {
title: "Area di Lavoro Database",
+4
View File
@@ -1323,6 +1323,10 @@ export default withEnglishFallback({
tablesCount: "{current} / {total} テーブル",
statementsCount: "{done}成功、{failed}失敗",
elapsed: "経過時間: {duration}",
showFailureDetails: "失敗の詳細({count})",
hideFailureDetails: "失敗の詳細を閉じる",
failureDetailTruncated: "エラー詳細は省略されました",
failureDetailsOmitted: "ほか {count} 件の失敗詳細は表示されていません",
},
welcome: {
title: "データベースワークスペース",
+4
View File
@@ -1324,6 +1324,10 @@ export default withEnglishFallback({
tablesCount: "{current} / {total} tabelas",
statementsCount: "{done} com sucesso, {failed} com falha",
elapsed: "Decorrido: {duration}",
showFailureDetails: "Detalhes das falhas ({count})",
hideFailureDetails: "Ocultar detalhes das falhas",
failureDetailTruncated: "Os detalhes do erro foram truncados",
failureDetailsOmitted: "{count} detalhes adicionais de falha não exibidos",
},
welcome: {
title: "Área de Trabalho do Banco de Dados",
+4
View File
@@ -1382,6 +1382,10 @@ export default withEnglishFallback({
tablesCount: "{current} / {total} 张表",
statementsCount: "成功 {done},失败 {failed}",
elapsed: "耗时:{duration}",
showFailureDetails: "失败详情({count})",
hideFailureDetails: "收起失败详情",
failureDetailTruncated: "错误详情已截断",
failureDetailsOmitted: "另有 {count} 条失败详情未显示",
},
welcome: {
title: "数据库工作台",
+4
View File
@@ -1323,6 +1323,10 @@ export default withEnglishFallback({
tablesCount: "{current} / {total} 張資料表",
statementsCount: "{done} 成功,{failed} 失敗",
elapsed: "耗時:{duration}",
showFailureDetails: "失敗詳情({count})",
hideFailureDetails: "收合失敗詳情",
failureDetailTruncated: "錯誤詳情已截斷",
failureDetailsOmitted: "另有 {count} 條失敗詳情未顯示",
},
welcome: {
title: "資料庫工作檯",
+1
View File
@@ -2501,6 +2501,7 @@ export interface TransferProgress {
status: "running" | "tableDone" | "done" | "error" | "cancelled";
error: string | null;
terminal: boolean;
transferFailuresOmitted?: number;
}
export async function startTransfer(request: TransferRequest, onProgress: (progress: TransferProgress) => void): Promise<void> {