fix(compare): cap per-table compare reads and flag truncation

This commit is contained in:
zipg authored and GitHub committed 2026-09-30 14:59:56 +08:00
1 parent 535417a763
commit 3dbaed310d
21 files changed
+471 -81

No files matched your search

@@ -190,6 +190,10 @@ const totalAdded = computed(() => batchResults.value.reduce((sum, item) => sum +
const totalRemoved = computed(() => batchResults.value.reduce((sum, item) => sum + item.removed, 0)); const totalRemoved = computed(() => batchResults.value.reduce((sum, item) => sum + item.removed, 0));
const totalModified = computed(() => batchResults.value.reduce((sum, item) => sum + item.modified, 0)); const totalModified = computed(() => batchResults.value.reduce((sum, item) => sum + item.modified, 0));
const hasResults = computed(() => batchResults.value.length > 0); const hasResults = computed(() => batchResults.value.length > 0);
// A compare reads both sides up to a fixed row budget, so a table larger than that budget
// is reported through `*Truncated`. Surfacing it here keeps a partial diff from looking
// like a complete one.
const truncatedResults = computed(() => batchResults.value.filter((item) => item.sourceTruncated || item.targetTruncated));
const visibleKinds = computed(() => [...(showAdded.value ? (["added"] as DiffKind[]) : []), ...(showRemoved.value ? (["removed"] as DiffKind[]) : []), ...(showModified.value ? (["modified"] as DiffKind[]) : [])]); const visibleKinds = computed(() => [...(showAdded.value ? (["added"] as DiffKind[]) : []), ...(showRemoved.value ? (["removed"] as DiffKind[]) : []), ...(showModified.value ? (["modified"] as DiffKind[]) : [])]);
const selectedAddedCount = computed(() => selectedDiffCount("added")); const selectedAddedCount = computed(() => selectedDiffCount("added"));
const selectedRemovedCount = computed(() => selectedDiffCount("removed")); const selectedRemovedCount = computed(() => selectedDiffCount("removed"));
@@ -1276,6 +1280,23 @@ onBeforeUnmount(() => {
<div class="text-xs text-muted-foreground">{{ selectedSummary }}</div> <div class="text-xs text-muted-foreground">{{ selectedSummary }}</div>
</div> </div>
<div v-if="truncatedResults.length" class="rounded-lg border border-amber-500/40 bg-amber-500/10 p-3 text-xs text-amber-700 space-y-1 dark:text-amber-400">
<div class="text-sm font-medium">{{ t("dataCompare.truncatedTitle") }}</div>
<div>{{ t("dataCompare.truncatedHint") }}</div>
<ul class="list-disc space-y-0.5 pl-4">
<li v-for="item in truncatedResults" :key="`truncated-${item.sourceTable}:${item.targetTable}`">
{{
t("dataCompare.truncatedTable", {
table: item.sourceTable,
source: item.sourceRowCount,
target: item.targetRowCount,
budget: item.rowBudget,
})
}}
</li>
</ul>
</div>
<div class="rounded-lg border p-3 space-y-3"> <div class="rounded-lg border p-3 space-y-3">
<div class="flex flex-wrap items-center gap-2"> <div class="flex flex-wrap items-center gap-2">
<Button size="sm" variant="outline" class="h-7 text-xs" :class="showAdded ? 'border-primary' : ''" @click="showAdded = !showAdded"> {{ t("diff.added") }} · {{ totalAdded }} </Button> <Button size="sm" variant="outline" class="h-7 text-xs" :class="showAdded ? 'border-primary' : ''" @click="showAdded = !showAdded"> {{ t("diff.added") }} · {{ totalAdded }} </Button>
@@ -1350,6 +1371,9 @@ onBeforeUnmount(() => {
}) })
}} }}
</div> </div>
<div v-if="item.sourceTruncated || item.targetTruncated" class="mt-1 text-amber-700 dark:text-amber-400">
{{ t("dataCompare.truncatedBadge", { budget: item.rowBudget }) }}
</div>
<div class="mt-1"> <div class="mt-1">
{{ t("dataCompare.keyColumnsInline", { columns: item.keyColumns.join(", ") }) }} {{ t("dataCompare.keyColumnsInline", { columns: item.keyColumns.join(", ") }) }}
</div> </div>
@@ -205,6 +205,7 @@ function preparationFor(table: string) {
targetRowCount: 0, targetRowCount: 0,
sourceTruncated: false, sourceTruncated: false,
targetTruncated: false, targetTruncated: false,
rowBudget: 100_000,
}; };
} }
@@ -129,6 +129,7 @@ function completedSession(): DataCompareSession {
targetRowCount: 0, targetRowCount: 0,
sourceTruncated: false, sourceTruncated: false,
targetTruncated: false, targetTruncated: false,
rowBudget: 100_000,
databaseType: "oracle", databaseType: "oracle",
diff: { diff: {
added: [ added: [
@@ -509,3 +510,28 @@ describe("DataCompareDialog results layout", () => {
expect(scroller?.contains(dialogFooter)).toBe(false); expect(scroller?.contains(dialogFooter)).toBe(false);
}); });
}); });
describe("DataCompareDialog truncation warning", () => {
it("warns that a table larger than the row budget was only partially compared", async () => {
const session = completedSession();
const result = session.batchResults[0]!;
result.sourceTruncated = true;
result.sourceRowCount = 4_000_000;
result.rowBudget = 100_000;
mountSessionDialog(session);
await flushAsyncSetup();
const text = document.body.textContent ?? "";
expect(text).toContain(i18n.global.t("dataCompare.truncatedTitle"));
expect(text).toContain(i18n.global.t("dataCompare.truncatedBadge", { budget: 100_000 }));
expect(text).toContain(i18n.global.t("dataCompare.truncatedTable", { table: "ORDERS", source: 4_000_000, target: 0, budget: 100_000 }));
});
it("keeps the warning out of the results when every table fit the budget", async () => {
mountSessionDialog(completedSession());
await flushAsyncSetup();
expect(document.body.textContent ?? "").not.toContain(i18n.global.t("dataCompare.truncatedTitle"));
});
});
@@ -44,6 +44,7 @@ function preparationFor(table: string) {
targetRowCount: 0, targetRowCount: 0,
sourceTruncated: false, sourceTruncated: false,
targetTruncated: false, targetTruncated: false,
rowBudget: 100_000,
}; };
} }
@@ -226,6 +227,30 @@ test("keeps the configured per-table match columns after cloning the session con
assert.deepEqual(session.config.keyColumnsByTable, { users: ["user_id", "email"] }); assert.deepEqual(session.config.keyColumnsByTable, { users: ["user_id", "email"] });
}); });
test("carries the row budget and truncation flags from the backend into the table result", async () => {
apiMock.prepareDataCompareFromTables.mockImplementation((options: { targetTable: string }) => Promise.resolve({ ...preparationFor(options.targetTable), sourceRowCount: 4_000_000, sourceTruncated: true, targetTruncated: true, rowBudget: 100_000 }));
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users" }), [{ sourceTable: "users", targetTable: "users" }]);
assert.equal(session.batchResults[0]?.rowBudget, 100_000);
assert.equal(session.batchResults[0]?.sourceTruncated, true);
assert.equal(session.batchResults[0]?.targetTruncated, true);
});
test("keeps a truncated missing-target table on the same row budget", async () => {
apiMock.prepareDataCompareMissingTarget.mockResolvedValue({
...preparationFor("users"),
sourceRowCount: 8_000_000,
sourceTruncated: true,
rowBudget: 100_000,
});
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users", targetTables: ["orders"] }), [{ sourceTable: "users", targetTable: "users" }]);
assert.equal(session.batchResults[0]?.rowBudget, 100_000);
assert.equal(session.batchResults[0]?.sourceTruncated, true);
});
test("normalizes a per-table match-column map", () => { test("normalizes a per-table match-column map", () => {
assert.deepEqual(normalizeKeyColumnOverrides({ users: [" id ", ""], orders: [], "": ["x"] }), { users: ["id"], orders: [] }); assert.deepEqual(normalizeKeyColumnOverrides({ users: [" id ", ""], orders: [], "": ["x"] }), { users: ["id"], orders: [] });
assert.deepEqual(normalizeKeyColumnOverrides(undefined), {}); assert.deepEqual(normalizeKeyColumnOverrides(undefined), {});
@@ -59,6 +59,8 @@ export interface DataCompareTableResult {
targetRowCount: number; targetRowCount: number;
sourceTruncated: boolean; sourceTruncated: boolean;
targetTruncated: boolean; targetTruncated: boolean;
/** Rows the backend was allowed to read per side; the diff only covers this many rows when a side is truncated. */
rowBudget: number;
databaseType?: DatabaseType; databaseType?: DatabaseType;
preSyncStatements?: string[]; preSyncStatements?: string[];
diff: SelectableDataCompareResult; diff: SelectableDataCompareResult;
@@ -264,6 +266,7 @@ function buildTableResult(task: DataCompareTableTask, preparation: DataCompareFr
targetRowCount: preparation.targetRowCount, targetRowCount: preparation.targetRowCount,
sourceTruncated: preparation.sourceTruncated, sourceTruncated: preparation.sourceTruncated,
targetTruncated: preparation.targetTruncated, targetTruncated: preparation.targetTruncated,
rowBudget: preparation.rowBudget,
databaseType, databaseType,
preSyncStatements: preparation.preSyncStatements, preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result), diff: toSelectableDiff(preparation.result),
@@ -287,6 +290,7 @@ function buildMissingTargetResult(task: DataCompareTableTask, preparation: DataC
targetRowCount: 0, targetRowCount: 0,
sourceTruncated: preparation.sourceTruncated, sourceTruncated: preparation.sourceTruncated,
targetTruncated: false, targetTruncated: false,
rowBudget: preparation.rowBudget,
databaseType, databaseType,
preSyncStatements: preparation.preSyncStatements, preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result), diff: toSelectableDiff(preparation.result),
@@ -310,6 +314,7 @@ function buildErrorResult(task: DataCompareTableTask, keyColumns: string[], data
targetRowCount: 0, targetRowCount: 0,
sourceTruncated: false, sourceTruncated: false,
targetTruncated: false, targetTruncated: false,
rowBudget: 0,
databaseType, databaseType,
preSyncStatements: [], preSyncStatements: [],
diff: { added: [], removed: [], modified: [] }, diff: { added: [], removed: [], modified: [] },
+4
View File
@@ -7058,6 +7058,10 @@ export default withEnglishFallback({
noComparableTables: "Müqayisə edilə bilən cədvəl tapılmadı. Eyni adlı hədəf cədvəllərin mövcudluğunu yoxlayın.", noComparableTables: "Müqayisə edilə bilən cədvəl tapılmadı. Eyni adlı hədəf cədvəllərin mövcudluğunu yoxlayın.",
noDifferences: "Verilənlər eynidir, sinxronlaşdırma SQL-inə ehtiyac yoxdur", noDifferences: "Verilənlər eynidir, sinxronlaşdırma SQL-inə ehtiyac yoxdur",
syncSuccess: "Verilənlərin sinxronlaşdırılması uğurla icra edildi", syncSuccess: "Verilənlərin sinxronlaşdırılması uğurla icra edildi",
truncatedTitle: "Bəzi cədvəllər tam müqayisə edilmədi",
truncatedHint: "Bir müqayisənin oxuya biləcəyindən çox sətri olan cədvəllər yalnız ilk sətirlər üzrə müqayisə edildi, ona görə də bu nöqtədən sonrakı əlavə, dəyişdirilmiş və ya silinmiş sətirlər aşkarlanmır. Aralığı filtr ilə daraldıb hissə-hissə müqayisə edin.",
truncatedTable: "{table}: mənbə {source} sətir, hədəf {target} sətir; yalnız ilk {budget} sətir müqayisə edildi",
truncatedBadge: "Yalnız ilk {budget} sətir müqayisə edildi",
}, },
settings: { settings: {
historyRetentionLimit: "Sorğu tarixçəsinin saxlanma həddi", historyRetentionLimit: "Sorğu tarixçəsinin saxlanma həddi",
+4
View File
@@ -7835,6 +7835,10 @@ export default {
noComparableTables: "No comparable tables found. Check whether same-name target tables exist.", noComparableTables: "No comparable tables found. Check whether same-name target tables exist.",
noDifferences: "Data is identical, no sync SQL needed", noDifferences: "Data is identical, no sync SQL needed",
syncSuccess: "Data sync executed successfully", syncSuccess: "Data sync executed successfully",
truncatedTitle: "Some tables were not compared in full",
truncatedHint: "Tables with more rows than a single compare can read were compared only over their first rows, so added, modified, or deleted rows beyond that point are not reported. Narrow the range with a filter and compare in batches.",
truncatedTable: "{table}: {source} source rows, {target} target rows, compared the first {budget} rows only",
truncatedBadge: "Compared the first {budget} rows only",
}, },
settings: { settings: {
historyRetentionLimit: "Query history retention", historyRetentionLimit: "Query history retention",
+4
View File
@@ -7100,6 +7100,10 @@ export default withEnglishFallback({
noComparableTables: "No se encontraron tablas comparables. Verifica si existen tablas de destino con el mismo nombre.", noComparableTables: "No se encontraron tablas comparables. Verifica si existen tablas de destino con el mismo nombre.",
noDifferences: "Los datos son idénticos, no se necesita SQL de sincronización", noDifferences: "Los datos son idénticos, no se necesita SQL de sincronización",
syncSuccess: "Sincronización de datos ejecutada exitosamente", syncSuccess: "Sincronización de datos ejecutada exitosamente",
truncatedTitle: "Algunas tablas no se compararon por completo",
truncatedHint: "Las tablas con más filas de las que puede leer una sola comparación solo se compararon en sus primeras filas, por lo que las filas añadidas, modificadas o eliminadas a partir de ahí no se detectan. Acota el rango con un filtro y compara por lotes.",
truncatedTable: "{table}: {source} filas de origen, {target} de destino; solo se compararon las primeras {budget} filas",
truncatedBadge: "Solo se compararon las primeras {budget} filas",
}, },
settings: { settings: {
historyRetentionLimit: "Límite del historial de consultas", historyRetentionLimit: "Límite del historial de consultas",
+4
View File
@@ -7024,6 +7024,10 @@ export default withEnglishFallback({
noComparableTables: "Nessuna tabella confrontabile trovata. Verifica se esistono tabelle di destinazione con lo stesso nome.", noComparableTables: "Nessuna tabella confrontabile trovata. Verifica se esistono tabelle di destinazione con lo stesso nome.",
noDifferences: "I dati sono identici, nessun SQL di sincronizzazione necessario", noDifferences: "I dati sono identici, nessun SQL di sincronizzazione necessario",
syncSuccess: "Sincronizzazione dei dati eseguita con successo", syncSuccess: "Sincronizzazione dei dati eseguita con successo",
truncatedTitle: "Alcune tabelle non sono state confrontate per intero",
truncatedHint: "Le tabelle con più righe di quante ne possa leggere un singolo confronto sono state confrontate solo sulle prime righe, quindi le righe aggiunte, modificate o eliminate oltre quel punto non vengono rilevate. Restringi l'intervallo con un filtro e confronta a blocchi.",
truncatedTable: "{table}: {source} righe nell'origine, {target} nella destinazione; confrontate solo le prime {budget} righe",
truncatedBadge: "Confrontate solo le prime {budget} righe",
}, },
settings: { settings: {
historyRetentionLimit: "Limite della cronologia query", historyRetentionLimit: "Limite della cronologia query",
+4
View File
@@ -7055,6 +7055,10 @@ export default withEnglishFallback({
noComparableTables: "比較可能なテーブルが見つかりません。同名のターゲットテーブルが存在するか確認してください。", noComparableTables: "比較可能なテーブルが見つかりません。同名のターゲットテーブルが存在するか確認してください。",
noDifferences: "データは同一です。同期SQLは不要です", noDifferences: "データは同一です。同期SQLは不要です",
syncSuccess: "データ同期が正常に実行されました", syncSuccess: "データ同期が正常に実行されました",
truncatedTitle: "一部のテーブルは全体を比較していません",
truncatedHint: "1 回の比較で読み込める行数を超えたテーブルは先頭の行だけを比較しているため、それ以降の追加・変更・削除は検出されません。フィルターで範囲を絞り、分割して比較してください。",
truncatedTable: "{table}:ソース {source} 行、ターゲット {target} 行、先頭 {budget} 行のみ比較",
truncatedBadge: "先頭 {budget} 行のみ比較",
}, },
settings: { settings: {
historyRetentionLimit: "クエリ履歴の保持件数", historyRetentionLimit: "クエリ履歴の保持件数",
+4
View File
@@ -6790,6 +6790,10 @@ export default withEnglishFallback({
noComparableTables: "비교 가능한 테이블이 없습니다. 같은 이름의 대상 테이블이 있는지 확인하세요.", noComparableTables: "비교 가능한 테이블이 없습니다. 같은 이름의 대상 테이블이 있는지 확인하세요.",
noDifferences: "데이터가 동일하여 동기화 SQL이 필요하지 않습니다", noDifferences: "데이터가 동일하여 동기화 SQL이 필요하지 않습니다",
syncSuccess: "데이터 동기화를 성공적으로 실행했습니다", syncSuccess: "데이터 동기화를 성공적으로 실행했습니다",
truncatedTitle: "일부 테이블은 전체를 비교하지 않았습니다",
truncatedHint: "한 번의 비교로 읽을 수 있는 행 수를 초과한 테이블은 앞부분만 비교했기 때문에 그 이후의 추가·수정·삭제는 감지되지 않습니다. 필터로 범위를 좁혀 나누어 비교하세요.",
truncatedTable: "{table}: 원본 {source}행, 대상 {target}행, 앞 {budget}행만 비교",
truncatedBadge: "앞 {budget}행만 비교",
}, },
settings: { settings: {
historyRetentionLimit: "쿼리 기록 보관 개수", historyRetentionLimit: "쿼리 기록 보관 개수",
+4
View File
@@ -7026,6 +7026,10 @@ export default withEnglishFallback({
noComparableTables: "Nenhuma tabela comparável encontrada. Verifique se existem tabelas de destino de mesmo nome.", noComparableTables: "Nenhuma tabela comparável encontrada. Verifique se existem tabelas de destino de mesmo nome.",
noDifferences: "Os dados são idênticos, nenhum SQL de sincronização é necessário", noDifferences: "Os dados são idênticos, nenhum SQL de sincronização é necessário",
syncSuccess: "Sincronização de dados executada com sucesso", syncSuccess: "Sincronização de dados executada com sucesso",
truncatedTitle: "Algumas tabelas não foram comparadas por completo",
truncatedHint: "Tabelas com mais linhas do que uma única comparação consegue ler foram comparadas apenas nas primeiras linhas, então linhas adicionadas, modificadas ou excluídas depois disso não são detectadas. Restrinja o intervalo com um filtro e compare em lotes.",
truncatedTable: "{table}: {source} linhas na origem, {target} no destino; apenas as primeiras {budget} linhas foram comparadas",
truncatedBadge: "Apenas as primeiras {budget} linhas comparadas",
}, },
settings: { settings: {
historyRetentionLimit: "Limite do histórico de consultas", historyRetentionLimit: "Limite do histórico de consultas",
+4
View File
@@ -8243,6 +8243,10 @@ export default withEnglishFallback({
noComparableTables: "Не найдено таблиц для сравнения. Проверьте наличие одноимённых целевых таблиц.", noComparableTables: "Не найдено таблиц для сравнения. Проверьте наличие одноимённых целевых таблиц.",
noDifferences: "Данные идентичны, SQL синхронизации не требуется", noDifferences: "Данные идентичны, SQL синхронизации не требуется",
syncSuccess: "Синхронизация данных выполнена успешно", syncSuccess: "Синхронизация данных выполнена успешно",
truncatedTitle: "Часть таблиц сравнена не полностью",
truncatedHint: "Таблицы, в которых строк больше, чем может прочитать одно сравнение, сравнены только по первым строкам, поэтому добавленные, изменённые и удалённые строки после этого не обнаруживаются. Сузьте диапазон фильтром и сравнивайте по частям.",
truncatedTable: "{table}: строк в источнике {source}, в приёмнике {target}; сравнены только первые {budget} строк",
truncatedBadge: "Сравнены только первые {budget} строк",
}, },
settings: { settings: {
historyRetentionLimit: "Хранение истории запросов", historyRetentionLimit: "Хранение истории запросов",
+4
View File
@@ -6940,6 +6940,10 @@ export default withEnglishFallback({
noComparableTables: "Karşılaştırılabilir tablo bulunamadı. Aynı adlı hedef tabloların var olup olmadığını denetleyin.", noComparableTables: "Karşılaştırılabilir tablo bulunamadı. Aynı adlı hedef tabloların var olup olmadığını denetleyin.",
noDifferences: "Veriler aynı, eşitleme SQL'i gerekmiyor", noDifferences: "Veriler aynı, eşitleme SQL'i gerekmiyor",
syncSuccess: "Veri eşitlemesi başarıyla çalıştırıldı", syncSuccess: "Veri eşitlemesi başarıyla çalıştırıldı",
truncatedTitle: "Bazı tablolar tam olarak karşılaştırılmadı",
truncatedHint: "Tek bir karşılaştırmanın okuyabileceğinden fazla satıra sahip tablolar yalnızca ilk satırları üzerinden karşılaştırıldı; bu noktadan sonraki eklenen, değişen veya silinen satırlar bildirilmez. Aralığı bir filtreyle daraltıp toplu olarak karşılaştırın.",
truncatedTable: "{table}: kaynak {source} satır, hedef {target} satır; yalnızca ilk {budget} satır karşılaştırıldı",
truncatedBadge: "Yalnızca ilk {budget} satır karşılaştırıldı",
}, },
settings: { settings: {
historyRetentionLimit: "Sorgu geçmişi saklama sınırı", historyRetentionLimit: "Sorgu geçmişi saklama sınırı",
+4
View File
@@ -7801,6 +7801,10 @@ export default withEnglishFallback({
noComparableTables: "没有可比较的表,请检查目标端是否存在同名表", noComparableTables: "没有可比较的表,请检查目标端是否存在同名表",
noDifferences: "数据完全一致,无需生成同步 SQL", noDifferences: "数据完全一致,无需生成同步 SQL",
syncSuccess: "数据同步执行成功", syncSuccess: "数据同步执行成功",
truncatedTitle: "部分表未完整比较",
truncatedHint: "行数超过单次比较上限的表只比较了前若干行,超出部分的新增、修改和删除不会被发现。建议加上过滤条件缩小范围后分批比较。",
truncatedTable: "{table}:源 {source} 行、目标 {target} 行,仅比较前 {budget} 行",
truncatedBadge: "仅比较前 {budget} 行",
}, },
settings: { settings: {
historyRetentionLimit: "查询历史保留条数", historyRetentionLimit: "查询历史保留条数",
+4
View File
@@ -6349,6 +6349,10 @@ export default withEnglishFallback({
noComparableTables: "沒有可比較的資料表。請檢查目標端是否存在同名資料表。", noComparableTables: "沒有可比較的資料表。請檢查目標端是否存在同名資料表。",
noDifferences: "資料完全一致,不需要產生同步 SQL", noDifferences: "資料完全一致,不需要產生同步 SQL",
syncSuccess: "資料同步執行成功", syncSuccess: "資料同步執行成功",
truncatedTitle: "部分表未完整比較",
truncatedHint: "資料筆數超過單次比較上限的表只比較了前幾筆,超出部分的新增、修改與刪除不會被發現。建議加上篩選條件縮小範圍後分批比較。",
truncatedTable: "{table}:來源 {source} 筆、目標 {target} 筆,僅比較前 {budget} 筆",
truncatedBadge: "僅比較前 {budget} 筆",
}, },
settings: { settings: {
historyRetentionLimit: "查詢歷史保留筆數", historyRetentionLimit: "查詢歷史保留筆數",
@@ -95,6 +95,8 @@ export interface DataCompareFromTablesPreparation extends DataComparePreparation
targetRowCount: number; targetRowCount: number;
sourceTruncated: boolean; sourceTruncated: boolean;
targetTruncated: boolean; targetTruncated: boolean;
/** Rows per side this compare was allowed to read; fewer than `sourceRowCount`/`targetRowCount` when truncated. */
rowBudget: number;
} }
export interface DataCompareSyncPlanTableOptions { export interface DataCompareSyncPlanTableOptions {
+1
View File
@@ -512,6 +512,7 @@ mod tests {
target_row_count: 2, target_row_count: 2,
source_truncated: false, source_truncated: false,
target_truncated: false, target_truncated: false,
row_budget: 100_000,
degradation_level: Some("full".to_string()), degradation_level: Some("full".to_string()),
sampling_rate: Some(1.0), sampling_rate: Some(1.0),
confidence_score: Some(1.0), confidence_score: Some(1.0),
+119 -81
View File
@@ -88,6 +88,8 @@ pub struct DataCompareMissingTargetOptions {
pub key_columns: Vec<String>, pub key_columns: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub fetch_batch_size: Option<usize>, pub fetch_batch_size: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub degradation_threshold: Option<DegradationThreshold>,
} }
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -179,6 +181,12 @@ pub struct DataCompareFromTablesPreparation {
pub target_row_count: u64, pub target_row_count: u64,
pub source_truncated: bool, pub source_truncated: bool,
pub target_truncated: bool, pub target_truncated: bool,
/// Rows per side this compare was allowed to read. When a side holds more rows than
/// this, the diff only covers the first `row_budget` rows and the matching
/// `*_truncated` flag is set, so the UI can warn instead of showing a partial plan
/// as if it were complete.
#[serde(default)]
pub row_budget: usize,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub degradation_level: Option<String>, pub degradation_level: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -363,68 +371,77 @@ pub async fn prepare_data_compare_from_tables(
let source_column_names = aligned_source_column_names(&options.columns, options.source_columns.as_ref()); let source_column_names = aligned_source_column_names(&options.columns, options.source_columns.as_ref());
let source_key_columns = aligned_source_key_columns(&options.columns, &source_column_names, &options.key_columns); let source_key_columns = aligned_source_key_columns(&options.columns, &source_column_names, &options.key_columns);
let (source_rows, target_rows, sampling_rate, verification_method) = match &degradation_level { // Every row is read into memory and then shipped to the UI, so even the "full" level
DegradationLevel::Full => { // is capped: the degradation thresholds decide the level, the budget bounds the fetch.
let (src, tgt) = tokio::try_join!( let row_budget = rows_read_budget(&degradation_level, &threshold);
fetch_compare_rows(
state, let (source_rows, target_rows, sampling_rate, verification_method, source_truncated, target_truncated) =
&options.source_connection_id, match &degradation_level {
&options.source_database, DegradationLevel::Full => {
&options.source_schema, let (src, tgt) = tokio::try_join!(
&options.source_table, fetch_compare_rows(
&source_column_names, state,
&source_key_columns, &options.source_connection_id,
source_database_type, &options.source_database,
fetch_batch_size, &options.source_schema,
), &options.source_table,
fetch_compare_rows( &source_column_names,
state, &source_key_columns,
&options.target_connection_id, source_database_type,
&options.target_database, fetch_batch_size,
&options.target_schema, row_budget,
&options.target_table, ),
&options.columns, fetch_compare_rows(
&options.key_columns, state,
target_database_type, &options.target_connection_id,
fetch_batch_size, &options.target_database,
) &options.target_schema,
)?; &options.target_table,
(src, tgt, 1.0, "full_compare".to_string()) &options.columns,
} &options.key_columns,
DegradationLevel::Sample => { target_database_type,
let (src, tgt) = tokio::try_join!( fetch_batch_size,
fetch_sampled_compare_rows( row_budget,
state, )
&options.source_connection_id, )?;
&options.source_database, let source_truncated = source_row_count > row_budget as u64;
&options.source_schema, let target_truncated = target_row_count > row_budget as u64;
&options.source_table, (src, tgt, 1.0, "full_compare".to_string(), source_truncated, target_truncated)
&source_column_names, }
&source_key_columns, DegradationLevel::Sample => {
source_database_type, let (src, tgt) = tokio::try_join!(
&sampling_strategy, fetch_sampled_compare_rows(
threshold.sample_size, state,
), &options.source_connection_id,
fetch_sampled_compare_rows( &options.source_database,
state, &options.source_schema,
&options.target_connection_id, &options.source_table,
&options.target_database, &source_column_names,
&options.target_schema, &source_key_columns,
&options.target_table, source_database_type,
&options.columns, &sampling_strategy,
&options.key_columns, threshold.sample_size,
target_database_type, ),
&sampling_strategy, fetch_sampled_compare_rows(
threshold.sample_size, state,
) &options.target_connection_id,
)?; &options.target_database,
let max_count = source_row_count.max(target_row_count); &options.target_schema,
let sample_rate = if max_count > 0 { threshold.sample_size as f64 / max_count as f64 } else { 1.0 }; &options.target_table,
let method = "sampled".to_string(); &options.columns,
(src, tgt, sample_rate.min(1.0), method) &options.key_columns,
} target_database_type,
DegradationLevel::SkipWithRisk => (Vec::new(), Vec::new(), 0.0, "skipped".to_string()), &sampling_strategy,
}; threshold.sample_size,
)
)?;
let max_count = source_row_count.max(target_row_count);
let sample_rate = if max_count > 0 { threshold.sample_size as f64 / max_count as f64 } else { 1.0 };
let method = "sampled".to_string();
(src, tgt, sample_rate.min(1.0), method, false, false)
}
DegradationLevel::SkipWithRisk => (Vec::new(), Vec::new(), 0.0, "skipped".to_string(), false, false),
};
let target_columns = get_columns_core( let target_columns = get_columns_core(
state, state,
@@ -468,8 +485,9 @@ pub async fn prepare_data_compare_from_tables(
pre_sync_statements: Vec::new(), pre_sync_statements: Vec::new(),
source_row_count, source_row_count,
target_row_count, target_row_count,
source_truncated: false, source_truncated,
target_truncated: false, target_truncated,
row_budget,
degradation_level: Some(degradation_level.to_string()), degradation_level: Some(degradation_level.to_string()),
sampling_rate: Some(sampling_rate), sampling_rate: Some(sampling_rate),
confidence_score: Some(confidence_score), confidence_score: Some(confidence_score),
@@ -509,6 +527,11 @@ pub async fn prepare_data_compare_missing_target(
) )
.await?; .await?;
let source_row_count = first_count(&source_count_result.rows)?; let source_row_count = first_count(&source_count_result.rows)?;
// A table that only exists on the source side has no count to compare against, but the
// sync plan still embeds one statement per row: reading the whole table freezes the
// client on large tables, so the plan covers at most one full-compare budget of rows.
let threshold = options.degradation_threshold.clone().unwrap_or_default();
let row_budget = threshold.full_compare_max_rows.max(1) as usize;
let source_rows = fetch_compare_rows( let source_rows = fetch_compare_rows(
state, state,
&options.source_connection_id, &options.source_connection_id,
@@ -519,8 +542,10 @@ pub async fn prepare_data_compare_missing_target(
&options.key_columns, &options.key_columns,
source_database_type, source_database_type,
fetch_batch_size, fetch_batch_size,
row_budget,
) )
.await?; .await?;
let source_truncated = source_row_count > row_budget as u64;
let result = missing_target_diff(&column_names, &options.key_columns, source_rows); let result = missing_target_diff(&column_names, &options.key_columns, source_rows);
let mut pre_sync_statements = Vec::new(); let mut pre_sync_statements = Vec::new();
pre_sync_statements.push(format!( pre_sync_statements.push(format!(
@@ -567,12 +592,15 @@ pub async fn prepare_data_compare_missing_target(
pre_sync_statements, pre_sync_statements,
source_row_count, source_row_count,
target_row_count: 0, target_row_count: 0,
source_truncated: false, source_truncated,
target_truncated: false, target_truncated: false,
degradation_level: Some("full".to_string()), row_budget,
sampling_rate: Some(1.0), degradation_level: Some(should_degrade(source_row_count, 0, &threshold).to_string()),
confidence_score: Some(1.0), sampling_rate: Some(if source_truncated { row_budget as f64 / source_row_count.max(1) as f64 } else { 1.0 }),
verification_method: Some("missing_target_full".to_string()), confidence_score: Some(if source_truncated { 0.5 } else { 1.0 }),
verification_method: Some(
if source_truncated { "missing_target_prefix" } else { "missing_target_full" }.to_string(),
),
source_checksums: None, source_checksums: None,
target_checksums: None, target_checksums: None,
}) })
@@ -1190,6 +1218,12 @@ fn build_rownum_data_compare_select_sql(
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
/// Fetches the rows a comparison needs, stopping after `row_budget` rows.
///
/// A compare reads both sides into memory and ships the diff to the UI, so an unbounded
/// fetch turns a large table into an out-of-memory freeze on the client. Callers derive
/// `source_truncated` / `target_truncated` from the table's row count versus `row_budget`
/// so a partial diff is never mistaken for a complete one.
async fn fetch_compare_rows( async fn fetch_compare_rows(
state: &AppState, state: &AppState,
connection_id: &str, connection_id: &str,
@@ -1200,20 +1234,15 @@ async fn fetch_compare_rows(
key_columns: &[String], key_columns: &[String],
database_type: DatabaseType, database_type: DatabaseType,
fetch_batch_size: usize, fetch_batch_size: usize,
row_budget: usize,
) -> Result<Vec<Vec<Value>>, String> { ) -> Result<Vec<Vec<Value>>, String> {
let mut rows = Vec::new(); let mut rows = Vec::new();
let mut offset = 0usize; let mut offset = 0usize;
loop { while rows.len() < row_budget {
let sql = build_data_compare_select_sql( let batch_size = fetch_batch_size.min(row_budget - rows.len());
database_type, let sql =
schema, build_data_compare_select_sql(database_type, schema, table_name, columns, key_columns, batch_size, offset);
table_name,
columns,
key_columns,
fetch_batch_size,
offset,
);
let result = execute_sql_statement_with_options( let result = execute_sql_statement_with_options(
state, state,
connection_id, connection_id,
@@ -1221,7 +1250,7 @@ async fn fetch_compare_rows(
&sql, &sql,
Some(schema), Some(schema),
None, None,
QueryExecutionOptions { max_rows: Some(fetch_batch_size), ..Default::default() }, QueryExecutionOptions { max_rows: Some(batch_size), ..Default::default() },
) )
.await?; .await?;
let fetched = result.rows.len(); let fetched = result.rows.len();
@@ -1229,7 +1258,7 @@ async fn fetch_compare_rows(
break; break;
} }
rows.extend(result.rows); rows.extend(result.rows);
if fetched < fetch_batch_size { if fetched < batch_size {
break; break;
} }
offset += fetched; offset += fetched;
@@ -1295,6 +1324,15 @@ fn data_grid_column_info(column: crate::types::ColumnInfo) -> DataGridColumnInfo
} }
} }
/// Rows per side a compare may read for a given degradation level.
fn rows_read_budget(level: &DegradationLevel, threshold: &DegradationThreshold) -> usize {
match level {
DegradationLevel::Full => threshold.full_compare_max_rows.max(1) as usize,
DegradationLevel::Sample => threshold.sample_size,
DegradationLevel::SkipWithRisk => 0,
}
}
fn should_degrade(source_row_count: u64, target_row_count: u64, threshold: &DegradationThreshold) -> DegradationLevel { fn should_degrade(source_row_count: u64, target_row_count: u64, threshold: &DegradationThreshold) -> DegradationLevel {
let max_count = source_row_count.max(target_row_count); let max_count = source_row_count.max(target_row_count);
if max_count <= threshold.full_compare_max_rows { if max_count <= threshold.full_compare_max_rows {
@@ -0,0 +1,221 @@
use dbx_core::connection::AppState;
use dbx_core::data_compare::{
prepare_data_compare_from_tables, prepare_data_compare_missing_target, DataCompareFromTablesOptions,
DataCompareMissingTargetOptions,
};
use dbx_core::db::mysql;
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
use serde_json::json;
use std::sync::Arc;
use std::time::{Duration, Instant};
fn live_mysql_config(id: &str) -> ConnectionConfig {
let host = std::env::var("DBX_LIVE_MYSQL_TRANSFER_HOST").expect("DBX_LIVE_MYSQL_TRANSFER_HOST");
let port =
std::env::var("DBX_LIVE_MYSQL_TRANSFER_PORT").ok().and_then(|value| value.parse::<u16>().ok()).unwrap_or(3306);
let username = std::env::var("DBX_LIVE_MYSQL_TRANSFER_USER").unwrap_or_else(|_| "root".to_string());
let password = std::env::var("DBX_LIVE_MYSQL_TRANSFER_PASSWORD").expect("DBX_LIVE_MYSQL_TRANSFER_PASSWORD");
serde_json::from_value(json!({
"id": id,
"name": id,
"db_type": DatabaseType::Mysql,
"host": host,
"port": port,
"username": username,
"password": password,
"database": null,
"connect_timeout_secs": 10,
"query_timeout_secs": 300,
"idle_timeout_secs": 60,
"keepalive_interval_secs": 0
}))
.expect("live MySQL compare config should deserialize")
}
fn mysql_url(config: &ConnectionConfig) -> String {
format!("mysql://{}:{}@{}:{}", config.username, config.password, config.host, config.port)
}
async fn live_state() -> Arc<AppState> {
let task_tmp = std::path::PathBuf::from(
std::env::var("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR").expect("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR"),
);
let dir = task_tmp.join(format!("data-compare-{}", uuid::Uuid::new_v4().simple()));
std::fs::create_dir_all(&dir).unwrap();
let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
Arc::new(AppState::new(storage))
}
fn big_source_table_ddl(database: &str, table: &str, rows: u64) -> String {
format!(
"CREATE DATABASE IF NOT EXISTS {database};\
SET SESSION cte_max_recursion_depth={}; \
CREATE TABLE {database}.{table} (\
id INT NOT NULL PRIMARY KEY, payload VARCHAR(120), amount DECIMAL(12,2)\
);\
INSERT INTO {database}.{table} (id, payload, amount)\
WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < {rows})\
SELECT n, CONCAT('row-', n), n * 1.5 FROM seq",
rows + 1
)
}
/// A source-only table must not drag the whole table into memory (and across the IPC
/// boundary) when the user compares it against a database that has no such table.
#[test]
#[ignore = "requires a disposable MySQL endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
fn live_mysql_data_compare_missing_target_is_bounded() {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(run_live_mysql_data_compare_missing_target_is_bounded());
}
async fn run_live_mysql_data_compare_missing_target_is_bounded() {
let suffix = uuid::Uuid::new_v4().simple().to_string();
let connection_id = format!("compare-conn-{}", &suffix[..12]);
let source_database = format!("dbx_cmp_src_{}", &suffix[..12]);
let target_database = format!("dbx_cmp_dst_{}", &suffix[..12]);
let config = live_mysql_config(&connection_id);
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
let rows: u64 = 300_000;
mysql::execute_query(&setup_pool, &big_source_table_ddl(&source_database, "big_source_only", rows), false)
.await
.unwrap();
mysql::execute_query(&setup_pool, &format!("CREATE DATABASE {target_database}"), false).await.unwrap();
let state = live_state().await;
state.configs.write().await.insert(connection_id.clone(), config);
let started = Instant::now();
let preparation = prepare_data_compare_missing_target(
&state,
DataCompareMissingTargetOptions {
source_connection_id: connection_id.clone(),
source_database: source_database.clone(),
source_schema: source_database.clone(),
source_table: "big_source_only".to_string(),
target_connection_id: connection_id.clone(),
target_database: target_database.clone(),
target_schema: target_database.clone(),
target_table: "big_source_only".to_string(),
key_columns: vec!["id".to_string()],
fetch_batch_size: None,
degradation_threshold: None,
},
)
.await
.unwrap();
let elapsed = started.elapsed();
println!(
"missing-target compare: rows={} added={} truncated={} level={:?} sql_bytes={} elapsed={:?}",
preparation.source_row_count,
preparation.result.added.len(),
preparation.source_truncated,
preparation.degradation_level,
preparation.sync_sql.len(),
elapsed
);
assert_eq!(
preparation.result.added.len(),
100_000,
"compare must stop at the full-compare row budget instead of materialising all {rows} rows"
);
assert!(preparation.source_truncated, "capped compare must flag source_truncated");
assert!(preparation.result.added.len() < rows as usize, "the capped plan must be smaller than the table");
assert!(elapsed < Duration::from_secs(30), "capped compare took {elapsed:?}");
}
/// The user-facing scenario: two databases on one connection, small tables, with real
/// differences in both directions.
#[test]
#[ignore = "requires a disposable MySQL endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
fn live_mysql_data_compare_two_databases_with_differences() {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(run_live_mysql_data_compare_two_databases_with_differences());
}
async fn run_live_mysql_data_compare_two_databases_with_differences() {
let suffix = uuid::Uuid::new_v4().simple().to_string();
let connection_id = format!("compare-conn-{}", &suffix[..12]);
let source_database = format!("dbx_cmp2_src_{}", &suffix[..12]);
let target_database = format!("dbx_cmp2_dst_{}", &suffix[..12]);
let config = live_mysql_config(&connection_id);
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
mysql::execute_query(
&setup_pool,
&format!(
"CREATE DATABASE {source_database};\
CREATE DATABASE {target_database};\
CREATE TABLE {source_database}.items (id INT NOT NULL PRIMARY KEY, name VARCHAR(60), qty INT);\
CREATE TABLE {target_database}.items (id INT NOT NULL PRIMARY KEY, name VARCHAR(60), qty INT);\
SET SESSION cte_max_recursion_depth=4000;\
INSERT INTO {source_database}.items (id, name, qty)\
WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < 2000)\
SELECT n, CONCAT('name-', n), n FROM seq;\
INSERT INTO {target_database}.items (id, name, qty)\
WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < 1500)\
SELECT n, CONCAT('name-', n * 2), n FROM seq"
),
false,
)
.await
.unwrap();
let state = live_state().await;
state.configs.write().await.insert(connection_id.clone(), config);
let started = Instant::now();
let preparation = prepare_data_compare_from_tables(
&state,
DataCompareFromTablesOptions {
source_connection_id: connection_id.clone(),
source_database: source_database.clone(),
source_schema: source_database.clone(),
source_table: "items".to_string(),
target_connection_id: connection_id.clone(),
target_database: target_database.clone(),
target_schema: target_database.clone(),
target_table: "items".to_string(),
columns: vec!["id".to_string(), "name".to_string(), "qty".to_string()],
key_columns: vec!["id".to_string()],
source_columns: None,
fetch_batch_size: None,
degradation_threshold: None,
sampling_strategy: None,
enable_checksum: None,
},
)
.await
.unwrap();
let elapsed = started.elapsed();
println!(
"two-database compare: source_rows={} target_rows={} added={} removed={} modified={} statements={} sql_bytes={} elapsed={:?}",
preparation.source_row_count,
preparation.target_row_count,
preparation.result.added.len(),
preparation.result.removed.len(),
preparation.result.modified.len(),
preparation.sync_statements.len(),
preparation.sync_sql.len(),
elapsed
);
assert_eq!(preparation.source_row_count, 2000);
assert_eq!(preparation.target_row_count, 1500);
assert_eq!(preparation.result.added.len(), 500, "rows 1501..2000 exist only on the source");
assert_eq!(preparation.result.removed.len(), 0, "the target has no extra rows");
assert!(!preparation.source_truncated && !preparation.target_truncated, "small compares are not truncated");
assert_eq!(preparation.result.modified.len(), 1500, "every shared row changed its name");
assert!(elapsed < Duration::from_secs(30), "compare took {elapsed:?}");
}
@@ -239,6 +239,7 @@ fn data_compare_from_tables_preparation_optional_fields_default() {
target_row_count: 0, target_row_count: 0,
source_truncated: false, source_truncated: false,
target_truncated: false, target_truncated: false,
row_budget: 100_000,
degradation_level: None, degradation_level: None,
sampling_rate: None, sampling_rate: None,
confidence_score: None, confidence_score: None,
@@ -252,6 +253,8 @@ fn data_compare_from_tables_preparation_optional_fields_default() {
// Core fields must be present // Core fields must be present
assert!(json.get("result").is_some(), "result must be present"); assert!(json.get("result").is_some(), "result must be present");
assert!(json.get("syncSql").is_some(), "syncSql must be present"); assert!(json.get("syncSql").is_some(), "syncSql must be present");
// The row budget travels with the preparation so the UI can flag a partial compare.
assert_eq!(json.get("rowBudget").and_then(|value| value.as_u64()), Some(100_000));
// New optional fields should be absent when None // New optional fields should be absent when None
assert!(json.get("degradationLevel").is_none(), "degradationLevel should be absent"); assert!(json.get("degradationLevel").is_none(), "degradationLevel should be absent");