diff --git a/apps/desktop/src/components/diff/DataCompareDialog.vue b/apps/desktop/src/components/diff/DataCompareDialog.vue
index 7f6b75551e..9b2384eead 100644
--- a/apps/desktop/src/components/diff/DataCompareDialog.vue
+++ b/apps/desktop/src/components/diff/DataCompareDialog.vue
@@ -190,6 +190,10 @@ const totalAdded = computed(() => batchResults.value.reduce((sum, item) => sum +
const totalRemoved = computed(() => batchResults.value.reduce((sum, item) => sum + item.removed, 0));
const totalModified = computed(() => batchResults.value.reduce((sum, item) => sum + item.modified, 0));
const hasResults = computed(() => batchResults.value.length > 0);
+// A compare reads both sides up to a fixed row budget, so a table larger than that budget
+// is reported through `*Truncated`. Surfacing it here keeps a partial diff from looking
+// like a complete one.
+const truncatedResults = computed(() => batchResults.value.filter((item) => item.sourceTruncated || item.targetTruncated));
const visibleKinds = computed(() => [...(showAdded.value ? (["added"] as DiffKind[]) : []), ...(showRemoved.value ? (["removed"] as DiffKind[]) : []), ...(showModified.value ? (["modified"] as DiffKind[]) : [])]);
const selectedAddedCount = computed(() => selectedDiffCount("added"));
const selectedRemovedCount = computed(() => selectedDiffCount("removed"));
@@ -1276,6 +1280,23 @@ onBeforeUnmount(() => {
@@ -1350,6 +1371,9 @@ onBeforeUnmount(() => {
})
}}
+
+ {{ t("dataCompare.truncatedBadge", { budget: item.rowBudget }) }}
+
{{ t("dataCompare.keyColumnsInline", { columns: item.keyColumns.join(", ") }) }}
diff --git a/apps/desktop/src/components/diff/__tests__/DataCompareDialog.keyColumns.spec.ts b/apps/desktop/src/components/diff/__tests__/DataCompareDialog.keyColumns.spec.ts
index a963c5d14e..6f7ab5ee13 100644
--- a/apps/desktop/src/components/diff/__tests__/DataCompareDialog.keyColumns.spec.ts
+++ b/apps/desktop/src/components/diff/__tests__/DataCompareDialog.keyColumns.spec.ts
@@ -205,6 +205,7 @@ function preparationFor(table: string) {
targetRowCount: 0,
sourceTruncated: false,
targetTruncated: false,
+ rowBudget: 100_000,
};
}
diff --git a/apps/desktop/src/components/diff/__tests__/DataCompareDialog.prefill.spec.ts b/apps/desktop/src/components/diff/__tests__/DataCompareDialog.prefill.spec.ts
index 1a6b5ef208..8e0ec5b345 100644
--- a/apps/desktop/src/components/diff/__tests__/DataCompareDialog.prefill.spec.ts
+++ b/apps/desktop/src/components/diff/__tests__/DataCompareDialog.prefill.spec.ts
@@ -129,6 +129,7 @@ function completedSession(): DataCompareSession {
targetRowCount: 0,
sourceTruncated: false,
targetTruncated: false,
+ rowBudget: 100_000,
databaseType: "oracle",
diff: {
added: [
@@ -509,3 +510,28 @@ describe("DataCompareDialog results layout", () => {
expect(scroller?.contains(dialogFooter)).toBe(false);
});
});
+
+describe("DataCompareDialog truncation warning", () => {
+ it("warns that a table larger than the row budget was only partially compared", async () => {
+ const session = completedSession();
+ const result = session.batchResults[0]!;
+ result.sourceTruncated = true;
+ result.sourceRowCount = 4_000_000;
+ result.rowBudget = 100_000;
+
+ mountSessionDialog(session);
+ await flushAsyncSetup();
+
+ const text = document.body.textContent ?? "";
+ expect(text).toContain(i18n.global.t("dataCompare.truncatedTitle"));
+ expect(text).toContain(i18n.global.t("dataCompare.truncatedBadge", { budget: 100_000 }));
+ expect(text).toContain(i18n.global.t("dataCompare.truncatedTable", { table: "ORDERS", source: 4_000_000, target: 0, budget: 100_000 }));
+ });
+
+ it("keeps the warning out of the results when every table fit the budget", async () => {
+ mountSessionDialog(completedSession());
+ await flushAsyncSetup();
+
+ expect(document.body.textContent ?? "").not.toContain(i18n.global.t("dataCompare.truncatedTitle"));
+ });
+});
diff --git a/apps/desktop/src/composables/__tests__/useDataCompareSession.spec.ts b/apps/desktop/src/composables/__tests__/useDataCompareSession.spec.ts
index 392924b13d..604123a11d 100644
--- a/apps/desktop/src/composables/__tests__/useDataCompareSession.spec.ts
+++ b/apps/desktop/src/composables/__tests__/useDataCompareSession.spec.ts
@@ -44,6 +44,7 @@ function preparationFor(table: string) {
targetRowCount: 0,
sourceTruncated: false,
targetTruncated: false,
+ rowBudget: 100_000,
};
}
@@ -226,6 +227,30 @@ test("keeps the configured per-table match columns after cloning the session con
assert.deepEqual(session.config.keyColumnsByTable, { users: ["user_id", "email"] });
});
+test("carries the row budget and truncation flags from the backend into the table result", async () => {
+ apiMock.prepareDataCompareFromTables.mockImplementation((options: { targetTable: string }) => Promise.resolve({ ...preparationFor(options.targetTable), sourceRowCount: 4_000_000, sourceTruncated: true, targetTruncated: true, rowBudget: 100_000 }));
+
+ const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users" }), [{ sourceTable: "users", targetTable: "users" }]);
+
+ assert.equal(session.batchResults[0]?.rowBudget, 100_000);
+ assert.equal(session.batchResults[0]?.sourceTruncated, true);
+ assert.equal(session.batchResults[0]?.targetTruncated, true);
+});
+
+test("keeps a truncated missing-target table on the same row budget", async () => {
+ apiMock.prepareDataCompareMissingTarget.mockResolvedValue({
+ ...preparationFor("users"),
+ sourceRowCount: 8_000_000,
+ sourceTruncated: true,
+ rowBudget: 100_000,
+ });
+
+ const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users", targetTables: ["orders"] }), [{ sourceTable: "users", targetTable: "users" }]);
+
+ assert.equal(session.batchResults[0]?.rowBudget, 100_000);
+ assert.equal(session.batchResults[0]?.sourceTruncated, true);
+});
+
test("normalizes a per-table match-column map", () => {
assert.deepEqual(normalizeKeyColumnOverrides({ users: [" id ", ""], orders: [], "": ["x"] }), { users: ["id"], orders: [] });
assert.deepEqual(normalizeKeyColumnOverrides(undefined), {});
diff --git a/apps/desktop/src/composables/useDataCompareSession.ts b/apps/desktop/src/composables/useDataCompareSession.ts
index 160e3e968e..848cd00019 100644
--- a/apps/desktop/src/composables/useDataCompareSession.ts
+++ b/apps/desktop/src/composables/useDataCompareSession.ts
@@ -59,6 +59,8 @@ export interface DataCompareTableResult {
targetRowCount: number;
sourceTruncated: boolean;
targetTruncated: boolean;
+ /** Rows the backend was allowed to read per side; the diff only covers this many rows when a side is truncated. */
+ rowBudget: number;
databaseType?: DatabaseType;
preSyncStatements?: string[];
diff: SelectableDataCompareResult;
@@ -264,6 +266,7 @@ function buildTableResult(task: DataCompareTableTask, preparation: DataCompareFr
targetRowCount: preparation.targetRowCount,
sourceTruncated: preparation.sourceTruncated,
targetTruncated: preparation.targetTruncated,
+ rowBudget: preparation.rowBudget,
databaseType,
preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result),
@@ -287,6 +290,7 @@ function buildMissingTargetResult(task: DataCompareTableTask, preparation: DataC
targetRowCount: 0,
sourceTruncated: preparation.sourceTruncated,
targetTruncated: false,
+ rowBudget: preparation.rowBudget,
databaseType,
preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result),
@@ -310,6 +314,7 @@ function buildErrorResult(task: DataCompareTableTask, keyColumns: string[], data
targetRowCount: 0,
sourceTruncated: false,
targetTruncated: false,
+ rowBudget: 0,
databaseType,
preSyncStatements: [],
diff: { added: [], removed: [], modified: [] },
diff --git a/apps/desktop/src/i18n/locales/az.ts b/apps/desktop/src/i18n/locales/az.ts
index d19cce11dd..5ab809cdbd 100644
--- a/apps/desktop/src/i18n/locales/az.ts
+++ b/apps/desktop/src/i18n/locales/az.ts
@@ -7058,6 +7058,10 @@ export default withEnglishFallback({
noComparableTables: "Müqayisə edilə bilən cədvəl tapılmadı. Eyni adlı hədəf cədvəllərin mövcudluğunu yoxlayın.",
noDifferences: "Verilənlər eynidir, sinxronlaşdırma SQL-inə ehtiyac yoxdur",
syncSuccess: "Verilənlərin sinxronlaşdırılması uğurla icra edildi",
+ truncatedTitle: "Bəzi cədvəllər tam müqayisə edilmədi",
+ truncatedHint: "Bir müqayisənin oxuya biləcəyindən çox sətri olan cədvəllər yalnız ilk sətirlər üzrə müqayisə edildi, ona görə də bu nöqtədən sonrakı əlavə, dəyişdirilmiş və ya silinmiş sətirlər aşkarlanmır. Aralığı filtr ilə daraldıb hissə-hissə müqayisə edin.",
+ truncatedTable: "{table}: mənbə {source} sətir, hədəf {target} sətir; yalnız ilk {budget} sətir müqayisə edildi",
+ truncatedBadge: "Yalnız ilk {budget} sətir müqayisə edildi",
},
settings: {
historyRetentionLimit: "Sorğu tarixçəsinin saxlanma həddi",
diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts
index 2e06f7c6df..fcd3b2f009 100644
--- a/apps/desktop/src/i18n/locales/en.ts
+++ b/apps/desktop/src/i18n/locales/en.ts
@@ -7835,6 +7835,10 @@ export default {
noComparableTables: "No comparable tables found. Check whether same-name target tables exist.",
noDifferences: "Data is identical, no sync SQL needed",
syncSuccess: "Data sync executed successfully",
+ truncatedTitle: "Some tables were not compared in full",
+ truncatedHint: "Tables with more rows than a single compare can read were compared only over their first rows, so added, modified, or deleted rows beyond that point are not reported. Narrow the range with a filter and compare in batches.",
+ truncatedTable: "{table}: {source} source rows, {target} target rows, compared the first {budget} rows only",
+ truncatedBadge: "Compared the first {budget} rows only",
},
settings: {
historyRetentionLimit: "Query history retention",
diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts
index 6ae4d5df8f..f12f98e481 100644
--- a/apps/desktop/src/i18n/locales/es.ts
+++ b/apps/desktop/src/i18n/locales/es.ts
@@ -7100,6 +7100,10 @@ export default withEnglishFallback({
noComparableTables: "No se encontraron tablas comparables. Verifica si existen tablas de destino con el mismo nombre.",
noDifferences: "Los datos son idénticos, no se necesita SQL de sincronización",
syncSuccess: "Sincronización de datos ejecutada exitosamente",
+ truncatedTitle: "Algunas tablas no se compararon por completo",
+ truncatedHint: "Las tablas con más filas de las que puede leer una sola comparación solo se compararon en sus primeras filas, por lo que las filas añadidas, modificadas o eliminadas a partir de ahí no se detectan. Acota el rango con un filtro y compara por lotes.",
+ truncatedTable: "{table}: {source} filas de origen, {target} de destino; solo se compararon las primeras {budget} filas",
+ truncatedBadge: "Solo se compararon las primeras {budget} filas",
},
settings: {
historyRetentionLimit: "Límite del historial de consultas",
diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts
index afaea24e7a..ea483d47da 100644
--- a/apps/desktop/src/i18n/locales/it.ts
+++ b/apps/desktop/src/i18n/locales/it.ts
@@ -7024,6 +7024,10 @@ export default withEnglishFallback({
noComparableTables: "Nessuna tabella confrontabile trovata. Verifica se esistono tabelle di destinazione con lo stesso nome.",
noDifferences: "I dati sono identici, nessun SQL di sincronizzazione necessario",
syncSuccess: "Sincronizzazione dei dati eseguita con successo",
+ truncatedTitle: "Alcune tabelle non sono state confrontate per intero",
+ truncatedHint: "Le tabelle con più righe di quante ne possa leggere un singolo confronto sono state confrontate solo sulle prime righe, quindi le righe aggiunte, modificate o eliminate oltre quel punto non vengono rilevate. Restringi l'intervallo con un filtro e confronta a blocchi.",
+ truncatedTable: "{table}: {source} righe nell'origine, {target} nella destinazione; confrontate solo le prime {budget} righe",
+ truncatedBadge: "Confrontate solo le prime {budget} righe",
},
settings: {
historyRetentionLimit: "Limite della cronologia query",
diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts
index 9bb3e4566c..79f51ea6e4 100644
--- a/apps/desktop/src/i18n/locales/ja.ts
+++ b/apps/desktop/src/i18n/locales/ja.ts
@@ -7055,6 +7055,10 @@ export default withEnglishFallback({
noComparableTables: "比較可能なテーブルが見つかりません。同名のターゲットテーブルが存在するか確認してください。",
noDifferences: "データは同一です。同期SQLは不要です",
syncSuccess: "データ同期が正常に実行されました",
+ truncatedTitle: "一部のテーブルは全体を比較していません",
+ truncatedHint: "1 回の比較で読み込める行数を超えたテーブルは先頭の行だけを比較しているため、それ以降の追加・変更・削除は検出されません。フィルターで範囲を絞り、分割して比較してください。",
+ truncatedTable: "{table}:ソース {source} 行、ターゲット {target} 行、先頭 {budget} 行のみ比較",
+ truncatedBadge: "先頭 {budget} 行のみ比較",
},
settings: {
historyRetentionLimit: "クエリ履歴の保持件数",
diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts
index b2706b405d..4f6997e7ea 100644
--- a/apps/desktop/src/i18n/locales/ko.ts
+++ b/apps/desktop/src/i18n/locales/ko.ts
@@ -6790,6 +6790,10 @@ export default withEnglishFallback({
noComparableTables: "비교 가능한 테이블이 없습니다. 같은 이름의 대상 테이블이 있는지 확인하세요.",
noDifferences: "데이터가 동일하여 동기화 SQL이 필요하지 않습니다",
syncSuccess: "데이터 동기화를 성공적으로 실행했습니다",
+ truncatedTitle: "일부 테이블은 전체를 비교하지 않았습니다",
+ truncatedHint: "한 번의 비교로 읽을 수 있는 행 수를 초과한 테이블은 앞부분만 비교했기 때문에 그 이후의 추가·수정·삭제는 감지되지 않습니다. 필터로 범위를 좁혀 나누어 비교하세요.",
+ truncatedTable: "{table}: 원본 {source}행, 대상 {target}행, 앞 {budget}행만 비교",
+ truncatedBadge: "앞 {budget}행만 비교",
},
settings: {
historyRetentionLimit: "쿼리 기록 보관 개수",
diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts
index eb34d9d8ad..600f62d20b 100644
--- a/apps/desktop/src/i18n/locales/pt-BR.ts
+++ b/apps/desktop/src/i18n/locales/pt-BR.ts
@@ -7026,6 +7026,10 @@ export default withEnglishFallback({
noComparableTables: "Nenhuma tabela comparável encontrada. Verifique se existem tabelas de destino de mesmo nome.",
noDifferences: "Os dados são idênticos, nenhum SQL de sincronização é necessário",
syncSuccess: "Sincronização de dados executada com sucesso",
+ truncatedTitle: "Algumas tabelas não foram comparadas por completo",
+ truncatedHint: "Tabelas com mais linhas do que uma única comparação consegue ler foram comparadas apenas nas primeiras linhas, então linhas adicionadas, modificadas ou excluídas depois disso não são detectadas. Restrinja o intervalo com um filtro e compare em lotes.",
+ truncatedTable: "{table}: {source} linhas na origem, {target} no destino; apenas as primeiras {budget} linhas foram comparadas",
+ truncatedBadge: "Apenas as primeiras {budget} linhas comparadas",
},
settings: {
historyRetentionLimit: "Limite do histórico de consultas",
diff --git a/apps/desktop/src/i18n/locales/ru.ts b/apps/desktop/src/i18n/locales/ru.ts
index d2909adde8..8006254f89 100644
--- a/apps/desktop/src/i18n/locales/ru.ts
+++ b/apps/desktop/src/i18n/locales/ru.ts
@@ -8243,6 +8243,10 @@ export default withEnglishFallback({
noComparableTables: "Не найдено таблиц для сравнения. Проверьте наличие одноимённых целевых таблиц.",
noDifferences: "Данные идентичны, SQL синхронизации не требуется",
syncSuccess: "Синхронизация данных выполнена успешно",
+ truncatedTitle: "Часть таблиц сравнена не полностью",
+ truncatedHint: "Таблицы, в которых строк больше, чем может прочитать одно сравнение, сравнены только по первым строкам, поэтому добавленные, изменённые и удалённые строки после этого не обнаруживаются. Сузьте диапазон фильтром и сравнивайте по частям.",
+ truncatedTable: "{table}: строк в источнике {source}, в приёмнике {target}; сравнены только первые {budget} строк",
+ truncatedBadge: "Сравнены только первые {budget} строк",
},
settings: {
historyRetentionLimit: "Хранение истории запросов",
diff --git a/apps/desktop/src/i18n/locales/tr.ts b/apps/desktop/src/i18n/locales/tr.ts
index 2f4056678c..3a14c4b7a5 100644
--- a/apps/desktop/src/i18n/locales/tr.ts
+++ b/apps/desktop/src/i18n/locales/tr.ts
@@ -6940,6 +6940,10 @@ export default withEnglishFallback({
noComparableTables: "Karşılaştırılabilir tablo bulunamadı. Aynı adlı hedef tabloların var olup olmadığını denetleyin.",
noDifferences: "Veriler aynı, eşitleme SQL'i gerekmiyor",
syncSuccess: "Veri eşitlemesi başarıyla çalıştırıldı",
+ truncatedTitle: "Bazı tablolar tam olarak karşılaştırılmadı",
+ truncatedHint: "Tek bir karşılaştırmanın okuyabileceğinden fazla satıra sahip tablolar yalnızca ilk satırları üzerinden karşılaştırıldı; bu noktadan sonraki eklenen, değişen veya silinen satırlar bildirilmez. Aralığı bir filtreyle daraltıp toplu olarak karşılaştırın.",
+ truncatedTable: "{table}: kaynak {source} satır, hedef {target} satır; yalnızca ilk {budget} satır karşılaştırıldı",
+ truncatedBadge: "Yalnızca ilk {budget} satır karşılaştırıldı",
},
settings: {
historyRetentionLimit: "Sorgu geçmişi saklama sınırı",
diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts
index 31f5c46189..23ecc3028d 100644
--- a/apps/desktop/src/i18n/locales/zh-CN.ts
+++ b/apps/desktop/src/i18n/locales/zh-CN.ts
@@ -7801,6 +7801,10 @@ export default withEnglishFallback({
noComparableTables: "没有可比较的表,请检查目标端是否存在同名表",
noDifferences: "数据完全一致,无需生成同步 SQL",
syncSuccess: "数据同步执行成功",
+ truncatedTitle: "部分表未完整比较",
+ truncatedHint: "行数超过单次比较上限的表只比较了前若干行,超出部分的新增、修改和删除不会被发现。建议加上过滤条件缩小范围后分批比较。",
+ truncatedTable: "{table}:源 {source} 行、目标 {target} 行,仅比较前 {budget} 行",
+ truncatedBadge: "仅比较前 {budget} 行",
},
settings: {
historyRetentionLimit: "查询历史保留条数",
diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts
index 62333d2a0c..ee9b944fe4 100644
--- a/apps/desktop/src/i18n/locales/zh-TW.ts
+++ b/apps/desktop/src/i18n/locales/zh-TW.ts
@@ -6349,6 +6349,10 @@ export default withEnglishFallback({
noComparableTables: "沒有可比較的資料表。請檢查目標端是否存在同名資料表。",
noDifferences: "資料完全一致,不需要產生同步 SQL",
syncSuccess: "資料同步執行成功",
+ truncatedTitle: "部分表未完整比較",
+ truncatedHint: "資料筆數超過單次比較上限的表只比較了前幾筆,超出部分的新增、修改與刪除不會被發現。建議加上篩選條件縮小範圍後分批比較。",
+ truncatedTable: "{table}:來源 {source} 筆、目標 {target} 筆,僅比較前 {budget} 筆",
+ truncatedBadge: "僅比較前 {budget} 筆",
},
settings: {
historyRetentionLimit: "查詢歷史保留筆數",
diff --git a/apps/desktop/src/lib/dataGrid/dataCompare.ts b/apps/desktop/src/lib/dataGrid/dataCompare.ts
index 235b4dcf30..08fb0c017c 100644
--- a/apps/desktop/src/lib/dataGrid/dataCompare.ts
+++ b/apps/desktop/src/lib/dataGrid/dataCompare.ts
@@ -95,6 +95,8 @@ export interface DataCompareFromTablesPreparation extends DataComparePreparation
targetRowCount: number;
sourceTruncated: boolean;
targetTruncated: boolean;
+ /** Rows per side this compare was allowed to read; fewer than `sourceRowCount`/`targetRowCount` when truncated. */
+ rowBudget: number;
}
export interface DataCompareSyncPlanTableOptions {
diff --git a/crates/dbx-core/src/data/correction.rs b/crates/dbx-core/src/data/correction.rs
index 66e9dcfc07..c03b7f313c 100644
--- a/crates/dbx-core/src/data/correction.rs
+++ b/crates/dbx-core/src/data/correction.rs
@@ -512,6 +512,7 @@ mod tests {
target_row_count: 2,
source_truncated: false,
target_truncated: false,
+ row_budget: 100_000,
degradation_level: Some("full".to_string()),
sampling_rate: Some(1.0),
confidence_score: Some(1.0),
diff --git a/crates/dbx-core/src/data/data_compare.rs b/crates/dbx-core/src/data/data_compare.rs
index bf36fd473f..a7705927a8 100644
--- a/crates/dbx-core/src/data/data_compare.rs
+++ b/crates/dbx-core/src/data/data_compare.rs
@@ -88,6 +88,8 @@ pub struct DataCompareMissingTargetOptions {
pub key_columns: Vec
,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fetch_batch_size: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub degradation_threshold: Option,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -179,6 +181,12 @@ pub struct DataCompareFromTablesPreparation {
pub target_row_count: u64,
pub source_truncated: bool,
pub target_truncated: bool,
+ /// Rows per side this compare was allowed to read. When a side holds more rows than
+ /// this, the diff only covers the first `row_budget` rows and the matching
+ /// `*_truncated` flag is set, so the UI can warn instead of showing a partial plan
+ /// as if it were complete.
+ #[serde(default)]
+ pub row_budget: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub degradation_level: Option,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -363,68 +371,77 @@ pub async fn prepare_data_compare_from_tables(
let source_column_names = aligned_source_column_names(&options.columns, options.source_columns.as_ref());
let source_key_columns = aligned_source_key_columns(&options.columns, &source_column_names, &options.key_columns);
- let (source_rows, target_rows, sampling_rate, verification_method) = match °radation_level {
- DegradationLevel::Full => {
- let (src, tgt) = tokio::try_join!(
- fetch_compare_rows(
- state,
- &options.source_connection_id,
- &options.source_database,
- &options.source_schema,
- &options.source_table,
- &source_column_names,
- &source_key_columns,
- source_database_type,
- fetch_batch_size,
- ),
- fetch_compare_rows(
- state,
- &options.target_connection_id,
- &options.target_database,
- &options.target_schema,
- &options.target_table,
- &options.columns,
- &options.key_columns,
- target_database_type,
- fetch_batch_size,
- )
- )?;
- (src, tgt, 1.0, "full_compare".to_string())
- }
- DegradationLevel::Sample => {
- let (src, tgt) = tokio::try_join!(
- fetch_sampled_compare_rows(
- state,
- &options.source_connection_id,
- &options.source_database,
- &options.source_schema,
- &options.source_table,
- &source_column_names,
- &source_key_columns,
- source_database_type,
- &sampling_strategy,
- threshold.sample_size,
- ),
- fetch_sampled_compare_rows(
- state,
- &options.target_connection_id,
- &options.target_database,
- &options.target_schema,
- &options.target_table,
- &options.columns,
- &options.key_columns,
- target_database_type,
- &sampling_strategy,
- threshold.sample_size,
- )
- )?;
- let max_count = source_row_count.max(target_row_count);
- let sample_rate = if max_count > 0 { threshold.sample_size as f64 / max_count as f64 } else { 1.0 };
- let method = "sampled".to_string();
- (src, tgt, sample_rate.min(1.0), method)
- }
- DegradationLevel::SkipWithRisk => (Vec::new(), Vec::new(), 0.0, "skipped".to_string()),
- };
+ // Every row is read into memory and then shipped to the UI, so even the "full" level
+ // is capped: the degradation thresholds decide the level, the budget bounds the fetch.
+ let row_budget = rows_read_budget(°radation_level, &threshold);
+
+ let (source_rows, target_rows, sampling_rate, verification_method, source_truncated, target_truncated) =
+ match °radation_level {
+ DegradationLevel::Full => {
+ let (src, tgt) = tokio::try_join!(
+ fetch_compare_rows(
+ state,
+ &options.source_connection_id,
+ &options.source_database,
+ &options.source_schema,
+ &options.source_table,
+ &source_column_names,
+ &source_key_columns,
+ source_database_type,
+ fetch_batch_size,
+ row_budget,
+ ),
+ fetch_compare_rows(
+ state,
+ &options.target_connection_id,
+ &options.target_database,
+ &options.target_schema,
+ &options.target_table,
+ &options.columns,
+ &options.key_columns,
+ target_database_type,
+ fetch_batch_size,
+ row_budget,
+ )
+ )?;
+ let source_truncated = source_row_count > row_budget as u64;
+ let target_truncated = target_row_count > row_budget as u64;
+ (src, tgt, 1.0, "full_compare".to_string(), source_truncated, target_truncated)
+ }
+ DegradationLevel::Sample => {
+ let (src, tgt) = tokio::try_join!(
+ fetch_sampled_compare_rows(
+ state,
+ &options.source_connection_id,
+ &options.source_database,
+ &options.source_schema,
+ &options.source_table,
+ &source_column_names,
+ &source_key_columns,
+ source_database_type,
+ &sampling_strategy,
+ threshold.sample_size,
+ ),
+ fetch_sampled_compare_rows(
+ state,
+ &options.target_connection_id,
+ &options.target_database,
+ &options.target_schema,
+ &options.target_table,
+ &options.columns,
+ &options.key_columns,
+ target_database_type,
+ &sampling_strategy,
+ threshold.sample_size,
+ )
+ )?;
+ let max_count = source_row_count.max(target_row_count);
+ let sample_rate = if max_count > 0 { threshold.sample_size as f64 / max_count as f64 } else { 1.0 };
+ let method = "sampled".to_string();
+ (src, tgt, sample_rate.min(1.0), method, false, false)
+ }
+ DegradationLevel::SkipWithRisk => (Vec::new(), Vec::new(), 0.0, "skipped".to_string(), false, false),
+ };
let target_columns = get_columns_core(
state,
@@ -468,8 +485,9 @@ pub async fn prepare_data_compare_from_tables(
pre_sync_statements: Vec::new(),
source_row_count,
target_row_count,
- source_truncated: false,
- target_truncated: false,
+ source_truncated,
+ target_truncated,
+ row_budget,
degradation_level: Some(degradation_level.to_string()),
sampling_rate: Some(sampling_rate),
confidence_score: Some(confidence_score),
@@ -509,6 +527,11 @@ pub async fn prepare_data_compare_missing_target(
)
.await?;
let source_row_count = first_count(&source_count_result.rows)?;
+ // A table that only exists on the source side has no count to compare against, but the
+ // sync plan still embeds one statement per row: reading the whole table freezes the
+ // client on large tables, so the plan covers at most one full-compare budget of rows.
+ let threshold = options.degradation_threshold.clone().unwrap_or_default();
+ let row_budget = threshold.full_compare_max_rows.max(1) as usize;
let source_rows = fetch_compare_rows(
state,
&options.source_connection_id,
@@ -519,8 +542,10 @@ pub async fn prepare_data_compare_missing_target(
&options.key_columns,
source_database_type,
fetch_batch_size,
+ row_budget,
)
.await?;
+ let source_truncated = source_row_count > row_budget as u64;
let result = missing_target_diff(&column_names, &options.key_columns, source_rows);
let mut pre_sync_statements = Vec::new();
pre_sync_statements.push(format!(
@@ -567,12 +592,15 @@ pub async fn prepare_data_compare_missing_target(
pre_sync_statements,
source_row_count,
target_row_count: 0,
- source_truncated: false,
+ source_truncated,
target_truncated: false,
- degradation_level: Some("full".to_string()),
- sampling_rate: Some(1.0),
- confidence_score: Some(1.0),
- verification_method: Some("missing_target_full".to_string()),
+ row_budget,
+ degradation_level: Some(should_degrade(source_row_count, 0, &threshold).to_string()),
+ sampling_rate: Some(if source_truncated { row_budget as f64 / source_row_count.max(1) as f64 } else { 1.0 }),
+ confidence_score: Some(if source_truncated { 0.5 } else { 1.0 }),
+ verification_method: Some(
+ if source_truncated { "missing_target_prefix" } else { "missing_target_full" }.to_string(),
+ ),
source_checksums: None,
target_checksums: None,
})
@@ -1190,6 +1218,12 @@ fn build_rownum_data_compare_select_sql(
}
#[allow(clippy::too_many_arguments)]
+/// Fetches the rows a comparison needs, stopping after `row_budget` rows.
+///
+/// A compare reads both sides into memory and ships the diff to the UI, so an unbounded
+/// fetch turns a large table into an out-of-memory freeze on the client. Callers derive
+/// `source_truncated` / `target_truncated` from the table's row count versus `row_budget`
+/// so a partial diff is never mistaken for a complete one.
async fn fetch_compare_rows(
state: &AppState,
connection_id: &str,
@@ -1200,20 +1234,15 @@ async fn fetch_compare_rows(
key_columns: &[String],
database_type: DatabaseType,
fetch_batch_size: usize,
+ row_budget: usize,
) -> Result>, String> {
let mut rows = Vec::new();
let mut offset = 0usize;
- loop {
- let sql = build_data_compare_select_sql(
- database_type,
- schema,
- table_name,
- columns,
- key_columns,
- fetch_batch_size,
- offset,
- );
+ while rows.len() < row_budget {
+ let batch_size = fetch_batch_size.min(row_budget - rows.len());
+ let sql =
+ build_data_compare_select_sql(database_type, schema, table_name, columns, key_columns, batch_size, offset);
let result = execute_sql_statement_with_options(
state,
connection_id,
@@ -1221,7 +1250,7 @@ async fn fetch_compare_rows(
&sql,
Some(schema),
None,
- QueryExecutionOptions { max_rows: Some(fetch_batch_size), ..Default::default() },
+ QueryExecutionOptions { max_rows: Some(batch_size), ..Default::default() },
)
.await?;
let fetched = result.rows.len();
@@ -1229,7 +1258,7 @@ async fn fetch_compare_rows(
break;
}
rows.extend(result.rows);
- if fetched < fetch_batch_size {
+ if fetched < batch_size {
break;
}
offset += fetched;
@@ -1295,6 +1324,15 @@ fn data_grid_column_info(column: crate::types::ColumnInfo) -> DataGridColumnInfo
}
}
+/// Rows per side a compare may read for a given degradation level.
+fn rows_read_budget(level: &DegradationLevel, threshold: &DegradationThreshold) -> usize {
+ match level {
+ DegradationLevel::Full => threshold.full_compare_max_rows.max(1) as usize,
+ DegradationLevel::Sample => threshold.sample_size,
+ DegradationLevel::SkipWithRisk => 0,
+ }
+}
+
fn should_degrade(source_row_count: u64, target_row_count: u64, threshold: &DegradationThreshold) -> DegradationLevel {
let max_count = source_row_count.max(target_row_count);
if max_count <= threshold.full_compare_max_rows {
diff --git a/crates/dbx-core/tests/live_mysql_data_compare.rs b/crates/dbx-core/tests/live_mysql_data_compare.rs
new file mode 100644
index 0000000000..784243ef99
--- /dev/null
+++ b/crates/dbx-core/tests/live_mysql_data_compare.rs
@@ -0,0 +1,221 @@
+use dbx_core::connection::AppState;
+use dbx_core::data_compare::{
+ prepare_data_compare_from_tables, prepare_data_compare_missing_target, DataCompareFromTablesOptions,
+ DataCompareMissingTargetOptions,
+};
+use dbx_core::db::mysql;
+use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
+use serde_json::json;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+fn live_mysql_config(id: &str) -> ConnectionConfig {
+ let host = std::env::var("DBX_LIVE_MYSQL_TRANSFER_HOST").expect("DBX_LIVE_MYSQL_TRANSFER_HOST");
+ let port =
+ std::env::var("DBX_LIVE_MYSQL_TRANSFER_PORT").ok().and_then(|value| value.parse::().ok()).unwrap_or(3306);
+ let username = std::env::var("DBX_LIVE_MYSQL_TRANSFER_USER").unwrap_or_else(|_| "root".to_string());
+ let password = std::env::var("DBX_LIVE_MYSQL_TRANSFER_PASSWORD").expect("DBX_LIVE_MYSQL_TRANSFER_PASSWORD");
+
+ serde_json::from_value(json!({
+ "id": id,
+ "name": id,
+ "db_type": DatabaseType::Mysql,
+ "host": host,
+ "port": port,
+ "username": username,
+ "password": password,
+ "database": null,
+ "connect_timeout_secs": 10,
+ "query_timeout_secs": 300,
+ "idle_timeout_secs": 60,
+ "keepalive_interval_secs": 0
+ }))
+ .expect("live MySQL compare config should deserialize")
+}
+
+fn mysql_url(config: &ConnectionConfig) -> String {
+ format!("mysql://{}:{}@{}:{}", config.username, config.password, config.host, config.port)
+}
+
+async fn live_state() -> Arc {
+ let task_tmp = std::path::PathBuf::from(
+ std::env::var("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR").expect("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR"),
+ );
+ let dir = task_tmp.join(format!("data-compare-{}", uuid::Uuid::new_v4().simple()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
+ Arc::new(AppState::new(storage))
+}
+
+fn big_source_table_ddl(database: &str, table: &str, rows: u64) -> String {
+ format!(
+ "CREATE DATABASE IF NOT EXISTS {database};\
+ SET SESSION cte_max_recursion_depth={}; \
+ CREATE TABLE {database}.{table} (\
+ id INT NOT NULL PRIMARY KEY, payload VARCHAR(120), amount DECIMAL(12,2)\
+ );\
+ INSERT INTO {database}.{table} (id, payload, amount)\
+ WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < {rows})\
+ SELECT n, CONCAT('row-', n), n * 1.5 FROM seq",
+ rows + 1
+ )
+}
+
+/// A source-only table must not drag the whole table into memory (and across the IPC
+/// boundary) when the user compares it against a database that has no such table.
+#[test]
+#[ignore = "requires a disposable MySQL endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
+fn live_mysql_data_compare_missing_target_is_bounded() {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(run_live_mysql_data_compare_missing_target_is_bounded());
+}
+
+async fn run_live_mysql_data_compare_missing_target_is_bounded() {
+ let suffix = uuid::Uuid::new_v4().simple().to_string();
+ let connection_id = format!("compare-conn-{}", &suffix[..12]);
+ let source_database = format!("dbx_cmp_src_{}", &suffix[..12]);
+ let target_database = format!("dbx_cmp_dst_{}", &suffix[..12]);
+ let config = live_mysql_config(&connection_id);
+ let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
+
+ let rows: u64 = 300_000;
+ mysql::execute_query(&setup_pool, &big_source_table_ddl(&source_database, "big_source_only", rows), false)
+ .await
+ .unwrap();
+ mysql::execute_query(&setup_pool, &format!("CREATE DATABASE {target_database}"), false).await.unwrap();
+
+ let state = live_state().await;
+ state.configs.write().await.insert(connection_id.clone(), config);
+
+ let started = Instant::now();
+ let preparation = prepare_data_compare_missing_target(
+ &state,
+ DataCompareMissingTargetOptions {
+ source_connection_id: connection_id.clone(),
+ source_database: source_database.clone(),
+ source_schema: source_database.clone(),
+ source_table: "big_source_only".to_string(),
+ target_connection_id: connection_id.clone(),
+ target_database: target_database.clone(),
+ target_schema: target_database.clone(),
+ target_table: "big_source_only".to_string(),
+ key_columns: vec!["id".to_string()],
+ fetch_batch_size: None,
+ degradation_threshold: None,
+ },
+ )
+ .await
+ .unwrap();
+ let elapsed = started.elapsed();
+
+ println!(
+ "missing-target compare: rows={} added={} truncated={} level={:?} sql_bytes={} elapsed={:?}",
+ preparation.source_row_count,
+ preparation.result.added.len(),
+ preparation.source_truncated,
+ preparation.degradation_level,
+ preparation.sync_sql.len(),
+ elapsed
+ );
+
+ assert_eq!(
+ preparation.result.added.len(),
+ 100_000,
+ "compare must stop at the full-compare row budget instead of materialising all {rows} rows"
+ );
+ assert!(preparation.source_truncated, "capped compare must flag source_truncated");
+ assert!(preparation.result.added.len() < rows as usize, "the capped plan must be smaller than the table");
+ assert!(elapsed < Duration::from_secs(30), "capped compare took {elapsed:?}");
+}
+
+/// The user-facing scenario: two databases on one connection, small tables, with real
+/// differences in both directions.
+#[test]
+#[ignore = "requires a disposable MySQL endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
+fn live_mysql_data_compare_two_databases_with_differences() {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(run_live_mysql_data_compare_two_databases_with_differences());
+}
+
+async fn run_live_mysql_data_compare_two_databases_with_differences() {
+ let suffix = uuid::Uuid::new_v4().simple().to_string();
+ let connection_id = format!("compare-conn-{}", &suffix[..12]);
+ let source_database = format!("dbx_cmp2_src_{}", &suffix[..12]);
+ let target_database = format!("dbx_cmp2_dst_{}", &suffix[..12]);
+ let config = live_mysql_config(&connection_id);
+ let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
+
+ mysql::execute_query(
+ &setup_pool,
+ &format!(
+ "CREATE DATABASE {source_database};\
+ CREATE DATABASE {target_database};\
+ CREATE TABLE {source_database}.items (id INT NOT NULL PRIMARY KEY, name VARCHAR(60), qty INT);\
+ CREATE TABLE {target_database}.items (id INT NOT NULL PRIMARY KEY, name VARCHAR(60), qty INT);\
+ SET SESSION cte_max_recursion_depth=4000;\
+ INSERT INTO {source_database}.items (id, name, qty)\
+ WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < 2000)\
+ SELECT n, CONCAT('name-', n), n FROM seq;\
+ INSERT INTO {target_database}.items (id, name, qty)\
+ WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < 1500)\
+ SELECT n, CONCAT('name-', n * 2), n FROM seq"
+ ),
+ false,
+ )
+ .await
+ .unwrap();
+
+ let state = live_state().await;
+ state.configs.write().await.insert(connection_id.clone(), config);
+
+ let started = Instant::now();
+ let preparation = prepare_data_compare_from_tables(
+ &state,
+ DataCompareFromTablesOptions {
+ source_connection_id: connection_id.clone(),
+ source_database: source_database.clone(),
+ source_schema: source_database.clone(),
+ source_table: "items".to_string(),
+ target_connection_id: connection_id.clone(),
+ target_database: target_database.clone(),
+ target_schema: target_database.clone(),
+ target_table: "items".to_string(),
+ columns: vec!["id".to_string(), "name".to_string(), "qty".to_string()],
+ key_columns: vec!["id".to_string()],
+ source_columns: None,
+ fetch_batch_size: None,
+ degradation_threshold: None,
+ sampling_strategy: None,
+ enable_checksum: None,
+ },
+ )
+ .await
+ .unwrap();
+ let elapsed = started.elapsed();
+
+ println!(
+ "two-database compare: source_rows={} target_rows={} added={} removed={} modified={} statements={} sql_bytes={} elapsed={:?}",
+ preparation.source_row_count,
+ preparation.target_row_count,
+ preparation.result.added.len(),
+ preparation.result.removed.len(),
+ preparation.result.modified.len(),
+ preparation.sync_statements.len(),
+ preparation.sync_sql.len(),
+ elapsed
+ );
+
+ assert_eq!(preparation.source_row_count, 2000);
+ assert_eq!(preparation.target_row_count, 1500);
+ assert_eq!(preparation.result.added.len(), 500, "rows 1501..2000 exist only on the source");
+ assert_eq!(preparation.result.removed.len(), 0, "the target has no extra rows");
+ assert!(!preparation.source_truncated && !preparation.target_truncated, "small compares are not truncated");
+ assert_eq!(preparation.result.modified.len(), 1500, "every shared row changed its name");
+ assert!(elapsed < Duration::from_secs(30), "compare took {elapsed:?}");
+}
diff --git a/crates/dbx-core/tests/regression_compatibility.rs b/crates/dbx-core/tests/regression_compatibility.rs
index 1fc5ac54aa..8ea0b190e3 100644
--- a/crates/dbx-core/tests/regression_compatibility.rs
+++ b/crates/dbx-core/tests/regression_compatibility.rs
@@ -239,6 +239,7 @@ fn data_compare_from_tables_preparation_optional_fields_default() {
target_row_count: 0,
source_truncated: false,
target_truncated: false,
+ row_budget: 100_000,
degradation_level: None,
sampling_rate: None,
confidence_score: None,
@@ -252,6 +253,8 @@ fn data_compare_from_tables_preparation_optional_fields_default() {
// Core fields must be present
assert!(json.get("result").is_some(), "result must be present");
assert!(json.get("syncSql").is_some(), "syncSql must be present");
+ // The row budget travels with the preparation so the UI can flag a partial compare.
+ assert_eq!(json.get("rowBudget").and_then(|value| value.as_u64()), Some(100_000));
// New optional fields should be absent when None
assert!(json.get("degradationLevel").is_none(), "degradationLevel should be absent");