fix(compare): cap per-table compare reads and flag truncation

This commit is contained in:
zipg authored and GitHub committed 2026-09-30 14:59:56 +08:00
1 parent 535417a763
commit 3dbaed310d
21 files changed
+471 -81

No files matched your search

@@ -44,6 +44,7 @@ function preparationFor(table: string) {
targetRowCount: 0,
sourceTruncated: false,
targetTruncated: false,
rowBudget: 100_000,
};
}
@@ -226,6 +227,30 @@ test("keeps the configured per-table match columns after cloning the session con
assert.deepEqual(session.config.keyColumnsByTable, { users: ["user_id", "email"] });
});
test("carries the row budget and truncation flags from the backend into the table result", async () => {
apiMock.prepareDataCompareFromTables.mockImplementation((options: { targetTable: string }) => Promise.resolve({ ...preparationFor(options.targetTable), sourceRowCount: 4_000_000, sourceTruncated: true, targetTruncated: true, rowBudget: 100_000 }));
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users" }), [{ sourceTable: "users", targetTable: "users" }]);
assert.equal(session.batchResults[0]?.rowBudget, 100_000);
assert.equal(session.batchResults[0]?.sourceTruncated, true);
assert.equal(session.batchResults[0]?.targetTruncated, true);
});
test("keeps a truncated missing-target table on the same row budget", async () => {
apiMock.prepareDataCompareMissingTarget.mockResolvedValue({
...preparationFor("users"),
sourceRowCount: 8_000_000,
sourceTruncated: true,
rowBudget: 100_000,
});
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users", targetTables: ["orders"] }), [{ sourceTable: "users", targetTable: "users" }]);
assert.equal(session.batchResults[0]?.rowBudget, 100_000);
assert.equal(session.batchResults[0]?.sourceTruncated, true);
});
test("normalizes a per-table match-column map", () => {
assert.deepEqual(normalizeKeyColumnOverrides({ users: [" id ", ""], orders: [], "": ["x"] }), { users: ["id"], orders: [] });
assert.deepEqual(normalizeKeyColumnOverrides(undefined), {});
@@ -59,6 +59,8 @@ export interface DataCompareTableResult {
targetRowCount: number;
sourceTruncated: boolean;
targetTruncated: boolean;
/** Rows the backend was allowed to read per side; the diff only covers this many rows when a side is truncated. */
rowBudget: number;
databaseType?: DatabaseType;
preSyncStatements?: string[];
diff: SelectableDataCompareResult;
@@ -264,6 +266,7 @@ function buildTableResult(task: DataCompareTableTask, preparation: DataCompareFr
targetRowCount: preparation.targetRowCount,
sourceTruncated: preparation.sourceTruncated,
targetTruncated: preparation.targetTruncated,
rowBudget: preparation.rowBudget,
databaseType,
preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result),
@@ -287,6 +290,7 @@ function buildMissingTargetResult(task: DataCompareTableTask, preparation: DataC
targetRowCount: 0,
sourceTruncated: preparation.sourceTruncated,
targetTruncated: false,
rowBudget: preparation.rowBudget,
databaseType,
preSyncStatements: preparation.preSyncStatements,
diff: toSelectableDiff(preparation.result),
@@ -310,6 +314,7 @@ function buildErrorResult(task: DataCompareTableTask, keyColumns: string[], data
targetRowCount: 0,
sourceTruncated: false,
targetTruncated: false,
rowBudget: 0,
databaseType,
preSyncStatements: [],
diff: { added: [], removed: [], modified: [] },