fix(compare): cap per-table compare reads and flag truncation
This commit is contained in:
21 files changed
+471
-81
No files matched your search
@@ -44,6 +44,7 @@ function preparationFor(table: string) {
|
||||
targetRowCount: 0,
|
||||
sourceTruncated: false,
|
||||
targetTruncated: false,
|
||||
rowBudget: 100_000,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -226,6 +227,30 @@ test("keeps the configured per-table match columns after cloning the session con
|
||||
assert.deepEqual(session.config.keyColumnsByTable, { users: ["user_id", "email"] });
|
||||
});
|
||||
|
||||
test("carries the row budget and truncation flags from the backend into the table result", async () => {
|
||||
apiMock.prepareDataCompareFromTables.mockImplementation((options: { targetTable: string }) => Promise.resolve({ ...preparationFor(options.targetTable), sourceRowCount: 4_000_000, sourceTruncated: true, targetTruncated: true, rowBudget: 100_000 }));
|
||||
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users" }), [{ sourceTable: "users", targetTable: "users" }]);
|
||||
|
||||
assert.equal(session.batchResults[0]?.rowBudget, 100_000);
|
||||
assert.equal(session.batchResults[0]?.sourceTruncated, true);
|
||||
assert.equal(session.batchResults[0]?.targetTruncated, true);
|
||||
});
|
||||
|
||||
test("keeps a truncated missing-target table on the same row budget", async () => {
|
||||
apiMock.prepareDataCompareMissingTarget.mockResolvedValue({
|
||||
...preparationFor("users"),
|
||||
sourceRowCount: 8_000_000,
|
||||
sourceTruncated: true,
|
||||
rowBudget: 100_000,
|
||||
});
|
||||
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users", targetTables: ["orders"] }), [{ sourceTable: "users", targetTable: "users" }]);
|
||||
|
||||
assert.equal(session.batchResults[0]?.rowBudget, 100_000);
|
||||
assert.equal(session.batchResults[0]?.sourceTruncated, true);
|
||||
});
|
||||
|
||||
test("normalizes a per-table match-column map", () => {
|
||||
assert.deepEqual(normalizeKeyColumnOverrides({ users: [" id ", ""], orders: [], "": ["x"] }), { users: ["id"], orders: [] });
|
||||
assert.deepEqual(normalizeKeyColumnOverrides(undefined), {});
|
||||
|
||||
@@ -59,6 +59,8 @@ export interface DataCompareTableResult {
|
||||
targetRowCount: number;
|
||||
sourceTruncated: boolean;
|
||||
targetTruncated: boolean;
|
||||
/** Rows the backend was allowed to read per side; the diff only covers this many rows when a side is truncated. */
|
||||
rowBudget: number;
|
||||
databaseType?: DatabaseType;
|
||||
preSyncStatements?: string[];
|
||||
diff: SelectableDataCompareResult;
|
||||
@@ -264,6 +266,7 @@ function buildTableResult(task: DataCompareTableTask, preparation: DataCompareFr
|
||||
targetRowCount: preparation.targetRowCount,
|
||||
sourceTruncated: preparation.sourceTruncated,
|
||||
targetTruncated: preparation.targetTruncated,
|
||||
rowBudget: preparation.rowBudget,
|
||||
databaseType,
|
||||
preSyncStatements: preparation.preSyncStatements,
|
||||
diff: toSelectableDiff(preparation.result),
|
||||
@@ -287,6 +290,7 @@ function buildMissingTargetResult(task: DataCompareTableTask, preparation: DataC
|
||||
targetRowCount: 0,
|
||||
sourceTruncated: preparation.sourceTruncated,
|
||||
targetTruncated: false,
|
||||
rowBudget: preparation.rowBudget,
|
||||
databaseType,
|
||||
preSyncStatements: preparation.preSyncStatements,
|
||||
diff: toSelectableDiff(preparation.result),
|
||||
@@ -310,6 +314,7 @@ function buildErrorResult(task: DataCompareTableTask, keyColumns: string[], data
|
||||
targetRowCount: 0,
|
||||
sourceTruncated: false,
|
||||
targetTruncated: false,
|
||||
rowBudget: 0,
|
||||
databaseType,
|
||||
preSyncStatements: [],
|
||||
diff: { added: [], removed: [], modified: [] },
|
||||
|
||||
Reference in new issue
Block a user