feat(data-compare): support per-table match column configuration
This commit is contained in:
1 parent
aa4e0a30fd
commit
d10216e302
18 files changed
+1359
-120
No files matched your search
@@ -1,5 +1,7 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { beforeEach, test, vi } from "vitest";
|
||||
import type { ColumnInfo } from "@/types/database";
|
||||
import type { DataCompareSession, DataCompareSessionConfig } from "../useDataCompareSession.ts";
|
||||
|
||||
const apiMock = vi.hoisted(() => ({
|
||||
buildDataCompareSyncPlan: vi.fn(),
|
||||
@@ -18,27 +20,76 @@ vi.mock("@/lib/backend/api", () => apiMock);
|
||||
vi.mock("@/composables/useDialogSources", () => ({ openDataCompareSession: openMock }));
|
||||
vi.mock("@/composables/useExportTracker", () => ({ useExportTracker: () => trackerMock }));
|
||||
|
||||
const { startDataCompareSession } = await import("../useDataCompareSession.ts");
|
||||
const { normalizeKeyColumnOverrides, startDataCompareSession } = await import("../useDataCompareSession.ts");
|
||||
|
||||
test("runs a data compare session independently of the dialog and retains its result", async () => {
|
||||
apiMock.getColumns.mockResolvedValue([
|
||||
{ name: "id", is_primary_key: true },
|
||||
{ name: "name", is_primary_key: false },
|
||||
]);
|
||||
apiMock.prepareDataCompareFromTables.mockResolvedValue({
|
||||
result: {
|
||||
added: [{ key: "1", keyValues: { id: 1 }, values: { id: 1, name: "new" } }],
|
||||
removed: [],
|
||||
modified: [],
|
||||
},
|
||||
syncStatements: ["INSERT INTO users (id, name) VALUES (1, 'new')"],
|
||||
syncSql: "INSERT INTO users (id, name) VALUES (1, 'new')",
|
||||
function column(name: string, isPrimaryKey = false): ColumnInfo {
|
||||
return { name, data_type: "TEXT", is_nullable: true, column_default: null, is_primary_key: isPrimaryKey, extra: null };
|
||||
}
|
||||
|
||||
const TABLES: Record<string, ColumnInfo[]> = {
|
||||
users: [column("user_id", true), column("email"), column("name")],
|
||||
orders: [column("tenant_id", true), column("order_id", true), column("amount")],
|
||||
payments: [column("payment_no", true), column("tenant_id"), column("amount")],
|
||||
logs: [column("category"), column("created_at")],
|
||||
legacy: [column("snid", true), column("username")],
|
||||
};
|
||||
|
||||
function preparationFor(table: string) {
|
||||
return {
|
||||
result: { added: [{ key: "1", keyValues: { id: 1 }, values: { id: 1 } }], removed: [], modified: [] },
|
||||
syncStatements: [`INSERT INTO ${table} (id) VALUES (1)`],
|
||||
syncSql: `INSERT INTO ${table} (id) VALUES (1)`,
|
||||
preSyncStatements: [],
|
||||
sourceRowCount: 1,
|
||||
targetRowCount: 0,
|
||||
sourceTruncated: false,
|
||||
targetTruncated: false,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function baseConfig(overrides: Partial<DataCompareSessionConfig> = {}): DataCompareSessionConfig {
|
||||
return {
|
||||
sourceConnectionId: "source",
|
||||
sourceDatabase: "app",
|
||||
sourceSchema: "public",
|
||||
sourceDatabases: ["app"],
|
||||
sourceSchemas: ["public"],
|
||||
sourceTables: Object.keys(TABLES),
|
||||
selectedSourceTables: [],
|
||||
targetConnectionId: "target",
|
||||
targetDatabase: "warehouse",
|
||||
targetSchema: "public",
|
||||
targetDatabases: ["warehouse"],
|
||||
targetSchemas: ["public"],
|
||||
targetTables: Object.keys(TABLES),
|
||||
targetTable: "",
|
||||
keyColumnsByTable: {},
|
||||
label: "app → warehouse",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const dependencies = {
|
||||
ensureConnected: () => Promise.resolve(undefined),
|
||||
getConfig: () => ({ db_type: "mysql" as const }),
|
||||
};
|
||||
|
||||
async function runSession(input: DataCompareSessionConfig, tasks: { sourceTable: string; targetTable: string }[]): Promise<DataCompareSession> {
|
||||
const session = startDataCompareSession(input, tasks, dependencies);
|
||||
for (let attempt = 0; attempt < 50 && session.status === "running"; attempt += 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
function keyColumnsOf(call: number): string[] {
|
||||
return apiMock.prepareDataCompareFromTables.mock.calls[call]?.[0]?.keyColumns as string[];
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
apiMock.getColumns.mockImplementation((_connectionId: string, _database: string, _schema: string, table: string) => Promise.resolve(TABLES[table] ?? []));
|
||||
apiMock.prepareDataCompareFromTables.mockImplementation((options: { targetTable: string }) => Promise.resolve(preparationFor(options.targetTable)));
|
||||
apiMock.buildDataCompareSyncPlan.mockResolvedValue({
|
||||
insertCount: 1,
|
||||
updateCount: 0,
|
||||
@@ -47,36 +98,10 @@ test("runs a data compare session independently of the dialog and retains its re
|
||||
syncStatements: ["INSERT"],
|
||||
syncSql: "INSERT",
|
||||
});
|
||||
});
|
||||
|
||||
const session = startDataCompareSession(
|
||||
{
|
||||
sourceConnectionId: "source",
|
||||
sourceDatabase: "app",
|
||||
sourceSchema: "public",
|
||||
sourceDatabases: ["app"],
|
||||
sourceSchemas: ["public"],
|
||||
sourceTables: ["users"],
|
||||
selectedSourceTables: ["users"],
|
||||
targetConnectionId: "target",
|
||||
targetDatabase: "warehouse",
|
||||
targetSchema: "public",
|
||||
targetDatabases: ["warehouse"],
|
||||
targetSchemas: ["public"],
|
||||
targetTables: ["users"],
|
||||
targetTable: "users",
|
||||
keyColumns: ["id"],
|
||||
label: "app → warehouse",
|
||||
},
|
||||
[{ sourceTable: "users", targetTable: "users" }],
|
||||
{
|
||||
ensureConnected: vi.fn().mockResolvedValue(undefined),
|
||||
getConfig: () => ({ db_type: "mysql" }),
|
||||
},
|
||||
);
|
||||
|
||||
for (let attempt = 0; attempt < 20 && session.status === "running"; attempt += 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
test("runs a data compare session independently of the dialog and retains its result", async () => {
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users" }), [{ sourceTable: "users", targetTable: "users" }]);
|
||||
|
||||
assert.equal(session.status, "completed");
|
||||
assert.equal(session.batchResults.length, 1);
|
||||
@@ -91,6 +116,117 @@ test("runs a data compare session independently of the dialog and retains its re
|
||||
assert.equal(openMock.mock.calls.at(-1)?.[0], session.id);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
test("infers the primary key of every table when no match columns are configured", async () => {
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users", "orders", "payments"] }), [
|
||||
{ sourceTable: "users", targetTable: "users" },
|
||||
{ sourceTable: "orders", targetTable: "orders" },
|
||||
{ sourceTable: "payments", targetTable: "payments" },
|
||||
]);
|
||||
|
||||
assert.equal(session.status, "completed");
|
||||
assert.deepEqual(
|
||||
session.batchResults.map((result) => result.keyColumns),
|
||||
[["user_id"], ["tenant_id", "order_id"], ["payment_no"]],
|
||||
);
|
||||
assert.deepEqual(
|
||||
apiMock.prepareDataCompareFromTables.mock.calls.map((call) => call[0].keyColumns),
|
||||
[["user_id"], ["tenant_id", "order_id"], ["payment_no"]],
|
||||
);
|
||||
});
|
||||
|
||||
test("gives one table its own match columns without changing the other tables", async () => {
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users", "orders", "payments"], keyColumnsByTable: { payments: ["tenant_id", "payment_no"] } }), [
|
||||
{ sourceTable: "users", targetTable: "users" },
|
||||
{ sourceTable: "orders", targetTable: "orders" },
|
||||
{ sourceTable: "payments", targetTable: "payments" },
|
||||
]);
|
||||
|
||||
assert.equal(session.status, "completed");
|
||||
assert.deepEqual(
|
||||
session.batchResults.map((result) => ({ table: result.sourceTable, keys: result.keyColumns })),
|
||||
[
|
||||
{ table: "users", keys: ["user_id"] },
|
||||
{ table: "orders", keys: ["tenant_id", "order_id"] },
|
||||
{ table: "payments", keys: ["tenant_id", "payment_no"] },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(keyColumnsOf(2), ["tenant_id", "payment_no"]);
|
||||
});
|
||||
|
||||
test("keeps a manual match-column selection on one table even when another table is overridden too", async () => {
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users", "orders"], keyColumnsByTable: { users: ["email"], orders: ["amount"] } }), [
|
||||
{ sourceTable: "users", targetTable: "users" },
|
||||
{ sourceTable: "orders", targetTable: "orders" },
|
||||
]);
|
||||
|
||||
assert.equal(session.status, "completed");
|
||||
assert.deepEqual(keyColumnsOf(0), ["email"]);
|
||||
assert.deepEqual(keyColumnsOf(1), ["amount"]);
|
||||
});
|
||||
|
||||
test("reports a missing key column only for the table that configures it", async () => {
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users", "orders"], keyColumnsByTable: { users: ["does_not_exist"] } }), [
|
||||
{ sourceTable: "users", targetTable: "users" },
|
||||
{ sourceTable: "orders", targetTable: "orders" },
|
||||
]);
|
||||
|
||||
assert.equal(session.status, "completed");
|
||||
assert.equal(session.batchResults[0]?.status, "error");
|
||||
assert.match(session.batchResults[0]?.error ?? "", /does_not_exist/);
|
||||
assert.equal(session.batchResults[1]?.status, "different");
|
||||
});
|
||||
|
||||
test("never falls back to the first column when a table has no primary key", async () => {
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["logs"], targetTable: "logs" }), [{ sourceTable: "logs", targetTable: "logs" }]);
|
||||
|
||||
assert.equal(session.batchResults.length, 1);
|
||||
assert.equal(session.batchResults[0]?.status, "error");
|
||||
assert.equal(apiMock.prepareDataCompareFromTables.mock.calls.length, 0);
|
||||
});
|
||||
|
||||
test("treats an explicitly cleared selection as missing match columns instead of restoring the primary key", async () => {
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users", keyColumnsByTable: { users: [] } }), [{ sourceTable: "users", targetTable: "users" }]);
|
||||
|
||||
assert.equal(session.batchResults[0]?.status, "error");
|
||||
assert.equal(apiMock.prepareDataCompareFromTables.mock.calls.length, 0);
|
||||
});
|
||||
|
||||
test("canonicalizes a manual match column against an upper-cased target column", async () => {
|
||||
apiMock.getColumns.mockImplementation((connectionId: string, _database: string, _schema: string, table: string) => {
|
||||
if (table !== "legacy") return Promise.resolve(TABLES[table] ?? []);
|
||||
return Promise.resolve(connectionId === "source" ? [column("snid", true), column("username")] : [column("SNID", true), column("USERNAME")]);
|
||||
});
|
||||
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["legacy"], targetTable: "legacy", keyColumnsByTable: { legacy: ["snid", "Username"] } }), [{ sourceTable: "legacy", targetTable: "legacy" }]);
|
||||
|
||||
assert.equal(session.batchResults[0]?.status, "different");
|
||||
assert.deepEqual(keyColumnsOf(0), ["SNID", "USERNAME"]);
|
||||
assert.deepEqual(apiMock.prepareDataCompareFromTables.mock.calls[0]?.[0].sourceColumns, ["snid", "username"]);
|
||||
assert.deepEqual(apiMock.prepareDataCompareFromTables.mock.calls[0]?.[0].columns, ["SNID", "USERNAME"]);
|
||||
});
|
||||
|
||||
test("uses a per-table override for a source table whose target table is missing", async () => {
|
||||
apiMock.prepareDataCompareMissingTarget.mockResolvedValue(preparationFor("users"));
|
||||
|
||||
const session = await runSession(baseConfig({ selectedSourceTables: ["users"], targetTable: "users", keyColumnsByTable: { users: ["email"] }, targetTables: ["orders"] }), [{ sourceTable: "users", targetTable: "users" }]);
|
||||
|
||||
assert.equal(session.batchResults[0]?.status, "different");
|
||||
assert.deepEqual(session.batchResults[0]?.keyColumns, ["email"]);
|
||||
assert.deepEqual(apiMock.prepareDataCompareMissingTarget.mock.calls[0]?.[0].keyColumns, ["email"]);
|
||||
});
|
||||
|
||||
test("keeps the configured per-table match columns after cloning the session config", async () => {
|
||||
const input = baseConfig({ selectedSourceTables: ["users"], targetTable: "users", keyColumnsByTable: { users: [" user_id ", "USER_ID", "email"] } });
|
||||
const session = await runSession(input, [{ sourceTable: "users", targetTable: "users" }]);
|
||||
|
||||
assert.deepEqual(session.config.keyColumnsByTable, { users: ["user_id", "email"] });
|
||||
assert.deepEqual(keyColumnsOf(0), ["user_id", "email"]);
|
||||
// The session must not share the caller's arrays.
|
||||
input.keyColumnsByTable.users?.push("name");
|
||||
assert.deepEqual(session.config.keyColumnsByTable, { users: ["user_id", "email"] });
|
||||
});
|
||||
|
||||
test("normalizes a per-table match-column map", () => {
|
||||
assert.deepEqual(normalizeKeyColumnOverrides({ users: [" id ", ""], orders: [], "": ["x"] }), { users: ["id"], orders: [] });
|
||||
assert.deepEqual(normalizeKeyColumnOverrides(undefined), {});
|
||||
});
|
||||
Reference in new issue
Block a user