feat(schema-diff): support multiple table exclusion rules
This commit is contained in:
1 parent
5433f067d7
commit
3ffff92c8f
5 files changed
+122
-7
No files matched your search
@@ -22,6 +22,41 @@ vi.mock("@/lib/schema/schemaDiffMetadataLoad", () => ({ loadSchemaDetails: vi.fn
|
|||||||
|
|
||||||
const { startSchemaDiffSession } = await import("../useSchemaDiffSession.ts");
|
const { startSchemaDiffSession } = await import("../useSchemaDiffSession.ts");
|
||||||
|
|
||||||
|
test("applies every exclude rule before loading source and target details", async () => {
|
||||||
|
const { loadSchemaDetails } = await import("@/lib/schema/schemaDiffMetadataLoad");
|
||||||
|
vi.mocked(loadSchemaDetails).mockClear();
|
||||||
|
apiMock.prepareSchemaDiff.mockResolvedValue({ diffs: [], renameCandidates: [], syncSql: "", rollbackSyncSql: "" });
|
||||||
|
const tableListLoader = {
|
||||||
|
load: vi.fn().mockResolvedValue(["im_users", "ib_orders", "audit_log", "users"].map((name) => ({ name, table_type: "BASE TABLE" }))),
|
||||||
|
};
|
||||||
|
const session = startSchemaDiffSession(
|
||||||
|
{
|
||||||
|
sourceConnectionId: "source",
|
||||||
|
sourceDatabase: "app",
|
||||||
|
sourceSchema: "public",
|
||||||
|
targetConnectionId: "target",
|
||||||
|
targetDatabase: "warehouse",
|
||||||
|
targetSchema: "public",
|
||||||
|
sourceDbType: "mysql",
|
||||||
|
targetDbType: "mysql",
|
||||||
|
options: { functions: false, tableExcludePattern: "^im_,^ib_,log$" },
|
||||||
|
ignoreComments: false,
|
||||||
|
label: "app → warehouse",
|
||||||
|
},
|
||||||
|
{ tableListLoader },
|
||||||
|
);
|
||||||
|
await waitForSession(session);
|
||||||
|
assert.equal(session.status, "completed");
|
||||||
|
assert.equal(vi.mocked(loadSchemaDetails).mock.calls.length, 2);
|
||||||
|
for (const [tables] of vi.mocked(loadSchemaDetails).mock.calls) {
|
||||||
|
assert.deepEqual(
|
||||||
|
tables.map((table) => table.name),
|
||||||
|
["users"],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
async function waitForSession(session: { status: string }) {
|
async function waitForSession(session: { status: string }) {
|
||||||
for (let attempt = 0; attempt < 40 && session.status === "running"; attempt += 1) {
|
for (let attempt = 0; attempt < 40 && session.status === "running"; attempt += 1) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
|||||||
@@ -7725,11 +7725,11 @@ export default {
|
|||||||
tableFilterHelp: "Table filter help",
|
tableFilterHelp: "Table filter help",
|
||||||
tableFilterHelpRule: "Matches full table names. Do not type /.../.",
|
tableFilterHelpRule: "Matches full table names. Do not type /.../.",
|
||||||
tableFilterHelpBlank: "Blank include means all tables. Blank skip means no exclusions.",
|
tableFilterHelpBlank: "Blank include means all tables. Blank skip means no exclusions.",
|
||||||
tableFilterHelpExample: "^user_ means starts with user_; _bak$ means ends with _bak; a|b means a or b.",
|
tableFilterHelpExample: "^user_ means starts with user_; _bak$ means ends with _bak; a|b means a or b. Separate skip rules with commas; escape a literal comma as \\,.",
|
||||||
tableIncludePattern: "Only compare matched tables",
|
tableIncludePattern: "Only compare matched tables",
|
||||||
tableIncludePatternPlaceholder: "e.g. ^user_",
|
tableIncludePatternPlaceholder: "e.g. ^user_",
|
||||||
tableExcludePattern: "Skip matched tables",
|
tableExcludePattern: "Skip matched tables",
|
||||||
tableExcludePatternPlaceholder: "e.g. _bak$",
|
tableExcludePatternPlaceholder: "Comma-separated rules, e.g. ^im_, ^ib_, log$",
|
||||||
tableFilterPriority: "When both match",
|
tableFilterPriority: "When both match",
|
||||||
tableFilterPriorityExclude: "Exclude wins",
|
tableFilterPriorityExclude: "Exclude wins",
|
||||||
tableFilterPriorityInclude: "Include wins",
|
tableFilterPriorityInclude: "Include wins",
|
||||||
|
|||||||
@@ -7691,11 +7691,11 @@ export default withEnglishFallback({
|
|||||||
tableFilterHelp: "表名过滤说明",
|
tableFilterHelp: "表名过滤说明",
|
||||||
tableFilterHelpRule: "匹配完整表名,不用写 /.../。",
|
tableFilterHelpRule: "匹配完整表名,不用写 /.../。",
|
||||||
tableFilterHelpBlank: "只比较为空表示全部表;跳过为空表示不排除。",
|
tableFilterHelpBlank: "只比较为空表示全部表;跳过为空表示不排除。",
|
||||||
tableFilterHelpExample: "^user_ 表示 user_ 开头;_bak$ 表示 _bak 结尾;a|b 表示 a 或 b。",
|
tableFilterHelpExample: "^user_ 表示 user_ 开头;_bak$ 表示 _bak 结尾;a|b 表示 a 或 b。多个跳过规则用逗号分隔,匹配逗号本身请写 \\,。",
|
||||||
tableIncludePattern: "只比较匹配的表",
|
tableIncludePattern: "只比较匹配的表",
|
||||||
tableIncludePatternPlaceholder: "如 ^user_",
|
tableIncludePatternPlaceholder: "如 ^user_",
|
||||||
tableExcludePattern: "跳过匹配的表",
|
tableExcludePattern: "跳过匹配的表",
|
||||||
tableExcludePatternPlaceholder: "如 _bak$",
|
tableExcludePatternPlaceholder: "多个规则用逗号分隔,如 ^im_, ^ib_, log$",
|
||||||
tableFilterPriority: "同时匹配时",
|
tableFilterPriority: "同时匹配时",
|
||||||
tableFilterPriorityExclude: "排除优先",
|
tableFilterPriorityExclude: "排除优先",
|
||||||
tableFilterPriorityInclude: "包含优先",
|
tableFilterPriorityInclude: "包含优先",
|
||||||
|
|||||||
@@ -10,6 +10,43 @@ const tables: TableInfo[] = [
|
|||||||
];
|
];
|
||||||
|
|
||||||
describe("schemaDiffTableFilter", () => {
|
describe("schemaDiffTableFilter", () => {
|
||||||
|
it("excludes tables matching any comma-separated rule on both sides", () => {
|
||||||
|
const names = ["im_users", "ib_orders", "audit_log", "users", "catalog"];
|
||||||
|
const input = names.map((name) => ({ name, table_type: "BASE TABLE" }));
|
||||||
|
const options = { ...DEFAULT_MYSQL_OPTIONS, tableExcludePattern: " ^im_, ^ib_, log$ " };
|
||||||
|
const result = filterSchemaDiffTables(input, input, compileSchemaDiffTableFilter(options), options);
|
||||||
|
expect(result.sourceTables.map((table) => table.name)).toEqual(["users"]);
|
||||||
|
expect(result.targetTables.map((table) => table.name)).toEqual(["users"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["^a{2,3}$, ^ib_", ["aa", "aaa", "ib_orders"], ["a", "aaaa", "users"]],
|
||||||
|
["^[a,b]+$, ^ib_", ["a,b", "ib_orders"], ["c", "users"]],
|
||||||
|
["^(a,b)$, ^ib_", ["a,b", "ib_orders"], ["a", "b", "users"]],
|
||||||
|
[String.raw`^a\,b$, ^ib_`, ["a,b", "ib_orders"], ["a", "b", "users"]],
|
||||||
|
[String.raw`^(a)\1$, ^(b)\1$`, ["aa", "bb"], ["ab", "ba"]],
|
||||||
|
])("preserves regex syntax and independent backreferences in %s", (pattern, excluded, kept) => {
|
||||||
|
const options = { ...DEFAULT_MYSQL_OPTIONS, tableExcludePattern: pattern };
|
||||||
|
const input = [...excluded, ...kept].map((name) => ({ name, table_type: "BASE TABLE" }));
|
||||||
|
expect(filterSchemaDiffTables(input, [], compileSchemaDiffTableFilter(options), options).sourceTables.map((table) => table.name)).toEqual(kept);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores empty rules without matching every table", () => {
|
||||||
|
const options = { ...DEFAULT_MYSQL_OPTIONS, tableExcludePattern: " , , " };
|
||||||
|
expect(filterSchemaDiffTables(tables, tables, compileSchemaDiffTableFilter(options), options).sourceTables).toEqual(tables);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps include priority when a table matches any exclude rule", () => {
|
||||||
|
const options = { ...DEFAULT_MYSQL_OPTIONS, tableIncludePattern: "^users$", tableExcludePattern: "^im_, users$", tableFilterPriority: "include" as const };
|
||||||
|
expect(filterSchemaDiffTables(tables, [], compileSchemaDiffTableFilter(options), options).sourceTables.map((table) => table.name)).toEqual(["users"]);
|
||||||
|
const excludePriority = { ...options, tableFilterPriority: "exclude" as const };
|
||||||
|
expect(filterSchemaDiffTables(tables, [], compileSchemaDiffTableFilter(excludePriority), excludePriority).sourceTables).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an invalid rule even if other rules are valid", () => {
|
||||||
|
expect(() => compileSchemaDiffTableFilter({ ...DEFAULT_MYSQL_OPTIONS, tableExcludePattern: "^im_, [" })).toThrow(/Invalid exclude table name regex/);
|
||||||
|
});
|
||||||
|
|
||||||
it("recognizes regular and materialized views", () => {
|
it("recognizes regular and materialized views", () => {
|
||||||
expect(isSchemaDiffView(tables[0])).toBe(false);
|
expect(isSchemaDiffView(tables[0])).toBe(false);
|
||||||
expect(isSchemaDiffView(tables[1])).toBe(true);
|
expect(isSchemaDiffView(tables[1])).toBe(true);
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import type { SchemaDiffCompareOptions } from "@/types/schemaDiff";
|
|||||||
|
|
||||||
export interface CompiledSchemaDiffTableFilter {
|
export interface CompiledSchemaDiffTableFilter {
|
||||||
include?: RegExp;
|
include?: RegExp;
|
||||||
exclude?: RegExp;
|
exclude?: RegExp[];
|
||||||
priority: SchemaDiffCompareOptions["tableFilterPriority"];
|
priority: SchemaDiffCompareOptions["tableFilterPriority"];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -24,17 +24,60 @@ function compilePattern(pattern: string, label: "include" | "exclude"): RegExp |
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function compileExcludePatterns(input: string): RegExp[] | undefined {
|
||||||
|
const patterns: RegExp[] = [];
|
||||||
|
let start = 0;
|
||||||
|
let escaped = false;
|
||||||
|
let inCharacterClass = false;
|
||||||
|
let groupDepth = 0;
|
||||||
|
let braceDepth = 0;
|
||||||
|
const appendPattern = (end: number) => {
|
||||||
|
const pattern = compilePattern(input.slice(start, end), "exclude");
|
||||||
|
if (pattern) patterns.push(pattern);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Split only outer commas; regex groups, character classes, quantifiers and
|
||||||
|
// escaped commas remain part of their rule. Compile rules separately so
|
||||||
|
// numbered backreferences do not change meaning across rules.
|
||||||
|
for (let index = 0; index < input.length; index++) {
|
||||||
|
const char = input[index];
|
||||||
|
if (escaped) {
|
||||||
|
escaped = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (char === "\\") {
|
||||||
|
escaped = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (inCharacterClass) {
|
||||||
|
if (char === "]") inCharacterClass = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (char === "[") inCharacterClass = true;
|
||||||
|
else if (char === "(") groupDepth++;
|
||||||
|
else if (char === ")") groupDepth = Math.max(0, groupDepth - 1);
|
||||||
|
else if (char === "{") braceDepth++;
|
||||||
|
else if (char === "}") braceDepth = Math.max(0, braceDepth - 1);
|
||||||
|
else if (char === "," && groupDepth === 0 && braceDepth === 0) {
|
||||||
|
appendPattern(index);
|
||||||
|
start = index + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
appendPattern(input.length);
|
||||||
|
return patterns.length > 0 ? patterns : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
export function compileSchemaDiffTableFilter(options: SchemaDiffCompareOptions): CompiledSchemaDiffTableFilter {
|
export function compileSchemaDiffTableFilter(options: SchemaDiffCompareOptions): CompiledSchemaDiffTableFilter {
|
||||||
return {
|
return {
|
||||||
include: compilePattern(options.tableIncludePattern, "include"),
|
include: compilePattern(options.tableIncludePattern, "include"),
|
||||||
exclude: compilePattern(options.tableExcludePattern, "exclude"),
|
exclude: compileExcludePatterns(options.tableExcludePattern),
|
||||||
priority: options.tableFilterPriority,
|
priority: options.tableFilterPriority,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function matchesSchemaDiffTableFilter(tableName: string, filter: CompiledSchemaDiffTableFilter): boolean {
|
export function matchesSchemaDiffTableFilter(tableName: string, filter: CompiledSchemaDiffTableFilter): boolean {
|
||||||
const includeMatches = filter.include ? filter.include.test(tableName) : true;
|
const includeMatches = filter.include ? filter.include.test(tableName) : true;
|
||||||
const excludeMatches = filter.exclude ? filter.exclude.test(tableName) : false;
|
const excludeMatches = filter.exclude?.some((pattern) => pattern.test(tableName)) ?? false;
|
||||||
|
|
||||||
if (filter.include && filter.exclude && includeMatches && excludeMatches) {
|
if (filter.include && filter.exclude && includeMatches && excludeMatches) {
|
||||||
return filter.priority === "include";
|
return filter.priority === "include";
|
||||||
|
|||||||
Reference in new issue
Block a user