feat(completion): support pinyin matching for Chinese identifiers
This commit is contained in:
1 parent
eafa98b771
commit
f700627675
12 files changed
+438
-10
No files matched your search
@@ -101,6 +101,32 @@ describe("QueryEditor completion Tab keymap", () => {
|
||||
expect(result).not.toHaveProperty("filter");
|
||||
});
|
||||
|
||||
it("keeps ASCII single-character filtering while allowing Han pinyin matches", () => {
|
||||
const source = [extractFunction("completionItemsForBypassedFilter"), extractFunction("shouldBypassCompletionFilter"), extractFunction("buildCompletionResult")].join("\n");
|
||||
const javascript = ts.transpileModule(source, {
|
||||
compilerOptions: { module: ts.ModuleKind.None, target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText;
|
||||
const buildCompletionResult = new Function("completionOptionForItem", "completionMatchRanges", `${javascript}\nreturn buildCompletionResult;`)(
|
||||
(item: { label: string }) => item,
|
||||
() => [],
|
||||
) as (items: Array<{ label: string }>, from: number, validFor?: RegExp, prefix?: string) => { filter?: boolean; options: Array<{ label: string }> } | null;
|
||||
|
||||
const result = buildCompletionResult([{ label: "amount" }, { label: "memo" }, { label: "明细" }], 0, undefined, "m");
|
||||
|
||||
expect(result?.filter).toBe(false);
|
||||
expect(result?.options.map((item) => item.label)).toEqual(["memo", "明细"]);
|
||||
});
|
||||
|
||||
it("bypasses CodeMirror filtering for Han substring matches", () => {
|
||||
const javascript = ts.transpileModule(extractFunction("shouldBypassCompletionFilter"), {
|
||||
compilerOptions: { module: ts.ModuleKind.None, target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText;
|
||||
const shouldBypassCompletionFilter = new Function(`${javascript}\nreturn shouldBypassCompletionFilter;`)() as (prefix: string, items: Array<{ label: string }>) => boolean;
|
||||
|
||||
expect(shouldBypassCompletionFilter("金", [{ label: "总租金" }])).toBe(true);
|
||||
expect(shouldBypassCompletionFilter("m", [{ label: "amount" }, { label: "memo" }])).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps normal Tab indentation when completion is inactive", () => {
|
||||
const harness = createHarness({ completionStatus: () => null });
|
||||
const view = createView();
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { containsHan, matchesPinyinInitials, orderedSubsequenceSpan, pinyinAwareMatchScore, pinyinFirstLetters } from "@/lib/common/pinyin";
|
||||
import { completionMatchRanges } from "@/lib/common/completionMatch";
|
||||
import { buildSqlCompletionItems } from "@/lib/sql/sqlCompletion";
|
||||
|
||||
describe("pinyinAwareMatchScore", () => {
|
||||
it("ranks prefix above pinyin prefix above substring above pinyin subsequence", () => {
|
||||
const prefix = pinyinAwareMatchScore("总租金", "总");
|
||||
const pinyinPrefix = pinyinAwareMatchScore("总租金", "zz");
|
||||
const substring = pinyinAwareMatchScore("总租金", "金");
|
||||
const subsequence = pinyinAwareMatchScore("总租金", "zj");
|
||||
expect(prefix).toBeGreaterThan(pinyinPrefix);
|
||||
expect(pinyinPrefix).toBeGreaterThan(substring);
|
||||
expect(substring).toBeGreaterThan(subsequence);
|
||||
expect(subsequence).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("returns -1 when nothing matches", () => {
|
||||
expect(pinyinAwareMatchScore("总租金", "jz")).toBe(-1);
|
||||
expect(pinyinAwareMatchScore("amount", "zz")).toBe(-1);
|
||||
expect(pinyinAwareMatchScore("amount", "")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ordered subsequence span", () => {
|
||||
it("matches initials subsequences", () => {
|
||||
expect(orderedSubsequenceSpan("zzj", "zj")).toEqual({ first: 0, span: 3 });
|
||||
expect(orderedSubsequenceSpan("zzj", "j")).toEqual({ first: 2, span: 1 });
|
||||
expect(orderedSubsequenceSpan("zzj", "jz")).toBeNull();
|
||||
expect(orderedSubsequenceSpan("zzj", "")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("completion match highlight ranges", () => {
|
||||
it("highlights a Han substring match", () => {
|
||||
expect(completionMatchRanges("总租金", "金")).toEqual([2, 3]);
|
||||
});
|
||||
|
||||
it("highlights every Han character matched by pinyin initials", () => {
|
||||
expect(completionMatchRanges("总租金", "zzj")).toEqual([0, 1, 1, 2, 2, 3]);
|
||||
expect(completionMatchRanges("总租金", "zz")).toEqual([0, 1, 1, 2]);
|
||||
});
|
||||
|
||||
it("highlights only the Han characters hit by an initials subsequence", () => {
|
||||
expect(completionMatchRanges("总租金", "zj")).toEqual([0, 1, 2, 3]);
|
||||
expect(completionMatchRanges("总租金", "j")).toEqual([2, 3]);
|
||||
});
|
||||
|
||||
it("highlights ASCII substrings and falls back to in-order fuzzy", () => {
|
||||
expect(completionMatchRanges("amount", "am")).toEqual([0, 2]);
|
||||
expect(completionMatchRanges("order_detail", "od")).toEqual([0, 1, 2, 3]);
|
||||
expect(completionMatchRanges("amount", "zz")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pinyin first letters", () => {
|
||||
it("maps Han characters to their pinyin initials", () => {
|
||||
expect(pinyinFirstLetters("总租金")).toBe("zzj");
|
||||
expect(pinyinFirstLetters("租赁日期")).toBe("zlrq");
|
||||
});
|
||||
|
||||
it("keeps ASCII alphanumerics and drops other characters", () => {
|
||||
expect(pinyinFirstLetters("order明细")).toBe("ordermx");
|
||||
expect(pinyinFirstLetters("Amount")).toBe("amount");
|
||||
});
|
||||
|
||||
it("matches only ASCII queries against Han candidates", () => {
|
||||
expect(matchesPinyinInitials("总租金", "zzj")).toBe(true);
|
||||
expect(matchesPinyinInitials("总租金", "zz")).toBe(true);
|
||||
expect(matchesPinyinInitials("总租金", "zj")).toBe(false);
|
||||
expect(matchesPinyinInitials("总租金", "租金")).toBe(false);
|
||||
expect(matchesPinyinInitials("amount", "am")).toBe(false);
|
||||
expect(containsHan("amount")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sqlCompletion Chinese column matching", () => {
|
||||
function completionItems(typedPrefix: string) {
|
||||
const sql = `SELECT * FROM orders WHERE ${typedPrefix}`;
|
||||
return buildSqlCompletionItems(sql, sql.length, {
|
||||
databaseType: "mysql",
|
||||
tables: [{ name: "orders", type: "table" }],
|
||||
columnsByTable: new Map([
|
||||
[
|
||||
"orders",
|
||||
[
|
||||
{ name: "总租金", table: "orders" },
|
||||
{ name: "租赁日期", table: "orders" },
|
||||
{ name: "amount", table: "orders" },
|
||||
],
|
||||
],
|
||||
]),
|
||||
});
|
||||
}
|
||||
|
||||
it("matches a single Han character anywhere in the column name", () => {
|
||||
const items = completionItems("金");
|
||||
expect(items).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
|
||||
});
|
||||
|
||||
it("matches pinyin initials like DataGrip", () => {
|
||||
const items = completionItems("zzj");
|
||||
expect(items).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
|
||||
expect(items.some((item) => item.label === "租赁日期")).toBe(false);
|
||||
expect(items.some((item) => item.label === "amount")).toBe(false);
|
||||
});
|
||||
|
||||
it("matches a single ASCII letter via pinyin initials", () => {
|
||||
const items = completionItems("z");
|
||||
const labels = items.filter((item) => item.type === "column").map((item) => item.label);
|
||||
expect(labels).toEqual(expect.arrayContaining(["总租金", "租赁日期"]));
|
||||
});
|
||||
|
||||
it("matches initials subsequences like DataGrip", () => {
|
||||
const subsequenceItems = completionItems("zj");
|
||||
expect(subsequenceItems).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
|
||||
expect(subsequenceItems.some((item) => item.label === "租赁日期")).toBe(false);
|
||||
|
||||
const lastInitialItems = completionItems("j");
|
||||
expect(lastInitialItems).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
|
||||
expect(lastInitialItems.some((item) => item.label === "租赁日期")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps plain English matching unchanged", () => {
|
||||
const items = completionItems("am");
|
||||
expect(items).toEqual(expect.arrayContaining([expect.objectContaining({ label: "amount", type: "column" })]));
|
||||
expect(items.some((item) => item.label === "总租金")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { pinyin } from "pinyin-pro";
|
||||
|
||||
const HAN_CHAR = /\p{Script=Han}/u;
|
||||
const ASCII_ALNUM = /[a-z0-9]/i;
|
||||
const ASCII_QUERY = /^[a-z0-9]+$/;
|
||||
|
||||
/**
|
||||
* Highlight ranges (flat [from, to, ...] UTF-16 offsets into `label`) for a
|
||||
* completion query, used when CodeMirror's built-in filter is bypassed.
|
||||
*
|
||||
* Tiers: case-insensitive substring → pinyin initials ("zzj" highlights each
|
||||
* Han character of 总租金) → in-order fuzzy. Returns [] when nothing matches.
|
||||
*/
|
||||
export function completionMatchRanges(label: string, query: string): number[] {
|
||||
if (!query) return [];
|
||||
const lowerLabel = label.toLowerCase();
|
||||
const lowerQuery = query.toLowerCase();
|
||||
|
||||
const direct = lowerLabel.indexOf(lowerQuery);
|
||||
if (direct >= 0) return [direct, direct + lowerQuery.length];
|
||||
|
||||
if (ASCII_QUERY.test(lowerQuery) && HAN_CHAR.test(label)) {
|
||||
const pinyinRanges = pinyinInitialMatchRanges(label, lowerQuery);
|
||||
if (pinyinRanges) return pinyinRanges;
|
||||
}
|
||||
|
||||
return fuzzyMatchRanges(lowerLabel, lowerQuery);
|
||||
}
|
||||
|
||||
function pinyinInitialMatchRanges(label: string, query: string): number[] | null {
|
||||
// Greedy earliest subsequence over the pinyin-initial positions, so both
|
||||
// prefix queries ("zz") and skipping queries ("zj") highlight the matched
|
||||
// Han characters.
|
||||
const ranges: number[] = [];
|
||||
let queryIndex = 0;
|
||||
let index = 0;
|
||||
for (const char of label) {
|
||||
if (queryIndex >= query.length) break;
|
||||
const isHan = HAN_CHAR.test(char);
|
||||
if (isHan || ASCII_ALNUM.test(char)) {
|
||||
const initial = isHan ? pinyin(char, { pattern: "first", toneType: "none" }) : char.toLowerCase();
|
||||
if (initial === query[queryIndex]) {
|
||||
ranges.push(index, index + char.length);
|
||||
queryIndex += 1;
|
||||
}
|
||||
}
|
||||
index += char.length;
|
||||
}
|
||||
return queryIndex === query.length ? ranges : null;
|
||||
}
|
||||
|
||||
function fuzzyMatchRanges(lowerLabel: string, lowerQuery: string): number[] {
|
||||
const ranges: number[] = [];
|
||||
let from = 0;
|
||||
for (const char of lowerQuery) {
|
||||
const position = lowerLabel.indexOf(char, from);
|
||||
if (position < 0) return [];
|
||||
ranges.push(position, position + char.length);
|
||||
from = position + char.length;
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { pinyin } from "pinyin-pro";
|
||||
|
||||
const HAN_CHAR = /\p{Script=Han}/u;
|
||||
const ASCII_ALNUM = /[a-z0-9]/i;
|
||||
|
||||
const firstLetterCache = new Map<string, string>();
|
||||
const MAX_CACHE_ENTRIES = 100_000;
|
||||
|
||||
function cacheFirstLetters(text: string, result: string): void {
|
||||
if (firstLetterCache.size >= MAX_CACHE_ENTRIES) {
|
||||
const oldest = firstLetterCache.keys().next().value;
|
||||
if (oldest !== undefined) firstLetterCache.delete(oldest);
|
||||
}
|
||||
firstLetterCache.set(text, result);
|
||||
}
|
||||
|
||||
export function containsHan(text: string): boolean {
|
||||
return HAN_CHAR.test(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* First pinyin letter of every Han character, with ASCII letters/digits kept
|
||||
* as-is (lowercased) and every other character dropped. Used for DataGrip-style
|
||||
* initials matching, e.g. pinyinFirstLetters("总租金") === "zzj".
|
||||
*
|
||||
* Only the default reading is used for polyphonic characters.
|
||||
*/
|
||||
export function pinyinFirstLetters(text: string): string {
|
||||
const cached = firstLetterCache.get(text);
|
||||
if (cached !== undefined) {
|
||||
firstLetterCache.delete(text);
|
||||
firstLetterCache.set(text, cached);
|
||||
return cached;
|
||||
}
|
||||
let result = "";
|
||||
for (const char of text) {
|
||||
if (HAN_CHAR.test(char)) {
|
||||
result += pinyin(char, { pattern: "first", toneType: "none" });
|
||||
} else if (ASCII_ALNUM.test(char)) {
|
||||
result += char.toLowerCase();
|
||||
}
|
||||
}
|
||||
cacheFirstLetters(text, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** True when an ASCII-letters/digits query can match `candidate` via pinyin initials. */
|
||||
export function matchesPinyinInitials(candidate: string, query: string): boolean {
|
||||
if (!/^[a-z0-9]+$/.test(query) || !containsHan(candidate)) return false;
|
||||
return pinyinFirstLetters(candidate).startsWith(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ordered-subsequence match of `query` against `text` (e.g. "zj" against the
|
||||
* initials "zzj"). Returns the first matched index and the span covering all
|
||||
* matched characters, or null when the query is not a subsequence.
|
||||
*/
|
||||
export function orderedSubsequenceSpan(text: string, query: string): { first: number; span: number } | null {
|
||||
if (!query) return null;
|
||||
let from = 0;
|
||||
let first = -1;
|
||||
let last = -1;
|
||||
for (const char of query) {
|
||||
const position = text.indexOf(char, from);
|
||||
if (position < 0) return null;
|
||||
if (first < 0) first = position;
|
||||
last = position;
|
||||
from = position + 1;
|
||||
}
|
||||
return { first, span: last - first + 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic matcher for small pick-lists (grid condition editor, ...):
|
||||
* prefix > pinyin-initials prefix > substring > pinyin-initials subsequence.
|
||||
* Returns -1 for no match; scores are per-tier constants so same-tier
|
||||
* candidates keep their original order. An empty query matches everything.
|
||||
*/
|
||||
export function pinyinAwareMatchScore(candidate: string, query: string): number {
|
||||
const text = candidate.toLowerCase();
|
||||
const normalized = query.trim().toLowerCase();
|
||||
if (!normalized) return 0;
|
||||
if (text.startsWith(normalized)) return 400;
|
||||
const asciiQuery = /^[a-z0-9]+$/.test(normalized);
|
||||
const han = containsHan(text);
|
||||
if (asciiQuery && han && pinyinFirstLetters(text).startsWith(normalized)) return 300;
|
||||
if (text.includes(normalized)) return 200;
|
||||
if (asciiQuery && han) {
|
||||
const subsequence = orderedSubsequenceSpan(pinyinFirstLetters(text), normalized);
|
||||
if (subsequence) return 100;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { findActiveSqlStatementSpan, tokenizeSqlSemantic } from "@/lib/sql/seman
|
||||
import type { SqlSemanticBuildOptions, SqlSemanticSpan } from "@/lib/sql/semantic/types";
|
||||
import { DEFAULT_SQL_SNIPPETS, MANTICORESEARCH_SQL_SNIPPETS, resolveSqlSnippetBodyForDatabase } from "@/lib/sql/sqlSnippetTemplates";
|
||||
import { requiresPostgresIdentifierQuote } from "@/lib/sql/sqlIdentifier";
|
||||
import { containsHan, orderedSubsequenceSpan, pinyinFirstLetters } from "@/lib/common/pinyin";
|
||||
|
||||
export { DEFAULT_SQL_SNIPPETS, resolveSqlSnippetBodyForDatabase } from "@/lib/sql/sqlSnippetTemplates";
|
||||
|
||||
@@ -4289,6 +4290,8 @@ function matchesPrefix(candidate: string, prefix: string): boolean {
|
||||
* Scoring tiers:
|
||||
* Exact match: 3000 - len
|
||||
* Initials match: 2400 + exactInitialsBonus - len
|
||||
* Pinyin initials: 2300 + exactInitialsBonus - len (ASCII query vs Han candidate, e.g. "zzj" → 总租金)
|
||||
* Pinyin subsequence: 1600 - penalties - len (ordered initials, e.g. "zj" → 总租金)
|
||||
* Prefix match: 2000 - len
|
||||
* Substring: 900 + boundaryBonus - len
|
||||
* Tight fuzzy: 1500 - gapPenalty + earlyMatchBonus - len (gaps < prefix length)
|
||||
@@ -4311,6 +4314,21 @@ function computeMatchScore(candidate: string, prefix: string): number {
|
||||
return 2400 + exactInitialsBonus - c.length;
|
||||
}
|
||||
|
||||
// DataGrip-style pinyin initials: an ASCII query matches the first pinyin
|
||||
// letters of Han characters — as a prefix ("zz" → 总租金) or as an ordered
|
||||
// subsequence ("zj" → 总租金, "j" → 总租金).
|
||||
if (/^[a-z0-9]+$/.test(p) && containsHan(c)) {
|
||||
const pinyinInitials = pinyinFirstLetters(c);
|
||||
if (pinyinInitials.startsWith(p)) {
|
||||
const exactInitialsBonus = pinyinInitials === p ? 300 : 0;
|
||||
return 2300 + exactInitialsBonus - c.length;
|
||||
}
|
||||
const subsequence = orderedSubsequenceSpan(pinyinInitials, p);
|
||||
if (subsequence) {
|
||||
return 1600 - subsequence.first * 30 - (subsequence.span - p.length) * 10 - c.length;
|
||||
}
|
||||
}
|
||||
|
||||
const substringIndex = c.indexOf(p);
|
||||
if (substringIndex >= 0) {
|
||||
const boundaryBonus = isIdentifierBoundary(candidate, substringIndex) ? 400 : Math.max(0, 180 - substringIndex * 12);
|
||||
|
||||
Reference in new issue
Block a user