feat(completion): support pinyin matching for Chinese identifiers

This commit is contained in:
Freedom authored and GitHub committed 2026-08-05 02:03:09 +08:00
1 parent eafa98b771
commit f700627675
12 files changed
+438 -10

No files matched your search

@@ -101,6 +101,32 @@ describe("QueryEditor completion Tab keymap", () => {
expect(result).not.toHaveProperty("filter");
});
it("keeps ASCII single-character filtering while allowing Han pinyin matches", () => {
const source = [extractFunction("completionItemsForBypassedFilter"), extractFunction("shouldBypassCompletionFilter"), extractFunction("buildCompletionResult")].join("\n");
const javascript = ts.transpileModule(source, {
compilerOptions: { module: ts.ModuleKind.None, target: ts.ScriptTarget.ES2022 },
}).outputText;
const buildCompletionResult = new Function("completionOptionForItem", "completionMatchRanges", `${javascript}\nreturn buildCompletionResult;`)(
(item: { label: string }) => item,
() => [],
) as (items: Array<{ label: string }>, from: number, validFor?: RegExp, prefix?: string) => { filter?: boolean; options: Array<{ label: string }> } | null;
const result = buildCompletionResult([{ label: "amount" }, { label: "memo" }, { label: "明细" }], 0, undefined, "m");
expect(result?.filter).toBe(false);
expect(result?.options.map((item) => item.label)).toEqual(["memo", "明细"]);
});
it("bypasses CodeMirror filtering for Han substring matches", () => {
const javascript = ts.transpileModule(extractFunction("shouldBypassCompletionFilter"), {
compilerOptions: { module: ts.ModuleKind.None, target: ts.ScriptTarget.ES2022 },
}).outputText;
const shouldBypassCompletionFilter = new Function(`${javascript}\nreturn shouldBypassCompletionFilter;`)() as (prefix: string, items: Array<{ label: string }>) => boolean;
expect(shouldBypassCompletionFilter("金", [{ label: "总租金" }])).toBe(true);
expect(shouldBypassCompletionFilter("m", [{ label: "amount" }, { label: "memo" }])).toBe(false);
});
it("keeps normal Tab indentation when completion is inactive", () => {
const harness = createHarness({ completionStatus: () => null });
const view = createView();
@@ -0,0 +1,129 @@
import { describe, expect, it } from "vitest";
import { containsHan, matchesPinyinInitials, orderedSubsequenceSpan, pinyinAwareMatchScore, pinyinFirstLetters } from "@/lib/common/pinyin";
import { completionMatchRanges } from "@/lib/common/completionMatch";
import { buildSqlCompletionItems } from "@/lib/sql/sqlCompletion";
describe("pinyinAwareMatchScore", () => {
it("ranks prefix above pinyin prefix above substring above pinyin subsequence", () => {
const prefix = pinyinAwareMatchScore("总租金", "总");
const pinyinPrefix = pinyinAwareMatchScore("总租金", "zz");
const substring = pinyinAwareMatchScore("总租金", "金");
const subsequence = pinyinAwareMatchScore("总租金", "zj");
expect(prefix).toBeGreaterThan(pinyinPrefix);
expect(pinyinPrefix).toBeGreaterThan(substring);
expect(substring).toBeGreaterThan(subsequence);
expect(subsequence).toBeGreaterThan(0);
});
it("returns -1 when nothing matches", () => {
expect(pinyinAwareMatchScore("总租金", "jz")).toBe(-1);
expect(pinyinAwareMatchScore("amount", "zz")).toBe(-1);
expect(pinyinAwareMatchScore("amount", "")).toBe(0);
});
});
describe("ordered subsequence span", () => {
it("matches initials subsequences", () => {
expect(orderedSubsequenceSpan("zzj", "zj")).toEqual({ first: 0, span: 3 });
expect(orderedSubsequenceSpan("zzj", "j")).toEqual({ first: 2, span: 1 });
expect(orderedSubsequenceSpan("zzj", "jz")).toBeNull();
expect(orderedSubsequenceSpan("zzj", "")).toBeNull();
});
});
describe("completion match highlight ranges", () => {
it("highlights a Han substring match", () => {
expect(completionMatchRanges("总租金", "金")).toEqual([2, 3]);
});
it("highlights every Han character matched by pinyin initials", () => {
expect(completionMatchRanges("总租金", "zzj")).toEqual([0, 1, 1, 2, 2, 3]);
expect(completionMatchRanges("总租金", "zz")).toEqual([0, 1, 1, 2]);
});
it("highlights only the Han characters hit by an initials subsequence", () => {
expect(completionMatchRanges("总租金", "zj")).toEqual([0, 1, 2, 3]);
expect(completionMatchRanges("总租金", "j")).toEqual([2, 3]);
});
it("highlights ASCII substrings and falls back to in-order fuzzy", () => {
expect(completionMatchRanges("amount", "am")).toEqual([0, 2]);
expect(completionMatchRanges("order_detail", "od")).toEqual([0, 1, 2, 3]);
expect(completionMatchRanges("amount", "zz")).toEqual([]);
});
});
describe("pinyin first letters", () => {
it("maps Han characters to their pinyin initials", () => {
expect(pinyinFirstLetters("总租金")).toBe("zzj");
expect(pinyinFirstLetters("租赁日期")).toBe("zlrq");
});
it("keeps ASCII alphanumerics and drops other characters", () => {
expect(pinyinFirstLetters("order明细")).toBe("ordermx");
expect(pinyinFirstLetters("Amount")).toBe("amount");
});
it("matches only ASCII queries against Han candidates", () => {
expect(matchesPinyinInitials("总租金", "zzj")).toBe(true);
expect(matchesPinyinInitials("总租金", "zz")).toBe(true);
expect(matchesPinyinInitials("总租金", "zj")).toBe(false);
expect(matchesPinyinInitials("总租金", "租金")).toBe(false);
expect(matchesPinyinInitials("amount", "am")).toBe(false);
expect(containsHan("amount")).toBe(false);
});
});
describe("sqlCompletion Chinese column matching", () => {
function completionItems(typedPrefix: string) {
const sql = `SELECT * FROM orders WHERE ${typedPrefix}`;
return buildSqlCompletionItems(sql, sql.length, {
databaseType: "mysql",
tables: [{ name: "orders", type: "table" }],
columnsByTable: new Map([
[
"orders",
[
{ name: "总租金", table: "orders" },
{ name: "租赁日期", table: "orders" },
{ name: "amount", table: "orders" },
],
],
]),
});
}
it("matches a single Han character anywhere in the column name", () => {
const items = completionItems("金");
expect(items).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
});
it("matches pinyin initials like DataGrip", () => {
const items = completionItems("zzj");
expect(items).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
expect(items.some((item) => item.label === "租赁日期")).toBe(false);
expect(items.some((item) => item.label === "amount")).toBe(false);
});
it("matches a single ASCII letter via pinyin initials", () => {
const items = completionItems("z");
const labels = items.filter((item) => item.type === "column").map((item) => item.label);
expect(labels).toEqual(expect.arrayContaining(["总租金", "租赁日期"]));
});
it("matches initials subsequences like DataGrip", () => {
const subsequenceItems = completionItems("zj");
expect(subsequenceItems).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
expect(subsequenceItems.some((item) => item.label === "租赁日期")).toBe(false);
const lastInitialItems = completionItems("j");
expect(lastInitialItems).toEqual(expect.arrayContaining([expect.objectContaining({ label: "总租金", type: "column" })]));
expect(lastInitialItems.some((item) => item.label === "租赁日期")).toBe(false);
});
it("keeps plain English matching unchanged", () => {
const items = completionItems("am");
expect(items).toEqual(expect.arrayContaining([expect.objectContaining({ label: "amount", type: "column" })]));
expect(items.some((item) => item.label === "总租金")).toBe(false);
});
});
@@ -0,0 +1,62 @@
import { pinyin } from "pinyin-pro";
const HAN_CHAR = /\p{Script=Han}/u;
const ASCII_ALNUM = /[a-z0-9]/i;
const ASCII_QUERY = /^[a-z0-9]+$/;
/**
* Highlight ranges (flat [from, to, ...] UTF-16 offsets into `label`) for a
* completion query, used when CodeMirror's built-in filter is bypassed.
*
* Tiers: case-insensitive substring → pinyin initials ("zzj" highlights each
* Han character of 总租金) → in-order fuzzy. Returns [] when nothing matches.
*/
export function completionMatchRanges(label: string, query: string): number[] {
if (!query) return [];
const lowerLabel = label.toLowerCase();
const lowerQuery = query.toLowerCase();
const direct = lowerLabel.indexOf(lowerQuery);
if (direct >= 0) return [direct, direct + lowerQuery.length];
if (ASCII_QUERY.test(lowerQuery) && HAN_CHAR.test(label)) {
const pinyinRanges = pinyinInitialMatchRanges(label, lowerQuery);
if (pinyinRanges) return pinyinRanges;
}
return fuzzyMatchRanges(lowerLabel, lowerQuery);
}
function pinyinInitialMatchRanges(label: string, query: string): number[] | null {
// Greedy earliest subsequence over the pinyin-initial positions, so both
// prefix queries ("zz") and skipping queries ("zj") highlight the matched
// Han characters.
const ranges: number[] = [];
let queryIndex = 0;
let index = 0;
for (const char of label) {
if (queryIndex >= query.length) break;
const isHan = HAN_CHAR.test(char);
if (isHan || ASCII_ALNUM.test(char)) {
const initial = isHan ? pinyin(char, { pattern: "first", toneType: "none" }) : char.toLowerCase();
if (initial === query[queryIndex]) {
ranges.push(index, index + char.length);
queryIndex += 1;
}
}
index += char.length;
}
return queryIndex === query.length ? ranges : null;
}
function fuzzyMatchRanges(lowerLabel: string, lowerQuery: string): number[] {
const ranges: number[] = [];
let from = 0;
for (const char of lowerQuery) {
const position = lowerLabel.indexOf(char, from);
if (position < 0) return [];
ranges.push(position, position + char.length);
from = position + char.length;
}
return ranges;
}
+93
View File
@@ -0,0 +1,93 @@
import { pinyin } from "pinyin-pro";
const HAN_CHAR = /\p{Script=Han}/u;
const ASCII_ALNUM = /[a-z0-9]/i;
const firstLetterCache = new Map<string, string>();
const MAX_CACHE_ENTRIES = 100_000;
function cacheFirstLetters(text: string, result: string): void {
if (firstLetterCache.size >= MAX_CACHE_ENTRIES) {
const oldest = firstLetterCache.keys().next().value;
if (oldest !== undefined) firstLetterCache.delete(oldest);
}
firstLetterCache.set(text, result);
}
export function containsHan(text: string): boolean {
return HAN_CHAR.test(text);
}
/**
* First pinyin letter of every Han character, with ASCII letters/digits kept
* as-is (lowercased) and every other character dropped. Used for DataGrip-style
* initials matching, e.g. pinyinFirstLetters("总租金") === "zzj".
*
* Only the default reading is used for polyphonic characters.
*/
export function pinyinFirstLetters(text: string): string {
const cached = firstLetterCache.get(text);
if (cached !== undefined) {
firstLetterCache.delete(text);
firstLetterCache.set(text, cached);
return cached;
}
let result = "";
for (const char of text) {
if (HAN_CHAR.test(char)) {
result += pinyin(char, { pattern: "first", toneType: "none" });
} else if (ASCII_ALNUM.test(char)) {
result += char.toLowerCase();
}
}
cacheFirstLetters(text, result);
return result;
}
/** True when an ASCII-letters/digits query can match `candidate` via pinyin initials. */
export function matchesPinyinInitials(candidate: string, query: string): boolean {
if (!/^[a-z0-9]+$/.test(query) || !containsHan(candidate)) return false;
return pinyinFirstLetters(candidate).startsWith(query);
}
/**
* Ordered-subsequence match of `query` against `text` (e.g. "zj" against the
* initials "zzj"). Returns the first matched index and the span covering all
* matched characters, or null when the query is not a subsequence.
*/
export function orderedSubsequenceSpan(text: string, query: string): { first: number; span: number } | null {
if (!query) return null;
let from = 0;
let first = -1;
let last = -1;
for (const char of query) {
const position = text.indexOf(char, from);
if (position < 0) return null;
if (first < 0) first = position;
last = position;
from = position + 1;
}
return { first, span: last - first + 1 };
}
/**
* Generic matcher for small pick-lists (grid condition editor, ...):
* prefix > pinyin-initials prefix > substring > pinyin-initials subsequence.
* Returns -1 for no match; scores are per-tier constants so same-tier
* candidates keep their original order. An empty query matches everything.
*/
export function pinyinAwareMatchScore(candidate: string, query: string): number {
const text = candidate.toLowerCase();
const normalized = query.trim().toLowerCase();
if (!normalized) return 0;
if (text.startsWith(normalized)) return 400;
const asciiQuery = /^[a-z0-9]+$/.test(normalized);
const han = containsHan(text);
if (asciiQuery && han && pinyinFirstLetters(text).startsWith(normalized)) return 300;
if (text.includes(normalized)) return 200;
if (asciiQuery && han) {
const subsequence = orderedSubsequenceSpan(pinyinFirstLetters(text), normalized);
if (subsequence) return 100;
}
return -1;
}
+18
View File
@@ -10,6 +10,7 @@ import { findActiveSqlStatementSpan, tokenizeSqlSemantic } from "@/lib/sql/seman
import type { SqlSemanticBuildOptions, SqlSemanticSpan } from "@/lib/sql/semantic/types";
import { DEFAULT_SQL_SNIPPETS, MANTICORESEARCH_SQL_SNIPPETS, resolveSqlSnippetBodyForDatabase } from "@/lib/sql/sqlSnippetTemplates";
import { requiresPostgresIdentifierQuote } from "@/lib/sql/sqlIdentifier";
import { containsHan, orderedSubsequenceSpan, pinyinFirstLetters } from "@/lib/common/pinyin";
export { DEFAULT_SQL_SNIPPETS, resolveSqlSnippetBodyForDatabase } from "@/lib/sql/sqlSnippetTemplates";
@@ -4289,6 +4290,8 @@ function matchesPrefix(candidate: string, prefix: string): boolean {
* Scoring tiers:
* Exact match: 3000 - len
* Initials match: 2400 + exactInitialsBonus - len
* Pinyin initials: 2300 + exactInitialsBonus - len (ASCII query vs Han candidate, e.g. "zzj" → 总租金)
* Pinyin subsequence: 1600 - penalties - len (ordered initials, e.g. "zj" → 总租金)
* Prefix match: 2000 - len
* Substring: 900 + boundaryBonus - len
* Tight fuzzy: 1500 - gapPenalty + earlyMatchBonus - len (gaps < prefix length)
@@ -4311,6 +4314,21 @@ function computeMatchScore(candidate: string, prefix: string): number {
return 2400 + exactInitialsBonus - c.length;
}
// DataGrip-style pinyin initials: an ASCII query matches the first pinyin
// letters of Han characters — as a prefix ("zz" → 总租金) or as an ordered
// subsequence ("zj" → 总租金, "j" → 总租金).
if (/^[a-z0-9]+$/.test(p) && containsHan(c)) {
const pinyinInitials = pinyinFirstLetters(c);
if (pinyinInitials.startsWith(p)) {
const exactInitialsBonus = pinyinInitials === p ? 300 : 0;
return 2300 + exactInitialsBonus - c.length;
}
const subsequence = orderedSubsequenceSpan(pinyinInitials, p);
if (subsequence) {
return 1600 - subsequence.first * 30 - (subsequence.span - p.length) * 10 - c.length;
}
}
const substringIndex = c.indexOf(p);
if (substringIndex >= 0) {
const boundaryBonus = isIdentifierBoundary(candidate, substringIndex) ? 400 : Math.max(0, 180 - substringIndex * 12);