fix(completion): replace unicode identifier prefixes correctly

Closes #7891
This commit is contained in:
jearhe authored and GitHub committed 2026-09-04 15:07:30 +08:00
1 parent 7b0a8cc111
commit 65abc74590
3 files changed
+73 -8

No files matched your search

@@ -13,6 +13,12 @@ describe("sqlSemanticTokens", () => {
expect(tokens.some((token) => token.kind === "string" && token.text === "'it''s ok'")).toBe(true); expect(tokens.some((token) => token.kind === "string" && token.text === "'it''s ok'")).toBe(true);
}); });
it("tokenizes Unicode identifier prefixes as complete word tokens", () => {
const tokens = tokenizeSqlSemantic("SELECT 名称 客户名称 𠀀字段", "sqlserver");
expect(tokens.filter((token) => token.kind === "word").map((token) => token.text)).toEqual(["SELECT", "名称", "客户名称", "𠀀字段"]);
});
it("marks comments and string literals as suppressed contexts", () => { it("marks comments and string literals as suppressed contexts", () => {
const sql = "select * from users -- user."; const sql = "select * from users -- user.";
const tokens = tokenizeSqlSemantic(sql); const tokens = tokenizeSqlSemantic(sql);
+21 -8
View File
@@ -1,7 +1,12 @@
import type { SqlSemanticSpan, SqlSemanticToken } from "@/lib/sql/semantic/types"; import type { SqlSemanticSpan, SqlSemanticToken } from "@/lib/sql/semantic/types";
const WORD_START = /[A-Za-z_@$#]/; const WORD_START = /[A-Za-z_@$#\p{ID_Start}]/u;
const WORD_PART = /[A-Za-z0-9_@$#]/; const WORD_PART = /[A-Za-z0-9_@$#\p{ID_Continue}]/u;
function characterAt(input: string, index: number): string {
const codePoint = input.codePointAt(index);
return codePoint === undefined ? "" : String.fromCodePoint(codePoint);
}
function token(kind: SqlSemanticToken["kind"], text: string, start: number, end: number, depth: number, quote?: string, closed?: boolean): SqlSemanticToken { function token(kind: SqlSemanticToken["kind"], text: string, start: number, end: number, depth: number, quote?: string, closed?: boolean): SqlSemanticToken {
return { return {
@@ -108,7 +113,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
while (index < input.length) { while (index < input.length) {
const start = index; const start = index;
const ch = input[index] ?? ""; const ch = characterAt(input, index);
const next = input[index + 1] ?? ""; const next = input[index + 1] ?? "";
if (/\s/.test(ch)) { if (/\s/.test(ch)) {
@@ -190,8 +195,12 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
} }
if (ch === ":" || ch === "?") { if (ch === ":" || ch === "?") {
index += 1; index += ch.length;
while (index < input.length && WORD_PART.test(input[index] ?? "")) index += 1; while (index < input.length) {
const part = characterAt(input, index);
if (!WORD_PART.test(part)) break;
index += part.length;
}
tokens.push(token("parameter", input.slice(start, index), start, index, depth)); tokens.push(token("parameter", input.slice(start, index), start, index, depth));
continue; continue;
} }
@@ -204,8 +213,12 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
} }
if (WORD_START.test(ch)) { if (WORD_START.test(ch)) {
index += 1; index += ch.length;
while (index < input.length && WORD_PART.test(input[index] ?? "") && !(dialectId === "postgres" && input[index] === "#")) index += 1; while (index < input.length) {
const part = characterAt(input, index);
if (!WORD_PART.test(part) || (dialectId === "postgres" && part === "#")) break;
index += part.length;
}
tokens.push(token("word", input.slice(start, index), start, index, depth)); tokens.push(token("word", input.slice(start, index), start, index, depth));
continue; continue;
} }
@@ -218,7 +231,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
continue; continue;
} }
index += 1; index += ch.length;
tokens.push(token("operator", ch, start, index, depth)); tokens.push(token("operator", ch, start, index, depth));
} }
+46
View File
@@ -969,6 +969,52 @@ test("replaces typed unquoted prefixes through semantic SQL completion", () => {
} }
}); });
test("replaces typed Unicode prefixes through semantic SQL Server completion", () => {
for (const fixture of [
{ prefix: "名", column: "名称" },
{ prefix: "客户", column: "客户名称" },
] as const) {
const sql = `select * from test where ${fixture.prefix}`;
const cursor = sql.length;
const options = { databaseType: "sqlserver", dialect: "sqlserver" } as const;
const semanticModel = buildSqlSemanticModel(sql, cursor, options);
const context = sqlCompletionContextFromSemantic(semanticModel, getSqlCompletionContext(sql, cursor, options));
const items = buildSqlCompletionItemsFromContext(context, {
tables: [{ name: "test", schema: "dbo", type: "table" }],
columnsByTable: new Map([["test", [{ name: fixture.column, table: "test", schema: "dbo" }]]]),
...options,
});
const replacement = prepareSqlCompletionReplacement(sql, cursor, context, items);
const column = replacement.items.find((item) => item.type === "column" && item.label === fixture.column);
assert.ok(column, fixture.prefix);
assert.deepEqual(semanticModel.cursorIntent.replacementRange, { start: cursor - fixture.prefix.length, end: cursor }, fixture.prefix);
assert.deepEqual(context.replacementRange, { start: cursor - fixture.prefix.length, end: cursor }, fixture.prefix);
assert.equal(replacement.from, cursor - fixture.prefix.length, fixture.prefix);
assert.equal(`${sql.slice(0, replacement.from)}${column.apply ?? column.label}${sql.slice(cursor)}`, `select * from test where ${fixture.column}`, fixture.prefix);
}
});
test("replaces a Unicode prefix inside an open SQL Server bracket identifier", () => {
const sql = "select * from test where [名";
const cursor = sql.length;
const options = { databaseType: "sqlserver", dialect: "sqlserver" } as const;
const semanticModel = buildSqlSemanticModel(sql, cursor, options);
const context = sqlCompletionContextFromSemantic(semanticModel, getSqlCompletionContext(sql, cursor, options));
const items = buildSqlCompletionItemsFromContext(context, {
tables: [{ name: "test", schema: "dbo", type: "table" }],
columnsByTable: new Map([["test", [{ name: "名称", table: "test", schema: "dbo" }]]]),
...options,
});
const replacement = prepareSqlCompletionReplacement(sql, cursor, context, items);
const column = replacement.items.find((item) => item.type === "column" && item.label === "名称");
assert.ok(column);
assert.deepEqual(context.replacementRange, { start: cursor - 2, end: cursor });
assert.equal(replacement.from, cursor - 2);
assert.equal(`${sql.slice(0, replacement.from)}${column.apply ?? column.label}${sql.slice(cursor)}`, "select * from test where [名称]");
});
test("suggests same-prefix tables while editing double-quoted Oracle-family identifiers", () => { test("suggests same-prefix tables while editing double-quoted Oracle-family identifiers", () => {
for (const databaseType of ["oracle", "dameng"] as const) { for (const databaseType of ["oracle", "dameng"] as const) {
const markedSql = 'SELECT * FROM "Fo|"'; const markedSql = 'SELECT * FROM "Fo|"';