diff --git a/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts b/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts index e92c973f9c..dcec531aec 100644 --- a/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts +++ b/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts @@ -13,6 +13,12 @@ describe("sqlSemanticTokens", () => { expect(tokens.some((token) => token.kind === "string" && token.text === "'it''s ok'")).toBe(true); }); + it("tokenizes Unicode identifier prefixes as complete word tokens", () => { + const tokens = tokenizeSqlSemantic("SELECT 名称 客户名称 𠀀字段", "sqlserver"); + + expect(tokens.filter((token) => token.kind === "word").map((token) => token.text)).toEqual(["SELECT", "名称", "客户名称", "𠀀字段"]); + }); + it("marks comments and string literals as suppressed contexts", () => { const sql = "select * from users -- user."; const tokens = tokenizeSqlSemantic(sql); diff --git a/apps/desktop/src/lib/sql/semantic/tokens.ts b/apps/desktop/src/lib/sql/semantic/tokens.ts index aa0ce6ef78..38ca9e0b5d 100644 --- a/apps/desktop/src/lib/sql/semantic/tokens.ts +++ b/apps/desktop/src/lib/sql/semantic/tokens.ts @@ -1,7 +1,12 @@ import type { SqlSemanticSpan, SqlSemanticToken } from "@/lib/sql/semantic/types"; -const WORD_START = /[A-Za-z_@$#]/; -const WORD_PART = /[A-Za-z0-9_@$#]/; +const WORD_START = /[A-Za-z_@$#\p{ID_Start}]/u; +const WORD_PART = /[A-Za-z0-9_@$#\p{ID_Continue}]/u; + +function characterAt(input: string, index: number): string { + const codePoint = input.codePointAt(index); + return codePoint === undefined ? "" : String.fromCodePoint(codePoint); +} function token(kind: SqlSemanticToken["kind"], text: string, start: number, end: number, depth: number, quote?: string, closed?: boolean): SqlSemanticToken { return { @@ -108,7 +113,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options? while (index < input.length) { const start = index; - const ch = input[index] ?? ""; + const ch = characterAt(input, index); const next = input[index + 1] ?? ""; if (/\s/.test(ch)) { @@ -190,8 +195,12 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options? } if (ch === ":" || ch === "?") { - index += 1; - while (index < input.length && WORD_PART.test(input[index] ?? "")) index += 1; + index += ch.length; + while (index < input.length) { + const part = characterAt(input, index); + if (!WORD_PART.test(part)) break; + index += part.length; + } tokens.push(token("parameter", input.slice(start, index), start, index, depth)); continue; } @@ -204,8 +213,12 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options? } if (WORD_START.test(ch)) { - index += 1; - while (index < input.length && WORD_PART.test(input[index] ?? "") && !(dialectId === "postgres" && input[index] === "#")) index += 1; + index += ch.length; + while (index < input.length) { + const part = characterAt(input, index); + if (!WORD_PART.test(part) || (dialectId === "postgres" && part === "#")) break; + index += part.length; + } tokens.push(token("word", input.slice(start, index), start, index, depth)); continue; } @@ -218,7 +231,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options? continue; } - index += 1; + index += ch.length; tokens.push(token("operator", ch, start, index, depth)); } diff --git a/packages/app-tests/sqlCompletion.test.ts b/packages/app-tests/sqlCompletion.test.ts index 8775ccdff2..0173e6979b 100644 --- a/packages/app-tests/sqlCompletion.test.ts +++ b/packages/app-tests/sqlCompletion.test.ts @@ -969,6 +969,52 @@ test("replaces typed unquoted prefixes through semantic SQL completion", () => { } }); +test("replaces typed Unicode prefixes through semantic SQL Server completion", () => { + for (const fixture of [ + { prefix: "名", column: "名称" }, + { prefix: "客户", column: "客户名称" }, + ] as const) { + const sql = `select * from test where ${fixture.prefix}`; + const cursor = sql.length; + const options = { databaseType: "sqlserver", dialect: "sqlserver" } as const; + const semanticModel = buildSqlSemanticModel(sql, cursor, options); + const context = sqlCompletionContextFromSemantic(semanticModel, getSqlCompletionContext(sql, cursor, options)); + const items = buildSqlCompletionItemsFromContext(context, { + tables: [{ name: "test", schema: "dbo", type: "table" }], + columnsByTable: new Map([["test", [{ name: fixture.column, table: "test", schema: "dbo" }]]]), + ...options, + }); + const replacement = prepareSqlCompletionReplacement(sql, cursor, context, items); + const column = replacement.items.find((item) => item.type === "column" && item.label === fixture.column); + + assert.ok(column, fixture.prefix); + assert.deepEqual(semanticModel.cursorIntent.replacementRange, { start: cursor - fixture.prefix.length, end: cursor }, fixture.prefix); + assert.deepEqual(context.replacementRange, { start: cursor - fixture.prefix.length, end: cursor }, fixture.prefix); + assert.equal(replacement.from, cursor - fixture.prefix.length, fixture.prefix); + assert.equal(`${sql.slice(0, replacement.from)}${column.apply ?? column.label}${sql.slice(cursor)}`, `select * from test where ${fixture.column}`, fixture.prefix); + } +}); + +test("replaces a Unicode prefix inside an open SQL Server bracket identifier", () => { + const sql = "select * from test where [名"; + const cursor = sql.length; + const options = { databaseType: "sqlserver", dialect: "sqlserver" } as const; + const semanticModel = buildSqlSemanticModel(sql, cursor, options); + const context = sqlCompletionContextFromSemantic(semanticModel, getSqlCompletionContext(sql, cursor, options)); + const items = buildSqlCompletionItemsFromContext(context, { + tables: [{ name: "test", schema: "dbo", type: "table" }], + columnsByTable: new Map([["test", [{ name: "名称", table: "test", schema: "dbo" }]]]), + ...options, + }); + const replacement = prepareSqlCompletionReplacement(sql, cursor, context, items); + const column = replacement.items.find((item) => item.type === "column" && item.label === "名称"); + + assert.ok(column); + assert.deepEqual(context.replacementRange, { start: cursor - 2, end: cursor }); + assert.equal(replacement.from, cursor - 2); + assert.equal(`${sql.slice(0, replacement.from)}${column.apply ?? column.label}${sql.slice(cursor)}`, "select * from test where [名称]"); +}); + test("suggests same-prefix tables while editing double-quoted Oracle-family identifiers", () => { for (const databaseType of ["oracle", "dameng"] as const) { const markedSql = 'SELECT * FROM "Fo|"';