fix(completion): replace unicode identifier prefixes correctly

Closes #7891

apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts

@@ -13,6 +13,12 @@ describe("sqlSemanticTokens", () => {
     expect(tokens.some((token) => token.kind === "string" && token.text === "'it''s ok'")).toBe(true);
   });
 
+  it("tokenizes Unicode identifier prefixes as complete word tokens", () => {
+    const tokens = tokenizeSqlSemantic("SELECT 名称 客户名称 𠀀字段", "sqlserver");
+
+    expect(tokens.filter((token) => token.kind === "word").map((token) => token.text)).toEqual(["SELECT", "名称", "客户名称", "𠀀字段"]);
+  });
+
   it("marks comments and string literals as suppressed contexts", () => {
     const sql = "select * from users -- user.";
     const tokens = tokenizeSqlSemantic(sql);

apps/desktop/src/lib/sql/semantic/tokens.ts

@@ -1,7 +1,12 @@
 import type { SqlSemanticSpan, SqlSemanticToken } from "@/lib/sql/semantic/types";
 
-const WORD_START = /[A-Za-z_@$#]/;
-const WORD_PART = /[A-Za-z0-9_@$#]/;
+const WORD_START = /[A-Za-z_@$#\p{ID_Start}]/u;
+const WORD_PART = /[A-Za-z0-9_@$#\p{ID_Continue}]/u;
+
+function characterAt(input: string, index: number): string {
+  const codePoint = input.codePointAt(index);
+  return codePoint === undefined ? "" : String.fromCodePoint(codePoint);
+}
 
 function token(kind: SqlSemanticToken["kind"], text: string, start: number, end: number, depth: number, quote?: string, closed?: boolean): SqlSemanticToken {
   return {
@@ -108,7 +113,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
 
   while (index < input.length) {
     const start = index;
-    const ch = input[index] ?? "";
+    const ch = characterAt(input, index);
     const next = input[index + 1] ?? "";
 
     if (/\s/.test(ch)) {
@@ -190,8 +195,12 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
     }
 
     if (ch === ":" || ch === "?") {
-      index += 1;
-      while (index < input.length && WORD_PART.test(input[index] ?? "")) index += 1;
+      index += ch.length;
+      while (index < input.length) {
+        const part = characterAt(input, index);
+        if (!WORD_PART.test(part)) break;
+        index += part.length;
+      }
       tokens.push(token("parameter", input.slice(start, index), start, index, depth));
       continue;
     }
@@ -204,8 +213,12 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
     }
 
     if (WORD_START.test(ch)) {
-      index += 1;
-      while (index < input.length && WORD_PART.test(input[index] ?? "") && !(dialectId === "postgres" && input[index] === "#")) index += 1;
+      index += ch.length;
+      while (index < input.length) {
+        const part = characterAt(input, index);
+        if (!WORD_PART.test(part) || (dialectId === "postgres" && part === "#")) break;
+        index += part.length;
+      }
       tokens.push(token("word", input.slice(start, index), start, index, depth));
       continue;
     }
@@ -218,7 +231,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?
       continue;
     }
 
-    index += 1;
+    index += ch.length;
     tokens.push(token("operator", ch, start, index, depth));
   }