From 4a3d63b09ad3e06f4c7dea4535841655d54ee520 Mon Sep 17 00:00:00 2001 From: Melody Ruch <396921921@qq.com> Date: Thu, 27 Aug 2026 09:21:54 +0800 Subject: [PATCH] fix(completion): mask strings and comments in clause detection --- .../lib/__tests__/sql/semantic/tokens.spec.ts | 25 ++ .../sql/sqlCompletion.context.spec.ts | 219 +++++++++++++++ apps/desktop/src/lib/sql/semantic/tokens.ts | 75 ++++- apps/desktop/src/lib/sql/sqlCompletion.ts | 256 +++++++++++++++--- 4 files changed, 530 insertions(+), 45 deletions(-) diff --git a/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts b/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts index 906180596e..e92c973f9c 100644 --- a/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts +++ b/apps/desktop/src/lib/__tests__/sql/semantic/tokens.spec.ts @@ -48,4 +48,29 @@ describe("sqlSemanticTokens", () => { expect(sql.slice(span.start, span.end)).toBe("select * from orders where id = 1"); }); + + it('treats "..." as identifier quoting by default, modeling MySQL\'s ANSI_QUOTES sql_mode', () => { + const sql = 'SELECT "col" FROM "orders"'; + const tokens = tokenizeSqlSemantic(sql, "mysql"); + + expect(tokens.some((token) => token.kind === "quoted_identifier" && unquoteSqlSemanticIdentifier(token) === "col")).toBe(true); + expect(tokens.some((token) => token.kind === "quoted_identifier" && unquoteSqlSemanticIdentifier(token) === "orders")).toBe(true); + expect(tokens.some((token) => token.kind === "string")).toBe(false); + }); + + it('opts into treating "..." as a string literal, modeling MySQL\'s default (non-ANSI_QUOTES) sql_mode', () => { + const sql = 'SELECT "col" FROM "orders"'; + const tokens = tokenizeSqlSemantic(sql, "mysql", { mysqlDoubleQuoteIsString: true }); + + expect(tokens.some((token) => token.kind === "string" && token.text === '"col"')).toBe(true); + expect(tokens.some((token) => token.kind === "string" && token.text === '"orders"')).toBe(true); + expect(tokens.some((token) => token.kind === "quoted_identifier")).toBe(false); + }); + + it('applies MySQL backslash escaping inside a mysqlDoubleQuoteIsString "..." string when opted in', () => { + const sql = 'SELECT "it\\"s ok"'; + const tokens = tokenizeSqlSemantic(sql, "mysql", { mysqlDoubleQuoteIsString: true, mysqlBackslashEscape: true }); + + expect(tokens.some((token) => token.kind === "string" && token.text === '"it\\"s ok"')).toBe(true); + }); }); diff --git a/apps/desktop/src/lib/__tests__/sql/sqlCompletion.context.spec.ts b/apps/desktop/src/lib/__tests__/sql/sqlCompletion.context.spec.ts index 379da9ef7c..a53a1ecfe6 100644 --- a/apps/desktop/src/lib/__tests__/sql/sqlCompletion.context.spec.ts +++ b/apps/desktop/src/lib/__tests__/sql/sqlCompletion.context.spec.ts @@ -730,6 +730,207 @@ describe("sqlCompletion scoped context classification", () => { expect(context.suggestRoutines).toBe(true); }); + it("classifies unqualified WHERE field input as column context for unquoted non-ASCII table/schema names", () => { + const options = { databaseType: "mysql" } as const; + + const tableOnly = getSqlCompletionContext("SELECT * FROM 用户表 WHERE ", "SELECT * FROM 用户表 WHERE ".length, options); + expect(tableOnly.contextKind).toBe("column"); + expect(tableOnly.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "用户表" })])); + expect(tableOnly.suggestColumns).toBe(true); + + const withSchema = getSqlCompletionContext("SELECT * FROM 中文库.用户表 WHERE ", "SELECT * FROM 中文库.用户表 WHERE ".length, options); + expect(withSchema.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ schema: "中文库", name: "用户表" })])); + expect(withSchema.suggestColumns).toBe(true); + }); + + it("does not treat a non-ASCII 'from' phrase inside a string literal as a referenced table", () => { + const sql = "SELECT * FROM real_table WHERE note = 'from 测试表' AND "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "测试表")).toBe(false); + }); + + it("does not treat a non-ASCII 'from' phrase inside a line comment as a referenced table", () => { + const sql = "SELECT *\nFROM real_table\n-- from 测试表\nWHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "测试表")).toBe(false); + }); + + it("does not treat a 'from' phrase inside a block comment as a referenced table", () => { + const sql = "SELECT *\nFROM real_table\n/* from ghost_table */\nWHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "ghost_table")).toBe(false); + }); + + it("does not let a quote inside a line comment desync literal/comment masking", () => { + const sql = "SELECT * FROM real_table -- it's a comment\nWHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + }); + + it("does not let a comment-like sequence inside a string literal swallow real SQL", () => { + const sql = "SELECT * FROM t1 WHERE note = '-- not a comment' AND "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "t1" })])); + }); + + it("does not match unquoted non-ASCII table names for ANSI-strict dialects", () => { + const sql = "SELECT * FROM 用户表 WHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "clickhouse" }); + expect(context.referencedTables.some((t) => t.name === "用户表")).toBe(false); + }); + + it("masks a double-quoted value right after an operator from being read as a table reference (mysql)", () => { + const sql = 'SELECT * FROM real_table WHERE note = "from 测试表" AND '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "测试表")).toBe(false); + }); + + it("treats a double-quoted FROM target as a table reference regardless of MySQL's sql_mode (position-aware masking)", () => { + // sql_mode (and therefore whether "..." is ANSI_QUOTES identifier quoting or a string literal) + // isn't observable at parse time, so "..." right after FROM/JOIN/UPDATE/etc. is always treated + // as a potential table identifier -- matching ANSI_QUOTES-enabled MySQL -- while "..." elsewhere + // (function args, CASE branches, operator-adjacent values) is still masked as a value, matching + // MySQL's actual default sql_mode. See maskSqlLiteralsAndComments's doc comment. + const sql = 'SELECT * FROM "orders" WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "orders" })])); + }); + + it("keeps resolving a double-quoted FROM target as a table when a comment sits between FROM and it (mysql)", () => { + const sql = 'SELECT * FROM /* c */ "orders" WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "orders" })])); + }); + + it("resolves a double-quoted, dotted schema-qualified table reference (mysql)", () => { + const sql = 'SELECT * FROM "db"."orders" WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ schema: "db", name: "orders" })])); + }); + + it("resolves both an unquoted and a double-quoted table across a JOIN, alongside a plain alias (mysql)", () => { + const sql = 'SELECT * FROM a JOIN "orders" o ON a.id = o.id WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "a" }), expect.objectContaining({ name: "orders", alias: "o" })])); + }); + + it("resolves a double-quoted UPDATE target as a referenced table while still masking a double-quoted value in SET (mysql)", () => { + const sql = 'UPDATE "orders" SET status = "x" WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "orders" })])); + expect(context.referencedTables.some((t) => t.name === "x")).toBe(false); + }); + + it("does not desync the token stream on a backslash-escaped quote inside a double-quoted value (mysql)", () => { + const sql = 'SELECT * FROM real_table WHERE note = "she said \\"hi\\"" '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.preferredKeywords).toEqual(expect.arrayContaining(["AND", "OR"])); + }); + + it("does not desync the token stream on a backslash-escaped quote inside a double-quoted value (hive)", () => { + // Hive isn't in MYSQL_DASH_COMMENT_DIALECTS, so "..." here is (and was, before this fix) always + // a quoted_identifier, masked only via the operator-adjacency fallback -- this test isn't about + // that masking decision, it's about whether reading the "..." span itself (now always routed + // through readQuotedString, see tokens.ts) still finds the true closing quote for a + // mysqlBackslashEscape dialect outside MYSQL_DASH_COMMENT_DIALECTS. Before this fix, hive read + // "..." with the doubled-quote-only reader regardless of mysqlBackslashEscape, so this span + // would desync at the first backslash-escaped quote and corrupt everything parsed after it. + const sql = 'SELECT * FROM real_table WHERE note = "she said \\"hi\\"" '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "hive" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.preferredKeywords).toEqual(expect.arrayContaining(["AND", "OR"])); + }); + + it("masks a double-quoted string used as a function argument from being read as a table reference (mysql)", () => { + const sql = 'SELECT CONCAT("from ghost_table", name) FROM real_table WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "ghost_table")).toBe(false); + }); + + it("masks a double-quoted CASE branch value from being read as a table reference (mysql)", () => { + const sql = 'SELECT CASE WHEN enabled THEN "from ghost_case" ELSE "ok" END FROM real_table WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "ghost_case")).toBe(false); + }); + + it("masks a double-quoted string nested inside a parenthesized expression from being read as a table reference (mysql)", () => { + const sql = 'SELECT * FROM real_table WHERE (status = "open" AND note = CONCAT("from ghost_nested", 1)) AND '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "ghost_nested")).toBe(false); + }); + + it("masks a double-quoted value right after an operator from being read as a table reference (postgres)", () => { + const sql = 'SELECT * FROM real_table WHERE note = "from 测试表" AND '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "postgres" }); + // Postgres treats "..." as a quoted identifier, not a string literal -- but a "..." span + // right after "=" is unambiguously a value position regardless of dialect, so this is masked + // the same way as mysql above, not left as a (harmless but avoidable) false-positive parse. + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "real_table" })])); + expect(context.referencedTables.some((t) => t.name === "测试表")).toBe(false); + }); + + it("keeps a double-quoted table name intact for dialects where they aren't string literals (postgres)", () => { + const sql = 'SELECT * FROM "orders" WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "postgres" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "orders", nameQuoted: true })])); + }); + + it("does not let a MySQL backslash-escaped quote desync the literal scan", () => { + const sql = "SELECT * FROM t1 WHERE note = 'it\\'s a test' UNION SELECT * FROM real_table2 WHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "t1" }), expect.objectContaining({ name: "real_table2" })])); + }); + + it("does not treat MySQL's unspaced '--' double-negation as a comment", () => { + const sql = "SELECT * FROM t1 WHERE x = 1--1 FROM real_table3 WHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "t1" }), expect.objectContaining({ name: "real_table3" })])); + }); + + it("still treats a bare '--' as a comment for non-MySQL dialects", () => { + const sql = "SELECT * FROM t1 WHERE x = 1--1 FROM real_table3 WHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "postgres" }); + expect(context.referencedTables.some((t) => t.name === "real_table3")).toBe(false); + }); + + it("does not let a doubled single-quote escape desync the literal scan", () => { + const sql = "SELECT * FROM t1 WHERE note = 'it''s a test' AND "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "t1" })])); + }); + + it("does not let a backslash-escaped quote desync the literal scan for MySQL-family dialects outside the dash-comment allowlist (hive)", () => { + const sql = "SELECT * FROM t1 WHERE note = 'it\\'s a test' UNION SELECT * FROM real_table2 WHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "hive" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ name: "t1" }), expect.objectContaining({ name: "real_table2" })])); + }); + + it("defaults to ASCII-only unquoted identifiers when databaseType is not yet known", () => { + const sql = "SELECT * FROM 用户表 WHERE "; + const context = getSqlCompletionContext(sql, sql.length, {}); + expect(context.referencedTables.some((t) => t.name === "用户表")).toBe(false); + }); + + it("does not let a special character inside a double-quoted qualified table name desync the scan (postgres)", () => { + const sql = 'SELECT * FROM "mydb"."orders" WHERE '; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "postgres" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ schema: "mydb", name: "orders" })])); + }); + + it("does not let a special character inside a backtick-quoted qualified table name desync the scan (mysql)", () => { + const sql = "SELECT * FROM `mydb`.`orders` WHERE "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.referencedTables).toEqual(expect.arrayContaining([expect.objectContaining({ schema: "mydb", name: "orders" })])); + }); + it("auto-opens column completion after WHERE whitespace before LIMIT", () => { const sql = "SELECT *\nFROM t_0001 AS t0 WHERE \nLIMIT 100;"; const cursor = "SELECT *\nFROM t_0001 AS t0 WHERE ".length; @@ -742,6 +943,24 @@ describe("sqlCompletion scoped context classification", () => { expect(shouldAutoOpenSqlCompletion(sql, cursor)).toBe(true); }); + it("does not let a trailing comment's stray 'and'/'or' text suppress the AND/OR keyword suggestion", () => { + const sql = "SELECT * FROM t WHERE id = 1 # mentions where and or\n"; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.preferredKeywords).toEqual(expect.arrayContaining(["AND", "OR"])); + }); + + it("does not let a special character inside a backtick-quoted table name break WHERE-clause keyword detection", () => { + const sql = "SELECT * FROM `my's table` t WHERE t.id = 1 "; + const context = getSqlCompletionContext(sql, sql.length, { databaseType: "mysql" }); + expect(context.preferredKeywords).toEqual(expect.arrayContaining(["AND", "OR"])); + }); + + it("does not let a trailing comment's stray 'where' text auto-open column completion with no active clause", () => { + const sql = "SELECT * FROM t_0001 t0 -- mentions where\n"; + const cursor = sql.length; + expect(shouldAutoOpenSqlCompletion(sql, cursor, { databaseType: "mysql" })).toBe(false); + }); + it("classifies CALL routine contexts", () => { const sql = "CALL usp_"; const context = getSqlCompletionContext(sql, sql.length); diff --git a/apps/desktop/src/lib/sql/semantic/tokens.ts b/apps/desktop/src/lib/sql/semantic/tokens.ts index 6b9e1c291f..aa0ce6ef78 100644 --- a/apps/desktop/src/lib/sql/semantic/tokens.ts +++ b/apps/desktop/src/lib/sql/semantic/tokens.ts @@ -30,6 +30,33 @@ function readQuoted(input: string, start: number, open: string, close: string): return { end: input.length, closed: false }; } +// Same doubled-quote escaping as readQuoted, plus (when allowBackslashEscape) MySQL-style `\x` +// escaping inside '...' strings. Backslash-escape recognition is NOT safe to apply +// unconditionally: a string that merely *ends* in a literal backslash right before the closing +// quote (e.g. a Windows path literal `'C:\Program Files\'` under Postgres, whose default +// standard_conforming_strings=on gives backslash no special meaning) would have its real closing +// quote misread as escaped, swallowing the rest of the query as a phantom string. So this is +// gated per-dialect by the caller (see tokenizeSqlSemantic's `mysqlBackslashEscape` option) rather +// than applied everywhere. +function readQuotedString(input: string, start: number, quote: string, allowBackslashEscape: boolean): { end: number; closed: boolean } { + let index = start + 1; + while (index < input.length) { + if (allowBackslashEscape && input[index] === "\\" && index + 1 < input.length) { + index += 2; + continue; + } + if (input[index] === quote) { + if (input[index + 1] === quote) { + index += 2; + continue; + } + return { end: index + 1, closed: true }; + } + index += 1; + } + return { end: input.length, closed: false }; +} + const DOLLAR_QUOTE_TAG_PATTERN = /\$[A-Za-z_0-9]*\$/y; /** @@ -43,8 +70,39 @@ export function matchDollarQuoteTag(input: string, index: number): string | unde return DOLLAR_QUOTE_TAG_PATTERN.exec(input)?.[0]; } -export function tokenizeSqlSemantic(input: string, dialectId = "mysql"): SqlSemanticToken[] { +/** + * options.mysqlDashCommentRequiresWhitespace opts into MySQL's rule that a bare "--" only starts + * a line comment when followed by whitespace/EOL, since MySQL reserves unspaced "--" for + * double-negation (e.g. `SELECT 1--1`). Defaults to false so every existing caller keeps today's + * dialect-generic "-- always starts a comment" behavior; only callers targeting a confirmed + * MySQL-family grammar should opt in. + * + * options.mysqlBackslashEscape opts into MySQL-style `\x` escaping inside '...' strings. Also + * dialect-gated (not a global default -- see readQuotedString's doc comment for why an + * unconditional default is unsafe): only dialects confirmed to actually use backslash escaping by + * convention (MySQL and its close wire-protocol/grammar clones) should opt in. + * + * By default "..." is always identifier quoting (never a string literal), matching MySQL's own + * dialect adapter (see semantic/dialect.ts), which lists '"' as one of its valid identifierQuotes + * -- this is correct for every dialect where "..." unconditionally means identifier (Postgres, + * SQL Server's ANSI mode, MySQL running with the ANSI_QUOTES sql_mode) and is what every existing + * caller other than sqlCompletion.ts's literal-masking layer wants. + * + * options.mysqlDoubleQuoteIsString flips the emitted *kind* of a "..." span to "string" instead of + * "quoted_identifier" -- it does not affect how the span itself is scanned (see readQuotedString's + * call below, gated only by mysqlBackslashEscape). Whether ANSI_QUOTES is actually enabled on the + * connected server is runtime state this tokenizer can't see, so callers that want to model MySQL's + * default (non-ANSI_QUOTES) sql_mode can opt in per call site instead of this being a global + * default; leaving it off keeps modeling the ANSI_QUOTES (identifier) interpretation. As of + * sqlCompletion.ts's maskSqlLiteralsAndComments, no caller passes this true anymore -- masking now + * decides string-vs-identifier per token position instead of per dialect (see that function's doc + * comment) -- but the option stays available as a general tokenizer capability. + */ +export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?: { mysqlDashCommentRequiresWhitespace?: boolean; mysqlBackslashEscape?: boolean; mysqlDoubleQuoteIsString?: boolean }): SqlSemanticToken[] { const tokens: SqlSemanticToken[] = []; + const mysqlDashCommentRequiresWhitespace = !!options?.mysqlDashCommentRequiresWhitespace; + const mysqlBackslashEscape = !!options?.mysqlBackslashEscape; + const mysqlDoubleQuoteIsString = !!options?.mysqlDoubleQuoteIsString; let index = 0; let depth = 0; @@ -58,7 +116,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql"): SqlSema continue; } - if (ch === "-" && next === "-") { + if (ch === "-" && next === "-" && (!mysqlDashCommentRequiresWhitespace || index + 2 >= input.length || /\s/.test(input[index + 2] ?? ""))) { index += 2; while (index < input.length && input[index] !== "\n" && input[index] !== "\r") index += 1; tokens.push(token("comment", input.slice(start, index), start, index, depth)); @@ -87,7 +145,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql"): SqlSema } if (ch === "'") { - const quoted = readQuoted(input, start, "'", "'"); + const quoted = readQuotedString(input, start, "'", mysqlBackslashEscape); index = quoted.end; tokens.push(token("string", input.slice(start, index), start, index, depth, "'", quoted.closed)); continue; @@ -104,9 +162,16 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql"): SqlSema } if (ch === '"') { - const quoted = readQuoted(input, start, '"', '"'); + // readQuotedString(...,false) is byte-for-byte identical to readQuoted for this quote pair + // (both use doubled-quote-only escaping), so routing every dialect through readQuotedString + // here is a no-op unless mysqlBackslashEscape is set -- which now matters independently of + // mysqlDoubleQuoteIsString: a MySQL-family "..." kept as a quoted_identifier (see that + // option's doc comment above) can still contain a backslash-escaped quote and must be read + // the same way a "..." string would be, or it desyncs the rest of the token stream. + const quoted = readQuotedString(input, start, '"', mysqlBackslashEscape); index = quoted.end; - tokens.push(token("quoted_identifier", input.slice(start, index), start, index, depth, '"', quoted.closed)); + const kind = mysqlDoubleQuoteIsString ? "string" : "quoted_identifier"; + tokens.push(token(kind, input.slice(start, index), start, index, depth, '"', quoted.closed)); continue; } diff --git a/apps/desktop/src/lib/sql/sqlCompletion.ts b/apps/desktop/src/lib/sql/sqlCompletion.ts index 895f847dae..2980f145e8 100644 --- a/apps/desktop/src/lib/sql/sqlCompletion.ts +++ b/apps/desktop/src/lib/sql/sqlCompletion.ts @@ -1643,7 +1643,7 @@ export function shouldAutoOpenSqlCompletion(sql: string, cursor: number, options const previousChar = sql[cursor - 1]; if (!previousChar) return false; if (/\bon\s+$/i.test(sql.slice(0, cursor))) return true; - if (isAfterJoinModifierContext(sql.slice(0, cursor))) return true; + if (isAfterJoinModifierContext(sql.slice(0, cursor), options.databaseType)) return true; if (/\bcall\s+(?:[A-Za-z_][\w$]*\.)?$/i.test(sql.slice(0, cursor))) return true; const context = getSqlCompletionContext(sql, cursor, options); if (previousChar === "(" && (context.insertTable || context.preferredValueKeywords?.length)) return true; @@ -1651,18 +1651,18 @@ export function shouldAutoOpenSqlCompletion(sql: string, cursor: number, options if (context.exclusiveTableSuggestions || context.exclusiveRoutineSuggestions || context.suggestTables) { return true; } - if (context.exclusiveColumnSuggestions || shouldAutoOpenColumnCompletion(context, sql, cursor)) return true; + if (context.exclusiveColumnSuggestions || shouldAutoOpenColumnCompletion(context, sql, cursor, options.databaseType)) return true; return /[A-Za-z_$@.]/.test(previousChar); } -function shouldAutoOpenColumnCompletion(context: SqlCompletionContext, sql: string, cursor: number): boolean { +function shouldAutoOpenColumnCompletion(context: SqlCompletionContext, sql: string, cursor: number, databaseType: DatabaseType | undefined): boolean { if (!context.suggestColumns || context.referencedTables.length === 0) return false; if (context.prefix.length > 0) return true; - return isColumnCompletionExpressionStart(sql.slice(0, cursor)); + return isColumnCompletionExpressionStart(sql.slice(0, cursor), databaseType); } -function isColumnCompletionExpressionStart(beforeCursor: string): boolean { - const cleaned = stripSqlLiterals(beforeCursor).trimEnd(); +function isColumnCompletionExpressionStart(beforeCursor: string, databaseType: DatabaseType | undefined): boolean { + const cleaned = maskSqlLiteralsAndComments(beforeCursor, databaseType).trimEnd(); if (!cleaned) return false; return /(?:\b(?:where|on|having|and|or|not|is|like|in|between|by)\b|[,(])$/i.test(cleaned); } @@ -2250,12 +2250,16 @@ export function getSqlCompletionContext(sql: string, cursor: number, options: Sq const insertInfo = detectInsertColumnListContext(beforeCursor); const updateInfo = detectUpdateCompletionContext(beforeCursor); const deleteInfo = detectDeleteCompletionContext(beforeCursor); - const oracleTableFunctionContext = detectOracleTableFunctionContext(beforeCursor); + const oracleTableFunctionContext = detectOracleTableFunctionContext(beforeCursor, options.databaseType); - const afterTableTrigger = isTableTriggerKeyword(lastWord, options) || (JOIN_MODIFIERS.has(lastWord) && isFollowedByJoin(beforeToken)) || isInTableListContext(beforeToken); - const exclusiveTableSuggestions = EXCLUSIVE_TABLE_TRIGGER_KEYWORDS.has(lastWord) || (JOIN_MODIFIERS.has(lastWord) && isFollowedByJoin(beforeToken)) || isInTableListContext(beforeToken); + // Hoisted out of the three checks below: they previously called isInTableListContext with + // identical arguments three times, which was cheap when it was regex-based but now runs a full + // tokenizeSqlSemantic pass, so the duplicate work is worth avoiding. + const inTableListContext = isInTableListContext(beforeToken, options.databaseType); + const afterTableTrigger = isTableTriggerKeyword(lastWord, options) || (JOIN_MODIFIERS.has(lastWord) && isFollowedByJoin(beforeToken)) || inTableListContext; + const exclusiveTableSuggestions = EXCLUSIVE_TABLE_TRIGGER_KEYWORDS.has(lastWord) || (JOIN_MODIFIERS.has(lastWord) && isFollowedByJoin(beforeToken)) || inTableListContext; const tableAliasAfterCursor = hasTableAliasAfterCursor(sql, cursor); - const autoAliasTableCompletions = (lastWord === "from" || lastWord === "join" || (JOIN_MODIFIERS.has(lastWord) && isFollowedByJoin(beforeToken)) || isInTableListContext(beforeToken)) && !tableAliasAfterCursor; + const autoAliasTableCompletions = (lastWord === "from" || lastWord === "join" || (JOIN_MODIFIERS.has(lastWord) && isFollowedByJoin(beforeToken)) || inTableListContext) && !tableAliasAfterCursor; const exclusiveColumnSuggestions = !!qualifier && !exclusiveTableSuggestions && !insertInfo; const activePrefixIsCte = cteDefs.some((cte) => normalizeIdentifierPart(cte.name) === normalizeIdentifierPart(prefix)); if (exclusiveTableSuggestions && prefix && !activePrefixIsCte && referencedTables.length > 1) { @@ -2273,9 +2277,9 @@ export function getSqlCompletionContext(sql: string, cursor: number, options: Sq const suggestRoutines = inCallRoutineContext || oracleTableFunctionContext || inPotentialPackageMemberContext || (!exclusiveTableSuggestions && !exclusiveColumnSuggestions && !insertInfo && !updateInfo?.inSetClause && prefix.length >= 2); const statementKind = detectStatementKind(beforeCursor || fullStatement); - const dataTypeContext = isCreateTableColumnTypeContext(beforeToken); + const dataTypeContext = isCreateTableColumnTypeContext(beforeToken, options.databaseType); const preferredValueKeywords = sqlServerDatepartCompletionValues(beforeCursor, options.databaseType); - const preferredKeywords = qualifier ? [] : preferredKeywordsForCompletion(beforeCursor, beforeToken, selectListColumnContext, exclusiveTableSuggestions, updateInfo, deleteInfo); + const preferredKeywords = qualifier ? [] : preferredKeywordsForCompletion(beforeCursor, beforeToken, selectListColumnContext, exclusiveTableSuggestions, updateInfo, deleteInfo, options.databaseType); const contextKind = detectCompletionContextKind({ qualifier, exclusiveTableSuggestions, @@ -2337,8 +2341,8 @@ function sqlServerDatepartCompletionValues(beforeCursor: string, databaseType?: return countTopLevelCommas(call.groupText) === 0 ? SQLSERVER_DATEPART_VALUES : undefined; } -function isCreateTableColumnTypeContext(beforeToken: string): boolean { - const cleaned = stripSqlLiterals(beforeToken); +function isCreateTableColumnTypeContext(beforeToken: string, databaseType: DatabaseType | undefined): boolean { + const cleaned = maskSqlLiteralsAndComments(beforeToken, databaseType); if (!/^\s*create\s+table\b/i.test(cleaned)) return false; const bodyStart = cleaned.indexOf("("); @@ -2727,33 +2731,41 @@ function detectDeleteCompletionContext(beforeCursor: string): { target?: { table return { target, afterTarget: !afterTargetText || /^[A-Za-z_][\w$]*$/i.test(afterTargetText) }; } -function detectOracleTableFunctionContext(beforeCursor: string): boolean { - const cleaned = beforeCursor.replace(/'[^']*'/g, "''").replace(/"[^"]*"/g, '""'); +function detectOracleTableFunctionContext(beforeCursor: string, databaseType: DatabaseType | undefined): boolean { + const cleaned = maskSqlLiteralsAndComments(beforeCursor, databaseType); return /\b(?:from|join)\s+table\s*\(\s*(?:(?:"[^"]+"|`[^`]+`|[A-Za-z_][\w$]*)\.){0,2}[A-Za-z_][\w$]*$/i.test(cleaned); } -function preferredKeywordsForCompletion(beforeCursor: string, beforeToken: string, selectListColumnContext: boolean, exclusiveTableSuggestions: boolean, updateInfo: ReturnType, deleteInfo: ReturnType): string[] { +function preferredKeywordsForCompletion( + beforeCursor: string, + beforeToken: string, + selectListColumnContext: boolean, + exclusiveTableSuggestions: boolean, + updateInfo: ReturnType, + deleteInfo: ReturnType, + databaseType: DatabaseType | undefined, +): string[] { const keywords: string[] = []; - if (selectListColumnContext && hasSelectListExpression(beforeCursor)) keywords.push("FROM"); - if (isAfterJoinModifierContext(beforeCursor)) keywords.push("JOIN"); - if (!exclusiveTableSuggestions && isAfterSelectBodyExpression(beforeToken)) keywords.push("LIMIT"); - if (isAfterConditionExpression(beforeToken)) keywords.push("AND", "OR"); + if (selectListColumnContext && hasSelectListExpression(beforeCursor, databaseType)) keywords.push("FROM"); + if (isAfterJoinModifierContext(beforeCursor, databaseType)) keywords.push("JOIN"); + if (!exclusiveTableSuggestions && isAfterSelectBodyExpression(beforeToken, databaseType)) keywords.push("LIMIT"); + if (isAfterConditionExpression(beforeToken, databaseType)) keywords.push("AND", "OR"); if (updateInfo?.afterTarget) keywords.push("SET"); if (updateInfo?.afterSetAssignments) keywords.push("WHERE"); if (deleteInfo?.afterTarget) keywords.push("WHERE"); return keywords; } -function hasSelectListExpression(beforeCursor: string): boolean { - const cleaned = stripSqlLiterals(beforeCursor).trimEnd(); +function hasSelectListExpression(beforeCursor: string, databaseType: DatabaseType | undefined): boolean { + const cleaned = maskSqlLiteralsAndComments(beforeCursor, databaseType).trimEnd(); const selectIndex = lastTopLevelKeywordIndex(cleaned, "select"); if (selectIndex < 0) return false; const afterSelect = cleaned.slice(selectIndex + "select".length).trim(); return !!afterSelect && !/^distinct\s*$/i.test(afterSelect); } -function isAfterSelectBodyExpression(beforeToken: string): boolean { - const cleaned = stripSqlLiterals(beforeToken).trimEnd(); +function isAfterSelectBodyExpression(beforeToken: string, databaseType: DatabaseType | undefined): boolean { + const cleaned = maskSqlLiteralsAndComments(beforeToken, databaseType).trimEnd(); if (!/^\s*(?:with\b[\s\S]*\bselect\b|select\b)/i.test(cleaned)) return false; if (!/\bfrom\b/i.test(cleaned)) return false; if (/\b(?:limit|offset|union|intersect|except)\b/i.test(cleaned)) return false; @@ -2766,16 +2778,16 @@ function isAfterSelectBodyExpression(beforeToken: string): boolean { const SELECT_BODY_INCOMPLETE_TAIL_KEYWORDS = new Set(["where", "and", "or", "not", "having", "group", "order", "by", "on", "is", "in", "like", "between", ...JOIN_MODIFIERS]); const CONDITION_INCOMPLETE_TAIL_KEYWORDS = new Set(["where", "and", "or", "not", "having", "on", "is", "in", "like", "between", "exists"]); -function isAfterConditionExpression(beforeToken: string): boolean { - const cleaned = stripSqlLiterals(beforeToken).trimEnd(); +function isAfterConditionExpression(beforeToken: string, databaseType: DatabaseType | undefined): boolean { + const cleaned = maskSqlLiteralsAndComments(beforeToken, databaseType).trimEnd(); if (!hasActiveConditionClause(cleaned)) return false; const lastKeyword = /\b([A-Za-z_][\w$]*)\s*$/.exec(cleaned)?.[1]?.toLowerCase(); if (lastKeyword && CONDITION_INCOMPLETE_TAIL_KEYWORDS.has(lastKeyword)) return false; return isExpressionTailComplete(cleaned); } -function isAfterJoinModifierContext(beforeCursor: string): boolean { - const cleaned = stripSqlLiterals(beforeCursor).trimEnd(); +function isAfterJoinModifierContext(beforeCursor: string, databaseType: DatabaseType | undefined): boolean { + const cleaned = maskSqlLiteralsAndComments(beforeCursor, databaseType).trimEnd(); const modifier = /\b([A-Za-z_][\w$]*)\s*$/.exec(cleaned)?.[1]?.toLowerCase(); if (!modifier || !JOIN_MODIFIERS.has(modifier)) return false; @@ -2822,10 +2834,6 @@ function isExpressionTailComplete(sql: string): boolean { return /(?:\)|\]|\b(?:true|false|null)\b|`[^`]+`|"[^"]*"|''|\b\d+(?:\.\d+)?\b|\b[A-Za-z_][\w$]*\b)$/i.test(trimmed); } -function stripSqlLiterals(sql: string): string { - return sql.replace(/'[^']*'/g, "''").replace(/"[^"]*"/g, '""'); -} - function lastTopLevelKeywordIndex(sql: string, keyword: string): number { const lower = sql.toLowerCase(); const target = keyword.toLowerCase(); @@ -2849,6 +2857,164 @@ function lastTopLevelKeywordIndex(sql: string, keyword: string): number { return lastIndex; } +// Dialects whose unquoted identifiers are ANSI/ASCII-only in practice; matching a Unicode +// unquoted token as a table reference there is a false-positive risk. Everything else defaults +// to Unicode support (MySQL/Postgres/SQL Server/SQLite/DuckDB and the many Chinese-vendor +// engines this product supports legitimately use non-ASCII unquoted identifiers). +const ASCII_ONLY_UNQUOTED_IDENTIFIER_DATABASES = new Set(["clickhouse", "snowflake", "bigquery", "hive", "spark", "trino", "prestosql", "impala", "db2", "teradata"]); + +// Dialects where a bare "--" needs trailing whitespace/EOL to start a comment, since MySQL +// reserves unspaced "--" for double-negation (e.g. `SELECT 1--1`). Scoped to mysql itself plus +// its wire-protocol-compatible clones confirmed to share this exact parser quirk; other +// MySQL-family members (hive, impala, spark, access, databend, tdengine, kyuubi) are left out +// since their grammars aren't confirmed to share it, and the safe default (unchanged behavior, +// bare "--" always starts a comment) is correct there. +const MYSQL_DASH_COMMENT_DIALECTS = new Set(["mysql", "doris", "starrocks"]); + +// Dialects that use MySQL-style backslash escaping (`\'`) inside '...' strings by convention. +// Deliberately a *different, wider* set than MYSQL_DASH_COMMENT_DIALECTS above: that one is +// narrowly scoped to a confirmed MySQL-only parser quirk (the "--" double-negation rule), while +// backslash escaping is a much more widely shared trait across MySQL's wire-protocol/grammar +// lineage -- confirmed by a real regression where Hive (not in MYSQL_DASH_COMMENT_DIALECTS) lost +// a table reference because its backslash-escaped string wasn't recognized. Not applied +// unconditionally to every dialect: see readQuotedString's doc comment in semantic/tokens.ts for +// why that's unsafe (a trailing backslash before a closing quote in a dialect that doesn't escape +// it, e.g. a Postgres Windows-path string literal, would misread the real closing quote as +// escaped and swallow the rest of the query). +const BACKSLASH_ESCAPE_STRING_DIALECTS = new Set(["mysql", "doris", "starrocks", "hive", "impala", "spark"]); + +// Table/schema/db unquoted-identifier continue class needs @ and # in addition to what +// SQL_IDENTIFIER_CONTINUE_CHAR covers, so splice its inner class body into a locally-built class +// instead of retyping the same Unicode escapes by hand. +const TABLE_REF_UNQUOTED_CONTINUE_SOURCE = `[@#${SQL_IDENTIFIER_CONTINUE_CHAR.source.slice(1, -1)}]`; +// The start class intentionally excludes '@' (unlike SQL_IDENTIFIER_START_CHAR, which includes +// it for a different purpose at its own call sites), so it's kept as its own small literal. +const TABLE_REF_UNQUOTED_START_SOURCE = "[_\\p{ID_Start}]"; +const TABLE_REF_UNQUOTED_IDENTIFIER_UNICODE = `${TABLE_REF_UNQUOTED_START_SOURCE}${TABLE_REF_UNQUOTED_CONTINUE_SOURCE}*`; +const TABLE_REF_UNQUOTED_IDENTIFIER_ASCII = "[A-Za-z_][\\w$@#]*"; +// Alias-group continue class is character-for-character identical to SQL_IDENTIFIER_CONTINUE_CHAR. +const TABLE_REF_ALIAS_UNICODE = `${TABLE_REF_UNQUOTED_START_SOURCE}${SQL_IDENTIFIER_CONTINUE_CHAR.source}*`; +const TABLE_REF_ALIAS_ASCII = "[A-Za-z_][\\w$]*"; + +const TABLE_REF_QUOTED_IDENTIFIER = `(?:"[^"]+"|\`[^\`]+\`|\\[[^\\]]+\\])`; + +// Keywords that introduce a table reference, deduplicated into one array so buildTableRefPattern's +// regex has a single source of truth instead of an inline copy. +const TABLE_REF_INTRODUCER_KEYWORDS = ["from", "join", "straight_join", "update", "apply"]; + +function buildTableRefPattern(unquotedIdentifier: string, aliasIdentifier: string, allowSqlServerDoubleDot: boolean): RegExp { + const identifier = `(?:${TABLE_REF_QUOTED_IDENTIFIER}|${unquotedIdentifier})`; + const qualifiedSeparator = allowSqlServerDoubleDot ? `\\.(?:${identifier}|\\.${identifier})` : `\\.${identifier}`; + return new RegExp(`\\b(?:${TABLE_REF_INTRODUCER_KEYWORDS.join("|")})\\s+(${identifier}(?:${qualifiedSeparator}){0,3})(?:\\s+(?:as\\s+)?(${aliasIdentifier}))?`, "giu"); +} + +// Precompiled once at module load (not per extractReferencedTables call) since this runs on +// effectively every keystroke through the SQL editor's completion pipeline. +const TABLE_REF_PATTERN_ASCII = buildTableRefPattern(TABLE_REF_UNQUOTED_IDENTIFIER_ASCII, TABLE_REF_ALIAS_ASCII, false); +const TABLE_REF_PATTERN_UNICODE_DEFAULT = buildTableRefPattern(TABLE_REF_UNQUOTED_IDENTIFIER_UNICODE, TABLE_REF_ALIAS_UNICODE, false); +const TABLE_REF_PATTERN_UNICODE_SQLSERVER = buildTableRefPattern(TABLE_REF_UNQUOTED_IDENTIFIER_UNICODE, TABLE_REF_ALIAS_UNICODE, true); + +/** + * Masks string literals and comments so keyword/table-reference scanning never mistakes text + * inside them for real SQL (e.g. `'from 测试表'` or `-- from 测试表`). Built on + * tokenizeSqlSemantic's own dialect-aware scan (the same one used for statement-span/lexical- + * context detection elsewhere in this file) rather than a hand-rolled lexer, so backtick- and + * bracket-quoted identifier spans are skipped correctly for free. + * + * `"..."` is dialect-ambiguous: Postgres/SQL Server/generic always treat it as identifier + * quoting, while MySQL's own dialect adapter (see semantic/dialect.ts) also lists '"' as a valid + * identifierQuotes entry -- but only because MySQL *can* be run with the ANSI_QUOTES sql_mode. + * Under MySQL's actual default sql_mode (ANSI_QUOTES disabled) it's a plain string literal instead, + * with the exact same rules as `'...'`, everywhere in a statement: inside function arguments + * (`CONCAT("from ghost_table", name)`), CASE branches (`WHEN x THEN "from ghost_case"`), nested + * expressions, anywhere. There's no way for this code to observe the connected server's *actual* + * runtime sql_mode, so rather than guessing a dialect-wide default (and being wrong for whichever + * sql_mode isn't guessed), this resolves the ambiguity by *position* instead, for dialects in + * MYSQL_DASH_COMMENT_DIALECTS (MySQL itself plus confirmed sql_mode-parity wire-protocol clones -- + * the same scope already used above for MySQL's other sql_mode-governed lexical quirks): a `"..."` + * span immediately in table-reference position -- right after one of + * TABLE_REF_INTRODUCER_KEYWORDS (`from`/`join`/`straight_join`/`update`/`apply`, exactly what + * extractReferencedTables's own regex looks for below), or a `.`-qualified continuation of one + * (`"db"."orders"`) -- is left unmasked as a potential identifier, matching ANSI_QUOTES-enabled + * MySQL. Every other `"..."` position (function args, CASE branches, operator-adjacent values, and + * everywhere else) is masked as a value, matching MySQL's actual default sql_mode. This is correct + * under *both* sql_modes at once: a real ANSI_QUOTES table name in `FROM "orders"` still resolves, + * while the ghost-table false positives this function exists to fix (`CONCAT("from ghost_table", + * ...)`, `CASE ... THEN "from ghost_case"`) stay masked regardless of the connected sql_mode. + * + * For every other dialect, where `"..."` unconditionally means identifier (Postgres, SQL Server's + * ANSI mode, generic), a `"..."` span right after an operator token (`=`, `<`, `<>`, `!=`, ...) is + * still masked as if it were a value: that position is unambiguous regardless of dialect (`col = + * "literal"` is never a quoted identifier there), so leaving it unmasked would misdetect e.g. + * Postgres's `WHERE note = "from ghost"` as a table reference. Every other position (right after + * FROM/JOIN, a `.` qualifier, a function's `(`, etc.) is left unmasked as a potential identifier for + * these dialects, same as before. + * + * The masked replacement doesn't need to preserve length or map back to offsets in `sql` -- + * callers only read captured regex groups (or do their own keyword search) off the masked copy. + * It DOES need to preserve a non-whitespace trailing marker for string/value tokens though: most + * callers of this helper trim trailing whitespace and then look at the last character to decide + * whether an expression is "complete" (e.g. `isAfterConditionExpression`'s + * `isExpressionTailComplete`). Collapsing a trailing value to a single space would get eaten by + * that trim, exposing whatever came before it (e.g. the `=` in `WHERE x = "value"`) as if it were + * the real tail, wrongly reporting an in-progress/incomplete expression -- confirmed to wrongly + * suppress the AND/OR keyword suggestion after a perfectly complete condition. So string/masked + * value tokens are replaced with their own quote character doubled (`''` / `""`), mirroring the + * pre-existing convention this helper's predecessor (the regex-based `stripSqlLiterals`) already + * used for exactly this reason. Comments don't represent a value and are still collapsed to a + * single space -- getting trimmed away and exposing the real preceding tail is the correct + * behavior there (already covered by existing tests). + */ +function maskSqlLiteralsAndComments(sql: string, databaseType?: DatabaseType): string { + const dialectId = resolveSqlDialectId({ databaseType }); + const mysqlFamily = !!databaseType && MYSQL_DASH_COMMENT_DIALECTS.has(databaseType); + const mysqlBackslashEscape = !!databaseType && BACKSLASH_ESCAPE_STRING_DIALECTS.has(databaseType); + // "..." always tokenizes as quoted_identifier here (never forced to "string") -- MySQL-family + // string-vs-identifier ambiguity is resolved below by token position instead, so the underlying + // scan doesn't need to guess a dialect-wide default. See this function's doc comment above. + const tokens = tokenizeSqlSemantic(sql, dialectId, { mysqlDashCommentRequiresWhitespace: mysqlFamily, mysqlBackslashEscape }); + + let out = ""; + let cursor = 0; + // Tracks "the previous non-comment token was an operator" for the unconditional-identifier + // dialects' operator-adjacency fallback below -- a comment sitting between an operator and its + // value (e.g. `= /* x */ "from ghost"`) must not reset this, or the value right after the + // comment escapes masking. + let afterValueOperator = false; + // Tracks whether the *next* token sits in table-reference position (right after + // TABLE_REF_INTRODUCER_KEYWORDS, or a `.`-qualified continuation of one) for the MySQL-family + // position-based rule below. Survives comment tokens the same way afterValueOperator does -- + // `FROM /* c */ "orders"` must still resolve "orders" as a table. + let tableRefState: "none" | "expectSegment" | "afterSegment" = "none"; + for (const t of tokens) { + out += sql.slice(cursor, t.span.start); + const isDoubleQuotedIdentifier = t.kind === "quoted_identifier" && t.quote === '"'; + const isTableRefSegment = tableRefState === "expectSegment" && (t.kind === "quoted_identifier" || t.kind === "word"); + const maskAsValue = t.kind === "string" || (isDoubleQuotedIdentifier && (afterValueOperator || (mysqlFamily && !isTableRefSegment))); + if (t.kind === "comment") { + out += " "; + } else if (maskAsValue) { + out += (t.quote ?? '"').repeat(2); + } else { + out += sql.slice(t.span.start, t.span.end); + } + cursor = t.span.end; + afterValueOperator = t.kind === "operator" || (t.kind === "comment" && afterValueOperator); + if (t.kind !== "comment") { + tableRefState = + tableRefState === "expectSegment" && (t.kind === "quoted_identifier" || t.kind === "word") + ? "afterSegment" + : tableRefState === "afterSegment" && t.kind === "punctuation" && t.text === "." + ? "expectSegment" + : t.kind === "word" && TABLE_REF_INTRODUCER_KEYWORDS.includes(t.normalized) + ? "expectSegment" + : "none"; + } + } + out += sql.slice(cursor); + return out; +} + function extractReferencedTables(sql: string, databaseType?: DatabaseType): SqlCompletionReferencedTable[] { // Keywords that should NOT be treated as table aliases const ALIAS_BLACKLIST = new Set([ @@ -2956,13 +3122,23 @@ function extractReferencedTables(sql: string, databaseType?: DatabaseType): SqlC ]); // STRAIGHT_JOIN is a standalone MySQL table introducer, not a modifier followed by JOIN. - const unquotedIdentifier = databaseType === "sqlserver" ? "[_\\p{ID_Start}][$@#_\\u200c\\u200d\\p{ID_Continue}]*" : "[A-Za-z_][\\w$@#]*"; - const identifier = `(?:"[^"]+"|\`[^\`]+\`|\\[[^\\]]+\\]|${unquotedIdentifier})`; - const qualifiedSeparator = databaseType === "sqlserver" ? `\\.(?:${identifier}|\\.${identifier})` : `\\.${identifier}`; - const pattern = new RegExp(`\\b(?:from|join|straight_join|update|apply)\\s+(${identifier}(?:${qualifiedSeparator}){0,3})(?:\\s+(?:as\\s+)?([A-Za-z_][\\w$]*))?`, databaseType === "sqlserver" ? "giu" : "gi"); + // Unquoted identifiers may contain non-ASCII letters (e.g. Chinese/Japanese/Korean database, + // schema and table names are valid unquoted MySQL/Postgres/etc. identifiers) for dialects that + // actually support it — ANSI-strict dialects (ClickHouse, Snowflake, BigQuery, Hive/Spark/ + // Trino/Presto/Impala, Db2, Teradata) keep the ASCII-only shape to avoid false-positive matches. + // An unresolved databaseType (e.g. a scratch editor before a connection is picked) also defaults + // to ASCII-only, matching this function's behavior before dialect-aware Unicode matching existed. + const pattern = !databaseType || ASCII_ONLY_UNQUOTED_IDENTIFIER_DATABASES.has(databaseType) ? TABLE_REF_PATTERN_ASCII : databaseType === "sqlserver" ? TABLE_REF_PATTERN_UNICODE_SQLSERVER : TABLE_REF_PATTERN_UNICODE_DEFAULT; + // These are shared module-level RegExp objects (built once for performance), so reset + // lastIndex before each scan rather than relying on a fresh object per call. + pattern.lastIndex = 0; + // Scan a masked copy so string literals and comments can never be mistaken for a real + // FROM/JOIN table reference; double-quoted table names survive masking (see + // maskSqlLiteralsAndComments), so the returned table info built from `match` is unaffected. + const scanSql = maskSqlLiteralsAndComments(sql, databaseType); const referenced: SqlCompletionReferencedTable[] = []; let match: RegExpExecArray | null; - while ((match = pattern.exec(sql)) !== null) { + while ((match = pattern.exec(scanSql)) !== null) { const rawName = match[1]; const alias = match[2]; if (alias && ALIAS_BLACKLIST.has(alias.toLowerCase())) { @@ -4092,9 +4268,9 @@ function isFollowedByJoin(beforeToken: string): boolean { return second === "join" || JOIN_MODIFIERS.has(second ?? ""); } -function isInTableListContext(beforeToken: string): boolean { +function isInTableListContext(beforeToken: string, databaseType: DatabaseType | undefined): boolean { if (isInOrderOrGroupByContext(beforeToken)) return false; - const cleaned = activeQueryBlockSql(stripSqlLiterals(beforeToken).trimEnd()); + const cleaned = activeQueryBlockSql(maskSqlLiteralsAndComments(beforeToken, databaseType).trimEnd()); if (!/,\s*$/.test(cleaned)) return false; // Only commas in the active top-level table segment should continue table completion.