fix(editor): bound SQL completion to active statement
This commit is contained in:
Melody Ruch authored and GitHub committed
1 parent
8271ba2a71
commit
41ffc969a1
12 files changed
+917
-210
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { findActiveSqlStatementSpan, tokenizeSqlSemantic, tokenIsIdentifier, unquoteSqlSemanticIdentifier } from "@/lib/sql/semantic/tokens";
|
||||
import { findActiveSqlStatementSpan, matchDollarQuoteTag, tokenizeSqlSemantic, tokenIsIdentifier, unquoteSqlSemanticIdentifier } from "@/lib/sql/semantic/tokens";
|
||||
import type { SqlSemanticSpan, SqlSemanticToken } from "@/lib/sql/semantic/types";
|
||||
|
||||
export interface InsertValueHint {
|
||||
@@ -228,188 +228,248 @@ function shiftClause(clause: InsertValuesClause, offset: number): InsertValuesCl
|
||||
}
|
||||
|
||||
/**
|
||||
* Expand [from, to) to the nearest top-level statement window (quote/comment aware).
|
||||
* Scans at most LOOKBACK bytes before `from` and LOOKAHEAD after `to` so large scripts
|
||||
* do not pay O(document) on every keystroke.
|
||||
* Expand [from, to) to the nearest top-level statement window (quote/comment/dollar-quote aware).
|
||||
* Scans at most LOOKBACK bytes before `from` and LOOKAHEAD after `to` so large scripts do not pay
|
||||
* O(document) on every keystroke in the common case (many statements, none of them huge).
|
||||
*
|
||||
* This is a bounded, best-effort heuristic operating on a plain string -- it has no notion of a
|
||||
* live document or incremental parse state. It's the correct (only) choice for this module's
|
||||
* pure-string callers (parseInsertValuesClauses* and their tests), which have no live editor to
|
||||
* consult. Callers that *do* have a live CodeMirror `EditorState` should prefer
|
||||
* `sqlSyntaxTreeWindow.ts`'s `resolveStatementWindowFromSyntaxTree`, which resolves the same
|
||||
* question against CodeMirror's own incrementally-parsed SQL syntax tree instead of re-deriving
|
||||
* lexical state locally -- see that module's doc comment for why it's provably correct for every
|
||||
* boundary class below except one narrow, disclosed gap (dollar-quoted bodies). sqlCompletion.ts's
|
||||
* `getSqlLexicalContext`/`activeSqlCompletionStatementSpan` already do this, falling back to the
|
||||
* scanner below only when there's no live state to consult.
|
||||
*
|
||||
* The backward scan starts at an arbitrary document offset, so its lexical state (in a string? in
|
||||
* a comment? inside a dollar-quoted body? how deep in nested parens?) cannot simply be assumed
|
||||
* clean -- if the cursor sits more than LOOKBACK bytes into such a construct, that assumption is
|
||||
* wrong and the scan can find a bogus statement boundary (e.g. a ';' inside an unclosed string or
|
||||
* a dollar-quoted function body). `resolveStatementStart` mitigates this with a best-effort check
|
||||
* instead of blind trust: it widens the backward window until two consecutive scans agree on the
|
||||
* same boundary, or until it reaches the true start of the document (index 0), where the
|
||||
* clean-state assumption is always correct.
|
||||
*
|
||||
* IMPORTANT: "two scans agree" is a heuristic, not a proof, and it has a known false-positive.
|
||||
* When the content between the two scan-start points is lexically inert (no quote/comment/paren/
|
||||
* dollar-quote characters -- e.g. a long run of uniform "SELECT n;" statements), both scans
|
||||
* converge on the identical answer regardless of whether the true state entering that content is
|
||||
* actually clean or is deep inside some unclosed construct opened much earlier. Confirmed
|
||||
* reproduction: an unterminated string containing many repeated ';' characters, or a large run of
|
||||
* "SELECT n;" statements inside a PostgreSQL `$$ ... $$` function body, both fool the check. This
|
||||
* is not fixable with a smarter local heuristic -- the two situations are byte-for-byte
|
||||
* indistinguishable from a purely local scan; making the check stricter (e.g. requiring a
|
||||
* lexically-significant character in the widened margin) would also reject the common, entirely
|
||||
* legitimate case of many small statements with no punctuation, which is the primary case this
|
||||
* windowing scheme exists to keep fast. This is a deliberate, documented tradeoff of the
|
||||
* pure-string fallback, not an oversight -- see `packages/app-tests/insertValueHints.test.ts`'s
|
||||
* test documenting it, and `sqlSyntaxTreeWindow.ts` for the provably-correct path used whenever a
|
||||
* live `EditorState` is available (i.e. for every real user keystroke in the editor).
|
||||
*
|
||||
* `dialectId` defaults to "mysql" (matching semantic/tokens.ts's tokenizeSqlSemantic default) and
|
||||
* only affects how `#` is treated -- see stepLexState. Callers that know the active SQL dialect
|
||||
* should pass it through so window boundaries agree with tokenizeSqlSemantic's own tokenization.
|
||||
*/
|
||||
const STATEMENT_LOOKBACK = 32 * 1024;
|
||||
const STATEMENT_LOOKAHEAD = 32 * 1024;
|
||||
const MAX_STATEMENT_START_WIDENINGS = 12; // 32KB * 2^12 = 128MB before we give up widening
|
||||
|
||||
export function expandToSqlStatementWindow(sql: string, from: number, to: number): TextRange {
|
||||
interface LexState {
|
||||
depth: number;
|
||||
inLineComment: boolean;
|
||||
inBlockComment: boolean;
|
||||
quote: string | null;
|
||||
dollarTag: string | null;
|
||||
}
|
||||
|
||||
function isLexStateClean(state: LexState): boolean {
|
||||
return state.depth === 0 && !state.inLineComment && !state.inBlockComment && !state.quote && !state.dollarTag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Advances `state` past the token starting at `sql[index]` and returns the next index.
|
||||
* `dialectId` only affects `#`: MySQL-family dialects treat it as a line-comment starter, but
|
||||
* PostgreSQL (and dialects that don't special-case it) use `#`/`#>`/`#>>`/`#-` as operators --
|
||||
* see the matching dialect check in semantic/tokens.ts's tokenizeSqlSemantic.
|
||||
*/
|
||||
function stepLexState(sql: string, index: number, state: LexState, dialectId: string): number {
|
||||
const ch = sql[index] ?? "";
|
||||
const next = sql[index + 1] ?? "";
|
||||
|
||||
if (state.inLineComment) {
|
||||
if (ch === "\n" || ch === "\r") state.inLineComment = false;
|
||||
return index + 1;
|
||||
}
|
||||
if (state.inBlockComment) {
|
||||
if (ch === "*" && next === "/") {
|
||||
state.inBlockComment = false;
|
||||
return index + 2;
|
||||
}
|
||||
return index + 1;
|
||||
}
|
||||
if (state.dollarTag) {
|
||||
if (sql.startsWith(state.dollarTag, index)) {
|
||||
const tagLength = state.dollarTag.length;
|
||||
state.dollarTag = null;
|
||||
return index + tagLength;
|
||||
}
|
||||
return index + 1;
|
||||
}
|
||||
if (state.quote) {
|
||||
// Backslash-escaping only applies to '...' and "...", matching getSqlLexicalContext's
|
||||
// scanner in sqlCompletion.ts -- backtick/bracket identifiers don't use it.
|
||||
if ((state.quote === "'" || state.quote === '"') && ch === "\\" && next) return index + 2;
|
||||
const closeCh = state.quote === "]" ? "]" : state.quote;
|
||||
if (ch === closeCh) {
|
||||
if (next === closeCh) return index + 2;
|
||||
state.quote = null;
|
||||
}
|
||||
return index + 1;
|
||||
}
|
||||
|
||||
if (ch === "-" && next === "-") {
|
||||
state.inLineComment = true;
|
||||
return index + 2;
|
||||
}
|
||||
if (ch === "#" && dialectId === "mysql") {
|
||||
state.inLineComment = true;
|
||||
return index + 1;
|
||||
}
|
||||
if (ch === "/" && next === "*") {
|
||||
state.inBlockComment = true;
|
||||
return index + 2;
|
||||
}
|
||||
if (ch === "'" || ch === '"' || ch === "`") {
|
||||
state.quote = ch;
|
||||
return index + 1;
|
||||
}
|
||||
if (ch === "[") {
|
||||
state.quote = "]";
|
||||
return index + 1;
|
||||
}
|
||||
if (ch === "$") {
|
||||
const marker = matchDollarQuoteTag(sql, index);
|
||||
if (marker) {
|
||||
state.dollarTag = marker;
|
||||
return index + marker.length;
|
||||
}
|
||||
return index + 1;
|
||||
}
|
||||
if (ch === "(") {
|
||||
state.depth += 1;
|
||||
return index + 1;
|
||||
}
|
||||
if (ch === ")") {
|
||||
state.depth = Math.max(0, state.depth - 1);
|
||||
return index + 1;
|
||||
}
|
||||
return index + 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans sql[from, hardStop) tracking the last top-level ';' strictly before `searchEndFrom` (when
|
||||
* `trackStart` is set) and the first top-level ';' at or after `searchEndFrom`. Starts from a copy
|
||||
* of `initialState` when given (for resuming a previous scan without re-walking the prefix it
|
||||
* already covered), otherwise from a known-clean state.
|
||||
*/
|
||||
function scanStatementBoundaries(sql: string, from: number, searchEndFrom: number, hardStop: number, trackStart: boolean, dialectId: string, initialState?: LexState): { start: number; startFound: boolean; end: number; endFound: boolean; finalState: LexState } {
|
||||
const state: LexState = initialState ? { ...initialState } : { depth: 0, inLineComment: false, inBlockComment: false, quote: null, dollarTag: null };
|
||||
let start = from;
|
||||
let startFound = false;
|
||||
let end = hardStop;
|
||||
let endFound = false;
|
||||
let index = from;
|
||||
|
||||
while (index < hardStop) {
|
||||
if (sql[index] === ";" && isLexStateClean(state)) {
|
||||
if (index >= searchEndFrom) {
|
||||
end = index;
|
||||
endFound = true;
|
||||
break;
|
||||
}
|
||||
if (trackStart) {
|
||||
start = index + 1;
|
||||
startFound = true;
|
||||
}
|
||||
}
|
||||
index = stepLexState(sql, index, state, dialectId);
|
||||
}
|
||||
|
||||
return { start, startFound, end, endFound, finalState: state };
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the nearest top-level statement boundary at or before `pos`. Best-effort, not a proof --
|
||||
* see the "IMPORTANT" note in the module comment above for the known false-positive case.
|
||||
*/
|
||||
function resolveStatementStart(sql: string, pos: number, dialectId: string): number {
|
||||
let lookback = STATEMENT_LOOKBACK;
|
||||
let previousCandidate: number | null = null;
|
||||
|
||||
for (let attempt = 0; attempt < MAX_STATEMENT_START_WIDENINGS; attempt += 1) {
|
||||
const scanFrom = Math.max(0, pos - lookback);
|
||||
const result = scanStatementBoundaries(sql, scanFrom, pos, pos, true, dialectId);
|
||||
if (scanFrom === 0) return result.start;
|
||||
if (result.startFound && result.start === previousCandidate) return result.start;
|
||||
previousCandidate = result.startFound ? result.start : null;
|
||||
lookback *= 2;
|
||||
}
|
||||
// Widening did not converge; ground at the true document start, where clean state is guaranteed.
|
||||
return scanStatementBoundaries(sql, 0, pos, pos, true, dialectId).start;
|
||||
}
|
||||
|
||||
function computeSqlStatementWindow(sql: string, from: number, to: number, dialectId: string): TextRange {
|
||||
const safeFrom = Math.max(0, Math.min(from, sql.length));
|
||||
const safeTo = Math.max(safeFrom, Math.min(to, sql.length));
|
||||
const scanFrom = Math.max(0, safeFrom - STATEMENT_LOOKBACK);
|
||||
const scanTo = Math.min(sql.length, safeTo + STATEMENT_LOOKAHEAD);
|
||||
const slice = scanFrom === 0 && scanTo === sql.length ? sql : sql.slice(scanFrom, scanTo);
|
||||
const localFrom = safeFrom - scanFrom;
|
||||
const localTo = safeTo - scanFrom;
|
||||
const start = findStatementStart(slice, localFrom) + scanFrom;
|
||||
const end = findStatementEnd(slice, Math.max(localFrom, localTo)) + scanFrom;
|
||||
const trimmed = trimSpan(sql, start, Math.min(end, scanTo));
|
||||
const start = resolveStatementStart(sql, safeFrom, dialectId);
|
||||
|
||||
// The forward end-search has no "unproven starting state" problem like resolveStatementStart --
|
||||
// it scans continuously from `start`, which is already verified clean, so it never needs to
|
||||
// second-guess itself. It just needs enough room: a single statement larger than one
|
||||
// STATEMENT_LOOKAHEAD (e.g. a huge multi-row INSERT) would otherwise be silently truncated at
|
||||
// an arbitrary hardStop instead of found. So on a miss, keep widening the lookahead and resume
|
||||
// scanning from where the previous pass stopped (carrying its LexState forward) rather than
|
||||
// rescanning from `start` -- common statements still cost one bounded scan; only a statement
|
||||
// that's actually larger than the lookahead pays for (and gets) a correct, wider one.
|
||||
let lookahead = STATEMENT_LOOKAHEAD;
|
||||
let hardStop = Math.min(sql.length, safeTo + lookahead);
|
||||
let result = scanStatementBoundaries(sql, start, safeTo, hardStop, false, dialectId);
|
||||
while (!result.endFound && hardStop < sql.length) {
|
||||
const resumeFrom = hardStop;
|
||||
lookahead *= 2;
|
||||
hardStop = Math.min(sql.length, safeTo + lookahead);
|
||||
result = scanStatementBoundaries(sql, resumeFrom, safeTo, hardStop, false, dialectId, result.finalState);
|
||||
}
|
||||
|
||||
// result.end is always <= hardStop by construction (scanStatementBoundaries only reports an
|
||||
// index it actually visited, or defaults to hardStop itself).
|
||||
const trimmed = trimSpan(sql, start, result.end);
|
||||
return { from: trimmed.start, to: trimmed.end };
|
||||
}
|
||||
|
||||
function findStatementStart(sql: string, pos: number): number {
|
||||
let index = 0;
|
||||
let start = 0;
|
||||
let depth = 0;
|
||||
let inLineComment = false;
|
||||
let inBlockComment = false;
|
||||
let quote: string | null = null;
|
||||
|
||||
while (index < pos) {
|
||||
const ch = sql[index] ?? "";
|
||||
const next = sql[index + 1] ?? "";
|
||||
|
||||
if (inLineComment) {
|
||||
if (ch === "\n") inLineComment = false;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (inBlockComment) {
|
||||
if (ch === "*" && next === "/") {
|
||||
inBlockComment = false;
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (quote) {
|
||||
if (ch === quote) {
|
||||
if (next === quote) {
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
quote = null;
|
||||
}
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === "-" && next === "-") {
|
||||
inLineComment = true;
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
if (ch === "#") {
|
||||
inLineComment = true;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === "/" && next === "*") {
|
||||
inBlockComment = true;
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
if (ch === "'" || ch === '"' || ch === "`") {
|
||||
quote = ch;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === "[") {
|
||||
quote = "]";
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === "(") {
|
||||
depth += 1;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === ")") {
|
||||
depth = Math.max(0, depth - 1);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === ";" && depth === 0) {
|
||||
start = index + 1;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
return start;
|
||||
interface StatementWindowMemo {
|
||||
sql: string;
|
||||
from: number;
|
||||
to: number;
|
||||
dialectId: string;
|
||||
result: TextRange;
|
||||
}
|
||||
|
||||
function findStatementEnd(sql: string, pos: number): number {
|
||||
let index = pos;
|
||||
let depth = 0;
|
||||
let inLineComment = false;
|
||||
let inBlockComment = false;
|
||||
let quote: string | null = null;
|
||||
// Single-entry cache: within one keystroke, several completion helpers (suppressed-context check,
|
||||
// active-statement-span lookup, ...) call this with the exact same (sql, from, to, dialectId).
|
||||
// String equality here is content equality (JS `===` on strings), so this is safe for a pure
|
||||
// function -- a cache hit is guaranteed to be the same answer, not a staleness risk.
|
||||
let statementWindowMemo: StatementWindowMemo | null = null;
|
||||
|
||||
while (index < sql.length) {
|
||||
const ch = sql[index] ?? "";
|
||||
const next = sql[index + 1] ?? "";
|
||||
|
||||
if (inLineComment) {
|
||||
if (ch === "\n") inLineComment = false;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (inBlockComment) {
|
||||
if (ch === "*" && next === "/") {
|
||||
inBlockComment = false;
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (quote) {
|
||||
if ((quote === "]" && ch === "]") || (quote !== "]" && ch === quote)) {
|
||||
if (next === (quote === "]" ? "]" : quote)) {
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
quote = null;
|
||||
}
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === "-" && next === "-") {
|
||||
inLineComment = true;
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
if (ch === "#") {
|
||||
inLineComment = true;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === "/" && next === "*") {
|
||||
inBlockComment = true;
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
if (ch === "'" || ch === '"' || ch === "`") {
|
||||
quote = ch;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === "[") {
|
||||
quote = "]";
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === "(") {
|
||||
depth += 1;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === ")") {
|
||||
depth = Math.max(0, depth - 1);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (ch === ";" && depth === 0) {
|
||||
return index;
|
||||
}
|
||||
index += 1;
|
||||
export function expandToSqlStatementWindow(sql: string, from: number, to: number, dialectId = "mysql"): TextRange {
|
||||
if (statementWindowMemo && statementWindowMemo.sql === sql && statementWindowMemo.from === from && statementWindowMemo.to === to && statementWindowMemo.dialectId === dialectId) {
|
||||
return statementWindowMemo.result;
|
||||
}
|
||||
return sql.length;
|
||||
const result = computeSqlStatementWindow(sql, from, to, dialectId);
|
||||
statementWindowMemo = { sql, from, to, dialectId, result };
|
||||
return result;
|
||||
}
|
||||
|
||||
function mergeTextRanges(ranges: readonly TextRange[]): TextRange[] {
|
||||
@@ -444,9 +504,9 @@ export function parseInsertValuesClausesInRanges(sql: string, ranges: readonly T
|
||||
}
|
||||
|
||||
/** Parse all INSERT ... VALUES clauses in `sql` (multi-statement aware). Prefer ranged parsing for editors. */
|
||||
export function parseInsertValuesClauses(sql: string): InsertValuesClause[] {
|
||||
export function parseInsertValuesClauses(sql: string, dialectId = "mysql"): InsertValuesClause[] {
|
||||
if (!sql.trim()) return [];
|
||||
const allTokens = tokenizeSqlSemantic(sql);
|
||||
const allTokens = tokenizeSqlSemantic(sql, dialectId);
|
||||
const clauses: InsertValuesClause[] = [];
|
||||
for (const { span, tokens } of statementTokenGroups(sql, allTokens)) {
|
||||
const clause = parseInsertClause(tokens, span);
|
||||
|
||||
@@ -194,3 +194,15 @@ export function sqlSemanticDialectFor(options: { databaseType?: DatabaseType; di
|
||||
return SQL_SEMANTIC_DIALECTS.generic;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the dialect id ("mysql", "postgres", ...) used to keep lexical/statement-boundary
|
||||
* scanning in sync with tokenizeSqlSemantic's own dialect-aware tokenization -- most importantly,
|
||||
* whether '#' starts a line comment (MySQL) or is an operator (PostgreSQL: #, #>, #>>, #-).
|
||||
* Defaults to "mysql" when no dialect info is available (rather than sqlSemanticDialectFor's own
|
||||
* "generic" default), matching tokenizeSqlSemantic's default and preserving the behavior callers
|
||||
* had before dialect-aware scanning existed.
|
||||
*/
|
||||
export function resolveSqlDialectId(options: { databaseType?: DatabaseType; dialect?: "mysql" | "postgres" | "sqlserver" | "clickhouse" }): string {
|
||||
return options.databaseType || options.dialect ? sqlSemanticDialectFor(options).id : "mysql";
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { sqlSemanticDialectFor, type SqlSemanticDialectAdapter } from "@/lib/sql/semantic/dialect";
|
||||
import { findActiveSqlStatementSpan, isSuppressedSqlSemanticContext, tokenIsIdentifier, tokenizeSqlSemantic, unquoteSqlSemanticIdentifier } from "@/lib/sql/semantic/tokens";
|
||||
import { resolveSqlStatementWindow } from "@/lib/sql/sqlSyntaxTreeWindow";
|
||||
import type {
|
||||
SqlSemanticBuildOptions,
|
||||
SqlSemanticClauseSpans,
|
||||
@@ -785,8 +786,16 @@ function buildScope(statement: SqlSemanticStatement, rowSources: SqlSemanticRowS
|
||||
export function buildSqlSemanticModel(sql: string, cursor: number, options: SqlSemanticBuildOptions = {}): SqlSemanticModel {
|
||||
const safeCursor = Math.max(0, Math.min(cursor, sql.length));
|
||||
const dialect = sqlSemanticDialectFor(options);
|
||||
const allTokens = tokenizeSqlSemantic(sql, dialect.id);
|
||||
const statementSpan = findActiveSqlStatementSpan(sql, allTokens, safeCursor);
|
||||
const window = resolveSqlStatementWindow(sql, safeCursor, options.editorState, dialect.id);
|
||||
const windowSql = sql.slice(window.from, window.to);
|
||||
const windowCursor = safeCursor - window.from;
|
||||
const localTokens = tokenizeSqlSemantic(windowSql, dialect.id);
|
||||
const localStatementSpan = findActiveSqlStatementSpan(windowSql, localTokens, windowCursor);
|
||||
const allTokens = localTokens.map((token) => ({
|
||||
...token,
|
||||
span: { start: token.span.start + window.from, end: token.span.end + window.from },
|
||||
}));
|
||||
const statementSpan = { start: localStatementSpan.start + window.from, end: localStatementSpan.end + window.from };
|
||||
const tokens = significantTokens(allTokens.filter((item) => item.span.end > statementSpan.start && item.span.start < statementSpan.end));
|
||||
const kind = statementKind(tokens);
|
||||
const statement: SqlSemanticStatement = {
|
||||
|
||||
@@ -29,6 +29,19 @@ function readQuoted(input: string, start: number, open: string, close: string):
|
||||
return input.length;
|
||||
}
|
||||
|
||||
const DOLLAR_QUOTE_TAG_PATTERN = /\$[A-Za-z_0-9]*\$/y;
|
||||
|
||||
/**
|
||||
* Matches a PostgreSQL dollar-quote tag (`$$` or `$tag$`) starting exactly at `index`, without
|
||||
* allocating a substring: a sticky ("y") regex anchors its match to `lastIndex` and scans the
|
||||
* original string in place, unlike `pattern.exec(input.slice(index))` which copies everything
|
||||
* from `index` to the end of `input` on every call.
|
||||
*/
|
||||
export function matchDollarQuoteTag(input: string, index: number): string | undefined {
|
||||
DOLLAR_QUOTE_TAG_PATTERN.lastIndex = index;
|
||||
return DOLLAR_QUOTE_TAG_PATTERN.exec(input)?.[0];
|
||||
}
|
||||
|
||||
export function tokenizeSqlSemantic(input: string, dialectId = "mysql"): SqlSemanticToken[] {
|
||||
const tokens: SqlSemanticToken[] = [];
|
||||
let index = 0;
|
||||
@@ -79,7 +92,7 @@ export function tokenizeSqlSemantic(input: string, dialectId = "mysql"): SqlSema
|
||||
}
|
||||
|
||||
if (ch === "$") {
|
||||
const marker = /^\$[A-Za-z_0-9]*\$/.exec(input.slice(start))?.[0];
|
||||
const marker = matchDollarQuoteTag(input, start);
|
||||
if (marker) {
|
||||
const closing = input.indexOf(marker, start + marker.length);
|
||||
index = closing < 0 ? input.length : closing + marker.length;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { EditorState } from "@codemirror/state";
|
||||
import type { DatabaseType } from "@/types/database";
|
||||
|
||||
export interface SqlSemanticSpan {
|
||||
@@ -128,4 +129,12 @@ export interface SqlSemanticModel {
|
||||
export interface SqlSemanticBuildOptions {
|
||||
databaseType?: DatabaseType;
|
||||
dialect?: "mysql" | "postgres" | "sqlserver";
|
||||
/**
|
||||
* Live CodeMirror state for the document being edited, when the caller has one (i.e. this is a
|
||||
* real editor completion request, not a pure-string test/utility call). When present and its
|
||||
* doc length matches the `sql` string being scanned, boundary-sensitive lookups prefer resolving
|
||||
* against its already-incrementally-parsed syntax tree (see sqlSyntaxTreeWindow.ts) instead of
|
||||
* the bounded heuristic scanner in insertValueHints.ts, which remains the fallback.
|
||||
*/
|
||||
editorState?: EditorState;
|
||||
}
|
||||
@@ -5,9 +5,11 @@ import { CLOUDFLARE_D1_COMMON_FUNCTION_NAMES } from "@/lib/sql/cloudflareD1";
|
||||
import { searchClickHouseFunctions } from "@/lib/sql/clickhouse/functionRegistry";
|
||||
import type { ClickHouseFunctionDefinition, ClickHouseFunctionKind } from "@/lib/sql/clickhouse/functionTypes";
|
||||
import type { SqlObjectNavigationType } from "@/lib/sql/sqlNavigation";
|
||||
import { sqlSemanticDialectFor } from "@/lib/sql/semantic/dialect";
|
||||
import { findActiveSqlStatementSpan, tokenizeSqlSemantic } from "@/lib/sql/semantic/tokens";
|
||||
import { resolveSqlDialectId } from "@/lib/sql/semantic/dialect";
|
||||
import { findActiveSqlStatementSpan, matchDollarQuoteTag, tokenizeSqlSemantic } from "@/lib/sql/semantic/tokens";
|
||||
import { expandToSqlStatementWindow } from "@/lib/sql/insertValueHints";
|
||||
import type { SqlSemanticBuildOptions, SqlSemanticSpan } from "@/lib/sql/semantic/types";
|
||||
import { isEditorStatePlausibleFor, resolveLexicalLeafFromSyntaxTree, resolveSqlStatementWindow } from "@/lib/sql/sqlSyntaxTreeWindow";
|
||||
import { DEFAULT_SQL_SNIPPETS, MANTICORESEARCH_SQL_SNIPPETS, resolveSqlSnippetBodyForDatabase } from "@/lib/sql/sqlSnippetTemplates";
|
||||
import { requiresMysqlIdentifierQuote, requiresPostgresIdentifierQuote } from "@/lib/sql/sqlIdentifier";
|
||||
import { identifierMatchScore, matchesIdentifierSearch } from "@/lib/sql/identifierSearch";
|
||||
@@ -1366,7 +1368,7 @@ export function buildSqlCompletionItems(
|
||||
autoAliasTables?: boolean;
|
||||
},
|
||||
): SqlCompletionItem[] {
|
||||
if (isSqlCompletionSuppressedContext(sql, cursor)) return [];
|
||||
if (isSqlCompletionSuppressedContext(sql, cursor, input)) return [];
|
||||
const context = getSqlCompletionContext(sql, cursor, input);
|
||||
return buildSqlCompletionItemsFromContext(context, input);
|
||||
}
|
||||
@@ -1521,7 +1523,7 @@ class SqlCompletionProvider {
|
||||
|
||||
export function shouldAutoOpenSqlCompletion(sql: string, cursor: number, options: SqlSemanticBuildOptions = {}): boolean {
|
||||
if (getPostgresSequenceLiteralCompletionContext(sql, cursor, options.databaseType)) return true;
|
||||
if (isSqlCompletionSuppressedContext(sql, cursor)) return false;
|
||||
if (isSqlCompletionSuppressedContext(sql, cursor, options)) return false;
|
||||
const previousChar = sql[cursor - 1];
|
||||
if (!previousChar) return false;
|
||||
if (/\bon\s+$/i.test(sql.slice(0, cursor))) return true;
|
||||
@@ -1549,11 +1551,20 @@ function isColumnCompletionExpressionStart(beforeCursor: string): boolean {
|
||||
return /(?:\b(?:where|on|having|and|or|not|is|like|in|between|by)\b|[,(])$/i.test(cleaned);
|
||||
}
|
||||
|
||||
export function isSqlCompletionSuppressedContext(sql: string, cursor: number): boolean {
|
||||
const context = getSqlLexicalContext(sql, cursor);
|
||||
export function isSqlCompletionSuppressedContext(sql: string, cursor: number, options: SqlSemanticBuildOptions = {}): boolean {
|
||||
const context = getSqlLexicalContext(sql, cursor, options);
|
||||
return context.inLineComment || context.inBlockComment || context.inStringLiteral;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the statement window backing a boundary-sensitive lookup, preferring CodeMirror's own
|
||||
* incrementally-parsed syntax tree (see sqlSyntaxTreeWindow.ts) when the caller has a live,
|
||||
* plausibly-matching `EditorState`, and falling back to the bounded heuristic scanner in
|
||||
* insertValueHints.ts otherwise -- e.g. pure-string test/utility callers, or a live editor whose
|
||||
* background parse hasn't caught up to `cursor` yet. The tree path never blocks and is never worse
|
||||
* than the scanner; see sqlSyntaxTreeWindow.ts's doc comment for the one known, disclosed gap
|
||||
* (dollar-quoted bodies, guarded rather than silently trusted).
|
||||
*/
|
||||
export function getPostgresSequenceLiteralCompletionContext(sql: string, cursor: number, databaseType?: DatabaseType): PostgresSequenceLiteralCompletionContext | null {
|
||||
if (databaseType !== "postgres") return null;
|
||||
const position = Math.max(0, Math.min(cursor, sql.length));
|
||||
@@ -1757,25 +1768,37 @@ function parsePostgresRegclassPrefix(raw: string): { nameStart: number; name: st
|
||||
};
|
||||
}
|
||||
|
||||
export function isSqlStringLiteralContext(sql: string, cursor: number): boolean {
|
||||
return getSqlLexicalContext(sql, cursor).inStringLiteral;
|
||||
export function isSqlStringLiteralContext(sql: string, cursor: number, options: SqlSemanticBuildOptions = {}): boolean {
|
||||
return getSqlLexicalContext(sql, cursor, options).inStringLiteral;
|
||||
}
|
||||
|
||||
export function isSqlCommentContext(sql: string, cursor: number): boolean {
|
||||
const context = getSqlLexicalContext(sql, cursor);
|
||||
export function isSqlCommentContext(sql: string, cursor: number, options: SqlSemanticBuildOptions = {}): boolean {
|
||||
const context = getSqlLexicalContext(sql, cursor, options);
|
||||
return context.inLineComment || context.inBlockComment;
|
||||
}
|
||||
|
||||
function getSqlLexicalContext(sql: string, cursor: number): { inLineComment: boolean; inBlockComment: boolean; inStringLiteral: boolean } {
|
||||
function getSqlLexicalContext(sql: string, cursor: number, options: SqlSemanticBuildOptions): { inLineComment: boolean; inBlockComment: boolean; inStringLiteral: boolean } {
|
||||
const end = Math.max(0, Math.min(cursor, sql.length));
|
||||
const editorState = options.editorState;
|
||||
if (editorState && isEditorStatePlausibleFor(editorState, sql)) {
|
||||
const treeContext = resolveLexicalLeafFromSyntaxTree(editorState, end);
|
||||
if (treeContext) return treeContext;
|
||||
}
|
||||
const dialectId = resolveSqlDialectId(options);
|
||||
// Bound the backward scan so huge documents do not pay O(document) on every keystroke (this
|
||||
// runs on every completion request). expandToSqlStatementWindow's `from` is verified rather
|
||||
// than assumed clean (it widens its own backward scan until the boundary is confirmed, or
|
||||
// grounds at the true document start), so scanning forward from it here is safe.
|
||||
const start = expandToSqlStatementWindow(sql, end, end, dialectId).from;
|
||||
let inSingleQuote = false;
|
||||
let inDoubleQuote = false;
|
||||
let inBacktick = false;
|
||||
let inBracket = false;
|
||||
let inLineComment = false;
|
||||
let inBlockComment = false;
|
||||
let dollarTag: string | null = null;
|
||||
|
||||
for (let index = 0; index < end; index += 1) {
|
||||
for (let index = start; index < end; index += 1) {
|
||||
const ch = sql[index] ?? "";
|
||||
const next = sql[index + 1] ?? "";
|
||||
|
||||
@@ -1791,6 +1814,13 @@ function getSqlLexicalContext(sql: string, cursor: number): { inLineComment: boo
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dollarTag) {
|
||||
if (sql.startsWith(dollarTag, index)) {
|
||||
index += dollarTag.length - 1;
|
||||
dollarTag = null;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (inSingleQuote) {
|
||||
if (ch === "\\" && next) {
|
||||
index += 1;
|
||||
@@ -1823,7 +1853,7 @@ function getSqlLexicalContext(sql: string, cursor: number): { inLineComment: boo
|
||||
if (ch === "-" && next === "-") {
|
||||
inLineComment = true;
|
||||
index += 1;
|
||||
} else if (ch === "#") {
|
||||
} else if (ch === "#" && dialectId === "mysql") {
|
||||
inLineComment = true;
|
||||
} else if (ch === "/" && next === "*") {
|
||||
inBlockComment = true;
|
||||
@@ -1836,16 +1866,22 @@ function getSqlLexicalContext(sql: string, cursor: number): { inLineComment: boo
|
||||
inBacktick = true;
|
||||
} else if (ch === "[") {
|
||||
inBracket = true;
|
||||
} else if (ch === "$") {
|
||||
const marker = matchDollarQuoteTag(sql, index);
|
||||
if (marker) {
|
||||
dollarTag = marker;
|
||||
index += marker.length - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only single-quoted text is a value literal here. Double quotes, backticks,
|
||||
// and brackets delimit identifiers in common SQL dialects, so they must not
|
||||
// Only single-quoted and dollar-quoted text is a value literal here. Double quotes,
|
||||
// backticks, and brackets delimit identifiers in common SQL dialects, so they must not
|
||||
// suppress identifier completion.
|
||||
return {
|
||||
inLineComment,
|
||||
inBlockComment,
|
||||
inStringLiteral: inSingleQuote,
|
||||
inStringLiteral: inSingleQuote || dollarTag !== null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1860,11 +1896,21 @@ export function isSqlLikeCompletionStatement(sql: string, cursor: number, option
|
||||
|
||||
function activeSqlCompletionStatementSpan(sql: string, cursor: number, options: SqlSemanticBuildOptions): SqlSemanticSpan {
|
||||
const safeCursor = Math.max(0, Math.min(cursor, sql.length));
|
||||
const dialectId = options.databaseType || options.dialect ? sqlSemanticDialectFor(options).id : "mysql";
|
||||
const tokens = tokenizeSqlSemantic(sql, dialectId);
|
||||
const statementSpan = findActiveSqlStatementSpan(sql, tokens, safeCursor);
|
||||
const dialectId = resolveSqlDialectId(options);
|
||||
// Tokenizing the full document on every keystroke is O(document) and, with
|
||||
// autocompletion's activateOnTyping, runs on every keystroke. Bound tokenization
|
||||
// to the statement window around the cursor (preferring the live syntax tree, see
|
||||
// resolveSqlStatementWindow) and translate spans back. Pass dialectId through so the
|
||||
// window boundary agrees with tokenizeSqlSemantic below on dialect-sensitive lexing
|
||||
// (e.g. '#' as a MySQL comment vs a PostgreSQL operator) when falling back to the scanner.
|
||||
const window = resolveSqlStatementWindow(sql, safeCursor, options.editorState, dialectId);
|
||||
const windowSql = sql.slice(window.from, window.to);
|
||||
const windowCursor = safeCursor - window.from;
|
||||
const tokens = tokenizeSqlSemantic(windowSql, dialectId);
|
||||
const statementSpan = findActiveSqlStatementSpan(windowSql, tokens, windowCursor);
|
||||
const firstStatementToken = tokens.find((token) => token.kind !== "comment" && token.span.end > statementSpan.start && token.span.start < statementSpan.end);
|
||||
return firstStatementToken ? { start: firstStatementToken.span.start, end: statementSpan.end } : statementSpan;
|
||||
const result = firstStatementToken ? { start: firstStatementToken.span.start, end: statementSpan.end } : statementSpan;
|
||||
return { start: result.start + window.from, end: result.end + window.from };
|
||||
}
|
||||
|
||||
function currentSqlLikeLineBlockSpan(sql: string, cursor: number, activeStatementSpan: SqlSemanticSpan): SqlSemanticSpan | null {
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { syntaxTree, syntaxTreeAvailable } from "@codemirror/language";
|
||||
import type { EditorState, Text } from "@codemirror/state";
|
||||
import { matchDollarQuoteTag } from "@/lib/sql/semantic/tokens";
|
||||
import { expandToSqlStatementWindow, type TextRange } from "@/lib/sql/insertValueHints";
|
||||
|
||||
// `@lezer/common` (the package that declares `SyntaxNode`) is a transitive dependency, not a
|
||||
// direct one, so pnpm's strict linking doesn't expose it for a direct type-only import here.
|
||||
// Derive the type structurally from `syntaxTree`'s own return type instead of importing it.
|
||||
type SyntaxNode = ReturnType<ReturnType<typeof syntaxTree>["resolve"]>;
|
||||
|
||||
/**
|
||||
* Statement-window resolution backed by CodeMirror's own incrementally-parsed SQL syntax tree
|
||||
* (mounted on the live editor via `@codemirror/lang-sql`'s `sql()` extension -- see
|
||||
* QueryEditor.vue's `buildSqlLanguageExtension`), used as the provably-correct alternative to the
|
||||
* bounded heuristic scanner in insertValueHints.ts for callers that have a live `EditorState`.
|
||||
*
|
||||
* Never forces a parse (`ensureSyntaxTree` is intentionally not used): both functions here only
|
||||
* consult whatever CodeMirror has already parsed in the background (`syntaxTree`/
|
||||
* `syntaxTreeAvailable`). If the parse hasn't caught up to `cursor` yet (e.g. immediately after
|
||||
* opening a huge file), or the tree can't be trusted for the reason below, they return `null` and
|
||||
* the caller falls back to the existing scanner -- i.e. never worse than today's baseline.
|
||||
*
|
||||
* Known gap, disclosed rather than hidden: this app disables `doubleDollarQuotedStrings` in its
|
||||
* CodeMirror SQL dialect config (see codemirrorSqlDialect.ts, commit a067ee8d7 / issue #788) so
|
||||
* that PL/pgSQL `$$ ... $$` function bodies highlight as nested SQL instead of one opaque string.
|
||||
* That means the tree does NOT recognize a dollar-quoted body as a single unit -- it parses `;`
|
||||
* characters inside it as real statement boundaries. `resolveStatementWindowFromSyntaxTree` guards
|
||||
* against this with a bounded backward scan (see DOLLAR_QUOTE_DISTRUST_LOOKBACK) for unmatched
|
||||
* dollar-quote markers and returns `null` (safe fallback) rather than silently returning a wrong
|
||||
* window when it finds one. This guard is itself bounded, not a proof: a dollar-quote tag opened
|
||||
* further back than the lookback budget is invisible to it, same class of limitation as the old
|
||||
* scanner's known-imperfect case, just narrower in practice (needs the resolved statement's start
|
||||
* to also be more than the lookback distance from the real opening tag). That narrower residual
|
||||
* gap is called out explicitly to the maintainer rather than presented as fully solved -- see the
|
||||
* PR reply plan. Every *other* boundary class raised across three review rounds (unterminated
|
||||
* strings, comments, backslash escapes, CR-only comments, oversized statements) is fixed outright
|
||||
* by delegating to the grammar instead of re-deriving it.
|
||||
*/
|
||||
|
||||
// Matches insertValueHints.ts's STATEMENT_LOOKBACK -- same budget, same tradeoff, used here only
|
||||
// as a safety check rather than as the primary boundary-finding mechanism.
|
||||
const DOLLAR_QUOTE_DISTRUST_LOOKBACK = 32 * 1024;
|
||||
|
||||
export function isEditorStatePlausibleFor(state: EditorState, sql: string): boolean {
|
||||
return state.doc.length === sql.length;
|
||||
}
|
||||
|
||||
function hasUnresolvedDollarQuoteBefore(doc: Text, from: number, lookback: number): boolean {
|
||||
const scanStart = Math.max(0, from - lookback);
|
||||
const slice = doc.sliceString(scanStart, from);
|
||||
let openTag: string | null = null;
|
||||
for (let index = 0; index < slice.length; ) {
|
||||
if (slice[index] === "$") {
|
||||
const marker = matchDollarQuoteTag(slice, index);
|
||||
if (marker) {
|
||||
if (openTag === null) openTag = marker;
|
||||
else if (marker === openTag) openTag = null;
|
||||
index += marker.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
// PostgreSQL closes a dollar-quoted body only with the exact opening delimiter. A different
|
||||
// tagged marker inside the body is ordinary text and must not accidentally balance the guard.
|
||||
return openTag !== null;
|
||||
}
|
||||
|
||||
export function resolveSqlStatementWindow(sql: string, cursor: number, editorState: EditorState | undefined, dialectId: string): TextRange {
|
||||
const safeCursor = Math.max(0, Math.min(cursor, sql.length));
|
||||
if (editorState && isEditorStatePlausibleFor(editorState, sql)) {
|
||||
const window = resolveStatementWindowFromSyntaxTree(editorState, safeCursor);
|
||||
if (window) return window;
|
||||
}
|
||||
return expandToSqlStatementWindow(sql, safeCursor, safeCursor, dialectId);
|
||||
}
|
||||
|
||||
/** Finds the nearest enclosing `Statement` node for `cursor`, resolving the trailing-whitespace
|
||||
* gap case (cursor after the last real token, before the next statement or end of document) via
|
||||
* `childBefore`, since `tree.resolve` on either side of such a gap lands on the root `Script`
|
||||
* node rather than a real node. */
|
||||
function statementNodeAt(state: EditorState, cursor: number): SyntaxNode | null {
|
||||
const tree = syntaxTree(state);
|
||||
let node: SyntaxNode | null = tree.resolve(cursor, -1);
|
||||
if (node.name === "Script") {
|
||||
node = tree.topNode.childBefore(cursor + 1);
|
||||
}
|
||||
if (!node || node.type.isError) return null;
|
||||
|
||||
let statementNode: SyntaxNode | null = node;
|
||||
while (statementNode && statementNode.name !== "Statement") {
|
||||
statementNode = statementNode.parent;
|
||||
}
|
||||
return statementNode;
|
||||
}
|
||||
|
||||
/** Tree-based replacement for `expandToSqlStatementWindow`'s outer boundary, for callers with a
|
||||
* live `EditorState`. Returns `null` when the tree hasn't parsed up to `cursor` yet, when the
|
||||
* resolved node looks like an error-recovery node, or when a dollar-quoted body might be hiding
|
||||
* the true boundary (see module doc comment) -- the caller should fall back to the scanner. */
|
||||
export function resolveStatementWindowFromSyntaxTree(state: EditorState, cursor: number): TextRange | null {
|
||||
if (!syntaxTreeAvailable(state, cursor)) return null;
|
||||
|
||||
const statementNode = statementNodeAt(state, cursor);
|
||||
if (!statementNode) return null;
|
||||
|
||||
const from = statementNode.from;
|
||||
const statementEnd = statementNode.to;
|
||||
if (cursor < from || (cursor > statementEnd && state.doc.sliceString(statementEnd, cursor).trim().length > 0)) return null;
|
||||
|
||||
let to = statementEnd;
|
||||
const last = statementNode.lastChild;
|
||||
if (last && last.name === ";") to = last.from;
|
||||
|
||||
// Error recovery can produce a partial Statement node that ends before the
|
||||
// completion cursor (for example `SELECT alias.`). Such a window is not an
|
||||
// enclosing statement and must fall back to the scanner.
|
||||
if (hasUnresolvedDollarQuoteBefore(state.doc, from, DOLLAR_QUOTE_DISTRUST_LOOKBACK)) return null;
|
||||
|
||||
return { from, to };
|
||||
}
|
||||
|
||||
/** Tree-based replacement for `getSqlLexicalContext`'s classification of the cursor position.
|
||||
* Classifies from the resolved leaf's node name alone: `String` (single- or dollar-quoted, per
|
||||
* the grammar's own tokenizer) is a literal; `LineComment`/`BlockComment` are comments;
|
||||
* `QuotedIdentifier` (double-quote/backtick/bracket) and everything else is neither, matching
|
||||
* getSqlLexicalContext's existing "only single/dollar-quoted text is a value literal" semantics. */
|
||||
export function resolveLexicalLeafFromSyntaxTree(state: EditorState, cursor: number): { inLineComment: boolean; inBlockComment: boolean; inStringLiteral: boolean } | null {
|
||||
if (!syntaxTreeAvailable(state, cursor)) return null;
|
||||
|
||||
const tree = syntaxTree(state);
|
||||
const node = tree.resolve(cursor, -1);
|
||||
if (node.type.isError) return null;
|
||||
|
||||
return {
|
||||
inLineComment: node.name === "LineComment",
|
||||
inBlockComment: node.name === "BlockComment",
|
||||
inStringLiteral: node.name === "String",
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user