fix(duckdb): preserve prefix aliases during SQL formatting
Closes #6977
This commit is contained in:
1 parent
351dc81a30
commit
add44cf39e
2 files changed
+208
-8
No files matched your search
@@ -1,6 +1,7 @@
|
|||||||
import { readFileSync } from "node:fs";
|
import { readFileSync } from "node:fs";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { canFormatSqlForDatabaseType, formatSqlForDisplay, formatSqlForEditing, formatSqlText, MAX_SQL_FORMAT_CHARS, sqlFormatDialectForDbType, UnsupportedStructuredInputError } from "@/lib/sql/sqlFormatter";
|
import { canFormatSqlForDatabaseType, formatSqlForDisplay, formatSqlForEditing, formatSqlText, MAX_SQL_FORMAT_CHARS, sqlFormatDialectForDbType, UnsupportedStructuredInputError } from "@/lib/sql/sqlFormatter";
|
||||||
|
import { extractSqlParameters } from "@/lib/sql/sqlParameters";
|
||||||
|
|
||||||
const sqlFormatterSource = readFileSync(new URL("../../sql/sqlFormatter.ts", import.meta.url), "utf8");
|
const sqlFormatterSource = readFileSync(new URL("../../sql/sqlFormatter.ts", import.meta.url), "utf8");
|
||||||
|
|
||||||
@@ -31,6 +32,81 @@ describe("sqlFormatter", () => {
|
|||||||
expect(sqlFormatDialectForDbType("dameng")).toBe("dameng");
|
expect(sqlFormatDialectForDbType("dameng")).toBe("dameng");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("maps DuckDB to its scoped formatter dialect", () => {
|
||||||
|
expect(sqlFormatDialectForDbType("duckdb")).toBe("duckdb");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps DuckDB prefix aliases out of formatted named parameters", async () => {
|
||||||
|
const cases = [
|
||||||
|
["select 日期:date(订单日期) from 订单;", []],
|
||||||
|
['select total:price * quantity, "order":sum(amount) from sales;', []],
|
||||||
|
["from r:range(:row_count) select total:r.range + :offset;", ["row_count", "offset"]],
|
||||||
|
["select res: col1 + col2, root: sqrt(col1) from tbl;", []],
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
for (const [sql, expectedParameters] of cases) {
|
||||||
|
const formatted = await formatSqlText(sql, sqlFormatDialectForDbType("duckdb"));
|
||||||
|
|
||||||
|
expect(formatted).not.toBe(sql);
|
||||||
|
expect(extractSqlParameters(formatted, { databaseType: "duckdb" })).toEqual(expectedParameters);
|
||||||
|
expect(formatted).not.toContain("__DBX_DUCKDB_PREFIX_ALIAS_COLON_");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps ordinary DuckDB named parameters enabled independently of prefix aliases", async () => {
|
||||||
|
const formatted = await formatSqlText("select :outside, total:price from sales;", sqlFormatDialectForDbType("duckdb"));
|
||||||
|
|
||||||
|
expect(extractSqlParameters(formatted, { databaseType: "duckdb" })).toEqual(["outside"]);
|
||||||
|
expect(extractSqlParameters(formatted, { databaseType: "duckdb", enabledSyntaxes: [] })).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not change generic formatting of compact colon syntax", async () => {
|
||||||
|
const formatted = await formatSqlText("select total:price from sales;", "generic");
|
||||||
|
|
||||||
|
expect(extractSqlParameters(formatted, { databaseType: "postgres" })).toEqual(["price"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves DuckDB strings and comments while protecting prefix aliases", async () => {
|
||||||
|
const sql = `select total:price, 'literal:date', $$dollar:date
|
||||||
|
AND inside
|
||||||
|
OR inside$$ as note
|
||||||
|
from sales /* alias:date */ /*__DBX_DUCKDB_PREFIX_ALIAS_COLON_0__*/ -- trailing:date`;
|
||||||
|
const formatted = await formatSqlText(sql, sqlFormatDialectForDbType("duckdb"), { logicalOperatorNewline: "none" });
|
||||||
|
|
||||||
|
expect(extractSqlParameters(formatted, { databaseType: "duckdb" })).toEqual([]);
|
||||||
|
expect(formatted).toContain("'literal:date'");
|
||||||
|
expect(formatted).toContain("$$dollar:date\nAND inside\nOR inside$$");
|
||||||
|
expect(formatted).toContain("/* alias:date */");
|
||||||
|
expect(formatted).toContain("/*__DBX_DUCKDB_PREFIX_ALIAS_COLON_0__*/");
|
||||||
|
expect(formatted).toContain("-- trailing:date");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps DuckDB casts, named arguments, and their real parameters intact", async () => {
|
||||||
|
const sql = "select struct_pack(key := :value), total:price, value::integer;";
|
||||||
|
const formatted = await formatSqlText(sql, sqlFormatDialectForDbType("duckdb"));
|
||||||
|
|
||||||
|
expect(extractSqlParameters(formatted, { databaseType: "duckdb" })).toEqual(["value"]);
|
||||||
|
expect(formatted).toContain("value::integer");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unsupported DuckDB struct literals unchanged without leaking formatter markers", async () => {
|
||||||
|
const sql = "select {'key': value, nested: {'inner': inner_value}}, total:price;";
|
||||||
|
const formatted = await formatSqlForEditing(sql, sqlFormatDialectForDbType("duckdb"));
|
||||||
|
|
||||||
|
expect(formatted).toBe(sql);
|
||||||
|
expect(extractSqlParameters(formatted, { databaseType: "duckdb" })).toEqual([]);
|
||||||
|
expect(formatted).not.toContain("__DBX_DUCKDB_PREFIX_ALIAS_COLON_");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns malformed DuckDB SQL unchanged without leaking formatter markers", async () => {
|
||||||
|
const sql = "select total:price from dbname.\n;";
|
||||||
|
|
||||||
|
const formatted = await formatSqlForEditing(sql, sqlFormatDialectForDbType("duckdb"));
|
||||||
|
|
||||||
|
expect(formatted).toBe(sql);
|
||||||
|
expect(formatted).not.toContain("__DBX_DUCKDB_PREFIX_ALIAS_COLON_");
|
||||||
|
});
|
||||||
|
|
||||||
it("preserves ClickHouse lambda arrows when formatting issue #3573 SQL", async () => {
|
it("preserves ClickHouse lambda arrows when formatting issue #3573 SQL", async () => {
|
||||||
const sql = `
|
const sql = `
|
||||||
WITH industry_code_donghua_id_RYCzfD AS (SELECT id
|
WITH industry_code_donghua_id_RYCzfD AS (SELECT id
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { DEFAULT_SQL_FORMATTER_SETTINGS, normalizeSqlFormatterSettings, sqlFormatterOptions, type SqlFormatterSettings } from "@/lib/sql/sqlFormatterConfig";
|
import { DEFAULT_SQL_FORMATTER_SETTINGS, normalizeSqlFormatterSettings, sqlFormatterOptions, type SqlFormatterSettings } from "@/lib/sql/sqlFormatterConfig";
|
||||||
import { looksLikeXml } from "@/lib/sql/autoFormat";
|
import { looksLikeXml } from "@/lib/sql/autoFormat";
|
||||||
|
|
||||||
export type SqlFormatDialect = "mysql" | "postgres" | "sqlite" | "sqlserver" | "clickhouse" | "dameng" | "generic";
|
export type SqlFormatDialect = "mysql" | "postgres" | "sqlite" | "sqlserver" | "clickhouse" | "dameng" | "duckdb" | "generic";
|
||||||
|
|
||||||
export const MAX_SQL_FORMAT_CHARS = 1_000_000;
|
export const MAX_SQL_FORMAT_CHARS = 1_000_000;
|
||||||
|
|
||||||
@@ -28,10 +28,10 @@ export class UnsupportedStructuredInputError extends Error {
|
|||||||
* Maps a connection's database type to the SQL-formatter dialect to use.
|
* Maps a connection's database type to the SQL-formatter dialect to use.
|
||||||
*
|
*
|
||||||
* Postgres-compatible engines (GaussDB/openGauss/Kingbase/...) reuse the
|
* Postgres-compatible engines (GaussDB/openGauss/Kingbase/...) reuse the
|
||||||
* "postgres" grammar, SQLite-compatible ones reuse "sqlite", and anything
|
* "postgres" grammar, SQLite-compatible ones reuse "sqlite", and DuckDB gets
|
||||||
* unrecognized falls back to the permissive "generic" dialect. Centralized
|
* a scoped preprocessing pass over the generic grammar. Anything unrecognized
|
||||||
* here so every surface that formats SQL (editor, object source, DDL viewers)
|
* falls back to the permissive "generic" dialect. Centralized here so every
|
||||||
* stays in sync.
|
* surface that formats SQL (editor, object source, DDL viewers) stays in sync.
|
||||||
*/
|
*/
|
||||||
export function sqlFormatDialectForDbType(dbType: string | null | undefined): SqlFormatDialect {
|
export function sqlFormatDialectForDbType(dbType: string | null | undefined): SqlFormatDialect {
|
||||||
switch (dbType) {
|
switch (dbType) {
|
||||||
@@ -59,6 +59,8 @@ export function sqlFormatDialectForDbType(dbType: string | null | undefined): Sq
|
|||||||
return "clickhouse";
|
return "clickhouse";
|
||||||
case "dameng":
|
case "dameng":
|
||||||
return "dameng";
|
return "dameng";
|
||||||
|
case "duckdb":
|
||||||
|
return "duckdb";
|
||||||
default:
|
default:
|
||||||
return "generic";
|
return "generic";
|
||||||
}
|
}
|
||||||
@@ -91,6 +93,126 @@ function splitTrailingStandaloneDot(sql: string): { body: string; suffix: string
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const DUCKDB_ALIAS_IDENTIFIER_START_RE = /[\p{L}_]/u;
|
||||||
|
const DUCKDB_ALIAS_IDENTIFIER_CHAR_RE = /[\p{L}\p{N}_]/u;
|
||||||
|
|
||||||
|
function hasDuckDbBarePrefixAliasBefore(sql: string, colonIndex: number): boolean {
|
||||||
|
let start = colonIndex;
|
||||||
|
while (start > 0 && DUCKDB_ALIAS_IDENTIFIER_CHAR_RE.test(sql[start - 1])) start -= 1;
|
||||||
|
return start < colonIndex && DUCKDB_ALIAS_IDENTIFIER_START_RE.test(sql[start]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function duckDbDollarQuoteMarkerAt(sql: string, index: number): string | null {
|
||||||
|
if (sql[index] !== "$" || DUCKDB_ALIAS_IDENTIFIER_CHAR_RE.test(sql[index - 1] ?? "")) return null;
|
||||||
|
if (sql[index + 1] === "$") return "$$";
|
||||||
|
if (!/[A-Za-z_]/.test(sql[index + 1] ?? "")) return null;
|
||||||
|
|
||||||
|
let end = index + 2;
|
||||||
|
while (/[A-Za-z0-9_]/.test(sql[end] ?? "")) end += 1;
|
||||||
|
return sql[end] === "$" ? sql.slice(index, end + 1) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function protectDuckDbPrefixAliasSeparators(sql: string): { sql: string; marker: string | null } {
|
||||||
|
let markerIndex = 0;
|
||||||
|
let marker = "";
|
||||||
|
do {
|
||||||
|
marker = `/*__DBX_DUCKDB_PREFIX_ALIAS_COLON_${markerIndex}__*/`;
|
||||||
|
markerIndex += 1;
|
||||||
|
} while (sql.includes(marker));
|
||||||
|
|
||||||
|
let protectedSql = "";
|
||||||
|
let quotedIdentifierEnd = -1;
|
||||||
|
let replaced = false;
|
||||||
|
let index = 0;
|
||||||
|
|
||||||
|
while (index < sql.length) {
|
||||||
|
const ch = sql[index];
|
||||||
|
const next = sql[index + 1];
|
||||||
|
|
||||||
|
if (ch === "-" && next === "-") {
|
||||||
|
const start = index;
|
||||||
|
index += 2;
|
||||||
|
while (index < sql.length && sql[index] !== "\n" && sql[index] !== "\r") index += 1;
|
||||||
|
protectedSql += sql.slice(start, index);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === "/" && next === "*") {
|
||||||
|
const start = index;
|
||||||
|
index += 2;
|
||||||
|
let depth = 1;
|
||||||
|
while (index < sql.length && depth > 0) {
|
||||||
|
if (sql[index] === "/" && sql[index + 1] === "*") {
|
||||||
|
depth += 1;
|
||||||
|
index += 2;
|
||||||
|
} else if (sql[index] === "*" && sql[index + 1] === "/") {
|
||||||
|
depth -= 1;
|
||||||
|
index += 2;
|
||||||
|
} else {
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
protectedSql += sql.slice(start, index);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dollarQuoteMarker = ch === "$" ? duckDbDollarQuoteMarkerAt(sql, index) : null;
|
||||||
|
if (dollarQuoteMarker) {
|
||||||
|
const start = index;
|
||||||
|
const end = sql.indexOf(dollarQuoteMarker, index + dollarQuoteMarker.length);
|
||||||
|
index = end < 0 ? sql.length : end + dollarQuoteMarker.length;
|
||||||
|
protectedSql += sql.slice(start, index);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === "'" || ch === '"' || ch === "`") {
|
||||||
|
const start = index;
|
||||||
|
const quote = ch;
|
||||||
|
let closed = false;
|
||||||
|
index += 1;
|
||||||
|
while (index < sql.length) {
|
||||||
|
if (sql[index] === "\\" && index + 1 < sql.length) {
|
||||||
|
index += 2;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (sql[index] === quote) {
|
||||||
|
if (sql[index + 1] === quote) {
|
||||||
|
index += 2;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
index += 1;
|
||||||
|
closed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
protectedSql += sql.slice(start, index);
|
||||||
|
if (quote === '"' && closed) quotedIdentifierEnd = index;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const prefixAlias = ch === ":" && sql[index - 1] !== ":" && next !== ":" && next !== "=" && (quotedIdentifierEnd === index || hasDuckDbBarePrefixAliasBefore(sql, index));
|
||||||
|
if (prefixAlias) {
|
||||||
|
// sql-formatter tokenizes a compact DuckDB prefix alias as a named
|
||||||
|
// parameter and inserts a space before `:`. Keeping an opaque separator
|
||||||
|
// through formatting preserves the boundary used by DBX's parameter scan.
|
||||||
|
protectedSql += marker;
|
||||||
|
replaced = true;
|
||||||
|
index += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
protectedSql += ch;
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return replaced ? { sql: protectedSql, marker } : { sql, marker: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
function restoreDuckDbPrefixAliasSeparators(sql: string, marker: string | null): string {
|
||||||
|
return marker ? sql.split(marker).join(":") : sql;
|
||||||
|
}
|
||||||
|
|
||||||
export async function formatSqlText(sql: string, dialect: SqlFormatDialect = "generic", settings: Partial<SqlFormatterSettings> = DEFAULT_SQL_FORMATTER_SETTINGS): Promise<string> {
|
export async function formatSqlText(sql: string, dialect: SqlFormatDialect = "generic", settings: Partial<SqlFormatterSettings> = DEFAULT_SQL_FORMATTER_SETTINGS): Promise<string> {
|
||||||
if (!sql.trim()) return sql;
|
if (!sql.trim()) return sql;
|
||||||
if (sql.length > MAX_SQL_FORMAT_CHARS) {
|
if (sql.length > MAX_SQL_FORMAT_CHARS) {
|
||||||
@@ -105,6 +227,7 @@ export async function formatSqlText(sql: string, dialect: SqlFormatDialect = "ge
|
|||||||
const normalizedSettings = normalizeSqlFormatterSettings(settings);
|
const normalizedSettings = normalizeSqlFormatterSettings(settings);
|
||||||
const options = sqlFormatterOptions(normalizedSettings);
|
const options = sqlFormatterOptions(normalizedSettings);
|
||||||
const language = formatterLanguage(dialect);
|
const language = formatterLanguage(dialect);
|
||||||
|
const protectedInput = dialect === "duckdb" ? protectDuckDbPrefixAliasSeparators(sql) : { sql, marker: null };
|
||||||
const formatterOptions =
|
const formatterOptions =
|
||||||
language === "postgresql"
|
language === "postgresql"
|
||||||
? {
|
? {
|
||||||
@@ -139,7 +262,8 @@ export async function formatSqlText(sql: string, dialect: SqlFormatDialect = "ge
|
|||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return applySqlFormatterLayout(formatWithFallback(sql), normalizedSettings, dialect);
|
const formatted = restoreDuckDbPrefixAliasSeparators(formatWithFallback(protectedInput.sql), protectedInput.marker);
|
||||||
|
return applySqlFormatterLayout(formatted, normalizedSettings, dialect);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const trailingDot = dialect === "dameng" ? splitTrailingStandaloneDot(sql) : null;
|
const trailingDot = dialect === "dameng" ? splitTrailingStandaloneDot(sql) : null;
|
||||||
if (!trailingDot) throw err;
|
if (!trailingDot) throw err;
|
||||||
@@ -165,7 +289,7 @@ export async function formatSqlText(sql: string, dialect: SqlFormatDialect = "ge
|
|||||||
*
|
*
|
||||||
* The scanner reuses the same dialect-aware token recognition as
|
* The scanner reuses the same dialect-aware token recognition as
|
||||||
* {@link compressSqlText}: block comments (with nesting for Postgres/SQL
|
* {@link compressSqlText}: block comments (with nesting for Postgres/SQL
|
||||||
* Server/ClickHouse), `--`/`#` line comments, Postgres dollar-quoted strings,
|
* Server/ClickHouse), `--`/`#` line comments, Postgres/DuckDB dollar-quoted strings,
|
||||||
* single-quoted strings (`''` and MySQL/`E'...'` backslash escapes), double-
|
* single-quoted strings (`''` and MySQL/`E'...'` backslash escapes), double-
|
||||||
* quoted identifiers (`""` escapes), MySQL backtick identifiers and SQL Server
|
* quoted identifiers (`""` escapes), MySQL backtick identifiers and SQL Server
|
||||||
* `[...]` identifiers. sql-formatter normalizes string literals to a single
|
* `[...]` identifiers. sql-formatter normalizes string literals to a single
|
||||||
@@ -240,7 +364,7 @@ function maskStringAndCommentSpans(sql: string, dialect: SqlFormatDialect): { ma
|
|||||||
}
|
}
|
||||||
|
|
||||||
// PostgreSQL dollar-quoted 字符串
|
// PostgreSQL dollar-quoted 字符串
|
||||||
if (dialect === "postgres" && ch === "$") {
|
if ((dialect === "postgres" || dialect === "duckdb") && ch === "$") {
|
||||||
const tag = dollarQuoteTagAt(i);
|
const tag = dollarQuoteTagAt(i);
|
||||||
if (tag) {
|
if (tag) {
|
||||||
const start = i;
|
const start = i;
|
||||||
|
|||||||
Reference in new issue
Block a user