feat(editor): show target columns for INSERT SELECT

Closes #6696

apps/desktop/src/lib/sql/insertValueHints.ts

@@ -14,7 +14,7 @@ export interface InsertValuesClause {
   database?: string;
   /** Explicit column list, or null when `INSERT INTO t VALUES` has no column list. */
   columns: string[] | null;
-  /** For each VALUES row, start offsets of top-level value expressions. */
+  /** For each VALUES row or SELECT projection list, start offsets of top-level expressions. */
   rows: number[][];
   span: SqlSemanticSpan;
 }
@@ -164,6 +164,82 @@ function parseValuesRows(tokens: readonly SqlSemanticToken[], valuesIndex: numbe
   return rows;
 }
 
+function findWordIndexAtDepth(tokens: readonly SqlSemanticToken[], word: string, from: number, depth: number): number {
+  const needle = word.toLowerCase();
+  for (let index = from; index < tokens.length; index += 1) {
+    const item = tokens[index];
+    if (item?.depth === depth && item.kind === "word" && item.normalized === needle) return index;
+  }
+  return -1;
+}
+
+function findClosingParenIndex(tokens: readonly SqlSemanticToken[], openIndex: number): number {
+  const open = tokens[openIndex];
+  if (!open || open.text !== "(") return -1;
+  for (let index = openIndex + 1; index < tokens.length; index += 1) {
+    const item = tokens[index];
+    if (item?.text === ")" && item.depth === open.depth) return index;
+  }
+  return -1;
+}
+
+const SELECT_CLAUSE_BOUNDARIES = new Set(["from", "into", "where", "group", "having", "order", "offset", "fetch", "for", "option", "union", "except", "intersect", "limit", "qualify", "window"]);
+
+function isSelectClauseBoundary(item: SqlSemanticToken | undefined, depth: number): boolean {
+  if (!item || item.depth !== depth || item.kind !== "word") return false;
+  return SELECT_CLAUSE_BOUNDARIES.has(item.normalized);
+}
+
+function isWildcardProjection(tokens: readonly SqlSemanticToken[], from: number, to: number, depth: number): boolean {
+  const topLevel = tokens.slice(from, to).filter((item) => item.depth === depth);
+  if (topLevel[topLevel.length - 1]?.text !== "*") return false;
+  if (topLevel.length === 1) return true;
+  return topLevel.length % 2 === 1 && topLevel.every((item, index) => (index % 2 === 0 ? (index === topLevel.length - 1 ? item.text === "*" : tokenIsIdentifier(item)) : item.text === "."));
+}
+
+function selectProjectionStarts(tokens: readonly SqlSemanticToken[], selectIndex: number): number[] {
+  const select = tokens[selectIndex];
+  if (!select) return [];
+  const depth = select.depth;
+  let index = selectIndex + 1;
+
+  if (tokens[index]?.depth === depth && tokens[index]?.kind === "word" && (tokens[index]?.normalized === "all" || tokens[index]?.normalized === "distinct")) {
+    index += 1;
+  }
+
+  const maybeTop = tokens[index];
+  const topValue = tokens[index + 1];
+  const hasTopModifier = maybeTop?.depth === depth && maybeTop.kind === "word" && maybeTop.normalized === "top" && topValue?.depth === depth && (topValue.text === "(" || topValue.kind === "number" || topValue.kind === "parameter" || (topValue.kind === "word" && topValue.text.startsWith("@")));
+  if (hasTopModifier) {
+    index += 1;
+    if (tokens[index]?.text === "(") {
+      const closeIndex = findClosingParenIndex(tokens, index);
+      if (closeIndex < 0) return [];
+      index = closeIndex + 1;
+    } else {
+      index += 1;
+    }
+    if (tokens[index]?.depth === depth && tokens[index]?.normalized === "percent") index += 1;
+    if (tokens[index]?.depth === depth && tokens[index]?.normalized === "with" && tokens[index + 1]?.depth === depth && tokens[index + 1]?.normalized === "ties") index += 2;
+  }
+
+  const starts: number[] = [];
+  let expressionStart = index;
+  for (; index < tokens.length; index += 1) {
+    const item = tokens[index];
+    if (!item) break;
+    if (isSelectClauseBoundary(item, depth)) break;
+    if (item.text !== "," || item.depth !== depth) continue;
+    if (index <= expressionStart || isWildcardProjection(tokens, expressionStart, index, depth)) return [];
+    starts.push(tokens[expressionStart]!.span.start);
+    expressionStart = index + 1;
+  }
+
+  if (index <= expressionStart || isWildcardProjection(tokens, expressionStart, index, depth)) return [];
+  starts.push(tokens[expressionStart]!.span.start);
+  return starts;
+}
+
 function parseInsertClause(tokens: readonly SqlSemanticToken[], span: SqlSemanticSpan): InsertValuesClause | null {
   const insertIndex = findWordIndex(tokens, "insert");
   if (insertIndex < 0) return null;
@@ -200,13 +276,14 @@ function parseInsertClause(tokens: readonly SqlSemanticToken[], span: SqlSemanti
     index = columnList.nextIndex;
   }
 
-  const valuesIndex = findWordIndex(tokens, "values", index);
+  const sourceDepth = tokens[insertIndex]?.depth ?? 0;
-  const selectIndex = findWordIndex(tokens, "select", index);
+  const valuesIndex = findWordIndexAtDepth(tokens, "values", index, sourceDepth);
-  if (valuesIndex < 0) return null;
+  const selectIndex = findWordIndexAtDepth(tokens, "select", index, sourceDepth);
-  if (selectIndex >= 0 && selectIndex < valuesIndex) return null;
+  if (valuesIndex < 0 && selectIndex < 0) return null;
 
-  const rows = parseValuesRows(tokens, valuesIndex);
+  const rows = selectIndex >= 0 && (valuesIndex < 0 || selectIndex < valuesIndex) ? [selectProjectionStarts(tokens, selectIndex)] : parseValuesRows(tokens, valuesIndex);
   if (rows.length === 0) return null;
+  if (rows[0]?.length === 0) return null;
 
   return {
     table: tableInfo.name,
@@ -488,7 +565,7 @@ function mergeTextRanges(ranges: readonly TextRange[]): TextRange[] {
   return merged;
 }
 
-/** Parse INSERT ... VALUES clauses only inside the given document ranges (expanded to statement windows). */
+/** Parse INSERT ... VALUES/SELECT clauses only inside the given document ranges (expanded to statement windows). */
 export function parseInsertValuesClausesInRanges(sql: string, ranges: readonly TextRange[]): InsertValuesClause[] {
   if (!sql.trim() || ranges.length === 0) return [];
   const windows = mergeTextRanges(ranges.map((range) => expandToSqlStatementWindow(sql, range.from, range.to)));
@@ -503,7 +580,7 @@ export function parseInsertValuesClausesInRanges(sql: string, ranges: readonly T
   return clauses;
 }
 
-/** Parse all INSERT ... VALUES clauses in `sql` (multi-statement aware). Prefer ranged parsing for editors. */
+/** Parse all INSERT ... VALUES/SELECT clauses in `sql` (multi-statement aware). Prefer ranged parsing for editors. */
 export function parseInsertValuesClauses(sql: string, dialectId = "mysql"): InsertValuesClause[] {
   if (!sql.trim()) return [];
   const allTokens = tokenizeSqlSemantic(sql, dialectId);

packages/app-tests/insertValueHints.test.ts

@@ -14,10 +14,7 @@ function assertSublinearScaling(measureAt: (scale: number) => number, options: {
   const { smallScale, bigScale, maxRatio, maxMs, label } = options;
   const smallMs = measureAt(smallScale);
   const bigMs = measureAt(bigScale);
-  assert.ok(
+  assert.ok(bigMs < Math.max(maxMs, smallMs * maxRatio), `${label}: ${smallScale}x took ${smallMs.toFixed(1)}ms, ${bigScale}x took ${bigMs.toFixed(1)}ms -- expected roughly bounded, not scaling with document size`);
-    bigMs < Math.max(maxMs, smallMs * maxRatio),
-    `${label}: ${smallScale}x took ${smallMs.toFixed(1)}ms, ${bigScale}x took ${bigMs.toFixed(1)}ms -- expected roughly bounded, not scaling with document size`,
-  );
 }
 
 test("maps explicit column list to single-row VALUES", () => {
@@ -88,11 +85,7 @@ test("resolves columns from table metadata when column list is omitted", () => {
 
 test("skips SQL Server identity columns when mapping multi-row VALUES without a column list", () => {
   const sql = "INSERT INTO dbo.users VALUES (N'A', 1), (N'B', 2)";
-  const columns = insertValueHintColumnNames("sqlserver", [
+  const columns = insertValueHintColumnNames("sqlserver", [{ name: "id", is_identity: true }, { name: "name" }, { name: "status" }]);
-    { name: "id", is_identity: true },
-    { name: "name" },
-    { name: "status" },
-  ]);
   const hints = parseInsertValueHints(sql, {
     resolveTableColumns: () => columns,
   });
@@ -116,30 +109,62 @@ test("skips SQL Server computed and temporal generated columns in positional hin
 });
 
 test("skips visible SQL Server generated columns in positional hints", () => {
-  assert.deepEqual(
+  assert.deepEqual(insertValueHintColumnNames("sqlserver", [{ name: "name" }, { name: "valid_from", generated_always_type: 1 }, { name: "valid_to", generated_always_type: 2 }]), ["name"]);
-    insertValueHintColumnNames("sqlserver", [
-      { name: "name" },
-      { name: "valid_from", generated_always_type: 1 },
-      { name: "valid_to", generated_always_type: 2 },
-    ]),
-    ["name"],
-  );
 });
 
 test("keeps identity columns in positional hints for databases other than SQL Server", () => {
+  assert.deepEqual(insertValueHintColumnNames("postgres", [{ name: "id", is_identity: true }, { name: "name" }]), ["id", "name"]);
+});
+
+test("maps INSERT ... SELECT projections to explicit target columns", () => {
+  const sql = "INSERT INTO dbo.users (id, name) SELECT source_id, source_name FROM staging";
+  const hints = parseInsertValueHints(sql);
   assert.deepEqual(
-    insertValueHintColumnNames("postgres", [
+    hints.map((hint) => ({ column: hint.column, text: sql.slice(hint.from).split(/[ ,]/u, 1)[0] })),
-      { name: "id", is_identity: true },
+    [
-      { name: "name" },
+      { column: "id", text: "source_id" },
-    ]),
+      { column: "name", text: "source_name" },
-    ["id", "name"],
+    ],
   );
 });
 
-test("returns no hints for INSERT ... SELECT", () => {
+test("skips SELECT modifiers and does not split nested projection expressions", () => {
-  const sql = "INSERT INTO users (id, name) SELECT id, name FROM staging";
+  const sql = "INSERT INTO dbo.users (name, row_count) SELECT DISTINCT TOP (10) COALESCE(first_name, last_name), COUNT(*) FROM staging";
-  assert.deepEqual(parseInsertValueHints(sql), []);
+  const hints = parseInsertValueHints(sql);
-  assert.deepEqual(parseInsertValuesClauses(sql), []);
+  assert.deepEqual(
+    hints.map((hint) => hint.column),
+    ["name", "row_count"],
+  );
+  assert.ok(sql.slice(hints[0]!.from).startsWith("COALESCE(first_name, last_name)"));
+  assert.ok(sql.slice(hints[1]!.from).startsWith("COUNT(*)"));
+});
+
+test("resolves filtered SQL Server target columns for INSERT ... SELECT without a column list", () => {
+  const sql = "INSERT INTO dbo.users SELECT source_name, source_status FROM staging";
+  const columns = insertValueHintColumnNames("sqlserver", [{ name: "id", is_identity: true }, { name: "name" }, { name: "doubled", is_computed: true }, { name: "status" }, { name: "valid_from", generated_always_type: 1 }]);
+  const hints = parseInsertValueHints(sql, { resolveTableColumns: () => columns });
+  assert.deepEqual(
+    hints.map((hint) => hint.column),
+    ["name", "status"],
+  );
+});
+
+test("returns no positional hints for wildcard INSERT ... SELECT projections", () => {
+  for (const sql of ["INSERT INTO users (id, name) SELECT * FROM staging", "INSERT INTO users (id, name) SELECT source.* FROM staging source"]) {
+    assert.deepEqual(parseInsertValueHints(sql), []);
+    assert.deepEqual(parseInsertValuesClauses(sql), []);
+  }
+});
+
+test("caps INSERT ... SELECT hints to the smaller target or projection count", () => {
+  assert.deepEqual(
+    parseInsertValueHints("INSERT INTO t (a, b) SELECT x, y, z FROM source").map((hint) => hint.column),
+    ["a", "b"],
+  );
+  assert.deepEqual(
+    parseInsertValueHints("INSERT INTO t (a, b, c) SELECT x, y FROM source").map((hint) => hint.column),
+    ["a", "b"],
+  );
 });
 
 test("caps hints when value count exceeds column count", () => {
@@ -252,11 +277,7 @@ test("expandToSqlStatementWindow proves clean state when the cursor is more than
   const sql = `CREATE FUNCTION f() RETURNS void AS $$ ${body} $$ LANGUAGE sql;`;
   const cursor = sql.indexOf(body) + 40_000;
   const window = expandToSqlStatementWindow(sql, cursor, cursor);
-  assert.equal(
+  assert.equal(window.from, 0, "semicolons inside a dollar-quoted function body are not statement boundaries, even past the lookback window");
-    window.from,
-    0,
-    "semicolons inside a dollar-quoted function body are not statement boundaries, even past the lookback window",
-  );
 });
 
 test("expandToSqlStatementWindow proves clean state when the cursor is more than 32KiB into deeply nested parens", () => {
@@ -266,11 +287,7 @@ test("expandToSqlStatementWindow proves clean state when the cursor is more than
   const sql = `SELECT ${opens}${junk}${closes};`;
   const cursor = sql.indexOf(junk) + junk.length - 100;
   const window = expandToSqlStatementWindow(sql, cursor, cursor);
-  assert.equal(
+  assert.equal(window.from, 0, "a ';' more than 32KiB past unclosed '(' characters is still nested, not a top-level statement boundary");
-    window.from,
-    0,
-    "a ';' more than 32KiB past unclosed '(' characters is still nested, not a top-level statement boundary",
-  );
 });
 
 test("expandToSqlStatementWindow stays bounded (fast path) for many small statements even far into the document", () => {
@@ -338,22 +355,14 @@ test("expandToSqlStatementWindow does not treat a backslash-escaped quote as clo
   const sql = "SELECT 'it\\'s a test; end' FROM t;";
   const cursor = sql.indexOf("FROM") + 2;
   const window = expandToSqlStatementWindow(sql, cursor, cursor);
-  assert.equal(
+  assert.equal(sql.slice(window.from, window.to), "SELECT 'it\\'s a test; end' FROM t", "the ';' inside the backslash-escaped string must not be mistaken for the statement boundary");
-    sql.slice(window.from, window.to),
-    "SELECT 'it\\'s a test; end' FROM t",
-    "the ';' inside the backslash-escaped string must not be mistaken for the statement boundary",
-  );
 });
 
 test("expandToSqlStatementWindow terminates a line comment at a bare '\\r' (no trailing '\\n')", () => {
   const sql = "SELECT 1; -- comment\rSELECT 2;";
   const cursor = sql.indexOf("SELECT 2") + 4;
   const window = expandToSqlStatementWindow(sql, cursor, cursor);
-  assert.equal(
+  assert.equal(sql.slice(window.from, window.to), "-- comment\rSELECT 2", "the comment must end at '\\r' so the trailing ';' is recognized as the real statement boundary");
-    sql.slice(window.from, window.to),
-    "-- comment\rSELECT 2",
-    "the comment must end at '\\r' so the trailing ';' is recognized as the real statement boundary",
-  );
 });
 
 test("expandToSqlStatementWindow finds the real end of a single statement larger than one lookahead window, instead of truncating", () => {