feat(editor): merge multi-database query results into one view
This commit is contained in:
1 parent
9d07708617
commit
8b822e974b
31 files changed
+3085
-87
No files matched your search
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { EXECUTION_CONFIRM_CODE_LENGTH, createExecutionConfirmCode, matchesExecutionConfirmCode } from "@/lib/query/executionConfirmCode";
|
||||
|
||||
describe("executionConfirmCode", () => {
|
||||
it("generates a code of the documented length from an unambiguous alphabet", () => {
|
||||
for (let run = 0; run < 50; run += 1) {
|
||||
const code = createExecutionConfirmCode();
|
||||
expect(code).toHaveLength(EXECUTION_CONFIRM_CODE_LENGTH);
|
||||
// No 0/O/1/I look-alikes: the operator has to retype it from the screen.
|
||||
expect(code).toMatch(/^[ABCDEFGHJKLMNPQRSTUVWXYZ23456789]+$/);
|
||||
}
|
||||
});
|
||||
|
||||
it("is deterministic for a given random source", () => {
|
||||
const sequence = () => {
|
||||
const values = [0, 0.5, 0.999, 0.25, 0.75, 0.1];
|
||||
let index = 0;
|
||||
return () => values[index++ % values.length]!;
|
||||
};
|
||||
expect(createExecutionConfirmCode(sequence())).toBe(createExecutionConfirmCode(sequence()));
|
||||
expect(createExecutionConfirmCode(() => 0)).toBe("AAAAAA");
|
||||
});
|
||||
|
||||
it("tolerates case and surrounding spaces when matching", () => {
|
||||
expect(matchesExecutionConfirmCode("ab12cd", "AB12CD")).toBe(true);
|
||||
expect(matchesExecutionConfirmCode(" AB12CD ", "ab12cd")).toBe(true);
|
||||
expect(matchesExecutionConfirmCode("", "AB12CD")).toBe(false);
|
||||
expect(matchesExecutionConfirmCode("AB12CE", "AB12CD")).toBe(false);
|
||||
});
|
||||
|
||||
it("never matches an empty code", () => {
|
||||
expect(matchesExecutionConfirmCode("", "")).toBe(false);
|
||||
expect(matchesExecutionConfirmCode("anything", "")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,199 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MULTI_SOURCE_MERGE_MAX_ROWS, isNumericSummaryCellValue, mergeMultiSourceResults, summarizeNumericColumns } from "@/lib/query/multiSourceResult";
|
||||
import type { QueryResult } from "@/types/database";
|
||||
|
||||
function result(columns: string[], rows: (string | number | boolean | null)[][], overrides: Partial<QueryResult> = {}): QueryResult {
|
||||
return {
|
||||
columns,
|
||||
rows,
|
||||
affected_rows: rows.length,
|
||||
execution_time_ms: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("mergeMultiSourceResults", () => {
|
||||
it("unions columns by name and injects the source column first", () => {
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "conn-a / sales", result: result(["id", "amount"], [[1, 10]]) },
|
||||
{ key: "b", label: "conn-b / sales", result: result(["amount", "region"], [[20, "north"]]) },
|
||||
],
|
||||
{ sourceColumnLabel: "Source" },
|
||||
);
|
||||
|
||||
expect(merged.columns).toEqual(["Source", "id", "amount", "region"]);
|
||||
expect(merged.rows).toEqual([
|
||||
["conn-a / sales", 1, 10, null],
|
||||
["conn-b / sales", null, 20, "north"],
|
||||
]);
|
||||
expect(merged.rowCount).toBe(2);
|
||||
expect(merged.truncated).toBe(false);
|
||||
expect(merged.skippedLabels).toEqual([]);
|
||||
expect(merged.sources).toEqual([
|
||||
{ key: "a", label: "conn-a / sales", rowCount: 1, columnCount: 2 },
|
||||
{ key: "b", label: "conn-b / sales", rowCount: 1, columnCount: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps duplicate column names of one source as separate value streams", () => {
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "a", result: result(["id", "id"], [[1, 2]]) },
|
||||
{ key: "b", label: "b", result: result(["id"], [[9]]) },
|
||||
],
|
||||
{ sourceColumnLabel: "Source" },
|
||||
);
|
||||
|
||||
expect(merged.columns).toEqual(["Source", "id", "id"]);
|
||||
expect(merged.rows).toEqual([
|
||||
["a", 1, 2],
|
||||
["b", 9, null],
|
||||
]);
|
||||
});
|
||||
|
||||
it("excludes hidden middle columns without shifting source values or numeric summaries", () => {
|
||||
const first = result(["name", "__DBX_PK_0", "amount", "region"], [["Alice", 17, 10, "north"]], { hidden_column_indexes: [1] });
|
||||
const second = result(["amount", "name", "__DBX_PK_0"], [[20, "Bob", 23]], { hidden_column_indexes: [2] });
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "a", result: first },
|
||||
{ key: "b", label: "b", result: second },
|
||||
],
|
||||
{ sourceColumnLabel: "Source" },
|
||||
);
|
||||
|
||||
expect(merged.columns).toEqual(["Source", "name", "amount", "region"]);
|
||||
expect(merged.rows).toEqual([
|
||||
["a", "Alice", 10, "north"],
|
||||
["b", "Bob", 20, null],
|
||||
]);
|
||||
expect(merged.summaries).toEqual([{ columnIndex: 2, column: "amount", sum: 30, count: 2 }]);
|
||||
expect(merged.sources.map((source) => source.columnCount)).toEqual([3, 2]);
|
||||
expect(first.columns).toEqual(["name", "__DBX_PK_0", "amount", "region"]);
|
||||
expect(first.rows).toEqual([["Alice", 17, 10, "north"]]);
|
||||
});
|
||||
|
||||
it("matches visible duplicate names and keeps unnamed columns at their source ordinals", () => {
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "a", result: result(["id", "id", "id", ""], [[99, 1, 2, 3]], { hidden_column_indexes: [0] }) },
|
||||
{ key: "b", label: "b", result: result(["id", "id", "__DBX_PK_0", ""], [[4, 5, 88, 6]], { hidden_column_indexes: [2] }) },
|
||||
],
|
||||
{ sourceColumnLabel: "Source" },
|
||||
);
|
||||
|
||||
expect(merged.columns).toEqual(["Source", "id", "id", "column_4"]);
|
||||
expect(merged.rows).toEqual([
|
||||
["a", 1, 2, 3],
|
||||
["b", 4, 5, 6],
|
||||
]);
|
||||
expect(merged.summaries.map((summary) => summary.sum)).toEqual([5, 7, 9]);
|
||||
});
|
||||
|
||||
it("skips sources without a tabular payload instead of failing", () => {
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "empty", result: result([], []) },
|
||||
{ key: "b", label: "failed", result: result(["Error"], [["boom"]], { execution_error: true }) },
|
||||
{ key: "c", label: "missing", result: undefined },
|
||||
{ key: "d", label: "ok", result: result(["id"], [[7]]) },
|
||||
],
|
||||
{ sourceColumnLabel: "Source" },
|
||||
);
|
||||
|
||||
expect(merged.sources.map((source) => source.label)).toEqual(["ok"]);
|
||||
expect(merged.skippedLabels).toEqual(["empty", "failed", "missing"]);
|
||||
expect(merged.rows).toEqual([["ok", 7]]);
|
||||
});
|
||||
|
||||
it("caps merged rows and reports truncation", () => {
|
||||
const rows = Array.from({ length: 5 }, (_, index) => [index]);
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "a", result: result(["n"], rows) },
|
||||
{ key: "b", label: "b", result: result(["n"], rows) },
|
||||
],
|
||||
{ sourceColumnLabel: "Source", maxRows: 4 },
|
||||
);
|
||||
|
||||
expect(merged.rowCount).toBe(4);
|
||||
expect(merged.truncated).toBe(true);
|
||||
expect(merged.rows.map((row) => row[0])).toEqual(["a", "a", "a", "a"]);
|
||||
});
|
||||
|
||||
it("names unnamed columns positionally and tolerates ragged rows", () => {
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "a", result: result(["", "b"], [[1]]) },
|
||||
{ key: "b", label: "b", result: result(["b"], [[2, 3, 4]]) },
|
||||
],
|
||||
{ sourceColumnLabel: "Source" },
|
||||
);
|
||||
|
||||
expect(merged.columns).toEqual(["Source", "column_1", "b"]);
|
||||
expect(merged.rows).toEqual([
|
||||
["a", 1, null],
|
||||
["b", null, 2],
|
||||
]);
|
||||
});
|
||||
|
||||
it("summarizes every all-numeric column of the merged table and never the source column", () => {
|
||||
const merged = mergeMultiSourceResults(
|
||||
[
|
||||
{ key: "a", label: "conn-a", result: result(["id", "amount", "label"], [[1, 10, "x"]]) },
|
||||
{ key: "b", label: "conn-b", result: result(["id", "amount", "label"], [[2, 32.5, "y"]]) },
|
||||
],
|
||||
{ sourceColumnLabel: "Source" },
|
||||
);
|
||||
|
||||
expect(merged.summaries).toEqual([
|
||||
{ columnIndex: 1, column: "id", sum: 3, count: 2 },
|
||||
{ columnIndex: 2, column: "amount", sum: 42.5, count: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses the documented row cap by default", () => {
|
||||
expect(MULTI_SOURCE_MERGE_MAX_ROWS).toBe(50_000);
|
||||
});
|
||||
|
||||
it("handles no usable input with an empty table", () => {
|
||||
const merged = mergeMultiSourceResults([{ key: "a", label: "a", result: undefined }], { sourceColumnLabel: "Source" });
|
||||
expect(merged.columns).toEqual(["Source"]);
|
||||
expect(merged.rows).toEqual([]);
|
||||
expect(merged.sources).toEqual([]);
|
||||
expect(merged.summaries).toEqual([]);
|
||||
expect(merged.skippedLabels).toEqual(["a"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("summarizeNumericColumns", () => {
|
||||
it("classifies numeric cells and rejects everything else", () => {
|
||||
expect(isNumericSummaryCellValue(1)).toBe(true);
|
||||
expect(isNumericSummaryCellValue(" 3.5 ")).toBe(true);
|
||||
expect(isNumericSummaryCellValue("")).toBe(false);
|
||||
expect(isNumericSummaryCellValue("12abc")).toBe(false);
|
||||
expect(isNumericSummaryCellValue("2026-07-24")).toBe(false);
|
||||
expect(isNumericSummaryCellValue(true)).toBe(false);
|
||||
expect(isNumericSummaryCellValue(null)).toBe(false);
|
||||
expect(isNumericSummaryCellValue(Number.NaN)).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores empty cells and drops a column once one value is not numeric", () => {
|
||||
expect(
|
||||
summarizeNumericColumns(
|
||||
["a", "b"],
|
||||
[
|
||||
[1, null],
|
||||
[null, "n/a"],
|
||||
["", 3],
|
||||
],
|
||||
),
|
||||
).toEqual([{ columnIndex: 0, column: "a", sum: 1, count: 1 }]);
|
||||
});
|
||||
|
||||
it("never sums boolean or all-empty columns", () => {
|
||||
expect(summarizeNumericColumns(["flag"], [[true], [false]])).toEqual([]);
|
||||
expect(summarizeNumericColumns(["a"], [[null], [null]])).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Confirmation code shown in the "confirm target execution" prompt of a
|
||||
* multi-database batch: the operator has to type it back before the batch is
|
||||
* allowed to run, so a mis-click cannot fire the same statement at every
|
||||
* connection.
|
||||
*/
|
||||
const CODE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789";
|
||||
|
||||
export const EXECUTION_CONFIRM_CODE_LENGTH = 6;
|
||||
|
||||
export function createExecutionConfirmCode(random: () => number = Math.random, length = EXECUTION_CONFIRM_CODE_LENGTH): string {
|
||||
const size = Math.max(1, Math.trunc(length));
|
||||
let code = "";
|
||||
for (let index = 0; index < size; index += 1) {
|
||||
const value = random();
|
||||
const normalized = Number.isFinite(value) ? Math.min(Math.max(value, 0), 0.999_999) : 0;
|
||||
code += CODE_ALPHABET[Math.floor(normalized * CODE_ALPHABET.length)] ?? CODE_ALPHABET[0];
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
/** Typed input matches the displayed code, ignoring case and surrounding spaces. */
|
||||
export function matchesExecutionConfirmCode(input: string, code: string): boolean {
|
||||
if (!code) return false;
|
||||
return input.trim().toUpperCase() === code.trim().toUpperCase();
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
import type { QueryResult } from "@/types/database";
|
||||
|
||||
/**
|
||||
* Rows read from a single target of a multi-database run. The merged view and
|
||||
* its Excel export need the query's body rather than just the editor's first
|
||||
* page, so every source is fetched up to this cap; the user's global result-row
|
||||
* limit still applies on top of it.
|
||||
*/
|
||||
export const MULTI_SOURCE_MAX_ROWS_PER_SOURCE = 10_000;
|
||||
|
||||
/**
|
||||
* Upper bound on merged rows. The merged view is a quick inspection surface, so
|
||||
* a fan-out over many connections must not be able to build an unbounded table;
|
||||
* hitting the cap is reported instead of silently dropping rows.
|
||||
*/
|
||||
export const MULTI_SOURCE_MERGE_MAX_ROWS = 50_000;
|
||||
|
||||
export type MultiSourceCellValue = string | number | boolean | null;
|
||||
|
||||
export interface MultiSourceResultInput {
|
||||
/** Stable identity of the contributing result (used as the row's origin key). */
|
||||
key: string;
|
||||
/** Human label written into the injected source column for every row. */
|
||||
label: string;
|
||||
result: QueryResult | undefined;
|
||||
}
|
||||
|
||||
export interface MultiSourceMergeOptions {
|
||||
/** Header of the injected first column; localized by the caller. */
|
||||
sourceColumnLabel: string;
|
||||
maxRows?: number;
|
||||
}
|
||||
|
||||
export interface MultiSourceColumnSummary {
|
||||
columnIndex: number;
|
||||
column: string;
|
||||
sum: number;
|
||||
/** Number of contributing values; empty cells never count. */
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface MultiSourceMergeSource {
|
||||
key: string;
|
||||
label: string;
|
||||
rowCount: number;
|
||||
columnCount: number;
|
||||
}
|
||||
|
||||
export interface MultiSourceMergeOutput {
|
||||
columns: string[];
|
||||
rows: MultiSourceCellValue[][];
|
||||
/** One entry per all-numeric column, aligned by column ordinal. */
|
||||
summaries: MultiSourceColumnSummary[];
|
||||
sources: MultiSourceMergeSource[];
|
||||
rowCount: number;
|
||||
truncated: boolean;
|
||||
/** Labels of inputs that carried no tabular payload and were left out. */
|
||||
skippedLabels: string[];
|
||||
}
|
||||
|
||||
interface MergePlan {
|
||||
input: MultiSourceResultInput;
|
||||
result: QueryResult;
|
||||
/** Merged column ordinal for each source column ordinal. */
|
||||
targetIndexes: Map<number, number>;
|
||||
}
|
||||
|
||||
function usableInput(input: MultiSourceResultInput): boolean {
|
||||
if (!input.result) return false;
|
||||
if (input.result.execution_error === true) return false;
|
||||
return (input.result.columns?.length ?? 0) > 0;
|
||||
}
|
||||
|
||||
function mergedColumnName(column: string | undefined, sourceIndex: number): string {
|
||||
const name = typeof column === "string" ? column.trim() : "";
|
||||
return name === "" ? `column_${sourceIndex + 1}` : name;
|
||||
}
|
||||
|
||||
function isEmptyCell(value: MultiSourceCellValue | undefined): boolean {
|
||||
if (value === null || value === undefined) return true;
|
||||
return typeof value === "string" && value.trim() === "";
|
||||
}
|
||||
|
||||
/**
|
||||
* A cell counts as numeric when it is a finite number or a non-empty string that
|
||||
* parses to one (drivers return DECIMAL/BIGINT as strings). Booleans are not
|
||||
* numeric, so a flag column never turns into a total.
|
||||
*/
|
||||
export function isNumericSummaryCellValue(value: MultiSourceCellValue | undefined): boolean {
|
||||
if (typeof value === "number") return Number.isFinite(value);
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
return trimmed !== "" && Number.isFinite(Number(trimmed));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple result-level aggregates: sums every column whose non-empty cells are
|
||||
* all numeric. A single non-numeric cell disqualifies the whole column, and a
|
||||
* column without values produces no total.
|
||||
*/
|
||||
export function summarizeNumericColumns(columns: readonly string[], rows: readonly (readonly MultiSourceCellValue[])[], options: { skipColumnIndexes?: readonly number[] } = {}): MultiSourceColumnSummary[] {
|
||||
const skipped = new Set(options.skipColumnIndexes ?? []);
|
||||
const sums: number[] = Array.from({ length: columns.length }, () => 0);
|
||||
const counts: number[] = Array.from({ length: columns.length }, () => 0);
|
||||
const stillNumeric: boolean[] = Array.from({ length: columns.length }, () => true);
|
||||
|
||||
for (const row of rows) {
|
||||
for (let columnIndex = 0; columnIndex < columns.length; columnIndex += 1) {
|
||||
if (skipped.has(columnIndex) || !stillNumeric[columnIndex]) continue;
|
||||
const value = row[columnIndex];
|
||||
if (isEmptyCell(value)) continue;
|
||||
if (!isNumericSummaryCellValue(value)) {
|
||||
stillNumeric[columnIndex] = false;
|
||||
continue;
|
||||
}
|
||||
sums[columnIndex] += typeof value === "number" ? value : Number(value);
|
||||
counts[columnIndex] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const summaries: MultiSourceColumnSummary[] = [];
|
||||
for (let columnIndex = 0; columnIndex < columns.length; columnIndex += 1) {
|
||||
if (skipped.has(columnIndex) || !stillNumeric[columnIndex] || counts[columnIndex] === 0) continue;
|
||||
summaries.push({ columnIndex, column: columns[columnIndex] ?? "", sum: sums[columnIndex]!, count: counts[columnIndex]! });
|
||||
}
|
||||
return summaries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges the results of one query executed against several connections /
|
||||
* databases into a single table.
|
||||
*
|
||||
* Columns are aligned by name (union, first-seen order) and every row keeps a
|
||||
* leading provenance column naming the source it came from. A column a source
|
||||
* does not have is padded with null. Duplicate column names inside one result
|
||||
* keep their own value streams by matching the Nth occurrence of a name.
|
||||
*/
|
||||
export function mergeMultiSourceResults(inputs: readonly MultiSourceResultInput[], options: MultiSourceMergeOptions): MultiSourceMergeOutput {
|
||||
const maxRows = Math.max(1, Math.trunc(options.maxRows ?? MULTI_SOURCE_MERGE_MAX_ROWS));
|
||||
const columns: string[] = [options.sourceColumnLabel];
|
||||
// A merged column is identified by (name, occurrence-in-source) so duplicate
|
||||
// names in one result still keep both of their value streams.
|
||||
const indexByLayoutKey = new Map<string, number>();
|
||||
const plans: MergePlan[] = [];
|
||||
const skippedLabels: string[] = [];
|
||||
|
||||
for (const input of inputs) {
|
||||
if (!usableInput(input)) {
|
||||
skippedLabels.push(input.label);
|
||||
continue;
|
||||
}
|
||||
const result = input.result!;
|
||||
const occurrences = new Map<string, number>();
|
||||
const targetIndexes = new Map<number, number>();
|
||||
const hiddenColumnIndexes = new Set(result.hidden_column_indexes ?? []);
|
||||
result.columns.forEach((column, sourceIndex) => {
|
||||
if (hiddenColumnIndexes.has(sourceIndex)) return;
|
||||
const name = mergedColumnName(column, sourceIndex);
|
||||
const occurrence = occurrences.get(name) ?? 0;
|
||||
occurrences.set(name, occurrence + 1);
|
||||
const layoutKey = `${name}\u0000${occurrence}`;
|
||||
let targetIndex = indexByLayoutKey.get(layoutKey);
|
||||
if (targetIndex === undefined) {
|
||||
targetIndex = columns.length;
|
||||
indexByLayoutKey.set(layoutKey, targetIndex);
|
||||
columns.push(name);
|
||||
}
|
||||
targetIndexes.set(sourceIndex, targetIndex);
|
||||
});
|
||||
plans.push({ input, result, targetIndexes });
|
||||
}
|
||||
|
||||
const rows: MultiSourceCellValue[][] = [];
|
||||
let truncated = false;
|
||||
for (const plan of plans) {
|
||||
for (const sourceRow of plan.result.rows) {
|
||||
if (rows.length >= maxRows) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
const row: MultiSourceCellValue[] = Array.from({ length: columns.length }, () => null);
|
||||
row[0] = plan.input.label;
|
||||
plan.targetIndexes.forEach((targetIndex, sourceIndex) => {
|
||||
row[targetIndex] = sourceRow[sourceIndex] ?? null;
|
||||
});
|
||||
rows.push(row);
|
||||
}
|
||||
if (truncated) break;
|
||||
}
|
||||
|
||||
return {
|
||||
columns,
|
||||
rows,
|
||||
// The injected provenance column is text by construction, but skip it
|
||||
// explicitly so a numeric-looking label can never become a total.
|
||||
summaries: summarizeNumericColumns(columns, rows, { skipColumnIndexes: [0] }),
|
||||
sources: plans.map((plan) => ({
|
||||
key: plan.input.key,
|
||||
label: plan.input.label,
|
||||
rowCount: plan.result.rows.length,
|
||||
columnCount: plan.targetIndexes.size,
|
||||
})),
|
||||
rowCount: rows.length,
|
||||
truncated,
|
||||
skippedLabels,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user