fix(grid): bound large table previews by byte budget
This commit is contained in:
19 files changed
+823
-23
No files matched your search
@@ -151,7 +151,7 @@ import { isCancelSearchShortcut, isCopyCurrentRowShortcut, isDeleteCurrentRowSho
|
|||||||
import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter";
|
import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter";
|
||||||
import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination";
|
import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination";
|
||||||
import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions";
|
import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions";
|
||||||
import { largeValueCellKey, largeValueCellMap } from "@/lib/dataGrid/dataGridLargeValues";
|
import { largeValueCellKey, largeValueCellMap, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
|
||||||
import { dataGridBottomScrollTop, dataGridScrollPosition, isDataGridAtScrollBottom, isDataGridNearScrollBottom, isDataGridPrefixAppend, shouldCheckInfiniteScrollAfterScroll, type DataGridScrollPosition } from "@/lib/dataGrid/dataGridInfiniteScroll";
|
import { dataGridBottomScrollTop, dataGridScrollPosition, isDataGridAtScrollBottom, isDataGridNearScrollBottom, isDataGridPrefixAppend, shouldCheckInfiniteScrollAfterScroll, type DataGridScrollPosition } from "@/lib/dataGrid/dataGridInfiniteScroll";
|
||||||
import { CANVAS_DATA_GRID_ROW_HEIGHT, MAX_CANVAS_DATA_GRID_PIXEL_RATIO, canvasDataGridActionOverlayWidth, canvasDataGridActionReservedWidth, dataGridSearchMatchKey, drawCanvasDataGrid, type CanvasDevicePixelSize } from "@/lib/dataGrid/canvasDataGridRenderer";
|
import { CANVAS_DATA_GRID_ROW_HEIGHT, MAX_CANVAS_DATA_GRID_PIXEL_RATIO, canvasDataGridActionOverlayWidth, canvasDataGridActionReservedWidth, dataGridSearchMatchKey, drawCanvasDataGrid, type CanvasDevicePixelSize } from "@/lib/dataGrid/canvasDataGridRenderer";
|
||||||
import { DATA_GRID_DARK_STRIPED_ROW_BG, DATA_GRID_LIGHT_STRIPED_ROW_BG, dataGridActiveRowBackground } from "@/lib/dataGrid/dataGridPaintTheme";
|
import { DATA_GRID_DARK_STRIPED_ROW_BG, DATA_GRID_LIGHT_STRIPED_ROW_BG, dataGridActiveRowBackground } from "@/lib/dataGrid/dataGridPaintTheme";
|
||||||
@@ -4141,7 +4141,7 @@ async function resolveLargeValueCells(rowIds: number[], columnIndexes: number[])
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (requestsByColumn.size === 0) return resolved;
|
if (requestsByColumn.size === 0) return resolved;
|
||||||
if (resolvedDatabaseType.value !== "mysql" || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) {
|
if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) {
|
||||||
throw new Error(t("grid.largeValueNeedsStableKey"));
|
throw new Error(t("grid.largeValueNeedsStableKey"));
|
||||||
}
|
}
|
||||||
const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex);
|
const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex);
|
||||||
@@ -5224,6 +5224,7 @@ async function applyOrderBySearch() {
|
|||||||
tableType: tableMeta.tableType,
|
tableType: tableMeta.tableType,
|
||||||
columns: tableMeta.columns.map((column) => column.name),
|
columns: tableMeta.columns.map((column) => column.name),
|
||||||
primaryKeys: tableMeta.primaryKeys,
|
primaryKeys: tableMeta.primaryKeys,
|
||||||
|
...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value),
|
||||||
orderBy: orderByClause,
|
orderBy: orderByClause,
|
||||||
limit: pageSize.value,
|
limit: pageSize.value,
|
||||||
whereInput: currentWhereInput(),
|
whereInput: currentWhereInput(),
|
||||||
@@ -5259,6 +5260,7 @@ async function applyWhereFilter() {
|
|||||||
tableType: tableMeta.tableType,
|
tableType: tableMeta.tableType,
|
||||||
columns: tableMeta.columns.map((column) => column.name),
|
columns: tableMeta.columns.map((column) => column.name),
|
||||||
primaryKeys: tableMeta.primaryKeys,
|
primaryKeys: tableMeta.primaryKeys,
|
||||||
|
...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value),
|
||||||
orderBy: orderByInput.value.trim() || (sortCol.value ? `${queryColumnRef(sortCol.value)} ${sortDir.value.toUpperCase()}` : undefined),
|
orderBy: orderByInput.value.trim() || (sortCol.value ? `${queryColumnRef(sortCol.value)} ${sortDir.value.toUpperCase()}` : undefined),
|
||||||
limit: pageSize.value,
|
limit: pageSize.value,
|
||||||
whereInput,
|
whereInput,
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import type { QueryTab } from "@/types/database";
|
|||||||
const mocks = vi.hoisted(() => ({
|
const mocks = vi.hoisted(() => ({
|
||||||
tabs: [] as QueryTab[],
|
tabs: [] as QueryTab[],
|
||||||
activeTabId: "",
|
activeTabId: "",
|
||||||
|
buildTableSelectSql: vi.fn(),
|
||||||
databaseType: "postgres" as string,
|
databaseType: "postgres" as string,
|
||||||
ensureConnected: vi.fn(),
|
ensureConnected: vi.fn(),
|
||||||
executeTabSql: vi.fn(),
|
executeTabSql: vi.fn(),
|
||||||
@@ -83,7 +84,7 @@ vi.mock("@/stores/settingsStore", () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("@/lib/table/tableSelectSql", () => ({
|
vi.mock("@/lib/table/tableSelectSql", () => ({
|
||||||
buildTableSelectSql: async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`,
|
buildTableSelectSql: mocks.buildTableSelectSql,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const dialogs = {
|
const dialogs = {
|
||||||
@@ -103,12 +104,13 @@ describe("useNavigationTargets openTableTarget", () => {
|
|||||||
mocks.tabs.length = 0;
|
mocks.tabs.length = 0;
|
||||||
mocks.databaseType = "postgres";
|
mocks.databaseType = "postgres";
|
||||||
mocks.ensureConnected.mockResolvedValue(undefined);
|
mocks.ensureConnected.mockResolvedValue(undefined);
|
||||||
|
mocks.buildTableSelectSql.mockImplementation(async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`);
|
||||||
mocks.executeTabSql.mockResolvedValue(undefined);
|
mocks.executeTabSql.mockResolvedValue(undefined);
|
||||||
mocks.getColumns.mockResolvedValue([column("id")]);
|
mocks.getColumns.mockResolvedValue([column("id")]);
|
||||||
mocks.listIndexes.mockResolvedValue([]);
|
mocks.listIndexes.mockResolvedValue([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("marks row identity pending until real metadata lands", async () => {
|
it("loads stable row identity before the first PostgreSQL table query", async () => {
|
||||||
let releaseColumns: (columns: unknown[]) => void = () => {};
|
let releaseColumns: (columns: unknown[]) => void = () => {};
|
||||||
mocks.getColumns.mockReturnValueOnce(
|
mocks.getColumns.mockReturnValueOnce(
|
||||||
new Promise((resolve) => {
|
new Promise((resolve) => {
|
||||||
@@ -124,10 +126,15 @@ describe("useNavigationTargets openTableTarget", () => {
|
|||||||
releaseColumns([column("id")]);
|
releaseColumns([column("id")]);
|
||||||
await open;
|
await open;
|
||||||
|
|
||||||
// 数据查询执行期间元数据未落地:行标识等待必须已挂起
|
expect(pendingDuringQuery).toBe(false);
|
||||||
expect(pendingDuringQuery).toBe(true);
|
|
||||||
expect(mocks.tabs[0]?.tableMeta?.primaryKeys).toEqual(["id"]);
|
expect(mocks.tabs[0]?.tableMeta?.primaryKeys).toEqual(["id"]);
|
||||||
expect(mocks.tabs[0]?.tableMetaPending).toBe(false);
|
expect(mocks.tabs[0]?.tableMetaPending).toBe(false);
|
||||||
|
expect(mocks.buildTableSelectSql).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
columns: ["id"],
|
||||||
|
primaryKeys: ["id"],
|
||||||
|
}),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("reuses cached metadata and force-refreshes it once per catalog after a structure save", async () => {
|
it("reuses cached metadata and force-refreshes it once per catalog after a structure save", async () => {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { useQueryStore } from "@/stores/queryStore";
|
|||||||
import { useSettingsStore } from "@/stores/settingsStore";
|
import { useSettingsStore } from "@/stores/settingsStore";
|
||||||
import { buildTableSelectSql, quoteTableDataIdentifier } from "@/lib/table/tableSelectSql";
|
import { buildTableSelectSql, quoteTableDataIdentifier } from "@/lib/table/tableSelectSql";
|
||||||
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
||||||
|
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
|
||||||
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
|
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
|
||||||
import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
|
import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
|
||||||
import * as api from "@/lib/backend/api";
|
import * as api from "@/lib/backend/api";
|
||||||
@@ -65,6 +66,7 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
|
|||||||
// 结果(可能是失败结果的 ["Error"]),进入 SQL 会生成非法投影;
|
// 结果(可能是失败结果的 ["Error"]),进入 SQL 会生成非法投影;
|
||||||
// 真实列缺失时省略 columns 让 builder 生成 SELECT *
|
// 真实列缺失时省略 columns 让 builder 生成 SELECT *
|
||||||
const realColumns = tab.tableMeta?.columns.length ? tab.tableMeta.columns : undefined;
|
const realColumns = tab.tableMeta?.columns.length ? tab.tableMeta.columns : undefined;
|
||||||
|
const limit = options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize);
|
||||||
return buildTableSelectSql({
|
return buildTableSelectSql({
|
||||||
databaseType: effectiveDbType,
|
databaseType: effectiveDbType,
|
||||||
driverProfile: config?.driver_profile,
|
driverProfile: config?.driver_profile,
|
||||||
@@ -76,8 +78,9 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
|
|||||||
catalog: tableMeta?.catalog,
|
catalog: tableMeta?.catalog,
|
||||||
columns: realColumns?.map((column) => column.name),
|
columns: realColumns?.map((column) => column.name),
|
||||||
primaryKeys,
|
primaryKeys,
|
||||||
|
...tableDataLargeValuePreviewOptions(effectiveDbType, realColumns ?? [], primaryKeys, limit),
|
||||||
includeRowId: useRowId,
|
includeRowId: useRowId,
|
||||||
limit: options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize),
|
limit,
|
||||||
...options,
|
...options,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { invalidateTableMetadataCache, loadTableMetadata } from "@/lib/metadata/
|
|||||||
import { canApplyDataTabMetadata } from "@/lib/sidebar/dataTabOpenPolicy";
|
import { canApplyDataTabMetadata } from "@/lib/sidebar/dataTabOpenPolicy";
|
||||||
import { isNoSnapshotErrorResult, isQueryExecutionErrorResult } from "@/lib/query/queryResultError";
|
import { isNoSnapshotErrorResult, isQueryExecutionErrorResult } from "@/lib/query/queryResultError";
|
||||||
import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
|
import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
|
||||||
|
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
|
||||||
import { editableRowIdentifierColumns, usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
|
import { editableRowIdentifierColumns, usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
|
||||||
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
||||||
import { uuid } from "@/lib/common/utils";
|
import { uuid } from "@/lib/common/utils";
|
||||||
@@ -136,6 +137,21 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
|
|||||||
await queryStore.executeTabSql(tabId, sql, { pagination: { limit: pageLimit, offset: 0 } });
|
await queryStore.executeTabSql(tabId, sql, { pagination: { limit: pageLimit, offset: 0 } });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const eagerMetadata =
|
||||||
|
effectiveDbType === "mysql" || effectiveDbType === "postgres"
|
||||||
|
? await loadTableMetadata({
|
||||||
|
connectionId: target.connectionId,
|
||||||
|
database: target.database,
|
||||||
|
schema: querySchema,
|
||||||
|
tableName: target.tableName,
|
||||||
|
tableType: targetTableType,
|
||||||
|
databaseType: effectiveDbType,
|
||||||
|
driverProfile: config.driver_profile || config.db_type,
|
||||||
|
catalog: target.catalog,
|
||||||
|
})
|
||||||
|
: undefined;
|
||||||
|
const eagerColumns = eagerMetadata?.metadata.columns ?? [];
|
||||||
|
const eagerPrimaryKeys = eagerMetadata?.metadata.primaryKeys ?? [];
|
||||||
const sql = await buildTableSelectSql({
|
const sql = await buildTableSelectSql({
|
||||||
databaseType: effectiveDbType,
|
databaseType: effectiveDbType,
|
||||||
driverProfile: config.driver_profile,
|
driverProfile: config.driver_profile,
|
||||||
@@ -145,6 +161,9 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
|
|||||||
database: target.database,
|
database: target.database,
|
||||||
tableName: target.tableName,
|
tableName: target.tableName,
|
||||||
tableType: targetTableType,
|
tableType: targetTableType,
|
||||||
|
columns: eagerColumns.map((column) => column.name),
|
||||||
|
primaryKeys: eagerPrimaryKeys,
|
||||||
|
...tableDataLargeValuePreviewOptions(effectiveDbType, eagerColumns, eagerPrimaryKeys, pageLimit),
|
||||||
whereInput: target.whereInput,
|
whereInput: target.whereInput,
|
||||||
limit: pageLimit,
|
limit: pageLimit,
|
||||||
});
|
});
|
||||||
@@ -156,8 +175,8 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
|
|||||||
database: target.database,
|
database: target.database,
|
||||||
tableName: target.tableName,
|
tableName: target.tableName,
|
||||||
tableType: targetTableType,
|
tableType: targetTableType,
|
||||||
columns: [],
|
columns: eagerColumns,
|
||||||
primaryKeys: [],
|
primaryKeys: eagerPrimaryKeys,
|
||||||
});
|
});
|
||||||
firstExecuteStarted = true;
|
firstExecuteStarted = true;
|
||||||
// 取消计数快照:isCancelling 是瞬态的(取消失败/查询先完成会被清掉),
|
// 取消计数快照:isCancelling 是瞬态的(取消失败/查询先完成会被清掉),
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import { hasTreeNodeDatabaseContext } from "@/lib/sidebar/treeNodeContext";
|
|||||||
import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
|
import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
|
||||||
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
|
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
|
||||||
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
||||||
|
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
|
||||||
import { canActivateExistingDataTableTab } from "@/lib/tabs/dataTabActivation";
|
import { canActivateExistingDataTableTab } from "@/lib/tabs/dataTabActivation";
|
||||||
import { beginDataTabNavigation, endDataTabNavigation, isCurrentDataTabNavigation } from "@/lib/tabs/dataTabNavigationGeneration";
|
import { beginDataTabNavigation, endDataTabNavigation, isCurrentDataTabNavigation } from "@/lib/tabs/dataTabNavigationGeneration";
|
||||||
|
|
||||||
@@ -293,18 +294,23 @@ export function useSidebarDataOpenRuntime() {
|
|||||||
} else if (deferTableMetaRefresh) {
|
} else if (deferTableMetaRefresh) {
|
||||||
logPhase("metadata-deferred", { tabId });
|
logPhase("metadata-deferred", { tabId });
|
||||||
} else {
|
} else {
|
||||||
void refreshTableMetaInBackground(tabId);
|
|
||||||
logPhase("metadata-started", { tabId });
|
logPhase("metadata-started", { tabId });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const metadataRefresh = shouldRefreshTableMeta && !deferTableMetaRefresh ? refreshTableMetaInBackground(tabId) : undefined;
|
||||||
|
if (!cachedTableMeta && (effectiveDbType === "mysql" || effectiveDbType === "postgres")) {
|
||||||
|
await metadataRefresh;
|
||||||
|
}
|
||||||
|
|
||||||
// Check if superseded by a newer openData call
|
// Check if superseded by a newer openData call
|
||||||
if (!isActive()) {
|
if (!isActive()) {
|
||||||
logPhase("superseded-before-build-sql", { tabId });
|
logPhase("superseded-before-build-sql", { tabId });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const columns = cachedTableMeta?.columns ?? [];
|
const loadedTableMeta = cachedTableMeta ?? queryStore.tabs.find((item) => item.id === tabId)?.tableMeta;
|
||||||
const primaryKeys = cachedTableMeta?.primaryKeys ?? [];
|
const columns = loadedTableMeta?.columns ?? [];
|
||||||
|
const primaryKeys = loadedTableMeta?.primaryKeys ?? [];
|
||||||
const includeRowId = usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableType);
|
const includeRowId = usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableType);
|
||||||
const sql = await buildTableSelectSql({
|
const sql = await buildTableSelectSql({
|
||||||
databaseType: effectiveDbType,
|
databaseType: effectiveDbType,
|
||||||
@@ -317,6 +323,7 @@ export function useSidebarDataOpenRuntime() {
|
|||||||
catalog: node.catalog,
|
catalog: node.catalog,
|
||||||
columns: columns.map((column) => column.name),
|
columns: columns.map((column) => column.name),
|
||||||
primaryKeys,
|
primaryKeys,
|
||||||
|
...tableDataLargeValuePreviewOptions(effectiveDbType, columns, primaryKeys, limit),
|
||||||
limit,
|
limit,
|
||||||
includeRowId,
|
includeRowId,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,8 +1,63 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues";
|
import { appendLargeValueCells, canUseTableDataLargeValuePreview, largeValueCellMap, remapLargeValueCells, TABLE_DATA_CELL_PREVIEW_SIZE, TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
|
||||||
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
|
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
|
||||||
|
import type { ColumnInfo } from "@/types/database";
|
||||||
|
|
||||||
|
function column(name: string, dataType: string, isPrimaryKey = false): ColumnInfo {
|
||||||
|
return {
|
||||||
|
name,
|
||||||
|
data_type: dataType,
|
||||||
|
is_nullable: false,
|
||||||
|
column_default: null,
|
||||||
|
is_primary_key: isPrimaryKey,
|
||||||
|
extra: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe("data grid large-value metadata", () => {
|
describe("data grid large-value metadata", () => {
|
||||||
|
it("enables bounded table previews only for supported databases with stable keys", () => {
|
||||||
|
const columns = [column("id", "bigint", true), column("payload", "longtext")];
|
||||||
|
|
||||||
|
expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({
|
||||||
|
columnTypes: ["bigint", "longtext"],
|
||||||
|
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
|
||||||
|
});
|
||||||
|
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({
|
||||||
|
columnTypes: ["bigint", "longtext"],
|
||||||
|
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
|
||||||
|
});
|
||||||
|
expect(tableDataLargeValuePreviewOptions("sqlite", columns, ["id"], 100)).toEqual({});
|
||||||
|
expect(tableDataLargeValuePreviewOptions("mysql", columns, [], 100)).toEqual({});
|
||||||
|
expect(tableDataLargeValuePreviewOptions("mysql", [columns[0]!], ["id"], 100)).toEqual({});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("disables marker parsing when a real column uses the reserved preview prefix", () => {
|
||||||
|
const columns = [column("id", "bigint", true), column("__dbx_large_value_bytes_t_0", "text")];
|
||||||
|
|
||||||
|
expect(canUseTableDataLargeValuePreview("postgres", columns, ["id"])).toBe(false);
|
||||||
|
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("shrinks each cell preview by serialized byte budget without changing page size", () => {
|
||||||
|
const columns = [column("id", "bigint", true), ...Array.from({ length: 256 }, (_, index) => column(`payload_${index}`, "text"))];
|
||||||
|
const options = tableDataLargeValuePreviewOptions("postgres", columns, ["ID"], 1_000);
|
||||||
|
|
||||||
|
expect(options).toEqual({
|
||||||
|
columnTypes: columns.map((item) => item.data_type),
|
||||||
|
largeValuePreviewSize: 16,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!("largeValuePreviewSize" in options)) throw new Error("expected preview options");
|
||||||
|
const previewSize = options.largeValuePreviewSize;
|
||||||
|
const previewCellCount = 1_000 * 256;
|
||||||
|
const serializedSamples = ["中".repeat(previewSize), "😀".repeat(previewSize), "\u0000".repeat(previewSize)];
|
||||||
|
for (const sample of serializedSamples) {
|
||||||
|
const serializedContent = JSON.stringify(sample).slice(1, -1);
|
||||||
|
const serializedBytes = new TextEncoder().encode(serializedContent).byteLength * previewCellCount;
|
||||||
|
expect(serializedBytes).toBeLessThanOrEqual(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
|
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
|
||||||
expect(
|
expect(
|
||||||
appendLargeValueCells(
|
appendLargeValueCells(
|
||||||
|
|||||||
@@ -929,6 +929,7 @@ export async function executeMulti(
|
|||||||
pageSize?: number;
|
pageSize?: number;
|
||||||
maxResultBytes?: number;
|
maxResultBytes?: number;
|
||||||
resultKeyColumns?: string[];
|
resultKeyColumns?: string[];
|
||||||
|
tableDataPreview?: boolean;
|
||||||
resultSessionId?: string;
|
resultSessionId?: string;
|
||||||
clientSessionId?: string;
|
clientSessionId?: string;
|
||||||
timeoutSecs?: number;
|
timeoutSecs?: number;
|
||||||
@@ -971,6 +972,7 @@ export async function executeMultiWithProgress(
|
|||||||
pageSize?: number;
|
pageSize?: number;
|
||||||
maxResultBytes?: number;
|
maxResultBytes?: number;
|
||||||
resultKeyColumns?: string[];
|
resultKeyColumns?: string[];
|
||||||
|
tableDataPreview?: boolean;
|
||||||
resultSessionId?: string;
|
resultSessionId?: string;
|
||||||
clientSessionId?: string;
|
clientSessionId?: string;
|
||||||
timeoutSecs?: number;
|
timeoutSecs?: number;
|
||||||
|
|||||||
@@ -1148,6 +1148,7 @@ export async function executeMulti(
|
|||||||
pageSize?: number;
|
pageSize?: number;
|
||||||
maxResultBytes?: number;
|
maxResultBytes?: number;
|
||||||
resultKeyColumns?: string[];
|
resultKeyColumns?: string[];
|
||||||
|
tableDataPreview?: boolean;
|
||||||
resultSessionId?: string;
|
resultSessionId?: string;
|
||||||
clientSessionId?: string;
|
clientSessionId?: string;
|
||||||
timeoutSecs?: number;
|
timeoutSecs?: number;
|
||||||
@@ -1194,6 +1195,7 @@ export async function executeMultiWithProgress(
|
|||||||
pageSize?: number;
|
pageSize?: number;
|
||||||
maxResultBytes?: number;
|
maxResultBytes?: number;
|
||||||
resultKeyColumns?: string[];
|
resultKeyColumns?: string[];
|
||||||
|
tableDataPreview?: boolean;
|
||||||
resultSessionId?: string;
|
resultSessionId?: string;
|
||||||
clientSessionId?: string;
|
clientSessionId?: string;
|
||||||
timeoutSecs?: number;
|
timeoutSecs?: number;
|
||||||
|
|||||||
@@ -1,6 +1,27 @@
|
|||||||
import type { QueryResult } from "@/types/database";
|
import type { ColumnInfo, DatabaseType, QueryResult } from "@/types/database";
|
||||||
|
|
||||||
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
|
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
|
||||||
|
export const TABLE_DATA_CELL_PREVIEW_MIN_SIZE = 1;
|
||||||
|
export const TABLE_DATA_CELL_PREVIEW_SIZE = 8 * 1024;
|
||||||
|
export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024;
|
||||||
|
export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6;
|
||||||
|
const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_";
|
||||||
|
|
||||||
|
export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean {
|
||||||
|
return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record<string, never> {
|
||||||
|
if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {};
|
||||||
|
const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase()));
|
||||||
|
const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
|
||||||
|
if (previewColumnCount === 0) return {};
|
||||||
|
const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER);
|
||||||
|
return {
|
||||||
|
columnTypes: columns.map((column) => column.data_type),
|
||||||
|
largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
|
export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
|
||||||
return `${rowIndex}:${columnIndex}`;
|
return `${rowIndex}:${columnIndex}`;
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ export interface BuildTableSelectSqlOptions {
|
|||||||
tableType?: string;
|
tableType?: string;
|
||||||
primaryKeys?: string[];
|
primaryKeys?: string[];
|
||||||
columns?: string[];
|
columns?: string[];
|
||||||
|
columnTypes?: string[];
|
||||||
|
largeValuePreviewSize?: number;
|
||||||
fallbackOrderColumns?: string[];
|
fallbackOrderColumns?: string[];
|
||||||
orderBy?: string;
|
orderBy?: string;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
|
|||||||
@@ -114,7 +114,9 @@ describe("queryStore table data refresh", () => {
|
|||||||
tableType: "TABLE",
|
tableType: "TABLE",
|
||||||
catalog: undefined,
|
catalog: undefined,
|
||||||
columns: ["id", "status", "created_at"],
|
columns: ["id", "status", "created_at"],
|
||||||
|
columnTypes: ["integer", "text", "timestamp"],
|
||||||
primaryKeys: ["id"],
|
primaryKeys: ["id"],
|
||||||
|
largeValuePreviewSize: 8192,
|
||||||
includeRowId: false,
|
includeRowId: false,
|
||||||
whereInput: "status = 'ACTIVE'",
|
whereInput: "status = 'ACTIVE'",
|
||||||
orderBy: "created_at DESC",
|
orderBy: "created_at DESC",
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ import { connectionObjectTreeNodeSchema, connectionQueryExecutionSchema, connect
|
|||||||
import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
|
import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
|
||||||
import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource";
|
import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource";
|
||||||
import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection";
|
import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection";
|
||||||
import { appendLargeValueCells, remapLargeValueCells, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues";
|
import { appendLargeValueCells, canUseTableDataLargeValuePreview, remapLargeValueCells, tableDataLargeValuePreviewOptions, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues";
|
||||||
import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort";
|
import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort";
|
||||||
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
|
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
|
||||||
import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit";
|
import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit";
|
||||||
@@ -2782,6 +2782,7 @@ export const useQueryStore = defineStore("query", () => {
|
|||||||
catalog: tableMeta.catalog,
|
catalog: tableMeta.catalog,
|
||||||
columns: tableMeta.columns.map((column) => column.name),
|
columns: tableMeta.columns.map((column) => column.name),
|
||||||
primaryKeys,
|
primaryKeys,
|
||||||
|
...tableDataLargeValuePreviewOptions(effectiveDbType, tableMeta.columns, primaryKeys, limit),
|
||||||
includeRowId: usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableMeta.tableType),
|
includeRowId: usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableMeta.tableType),
|
||||||
whereInput: tab.whereInput,
|
whereInput: tab.whereInput,
|
||||||
orderBy,
|
orderBy,
|
||||||
@@ -4573,6 +4574,8 @@ export const useQueryStore = defineStore("query", () => {
|
|||||||
queryExecutionLog("info", "execute-multi:start", { traceId, elapsed: elapsed() });
|
queryExecutionLog("info", "execute-multi:start", { traceId, elapsed: elapsed() });
|
||||||
// Query and data tabs use a tab-scoped pool so repeated executions keep
|
// Query and data tabs use a tab-scoped pool so repeated executions keep
|
||||||
// connection-local state and avoid MySQL pool resets on every refresh.
|
// connection-local state and avoid MySQL pool resets on every refresh.
|
||||||
|
const dataTabMeta = tab.mode === "data" ? tableMetaForDataTab(tab) : undefined;
|
||||||
|
const useTableDataPreview = canUseTableDataLargeValuePreview(effectiveDbType, dataTabMeta?.columns ?? [], dataTabMeta?.primaryKeys ?? []);
|
||||||
const executionOptions = {
|
const executionOptions = {
|
||||||
...(typeof pageLimit === "number"
|
...(typeof pageLimit === "number"
|
||||||
? useAgentResultSession
|
? useAgentResultSession
|
||||||
@@ -4586,10 +4589,11 @@ export const useQueryStore = defineStore("query", () => {
|
|||||||
: { maxRows: pageLimit, fetchSize: pageLimit }
|
: { maxRows: pageLimit, fetchSize: pageLimit }
|
||||||
: { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }),
|
: { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }),
|
||||||
...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}),
|
...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}),
|
||||||
...(tab.mode === "data" && effectiveDbType === "mysql"
|
...(tab.mode === "data" && (effectiveDbType === "mysql" || effectiveDbType === "postgres")
|
||||||
? {
|
? {
|
||||||
maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES,
|
maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES,
|
||||||
resultKeyColumns: tableMetaForDataTab(tab)?.primaryKeys ?? [],
|
resultKeyColumns: dataTabMeta?.primaryKeys ?? [],
|
||||||
|
tableDataPreview: useTableDataPreview,
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
timeoutSecs: queryTimeoutSecs,
|
timeoutSecs: queryTimeoutSecs,
|
||||||
|
|||||||
@@ -266,15 +266,43 @@ impl ExecuteMultiResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn success_with_index_and_large_values(
|
fn success_with_index_and_large_values(
|
||||||
|
mut result: db::QueryResult,
|
||||||
|
statement_index: usize,
|
||||||
|
mut large_value_cells: Vec<db::LargeValueCell>,
|
||||||
|
table_data_preview: bool,
|
||||||
|
) -> Self {
|
||||||
|
let server_cells = if table_data_preview {
|
||||||
|
remap_large_value_cells_around_server_markers(&result, &mut large_value_cells);
|
||||||
|
extract_server_large_value_markers(&mut result)
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
};
|
||||||
|
Self {
|
||||||
|
result,
|
||||||
|
large_value_cells: merge_large_value_cells(large_value_cells, server_cells),
|
||||||
|
execution_error: false,
|
||||||
|
statement_index: Some(statement_index),
|
||||||
|
error: None,
|
||||||
|
server_message: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn success_with_index_and_optional_server_large_values(
|
||||||
result: db::QueryResult,
|
result: db::QueryResult,
|
||||||
statement_index: usize,
|
statement_index: usize,
|
||||||
large_value_cells: Vec<db::LargeValueCell>,
|
table_data_preview: bool,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
|
Self::success_with_index_and_large_values(result, statement_index, Vec::new(), table_data_preview)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn success_with_optional_server_large_values(mut result: db::QueryResult, table_data_preview: bool) -> Self {
|
||||||
|
let large_value_cells =
|
||||||
|
if table_data_preview { extract_server_large_value_markers(&mut result) } else { Vec::new() };
|
||||||
Self {
|
Self {
|
||||||
result,
|
result,
|
||||||
large_value_cells,
|
large_value_cells,
|
||||||
execution_error: false,
|
execution_error: false,
|
||||||
statement_index: Some(statement_index),
|
statement_index: None,
|
||||||
error: None,
|
error: None,
|
||||||
server_message: false,
|
server_message: false,
|
||||||
}
|
}
|
||||||
@@ -290,6 +318,228 @@ impl ExecuteMultiResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const SERVER_LARGE_VALUE_UNKNOWN_BYTES: usize = 64 * 1024 * 1024;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum ServerLargeValuePreviewKind {
|
||||||
|
Text,
|
||||||
|
Binary,
|
||||||
|
Vector,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct ServerLargeValueMarker {
|
||||||
|
result_index: usize,
|
||||||
|
source_index: usize,
|
||||||
|
preview_kind: Option<ServerLargeValuePreviewKind>,
|
||||||
|
source_type: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn server_large_value_alias(
|
||||||
|
suffix: &str,
|
||||||
|
) -> Option<(usize, Option<ServerLargeValuePreviewKind>, Option<&'static str>)> {
|
||||||
|
if let Ok(source_index) = suffix.parse::<usize>() {
|
||||||
|
return Some((source_index, None, None));
|
||||||
|
}
|
||||||
|
let (kind, source_index) = suffix.split_once('_')?;
|
||||||
|
let source_index = source_index.parse::<usize>().ok()?;
|
||||||
|
let (preview_kind, source_type) = match kind {
|
||||||
|
"T" => (ServerLargeValuePreviewKind::Text, None),
|
||||||
|
"B" => (ServerLargeValuePreviewKind::Binary, None),
|
||||||
|
"V" => (ServerLargeValuePreviewKind::Vector, Some("vector")),
|
||||||
|
"J" => (ServerLargeValuePreviewKind::Text, Some("json")),
|
||||||
|
"K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")),
|
||||||
|
"S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")),
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
Some((source_index, Some(preview_kind), source_type))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeValuePreviewKind, usize)> {
|
||||||
|
let (kind, preview_size) = value.as_str()?.split_once(':')?;
|
||||||
|
let kind = match kind {
|
||||||
|
"T" => ServerLargeValuePreviewKind::Text,
|
||||||
|
"B" => ServerLargeValuePreviewKind::Binary,
|
||||||
|
"V" => ServerLargeValuePreviewKind::Vector,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
Some((kind, preview_size.parse::<usize>().ok()?.max(1)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate_server_large_value_preview(
|
||||||
|
value: &mut serde_json::Value,
|
||||||
|
kind: ServerLargeValuePreviewKind,
|
||||||
|
preview_size: usize,
|
||||||
|
) -> bool {
|
||||||
|
let serde_json::Value::String(text) = value else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if matches!(kind, ServerLargeValuePreviewKind::Vector) {
|
||||||
|
let truncated = text.chars().count() > preview_size;
|
||||||
|
let vector_text = if truncated {
|
||||||
|
let prefix: String = text.chars().take(preview_size).collect();
|
||||||
|
let Some(last_separator) = prefix.rfind(',') else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
format!("{}]", &prefix[..last_separator])
|
||||||
|
} else {
|
||||||
|
text.clone()
|
||||||
|
};
|
||||||
|
let Ok(vector) = serde_json::from_str::<Vec<serde_json::Value>>(&vector_text) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
*value = serde_json::Value::Array(vector);
|
||||||
|
return truncated;
|
||||||
|
}
|
||||||
|
let truncate_at = match kind {
|
||||||
|
ServerLargeValuePreviewKind::Text => text.char_indices().nth(preview_size).map(|(index, _)| index),
|
||||||
|
ServerLargeValuePreviewKind::Binary => text
|
||||||
|
.strip_prefix("0x")
|
||||||
|
.filter(|hex| hex.len() > preview_size.saturating_mul(2))
|
||||||
|
.map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))),
|
||||||
|
ServerLargeValuePreviewKind::Vector => unreachable!(),
|
||||||
|
};
|
||||||
|
let Some(truncate_at) = truncate_at else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
text.truncate(truncate_at);
|
||||||
|
text.push_str("...");
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn server_large_value_markers(result: &db::QueryResult) -> Vec<ServerLargeValueMarker> {
|
||||||
|
let mut markers = Vec::new();
|
||||||
|
for (result_index, column) in result.columns.iter().enumerate() {
|
||||||
|
let Some((source_index, preview_kind, source_type)) = column
|
||||||
|
.strip_prefix(crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX)
|
||||||
|
.and_then(server_large_value_alias)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let expected_source_index = result_index.checked_sub(markers.len() + 1);
|
||||||
|
if expected_source_index == Some(source_index) {
|
||||||
|
markers.push(ServerLargeValueMarker { result_index, source_index, preview_kind, source_type });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
markers
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec<db::LargeValueCell> {
|
||||||
|
let markers = server_large_value_markers(result);
|
||||||
|
if markers.is_empty() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut large_value_cells = Vec::new();
|
||||||
|
for marker in &markers {
|
||||||
|
let source_result_index = marker.result_index.saturating_sub(1);
|
||||||
|
let row_kind = result
|
||||||
|
.rows
|
||||||
|
.iter()
|
||||||
|
.find_map(|row| row.get(marker.result_index).and_then(server_large_value_marker).map(|value| value.0));
|
||||||
|
let source_type = marker.source_type.or_else(|| {
|
||||||
|
(marker.preview_kind.or(row_kind) == Some(ServerLargeValuePreviewKind::Vector)).then_some("vector")
|
||||||
|
});
|
||||||
|
if let (Some(source_type), Some(column_type)) = (source_type, result.column_types.get_mut(source_result_index))
|
||||||
|
{
|
||||||
|
*column_type = source_type.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (row_index, row) in result.rows.iter_mut().enumerate() {
|
||||||
|
for marker in &markers {
|
||||||
|
let marker_value = row.get(marker.result_index).and_then(server_large_value_marker);
|
||||||
|
let source_result_index = marker.result_index.saturating_sub(1);
|
||||||
|
if marker_value.is_some_and(|(kind, preview_size)| {
|
||||||
|
row.get_mut(source_result_index)
|
||||||
|
.is_some_and(|value| truncate_server_large_value_preview(value, kind, preview_size))
|
||||||
|
}) {
|
||||||
|
large_value_cells.push(db::LargeValueCell {
|
||||||
|
row_index,
|
||||||
|
column_index: marker.source_index,
|
||||||
|
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let removed: std::collections::HashSet<usize> = markers.iter().map(|marker| marker.result_index).collect();
|
||||||
|
let retained_index = |index: usize| index - removed.iter().filter(|removed_index| **removed_index < index).count();
|
||||||
|
result.spatial_columns.retain_mut(|column| {
|
||||||
|
if removed.contains(&column.column_index) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
column.column_index = retained_index(column.column_index);
|
||||||
|
true
|
||||||
|
});
|
||||||
|
for row in &mut result.rows {
|
||||||
|
let mut index = 0;
|
||||||
|
row.retain(|_| {
|
||||||
|
let keep = !removed.contains(&index);
|
||||||
|
index += 1;
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for row in &mut result.spatial_values {
|
||||||
|
let mut index = 0;
|
||||||
|
row.retain(|_| {
|
||||||
|
let keep = !removed.contains(&index);
|
||||||
|
index += 1;
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let mut index = 0;
|
||||||
|
result.columns.retain(|_| {
|
||||||
|
let keep = !removed.contains(&index);
|
||||||
|
index += 1;
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
let mut index = 0;
|
||||||
|
result.column_types.retain(|_| {
|
||||||
|
let keep = !removed.contains(&index);
|
||||||
|
index += 1;
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
let mut index = 0;
|
||||||
|
result.column_sortables.retain(|_| {
|
||||||
|
let keep = !removed.contains(&index);
|
||||||
|
index += 1;
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
large_value_cells
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remap_large_value_cells_around_server_markers(result: &db::QueryResult, cells: &mut Vec<db::LargeValueCell>) {
|
||||||
|
let removed: std::collections::HashSet<usize> =
|
||||||
|
server_large_value_markers(result).into_iter().map(|marker| marker.result_index).collect();
|
||||||
|
if removed.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cells.retain_mut(|cell| {
|
||||||
|
if removed.contains(&cell.column_index) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
cell.column_index -= removed.iter().filter(|index| **index < cell.column_index).count();
|
||||||
|
true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn merge_large_value_cells(
|
||||||
|
mut driver_cells: Vec<db::LargeValueCell>,
|
||||||
|
server_cells: Vec<db::LargeValueCell>,
|
||||||
|
) -> Vec<db::LargeValueCell> {
|
||||||
|
for server_cell in server_cells {
|
||||||
|
if let Some(driver_cell) = driver_cells
|
||||||
|
.iter_mut()
|
||||||
|
.find(|cell| cell.row_index == server_cell.row_index && cell.column_index == server_cell.column_index)
|
||||||
|
{
|
||||||
|
*driver_cell = server_cell;
|
||||||
|
} else {
|
||||||
|
driver_cells.push(server_cell);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
driver_cells
|
||||||
|
}
|
||||||
|
|
||||||
impl From<db::QueryResult> for ExecuteMultiResult {
|
impl From<db::QueryResult> for ExecuteMultiResult {
|
||||||
fn from(result: db::QueryResult) -> Self {
|
fn from(result: db::QueryResult) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -639,6 +889,9 @@ pub struct QueryExecutionOptions {
|
|||||||
/// Result columns that must stay exact because clients use them as stable
|
/// Result columns that must stay exact because clients use them as stable
|
||||||
/// row identifiers when fetching full large-cell values on demand.
|
/// row identifiers when fetching full large-cell values on demand.
|
||||||
pub result_key_columns: Vec<String>,
|
pub result_key_columns: Vec<String>,
|
||||||
|
/// Enables extraction of hidden server-side preview metadata. This must be
|
||||||
|
/// set only for generated table-data preview SQL, never arbitrary queries.
|
||||||
|
pub table_data_preview: bool,
|
||||||
/// Doris / StarRocks catalog selected for this query tab.
|
/// Doris / StarRocks catalog selected for this query tab.
|
||||||
pub catalog: Option<String>,
|
pub catalog: Option<String>,
|
||||||
pub result_session_id: Option<String>,
|
pub result_session_id: Option<String>,
|
||||||
@@ -2252,6 +2505,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
|||||||
// HTTP SQLite providers send all statements in one request so the provider
|
// HTTP SQLite providers send all statements in one request so the provider
|
||||||
// can preserve batch ordering and atomicity.
|
// can preserve batch ordering and atomicity.
|
||||||
if is_http_sqlite {
|
if is_http_sqlite {
|
||||||
|
let table_data_preview = options.table_data_preview;
|
||||||
return single_statement_multi_result(
|
return single_statement_multi_result(
|
||||||
execute_sql_statement_with_options_typed(
|
execute_sql_statement_with_options_typed(
|
||||||
state,
|
state,
|
||||||
@@ -2263,6 +2517,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
|||||||
options,
|
options,
|
||||||
)
|
)
|
||||||
.await,
|
.await,
|
||||||
|
table_data_preview,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2306,6 +2561,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
|||||||
)
|
)
|
||||||
{
|
{
|
||||||
let single_sql = statements.into_iter().next().unwrap_or_default();
|
let single_sql = statements.into_iter().next().unwrap_or_default();
|
||||||
|
let table_data_preview = options.table_data_preview;
|
||||||
return single_statement_multi_result(
|
return single_statement_multi_result(
|
||||||
execute_sql_statement_with_options_typed(
|
execute_sql_statement_with_options_typed(
|
||||||
state,
|
state,
|
||||||
@@ -2317,6 +2573,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
|||||||
options,
|
options,
|
||||||
)
|
)
|
||||||
.await,
|
.await,
|
||||||
|
table_data_preview,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2372,7 +2629,11 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
|||||||
{
|
{
|
||||||
Ok(r) => {
|
Ok(r) => {
|
||||||
report_execute_multi_progress(progress.as_ref(), statement_index, statements.len(), &r, true, None);
|
report_execute_multi_progress(progress.as_ref(), statement_index, statements.len(), &r, true, None);
|
||||||
results.push(ExecuteMultiResult::success_with_index(r, statement_index));
|
results.push(ExecuteMultiResult::success_with_index_and_optional_server_large_values(
|
||||||
|
r,
|
||||||
|
statement_index,
|
||||||
|
options.table_data_preview,
|
||||||
|
));
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let action = query_execution_error_action(db_type, stmt, &error);
|
let action = query_execution_error_action(db_type, stmt, &error);
|
||||||
@@ -2403,8 +2664,9 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
|||||||
|
|
||||||
fn single_statement_multi_result(
|
fn single_statement_multi_result(
|
||||||
result: Result<db::QueryResult, QueryExecutionError>,
|
result: Result<db::QueryResult, QueryExecutionError>,
|
||||||
|
table_data_preview: bool,
|
||||||
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
|
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
|
||||||
result.map(|result| vec![result.into()])
|
result.map(|result| vec![ExecuteMultiResult::success_with_optional_server_large_values(result, table_data_preview)])
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mysql_single_statement_uses_batch_route(
|
fn mysql_single_statement_uses_batch_route(
|
||||||
@@ -2420,6 +2682,10 @@ fn mysql_single_statement_uses_batch_route(
|
|||||||
}
|
}
|
||||||
|
|
||||||
trait MysqlBatchStatementExecutor {
|
trait MysqlBatchStatementExecutor {
|
||||||
|
fn table_data_preview(&self) -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String>;
|
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String>;
|
||||||
|
|
||||||
async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome {
|
async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome {
|
||||||
@@ -2450,10 +2716,15 @@ struct MysqlBatchConnection<'a> {
|
|||||||
max_rows: Option<usize>,
|
max_rows: Option<usize>,
|
||||||
max_result_bytes: Option<usize>,
|
max_result_bytes: Option<usize>,
|
||||||
result_key_columns: &'a [String],
|
result_key_columns: &'a [String],
|
||||||
|
table_data_preview: bool,
|
||||||
dialect: db::mysql::MySqlQueryDialect,
|
dialect: db::mysql::MySqlQueryDialect,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> {
|
impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> {
|
||||||
|
fn table_data_preview(&self) -> bool {
|
||||||
|
self.table_data_preview
|
||||||
|
}
|
||||||
|
|
||||||
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String> {
|
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String> {
|
||||||
wait_for_result_opt(
|
wait_for_result_opt(
|
||||||
self.cancel_token.clone(),
|
self.cancel_token.clone(),
|
||||||
@@ -2553,6 +2824,7 @@ where
|
|||||||
{
|
{
|
||||||
let mut results = Vec::with_capacity(statements.len());
|
let mut results = Vec::with_capacity(statements.len());
|
||||||
let mut statement_index = 0usize;
|
let mut statement_index = 0usize;
|
||||||
|
let table_data_preview = executor.table_data_preview();
|
||||||
while statement_index < statements.len() {
|
while statement_index < statements.len() {
|
||||||
if is_canceled(&cancel_token) {
|
if is_canceled(&cancel_token) {
|
||||||
results.push(ExecuteMultiResult::execution_error(error_query_result(canceled_error())));
|
results.push(ExecuteMultiResult::execution_error(error_query_result(canceled_error())));
|
||||||
@@ -2618,6 +2890,7 @@ where
|
|||||||
result.result,
|
result.result,
|
||||||
statement_index,
|
statement_index,
|
||||||
result.large_value_cells,
|
result.large_value_cells,
|
||||||
|
table_data_preview,
|
||||||
)
|
)
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
@@ -2711,6 +2984,7 @@ async fn execute_multi_mysql(
|
|||||||
max_rows,
|
max_rows,
|
||||||
max_result_bytes,
|
max_result_bytes,
|
||||||
result_key_columns: &options.result_key_columns,
|
result_key_columns: &options.result_key_columns,
|
||||||
|
table_data_preview: options.table_data_preview,
|
||||||
dialect,
|
dialect,
|
||||||
};
|
};
|
||||||
let (results, error_action) = execute_mysql_batch_statements(
|
let (results, error_action) = execute_mysql_batch_statements(
|
||||||
@@ -5766,6 +6040,7 @@ for line in sys.stdin:
|
|||||||
empty_query_result(1),
|
empty_query_result(1),
|
||||||
0,
|
0,
|
||||||
vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }],
|
vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }],
|
||||||
|
false,
|
||||||
);
|
);
|
||||||
|
|
||||||
let serialized = serde_json::to_value(result).unwrap();
|
let serialized = serde_json::to_value(result).unwrap();
|
||||||
@@ -5901,7 +6176,7 @@ for line in sys.stdin:
|
|||||||
)
|
)
|
||||||
.with_omitted_sql_context("SELECT * FROM dbx_table_that_does_not_exist");
|
.with_omitted_sql_context("SELECT * FROM dbx_table_that_does_not_exist");
|
||||||
|
|
||||||
let error = single_statement_multi_result(Err(error)).unwrap_err();
|
let error = single_statement_multi_result(Err(error), false).unwrap_err();
|
||||||
let backend_error = error.into_backend_error();
|
let backend_error = error.into_backend_error();
|
||||||
|
|
||||||
assert_eq!(backend_error.code(), "DBX-JDBC-4001");
|
assert_eq!(backend_error.code(), "DBX-JDBC-4001");
|
||||||
@@ -6900,6 +7175,168 @@ for line in sys.stdin:
|
|||||||
assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"]));
|
assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_server_large_value_markers_and_restores_source_columns() {
|
||||||
|
let mut result = db::QueryResult {
|
||||||
|
columns: vec![
|
||||||
|
"id".to_string(),
|
||||||
|
"payload".to_string(),
|
||||||
|
format!("{}T_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||||
|
"note".to_string(),
|
||||||
|
],
|
||||||
|
column_types: vec!["integer".to_string(), "text".to_string(), "bigint".to_string(), "text".to_string()],
|
||||||
|
column_sortables: vec![true, true, true, true],
|
||||||
|
spatial_columns: Vec::new(),
|
||||||
|
spatial_values: Vec::new(),
|
||||||
|
rows: vec![
|
||||||
|
vec![
|
||||||
|
serde_json::json!(1),
|
||||||
|
serde_json::json!("预览文本多"),
|
||||||
|
serde_json::json!("T:4"),
|
||||||
|
serde_json::json!("a"),
|
||||||
|
],
|
||||||
|
vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("T:4"), serde_json::json!("b")],
|
||||||
|
],
|
||||||
|
affected_rows: 0,
|
||||||
|
execution_time_ms: 0,
|
||||||
|
truncated: false,
|
||||||
|
session_id: None,
|
||||||
|
has_more: false,
|
||||||
|
elasticsearch_raw_body: None,
|
||||||
|
messages: Vec::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let cells = extract_server_large_value_markers(&mut result);
|
||||||
|
|
||||||
|
assert_eq!(result.columns, vec!["id", "payload", "note"]);
|
||||||
|
assert_eq!(result.column_types, vec!["integer", "text", "text"]);
|
||||||
|
assert_eq!(
|
||||||
|
result.rows[0],
|
||||||
|
vec![serde_json::json!(1), serde_json::json!("预览文本..."), serde_json::json!("a")]
|
||||||
|
);
|
||||||
|
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("b")]);
|
||||||
|
assert_eq!(
|
||||||
|
cells,
|
||||||
|
vec![db::LargeValueCell {
|
||||||
|
row_index: 0,
|
||||||
|
column_index: 1,
|
||||||
|
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncates_server_binary_preview_after_the_configured_byte_count() {
|
||||||
|
let mut result = db::QueryResult {
|
||||||
|
columns: vec![
|
||||||
|
"raw_value".to_string(),
|
||||||
|
format!("{}B_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||||
|
],
|
||||||
|
column_types: vec!["bytea".to_string(), "text".to_string()],
|
||||||
|
column_sortables: vec![true, true],
|
||||||
|
spatial_columns: Vec::new(),
|
||||||
|
spatial_values: Vec::new(),
|
||||||
|
rows: vec![vec![serde_json::json!("0x0102030405"), serde_json::json!("B:4")]],
|
||||||
|
affected_rows: 0,
|
||||||
|
execution_time_ms: 0,
|
||||||
|
truncated: false,
|
||||||
|
session_id: None,
|
||||||
|
has_more: false,
|
||||||
|
elasticsearch_raw_body: None,
|
||||||
|
messages: Vec::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let cells = extract_server_large_value_markers(&mut result);
|
||||||
|
|
||||||
|
assert_eq!(result.rows, vec![vec![serde_json::json!("0x01020304...")]]);
|
||||||
|
assert_eq!(cells.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() {
|
||||||
|
let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
|
||||||
|
let mut result = db::QueryResult {
|
||||||
|
columns: vec!["embedding".to_string(), marker],
|
||||||
|
column_types: vec!["text".to_string(), "text".to_string()],
|
||||||
|
column_sortables: vec![true, true],
|
||||||
|
spatial_columns: Vec::new(),
|
||||||
|
spatial_values: Vec::new(),
|
||||||
|
rows: vec![
|
||||||
|
vec![serde_json::json!("[0.1,0.2,0.3]"), serde_json::json!("V:9")],
|
||||||
|
vec![serde_json::json!("[0.1,0.2]"), serde_json::json!("V:20")],
|
||||||
|
],
|
||||||
|
affected_rows: 0,
|
||||||
|
execution_time_ms: 0,
|
||||||
|
truncated: false,
|
||||||
|
session_id: None,
|
||||||
|
has_more: false,
|
||||||
|
elasticsearch_raw_body: None,
|
||||||
|
messages: Vec::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let cells = extract_server_large_value_markers(&mut result);
|
||||||
|
|
||||||
|
assert_eq!(result.column_types, vec!["vector"]);
|
||||||
|
assert_eq!(result.rows[0][0], serde_json::json!([0.1, 0.2]));
|
||||||
|
assert_eq!(result.rows[1][0], serde_json::json!([0.1, 0.2]));
|
||||||
|
assert_eq!(cells.len(), 1);
|
||||||
|
assert_eq!(cells[0].row_index, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restores_server_preview_types_without_rows() {
|
||||||
|
let mut result = db::QueryResult {
|
||||||
|
columns: vec![
|
||||||
|
"embedding".to_string(),
|
||||||
|
format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||||
|
"metadata".to_string(),
|
||||||
|
format!("{}K_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||||
|
],
|
||||||
|
column_types: vec!["text".to_string(), "text".to_string(), "text".to_string(), "text".to_string()],
|
||||||
|
column_sortables: vec![true; 4],
|
||||||
|
spatial_columns: Vec::new(),
|
||||||
|
spatial_values: Vec::new(),
|
||||||
|
rows: Vec::new(),
|
||||||
|
affected_rows: 0,
|
||||||
|
execution_time_ms: 0,
|
||||||
|
truncated: false,
|
||||||
|
session_id: None,
|
||||||
|
has_more: false,
|
||||||
|
elasticsearch_raw_body: None,
|
||||||
|
messages: Vec::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let cells = extract_server_large_value_markers(&mut result);
|
||||||
|
|
||||||
|
assert_eq!(result.columns, vec!["embedding", "metadata"]);
|
||||||
|
assert_eq!(result.column_types, vec!["vector", "jsonb"]);
|
||||||
|
assert!(cells.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_queries_preserve_columns_that_resemble_preview_markers() {
|
||||||
|
let marker = format!("{}0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
|
||||||
|
let mut result = empty_query_result(1);
|
||||||
|
result.columns = vec!["payload".to_string(), marker.clone()];
|
||||||
|
result.column_types = vec!["text".to_string(), "bigint".to_string()];
|
||||||
|
result.column_sortables = vec![true, true];
|
||||||
|
result.rows = vec![vec![serde_json::json!("value"), serde_json::json!(123)]];
|
||||||
|
|
||||||
|
let ordinary = ExecuteMultiResult::success_with_index_and_optional_server_large_values(result, 0, false);
|
||||||
|
|
||||||
|
assert_eq!(ordinary.result.columns, vec!["payload".to_string(), marker]);
|
||||||
|
assert_eq!(ordinary.result.rows[0], vec![serde_json::json!("value"), serde_json::json!(123)]);
|
||||||
|
assert!(ordinary.large_value_cells.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn single_statement_preview_preserves_absent_statement_index() {
|
||||||
|
let result = single_statement_multi_result(Ok(empty_query_result(1)), true).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.len(), 1);
|
||||||
|
assert_eq!(result[0].statement_index, None);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn query_execution_options_default_use_transaction_is_none() {
|
fn query_execution_options_default_use_transaction_is_none() {
|
||||||
let opts = QueryExecutionOptions::default();
|
let opts = QueryExecutionOptions::default();
|
||||||
|
|||||||
@@ -38,7 +38,9 @@ pub use identifiers::{
|
|||||||
normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier,
|
normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier,
|
||||||
};
|
};
|
||||||
pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier};
|
pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier};
|
||||||
pub use table_select::{build_count_table_sql, build_table_data_select_sql, build_table_select_sql};
|
pub use table_select::{
|
||||||
|
build_count_table_sql, build_table_data_select_sql, build_table_select_sql, DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX,
|
||||||
|
};
|
||||||
pub(crate) use table_select::{
|
pub(crate) use table_select::{
|
||||||
quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote,
|
quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -12,6 +12,131 @@ use super::types::{
|
|||||||
DBX_TDENGINE_TBNAME_COLUMN,
|
DBX_TDENGINE_TBNAME_COLUMN,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub const DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX: &str = "__DBX_LARGE_VALUE_BYTES_";
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum LargeValuePreviewKind {
|
||||||
|
Text,
|
||||||
|
Binary,
|
||||||
|
TextCast,
|
||||||
|
Vector,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn large_value_marker_alias_kind(kind: LargeValuePreviewKind, data_type: &str) -> &'static str {
|
||||||
|
match kind {
|
||||||
|
LargeValuePreviewKind::Binary => "B",
|
||||||
|
LargeValuePreviewKind::Vector => "V",
|
||||||
|
LargeValuePreviewKind::TextCast => match normalized_data_type_base(data_type).as_str() {
|
||||||
|
"json" => "J",
|
||||||
|
"jsonb" => "K",
|
||||||
|
"tsvector" => "S",
|
||||||
|
_ => "T",
|
||||||
|
},
|
||||||
|
LargeValuePreviewKind::Text => "T",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_data_type_base(data_type: &str) -> String {
|
||||||
|
data_type.trim().split(['(', '[']).next().unwrap_or_default().trim().to_ascii_lowercase()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn large_value_preview_kind(database_type: Option<DatabaseType>, data_type: &str) -> Option<LargeValuePreviewKind> {
|
||||||
|
let normalized = data_type.trim().to_ascii_lowercase();
|
||||||
|
let base = normalized_data_type_base(data_type);
|
||||||
|
match database_type {
|
||||||
|
Some(DatabaseType::Mysql) => {
|
||||||
|
if matches!(base.as_str(), "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob") {
|
||||||
|
Some(LargeValuePreviewKind::Binary)
|
||||||
|
} else if base == "json" {
|
||||||
|
Some(LargeValuePreviewKind::TextCast)
|
||||||
|
} else if matches!(base.as_str(), "char" | "varchar" | "tinytext" | "text" | "mediumtext" | "longtext") {
|
||||||
|
Some(LargeValuePreviewKind::Text)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(DatabaseType::Postgres) => {
|
||||||
|
if base == "bytea" {
|
||||||
|
Some(LargeValuePreviewKind::Binary)
|
||||||
|
} else if matches!(base.as_str(), "char" | "character" | "varchar" | "text" | "citext" | "name" | "xml")
|
||||||
|
|| normalized.starts_with("character varying")
|
||||||
|
{
|
||||||
|
Some(LargeValuePreviewKind::Text)
|
||||||
|
} else if base == "vector" {
|
||||||
|
Some(LargeValuePreviewKind::Vector)
|
||||||
|
} else if matches!(base.as_str(), "json" | "jsonb" | "tsvector") {
|
||||||
|
Some(LargeValuePreviewKind::TextCast)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Option<String> {
|
||||||
|
let database_type = options.database_type;
|
||||||
|
let preview_size = options.large_value_preview_size?.max(1);
|
||||||
|
if options.columns.is_empty()
|
||||||
|
|| options.columns.len() != options.column_types.len()
|
||||||
|
|| options.primary_keys.is_empty()
|
||||||
|
|| options
|
||||||
|
.columns
|
||||||
|
.iter()
|
||||||
|
.any(|column| column.to_ascii_uppercase().starts_with(DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX))
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let protected: std::collections::HashSet<String> =
|
||||||
|
options.primary_keys.iter().map(|column| column.to_ascii_lowercase()).collect();
|
||||||
|
let mut projections = Vec::with_capacity(options.columns.len() * 2);
|
||||||
|
let mut marker_count = 0;
|
||||||
|
for (column_index, (column, data_type)) in options.columns.iter().zip(&options.column_types).enumerate() {
|
||||||
|
let quoted = if uses_connection_identifier_quote(database_type, options.identifier_quote.as_deref()) {
|
||||||
|
quote_table_data_identifier(database_type, column, options.identifier_quote.as_deref())
|
||||||
|
} else {
|
||||||
|
quote_table_identifier(database_type, column)
|
||||||
|
};
|
||||||
|
let kind = (!protected.contains(&column.to_ascii_lowercase()))
|
||||||
|
.then(|| large_value_preview_kind(database_type, data_type))
|
||||||
|
.flatten();
|
||||||
|
let Some(kind) = kind else {
|
||||||
|
projections.push(quoted);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let alias_kind = large_value_marker_alias_kind(kind, data_type);
|
||||||
|
let marker_alias = quote_table_identifier(
|
||||||
|
database_type,
|
||||||
|
&format!("{DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX}{alias_kind}_{column_index}"),
|
||||||
|
);
|
||||||
|
let prefix_size = preview_size.saturating_add(1);
|
||||||
|
let (preview, marker_kind) = match database_type {
|
||||||
|
Some(DatabaseType::Mysql) if kind == LargeValuePreviewKind::Binary => {
|
||||||
|
(format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "B")
|
||||||
|
}
|
||||||
|
Some(DatabaseType::Mysql) => (format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "T"),
|
||||||
|
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Binary => {
|
||||||
|
(format!("substring({quoted} from 1 for {prefix_size}) AS {quoted}"), "B")
|
||||||
|
}
|
||||||
|
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::TextCast => {
|
||||||
|
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "T")
|
||||||
|
}
|
||||||
|
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Vector => {
|
||||||
|
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "V")
|
||||||
|
}
|
||||||
|
Some(DatabaseType::Postgres) => (format!("left({quoted}, {prefix_size}) AS {quoted}"), "T"),
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let marker = format!("'{marker_kind}:{preview_size}' AS {marker_alias}");
|
||||||
|
projections.push(preview);
|
||||||
|
projections.push(marker);
|
||||||
|
marker_count += 1;
|
||||||
|
}
|
||||||
|
(marker_count > 0).then(|| projections.join(", "))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn build_count_table_sql(database_type: Option<DatabaseType>, schema: Option<&str>, table_name: &str) -> String {
|
pub fn build_count_table_sql(database_type: Option<DatabaseType>, schema: Option<&str>, table_name: &str) -> String {
|
||||||
if database_type == Some(DatabaseType::VictoriaMetrics) {
|
if database_type == Some(DatabaseType::VictoriaMetrics) {
|
||||||
return format!("count({})", victoriametrics_metric_selector(table_name));
|
return format!("count({})", victoriametrics_metric_selector(table_name));
|
||||||
@@ -77,6 +202,8 @@ pub fn build_table_data_select_sql(options: TableDataSelectSqlOptions) -> String
|
|||||||
|
|
||||||
let select_columns = if include_oracle_row_id {
|
let select_columns = if include_oracle_row_id {
|
||||||
format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*")
|
format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*")
|
||||||
|
} else if let Some(preview_columns) = build_large_value_preview_columns(&options) {
|
||||||
|
preview_columns
|
||||||
} else {
|
} else {
|
||||||
build_select_columns(
|
build_select_columns(
|
||||||
database_type,
|
database_type,
|
||||||
@@ -521,6 +648,8 @@ mod tests {
|
|||||||
table_type: None,
|
table_type: None,
|
||||||
primary_keys: Vec::new(),
|
primary_keys: Vec::new(),
|
||||||
columns: Vec::new(),
|
columns: Vec::new(),
|
||||||
|
column_types: Vec::new(),
|
||||||
|
large_value_preview_size: None,
|
||||||
fallback_order_columns: Vec::new(),
|
fallback_order_columns: Vec::new(),
|
||||||
order_by: None,
|
order_by: None,
|
||||||
limit: Some(10),
|
limit: Some(10),
|
||||||
|
|||||||
@@ -641,6 +641,97 @@ fn builds_table_data_where_and_schema_queries() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_mysql_table_data_large_value_previews_without_truncating_keys() {
|
||||||
|
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||||
|
database_type: Some(DatabaseType::Mysql),
|
||||||
|
table_name: "large_rows".to_string(),
|
||||||
|
primary_keys: vec!["id".to_string()],
|
||||||
|
columns: vec!["id".to_string(), "payload".to_string(), "raw_value".to_string(), "metadata".to_string()],
|
||||||
|
column_types: vec!["bigint".to_string(), "longtext".to_string(), "longblob".to_string(), "json".to_string()],
|
||||||
|
large_value_preview_size: Some(4096),
|
||||||
|
limit: Some(100),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(sql.starts_with("SELECT `id`, LEFT(`payload`, 4097) AS `payload`"));
|
||||||
|
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_T_1`"));
|
||||||
|
assert!(sql.contains("LEFT(`raw_value`, 4097) AS `raw_value`"));
|
||||||
|
assert!(sql.contains("'B:4096' AS `__DBX_LARGE_VALUE_BYTES_B_2`"));
|
||||||
|
assert!(sql.contains("LEFT(`metadata`, 4097) AS `metadata`"));
|
||||||
|
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_J_3`"));
|
||||||
|
assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_0"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_postgres_table_data_large_value_previews() {
|
||||||
|
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||||
|
database_type: Some(DatabaseType::Postgres),
|
||||||
|
schema: Some("public".to_string()),
|
||||||
|
table_name: "large_rows".to_string(),
|
||||||
|
primary_keys: vec!["id".to_string()],
|
||||||
|
columns: vec!["id".to_string(), "payload".to_string(), "metadata".to_string()],
|
||||||
|
column_types: vec!["integer".to_string(), "text".to_string(), "jsonb".to_string()],
|
||||||
|
large_value_preview_size: Some(8192),
|
||||||
|
limit: Some(100),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(sql.starts_with("SELECT \"id\", left(\"payload\", 8193) AS \"payload\""));
|
||||||
|
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_T_1\""));
|
||||||
|
assert!(sql.contains("left(\"metadata\"::text, 8193) AS \"metadata\""));
|
||||||
|
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_K_2\""));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_pgvector_table_data_preview_with_compatible_text_cast() {
|
||||||
|
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||||
|
database_type: Some(DatabaseType::Postgres),
|
||||||
|
schema: Some("public".to_string()),
|
||||||
|
table_name: "embeddings".to_string(),
|
||||||
|
primary_keys: vec!["id".to_string()],
|
||||||
|
columns: vec!["id".to_string(), "embedding".to_string()],
|
||||||
|
column_types: vec!["integer".to_string(), "vector(16000)".to_string()],
|
||||||
|
large_value_preview_size: Some(8192),
|
||||||
|
limit: Some(100),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(sql.contains("left(\"embedding\"::text, 8193) AS \"embedding\""));
|
||||||
|
assert!(sql.contains("'V:8192' AS \"__DBX_LARGE_VALUE_BYTES_V_1\""));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn table_data_preview_requires_stable_keys_and_parallel_types() {
|
||||||
|
let base = TableDataSelectSqlOptions {
|
||||||
|
database_type: Some(DatabaseType::Postgres),
|
||||||
|
table_name: "large_rows".to_string(),
|
||||||
|
columns: vec!["payload".to_string()],
|
||||||
|
column_types: vec!["text".to_string()],
|
||||||
|
large_value_preview_size: Some(8192),
|
||||||
|
limit: Some(100),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(build_table_data_select_sql(base.clone()), "SELECT * FROM \"large_rows\" LIMIT 100;");
|
||||||
|
assert_eq!(
|
||||||
|
build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||||
|
primary_keys: vec!["id".to_string()],
|
||||||
|
column_types: Vec::new(),
|
||||||
|
..base.clone()
|
||||||
|
}),
|
||||||
|
"SELECT * FROM \"large_rows\" LIMIT 100;"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||||
|
primary_keys: vec!["id".to_string()],
|
||||||
|
columns: vec!["id".to_string(), "__dbx_large_value_bytes_t_0".to_string()],
|
||||||
|
column_types: vec!["integer".to_string(), "text".to_string()],
|
||||||
|
..base
|
||||||
|
}),
|
||||||
|
"SELECT * FROM \"large_rows\" LIMIT 100;"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn builds_informix_table_data_with_skip_first_pagination() {
|
fn builds_informix_table_data_with_skip_first_pagination() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -44,6 +44,15 @@ pub struct TableDataSelectSqlOptions {
|
|||||||
pub primary_keys: Vec<String>,
|
pub primary_keys: Vec<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub columns: Vec<String>,
|
pub columns: Vec<String>,
|
||||||
|
/// Database type names parallel to `columns`. Table-data callers can use
|
||||||
|
/// these to request bounded server-side projections for variable-length
|
||||||
|
/// values without changing arbitrary query execution.
|
||||||
|
#[serde(default)]
|
||||||
|
pub column_types: Vec<String>,
|
||||||
|
/// Maximum characters (text-like values) or bytes (binary values) kept in
|
||||||
|
/// a table-data preview. Omitted for exports and ordinary SQL queries.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub large_value_preview_size: Option<usize>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub fallback_order_columns: Vec<String>,
|
pub fallback_order_columns: Vec<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ pub struct ExecuteQueryRequest {
|
|||||||
pub max_result_bytes: Option<usize>,
|
pub max_result_bytes: Option<usize>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub result_key_columns: Vec<String>,
|
pub result_key_columns: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub table_data_preview: bool,
|
||||||
pub result_session_id: Option<String>,
|
pub result_session_id: Option<String>,
|
||||||
pub client_session_id: Option<String>,
|
pub client_session_id: Option<String>,
|
||||||
pub timeout_secs: Option<u64>,
|
pub timeout_secs: Option<u64>,
|
||||||
@@ -363,6 +365,7 @@ pub async fn execute_query(
|
|||||||
page_size: req.page_size,
|
page_size: req.page_size,
|
||||||
max_result_bytes: req.max_result_bytes,
|
max_result_bytes: req.max_result_bytes,
|
||||||
result_key_columns: req.result_key_columns,
|
result_key_columns: req.result_key_columns,
|
||||||
|
table_data_preview: req.table_data_preview,
|
||||||
catalog: req.catalog,
|
catalog: req.catalog,
|
||||||
result_session_id: req.result_session_id,
|
result_session_id: req.result_session_id,
|
||||||
client_session_id: req.client_session_id,
|
client_session_id: req.client_session_id,
|
||||||
@@ -411,6 +414,7 @@ pub async fn execute_multi(
|
|||||||
page_size: req.page_size,
|
page_size: req.page_size,
|
||||||
max_result_bytes: req.max_result_bytes,
|
max_result_bytes: req.max_result_bytes,
|
||||||
result_key_columns: req.result_key_columns,
|
result_key_columns: req.result_key_columns,
|
||||||
|
table_data_preview: req.table_data_preview,
|
||||||
catalog: req.catalog,
|
catalog: req.catalog,
|
||||||
result_session_id: req.result_session_id,
|
result_session_id: req.result_session_id,
|
||||||
client_session_id: req.client_session_id,
|
client_session_id: req.client_session_id,
|
||||||
|
|||||||
@@ -90,6 +90,7 @@ pub async fn execute_multi(
|
|||||||
page_size: Option<usize>,
|
page_size: Option<usize>,
|
||||||
max_result_bytes: Option<usize>,
|
max_result_bytes: Option<usize>,
|
||||||
result_key_columns: Option<Vec<String>>,
|
result_key_columns: Option<Vec<String>>,
|
||||||
|
table_data_preview: Option<bool>,
|
||||||
result_session_id: Option<String>,
|
result_session_id: Option<String>,
|
||||||
client_session_id: Option<String>,
|
client_session_id: Option<String>,
|
||||||
timeout_secs: Option<u64>,
|
timeout_secs: Option<u64>,
|
||||||
@@ -148,6 +149,7 @@ pub async fn execute_multi(
|
|||||||
page_size,
|
page_size,
|
||||||
max_result_bytes,
|
max_result_bytes,
|
||||||
result_key_columns: result_key_columns.unwrap_or_default(),
|
result_key_columns: result_key_columns.unwrap_or_default(),
|
||||||
|
table_data_preview: table_data_preview.unwrap_or(false),
|
||||||
catalog,
|
catalog,
|
||||||
result_session_id,
|
result_session_id,
|
||||||
client_session_id,
|
client_session_id,
|
||||||
|
|||||||
Reference in new issue
Block a user