From c27a4dd83d9eb628c2f854d396792eb1e149a6d9 Mon Sep 17 00:00:00 2001 From: zipg Date: Wed, 12 Aug 2026 14:06:33 +0800 Subject: [PATCH] perf(mysql): limit large table result memory --- apps/desktop/src/components/grid/DataGrid.vue | 436 +++++++++++++-- .../__tests__/useDataGridActions.spec.ts | 32 +- .../__tests__/useDataGridEditor.spec.ts | 9 + .../__tests__/useDataGridExport.spec.ts | 97 ++++ .../useSidebarDataOpenRuntime.spec.ts | 2 + .../src/composables/useDataGridEditor.ts | 13 +- .../src/composables/useDataGridExport.ts | 33 +- .../src/composables/useDataGridExtractor.ts | 63 ++- .../composables/useSidebarDataOpenRuntime.ts | 1 + apps/desktop/src/i18n/locales/en.ts | 6 + apps/desktop/src/i18n/locales/es.ts | 6 + apps/desktop/src/i18n/locales/it.ts | 6 + apps/desktop/src/i18n/locales/ja.ts | 6 + apps/desktop/src/i18n/locales/ko.ts | 6 + apps/desktop/src/i18n/locales/pt-BR.ts | 6 + apps/desktop/src/i18n/locales/zh-CN.ts | 6 + apps/desktop/src/i18n/locales/zh-TW.ts | 6 + .../dataGrid/dataGridLargeValues.spec.ts | 59 +++ .../lib/__tests__/tabs/tabResultCache.spec.ts | 19 + apps/desktop/src/lib/backend/http.ts | 4 + apps/desktop/src/lib/backend/tauri.ts | 4 + .../src/lib/dataGrid/dataGridDetail.ts | 7 +- .../src/lib/dataGrid/dataGridLargeValues.ts | 29 + apps/desktop/src/lib/tabs/tabResultCache.ts | 6 + .../queryStore.tableDataRefresh.spec.ts | 54 ++ apps/desktop/src/stores/queryStore.ts | 26 +- apps/desktop/src/types/database.ts | 4 + crates/dbx-core/src/db/mysql.rs | 496 +++++++++++++++--- crates/dbx-core/src/query.rs | 155 +++++- crates/dbx-core/src/types.rs | 9 + crates/dbx-web/src/routes/query.rs | 8 + src-tauri/src/commands/query.rs | 4 + 32 files changed, 1441 insertions(+), 177 deletions(-) create mode 100644 apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts create mode 100644 apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index 56134c3694..b1c078d571 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -149,6 +149,7 @@ import { isCancelSearchShortcut, isCopyCurrentRowShortcut, isDeleteCurrentRowSho import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter"; import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination"; import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions"; +import { largeValueCellKey, largeValueCellMap } from "@/lib/dataGrid/dataGridLargeValues"; import { dataGridBottomScrollTop, dataGridScrollPosition, isDataGridAtScrollBottom, isDataGridNearScrollBottom, isDataGridPrefixAppend, shouldCheckInfiniteScrollAfterScroll, type DataGridScrollPosition } from "@/lib/dataGrid/dataGridInfiniteScroll"; import { CANVAS_DATA_GRID_ROW_HEIGHT, MAX_CANVAS_DATA_GRID_PIXEL_RATIO, canvasDataGridActionOverlayWidth, canvasDataGridActionReservedWidth, dataGridSearchMatchKey, drawCanvasDataGrid, type CanvasDevicePixelSize } from "@/lib/dataGrid/canvasDataGridRenderer"; import { DATA_GRID_DARK_STRIPED_ROW_BG, DATA_GRID_LIGHT_STRIPED_ROW_BG, dataGridActiveRowBackground } from "@/lib/dataGrid/dataGridPaintTheme"; @@ -195,6 +196,7 @@ import { import { buildColumnForeignKeyMap, combineForeignKeyConditions, foreignKeyAssociationCells, foreignKeyMetadataRequestCurrent, foreignKeyNavigationTarget, foreignKeySourceColumnName, foreignKeyTableIdentity, type ForeignKeyAssociation } from "@/lib/dataGrid/dataGridForeignKeyNavigation"; import { useToast } from "@/composables/useToast"; +import { translateBackendError } from "@/i18n/backend-errors"; import { useNavigationTargets } from "@/composables/useNavigationTargets"; import { useDataGridExport, type MongoCopyUpdateTarget } from "@/composables/useDataGridExport"; import { eventTargetAllowsNativeClipboard, isPlainClipboardShortcut, readTextFromClipboard } from "@/lib/common/clipboard"; @@ -256,6 +258,7 @@ import { dataGridConditionColumnOptions, dataGridConditionIdentifierQuote } from import { isMacOS } from "@/lib/backend/platform"; import { appendDebugLog, isDebugLoggingEnabled } from "@/lib/backend/debugLog"; import { formatShortcut } from "@/lib/editor/shortcutRegistry"; +import { queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout"; import { SearchableSelect } from "@/components/ui/searchable-select"; import timezone from "dayjs/plugin/timezone"; import utc from "dayjs/plugin/utc"; @@ -283,6 +286,7 @@ const queryStore = useQueryStore(); const settingsStore = useSettingsStore(); const tableFontSize = computed(() => settingsStore.editorSettings.tableFontSize); const rowNumberWidth = ref(DATA_GRID_ROW_NUM_WIDTH); +const largeValueResolutionVersion = ref(0); const multiRowTranspose = computed(() => settingsStore.editorSettings.dataGridMultiRowTranspose); const hideNullColumns = computed(() => settingsStore.editorSettings.dataGridHideNullColumns); const booleanCellsUseCheckbox = computed(() => settingsStore.editorSettings.dataGridBooleanDisplayMode === "checkbox"); @@ -3146,7 +3150,7 @@ const { onEditKeydown, addRows: addEditorRows, appendPastedRowsToNewRow, - cloneRow, + cloneRow: cloneEditorRow, showDeleteRowConfirm, requestDeleteRow, confirmDeleteRow, @@ -3154,7 +3158,7 @@ const { restoreRows, pendingDeleteRowIds, requestDeleteRows, - cloneRows, + cloneRows: cloneEditorRows, saveChanges, discardChanges, canUndoPendingChange, @@ -3301,15 +3305,17 @@ function cellUsesExpandedEditor(rowId: number | undefined, columnIndex: number): return !!expandedCellEditor.value && expandedCellEditor.value.rowId === rowId && expandedCellEditor.value.col === columnIndex; } -function startCellEdit(rowId: number, columnIndex: number, expanded: boolean) { +async function startCellEdit(rowId: number, columnIndex: number, expanded: boolean) { + if (!(await hydrateLargeValueCell(rowId, columnIndex))) return; expandedCellEditor.value = expanded ? { rowId, col: columnIndex } : null; startEdit(rowId, columnIndex); } -function startDomCellEdit(rowId: number, columnIndex: number, displayText: string, event: MouseEvent) { +async function startDomCellEdit(rowId: number, columnIndex: number, displayText: string, event: MouseEvent) { + if (!(await hydrateLargeValueCell(rowId, columnIndex))) return; const item = getRowItem(rowId); const editText = item ? cellEditorTextForValue(item.data[columnIndex], columnIndex) : displayText; - startCellEdit( + await startCellEdit( rowId, columnIndex, cellEditContentNeedsExpandedEditor({ @@ -3326,10 +3332,10 @@ function showReadonlyCellDetailsOnDblClick(item: RowItem, rowIndex: number, visi return true; } -function onDomCellDblClick(item: RowItem, rowIndex: number, visibleColIdx: number, actualColIdx: number, event: MouseEvent) { +async function onDomCellDblClick(item: RowItem, rowIndex: number, visibleColIdx: number, actualColIdx: number, event: MouseEvent) { if (booleanCellsUseCheckbox.value && isBooleanGridCell(item, actualColIdx) && canEditCellItem(item, actualColIdx)) return; if (showReadonlyCellDetailsOnDblClick(item, rowIndex, visibleColIdx, actualColIdx)) return; - startDomCellEdit(item.id, actualColIdx, formatCellCached(item.data[actualColIdx], actualColIdx), event); + await startDomCellEdit(item.id, actualColIdx, formatCellCached(item.data[actualColIdx], actualColIdx), event); } function cellEditContentNeedsExpandedEditor(options: { displayText: string; editText: string; target: EventTarget | null }): boolean { @@ -3775,6 +3781,7 @@ const displayRowIndexByIdLookup = computed(() => { }); function rowItemFromDisplayRef(ref: DisplayRowRef): RowItem { + largeValueResolutionVersion.value; if (ref.isNew) { return { ...ref, @@ -3954,6 +3961,215 @@ const visibleDisplayItems = computed(() => isDirtyCol: visibleDirtyColumns(item.isDirtyCol), })), ); + +type ResolvedLargeValueCells = Map>; +type LargeValueCellRequest = { + item: RowItem; + sourceIndex: number; + columnIndex: number; + originalBytes: number; +}; + +const LARGE_VALUE_FETCH_MAX_ROWS = 200; +const LARGE_VALUE_FETCH_TARGET_BYTES = 64 * 1024 * 1024; +const pendingLargeValueHydrations = new Map>(); +const largeValueCellsByKey = computed(() => largeValueCellMap(props.result)); + +function isLargeValuePreview(item: RowItem | undefined, columnIndex: number): boolean { + if (!item || item.isNew || item.isDraft || item.sourceIndex === undefined || item.isDirtyCol[columnIndex]) return false; + return largeValueCellsByKey.value.has(largeValueCellKey(item.sourceIndex, columnIndex)); +} + +function normalizedLargeValueIdentityPart(value: CellValue): string { + if (value === null) return "null"; + return `${typeof value}:${JSON.stringify(value)}`; +} + +function largeValueIdentityKey(row: readonly CellValue[], indexes: readonly number[]): string { + return indexes.map((index) => normalizedLargeValueIdentityPart(row[index] ?? null)).join("\u001f"); +} + +function largeValueSourceColumnIndex(columnName: string): number { + const normalized = columnName.toLocaleLowerCase(); + return resultSourceColumns.value.findIndex((column) => column.toLocaleLowerCase() === normalized); +} + +function chunkLargeValueRequests(requests: LargeValueCellRequest[]): LargeValueCellRequest[][] { + const chunks: LargeValueCellRequest[][] = []; + let chunk: LargeValueCellRequest[] = []; + let bytes = 0; + for (const request of requests) { + if (chunk.length > 0 && (chunk.length >= LARGE_VALUE_FETCH_MAX_ROWS || bytes + request.originalBytes > LARGE_VALUE_FETCH_TARGET_BYTES)) { + chunks.push(chunk); + chunk = []; + bytes = 0; + } + chunk.push(request); + bytes += request.originalBytes; + } + if (chunk.length > 0) chunks.push(chunk); + return chunks; +} + +async function largeValueRowPredicate(item: RowItem, primaryKeyIndexes: number[]): Promise { + const originalRow = item.sourceIndex === undefined ? undefined : props.result.rows[item.sourceIndex]; + if (!originalRow) throw new Error(t("grid.largeValueRowUnavailable")); + const tableMeta = props.tableMeta!; + const conditions = await Promise.all( + tableMeta.primaryKeys.map((columnName, index) => + buildDataGridContextFilterCondition({ + databaseType: resolvedDatabaseType.value, + identifierQuote: connectionStore.connectionIdentifierQuote?.(props.connectionId), + columnName, + columnInfo: tableMeta.columns.find((column) => column.name.toLocaleLowerCase() === columnName.toLocaleLowerCase()), + mode: "equals", + value: originalRow[primaryKeyIndexes[index]!] ?? null, + }), + ), + ); + if (conditions.some((condition) => !condition)) throw new Error(t("grid.largeValueRowUnavailable")); + return conditions.map((condition) => `(${condition})`).join(" AND "); +} + +async function fetchLargeValueRequestChunk(columnIndex: number, requests: LargeValueCellRequest[], primaryKeyIndexes: number[], resolved: ResolvedLargeValueCells) { + const tableMeta = props.tableMeta!; + const sourceColumn = resultSourceColumns.value[columnIndex]; + if (!sourceColumn) throw new Error(t("grid.largeValueColumnUnavailable")); + const predicates = await Promise.all(requests.map((request) => largeValueRowPredicate(request.item, primaryKeyIndexes))); + const selectedColumns = [...new Set([...tableMeta.primaryKeys, sourceColumn])]; + const sql = await buildTableSelectSql({ + databaseType: resolvedDatabaseType.value, + identifierQuote: connectionStore.connectionIdentifierQuote?.(props.connectionId), + database: tableMeta.database, + schema: tableMeta.schema, + tableName: tableMeta.tableName, + tableType: tableMeta.tableType, + catalog: tableMeta.catalog, + columns: selectedColumns, + primaryKeys: tableMeta.primaryKeys, + whereInput: predicates.map((predicate) => `(${predicate})`).join(" OR "), + limit: requests.length, + offset: 0, + }); + const connection = props.connectionId ? connectionStore.getConfig(props.connectionId) : undefined; + const results = await api.executeMulti(props.connectionId!, props.executionDatabase ?? props.database ?? "", sql, undefined, uuid(), { + maxRows: requests.length, + fetchSize: requests.length, + timeoutSecs: queryTimeoutSecsForConnection(connection, settingsStore.editorSettings.globalQueryTimeoutSecs), + }); + const result = results[0]; + if (!result || result.execution_error) { + throw new Error(result?.error ? translateBackendError(t, result.error) : String(result?.rows?.[0]?.[0] ?? t("grid.largeValueLoadFailed"))); + } + const resultColumnIndexes = selectedColumns.map((column) => { + const normalized = column.toLocaleLowerCase(); + return result.columns.findIndex((resultColumn) => resultColumn.toLocaleLowerCase() === normalized); + }); + if (resultColumnIndexes.some((index) => index < 0)) throw new Error(t("grid.largeValueColumnUnavailable")); + const resultPrimaryKeyIndexes = resultColumnIndexes.slice(0, tableMeta.primaryKeys.length); + const resultValueIndex = resultColumnIndexes[selectedColumns.indexOf(sourceColumn)]; + const valuesByIdentity = new Map(result.rows.map((row) => [largeValueIdentityKey(row, resultPrimaryKeyIndexes), row[resultValueIndex!] ?? null])); + for (const request of requests) { + const originalRow = props.result.rows[request.sourceIndex]; + const identity = originalRow ? largeValueIdentityKey(originalRow, primaryKeyIndexes) : ""; + if (!valuesByIdentity.has(identity)) throw new Error(t("grid.largeValueRowUnavailable")); + const rowValues = resolved.get(request.item.id) ?? new Map(); + rowValues.set(columnIndex, valuesByIdentity.get(identity) ?? null); + resolved.set(request.item.id, rowValues); + } +} + +async function resolveLargeValueCells(rowIds: number[], columnIndexes: number[]): Promise { + const resolved: ResolvedLargeValueCells = new Map(); + const requestedColumns = new Set(columnIndexes); + const requestsByColumn = new Map(); + for (const rowId of new Set(rowIds)) { + const item = getRowItem(rowId); + if (!item || item.sourceIndex === undefined) continue; + for (const columnIndex of requestedColumns) { + if (!isLargeValuePreview(item, columnIndex)) continue; + const metadata = largeValueCellsByKey.value.get(largeValueCellKey(item.sourceIndex, columnIndex)); + if (!metadata) continue; + const requests = requestsByColumn.get(columnIndex) ?? []; + requests.push({ item, sourceIndex: item.sourceIndex, columnIndex, originalBytes: metadata.original_bytes }); + requestsByColumn.set(columnIndex, requests); + } + } + if (requestsByColumn.size === 0) return resolved; + if (resolvedDatabaseType.value !== "mysql" || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) { + throw new Error(t("grid.largeValueNeedsStableKey")); + } + const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex); + if (primaryKeyIndexes.some((index) => index < 0)) throw new Error(t("grid.largeValueNeedsStableKey")); + + for (const [columnIndex, requests] of requestsByColumn) { + for (const chunk of chunkLargeValueRequests(requests)) { + await fetchLargeValueRequestChunk(columnIndex, chunk, primaryKeyIndexes, resolved); + } + } + return resolved; +} + +function reportLargeValueLoadError(error: unknown) { + toast(t("grid.largeValueLoadFailedWithMessage", { message: translateBackendError(t, error) }), 5000); +} + +async function cloneRow(rowId: number) { + try { + const resolved = await resolveLargeValueCells( + [rowId], + props.result.columns.map((_, index) => index), + ); + cloneEditorRow(rowId, resolved.get(rowId)); + } catch (error) { + reportLargeValueLoadError(error); + } +} + +async function cloneRows(rowIds: number[]) { + try { + const resolved = await resolveLargeValueCells( + rowIds, + props.result.columns.map((_, index) => index), + ); + cloneEditorRows(rowIds, resolved); + } catch (error) { + reportLargeValueLoadError(error); + } +} + +async function hydrateLargeValueCell(rowId: number, columnIndex: number): Promise { + const item = getRowItem(rowId); + if (!isLargeValuePreview(item, columnIndex) || item?.sourceIndex === undefined) return true; + const hydrationKey = largeValueCellKey(item.sourceIndex, columnIndex); + const pending = pendingLargeValueHydrations.get(hydrationKey); + if (pending) return pending; + const hydration = (async () => { + try { + const resolved = await resolveLargeValueCells([rowId], [columnIndex]); + const value = resolved.get(rowId)?.get(columnIndex); + if (value === undefined && !resolved.get(rowId)?.has(columnIndex)) return false; + const row = [...(props.result.rows[item.sourceIndex!] ?? [])]; + row[columnIndex] = value ?? null; + const rows = props.result.rows.slice(); + rows[item.sourceIndex!] = row; + props.result.rows = rows; + props.result.large_value_cells = props.result.large_value_cells?.filter((cell) => cell.row_index !== item.sourceIndex || cell.column_index !== columnIndex); + largeValueResolutionVersion.value += 1; + clearCellFormatCache(); + queryStore.invalidateResultEstimateForPayload(props.result); + return true; + } catch (error) { + reportLargeValueLoadError(error); + return false; + } finally { + pendingLargeValueHydrations.delete(hydrationKey); + } + })(); + pendingLargeValueHydrations.set(hydrationKey, hydration); + return hydration; +} + const exportContextCell = computed(() => { if (!contextCell.value) return null; const visibleCol = visibleColumnIndexes.value.indexOf(contextCell.value.col); @@ -4330,6 +4546,7 @@ function cellDetailFor(rowIndex: number, columnIndex: number): DataGridCellDetai canEditCell: canEditCellItem(item, columnIndex), isDraft: !!item.isDraft, }), + isValuePreviewTruncated: isLargeValuePreview(item, columnIndex), }); } @@ -4402,6 +4619,7 @@ const rowDetail = computed(() => { canEditCell: canEditCellItem(item, columnIndex), isDraft: !!item.isDraft, }), + isValuePreviewTruncated: (columnIndex) => isLargeValuePreview(item, columnIndex), }); }); @@ -4419,6 +4637,7 @@ const columnDetail = computed(() => { canEditCell: canEditCellItem(item, columnIndex), isDraft: !!item.isDraft, }), + isValuePreviewTruncated: isLargeValuePreview(item, columnIndex), })), columns: props.result.columns, columnIndex, @@ -4477,7 +4696,7 @@ watch(activeCellDetailTabs, (tabs) => { watch(activeCellDetailTab, (tab) => { if (tab === "valueEditor") { - startDetailEdit(); + void startDetailEdit(); } else { resetDetailEdit(); } @@ -4541,6 +4760,8 @@ const detailSqlConditionKey = computed(() => { }); function canCopyPreparedDetailSqlCondition(): boolean { + const detail = activeCellDetail.value; + if (detail && isLargeValuePreview(getRowItem(detail.rowId), detail.colIndex)) return false; return detailSqlConditionCopy.value.ready && detailSqlConditionCopy.value.key === detailSqlConditionKey.value; } @@ -4726,7 +4947,10 @@ function toggleCellDetailMetadataCollapsed() { }); } -function startDetailEdit() { +async function startDetailEdit() { + const initialDetail = activeCellDetail.value; + if (!initialDetail || !initialDetail.isEditable) return; + if (!(await hydrateLargeValueCell(initialDetail.rowId, initialDetail.colIndex))) return; const detail = activeCellDetail.value; if (!detail || !detail.isEditable) return; warnFormattedJsonEditIfNeeded(detail); @@ -4843,6 +5067,10 @@ function setDetailNull() { function applyColumnSort(column: string, columnIndex: number, direction: "asc" | "desc" | null, mode: DataGridSortMode = "database") { if (getIsResizing()) return; + if (mode === "local" && direction && props.result.large_value_cells?.some((cell) => cell.column_index === columnIndex)) { + toast(t("grid.largeValueLocalSortUnavailable"), 5000); + return; + } if (mode === "database" && infiniteScrollEnabled.value) { resetInfiniteScrollState(); } else { @@ -4879,6 +5107,9 @@ function applyContextSort(direction: "asc" | "desc" | null, mode: DataGridSortMo async function contextFilterCondition(mode: FilterMode): Promise { if (!contextColumn.value) return null; + if (mode !== "is-null" && mode !== "is-not-null" && contextCell.value) { + if (!(await hydrateLargeValueCell(contextCell.value.rowId, contextCell.value.col))) return null; + } return ( (await buildDataGridContextFilterCondition({ databaseType: resolvedDatabaseType.value, @@ -4998,6 +5229,7 @@ async function applyWhereFilter() { const CELL_FORMAT_CACHE_LIMIT = 20_000; const CELL_FORMAT_CACHE_PRUNE_COUNT = 5_000; +const CELL_FORMAT_CACHE_STRING_KEY_MAX_LENGTH = 512; const resolvedColumnFormatters = computed(() => props.result.columns.map((_, columnIndex) => columnFormatter(columnIndex))); const columnFormatterSignatures = computed(() => resolvedColumnFormatters.value.map(formatterSignature)); @@ -5069,6 +5301,13 @@ function formatCellCached(value: CellValue, columnIndex?: number): string { return display; } + // Avoid retaining a second copy of large text values in Map keys. Long + // values are truncated for display anyway, so formatting them on demand is + // cheaper than keeping the full source string in the primitive cache. + if (typeof value === "string" && value.length > CELL_FORMAT_CACHE_STRING_KEY_MAX_LENGTH) { + return formatCell(value, columnIndex); + } + const key = primitiveCellFormatKey(value, columnIndex); const cached = primitiveCellFormatCache.get(key); if (cached !== undefined) return cached; @@ -5635,7 +5874,7 @@ function onCanvasContext(event: MouseEvent) { onCellContext(item.id, item.displayIndex, actualColIdx, hit.visibleColIdx, event); } -function onCanvasDblClick(event: MouseEvent) { +async function onCanvasDblClick(event: MouseEvent) { const hit = canvasHitTest(event); if (!hit) return; if (hit.rowNumber) { @@ -5648,7 +5887,7 @@ function onCanvasDblClick(event: MouseEvent) { if (!item || actualColIdx === undefined) return; if (showReadonlyCellDetailsOnDblClick(item, item.displayIndex, hit.visibleColIdx, actualColIdx)) return; if (booleanCellsUseCheckbox.value && isBooleanGridCell(item, actualColIdx) && canEditCellItem(item, actualColIdx)) return; - startCellEdit(item.id, actualColIdx, canvasCellContentOverflows(item, actualColIdx, hit.visibleColIdx)); + await startCellEdit(item.id, actualColIdx, canvasCellContentOverflows(item, actualColIdx, hit.visibleColIdx)); } function canvasCellContentOverflows(item: RowItem, actualColIdx: number, visibleColIdx: number): boolean { @@ -5961,6 +6200,7 @@ onMounted(() => { onDeactivated(pauseCanvasGridWork); onUnmounted(() => { dataGridRuntimeScope.dispose(); + clearCellFormatCache(); pauseCanvasGridWork(); canvasRuntime.dispose(); gridScrollbarsRuntime.dispose(); @@ -6070,6 +6310,7 @@ const { getRowItem: (rowId: number) => visibleDisplayItems.value.find((item) => item.id === rowId), selectedRowIds, hasRowSelection, + resolveSourceValues: resolveLargeValueCells, fullExportResult: props.fullExportResult, queryResultExportRequest: props.queryResultExportRequest, hasCompleteLocalResult, @@ -6300,6 +6541,8 @@ function showCellDetails(rowIndex: number, colIndex: number) { detailCell.value = { rowIndex, col: colIndex }; activeCellDetailTab.value = defaultCellDetailTab(); showCellDetail.value = true; + const item = displayItemAt(rowIndex); + if (item && isLargeValuePreview(item, colIndex)) void hydrateLargeValueCell(item.id, colIndex); } function showCellDetailsForVisibleCell(rowIndex: number, visibleColIdx: number, actualColIdx: number) { @@ -6312,6 +6555,8 @@ function showCellDetailsForVisibleCell(rowIndex: number, visibleColIdx: number, function openCellDetailDialog(rowIndex: number, columnIndex: number) { cellDetailDialogTarget.value = { rowIndex, col: columnIndex }; cellDetailDialogOpen.value = true; + const item = displayItemAt(rowIndex); + if (item && isLargeValuePreview(item, columnIndex)) void hydrateLargeValueCell(item.id, columnIndex); } function openColumnDetailDialog(columnIndex: number) { @@ -6416,14 +6661,14 @@ function showTransposeCellDetails(rowIndex: number, actualColIdx: number) { gridRef.value?.focus({ preventScroll: true }); } -function onTransposeCellDblClick(rowIndex: number, actualColIdx: number, displayText: string, event: MouseEvent) { +async function onTransposeCellDblClick(rowIndex: number, actualColIdx: number, displayText: string, event: MouseEvent) { const item = displayItemAt(rowIndex); if (!item) return; if (!canEditCellItem(item, actualColIdx)) { showTransposeCellDetails(rowIndex, actualColIdx); return; } - startDomCellEdit(item.id, actualColIdx, displayText, event); + await startDomCellEdit(item.id, actualColIdx, displayText, event); } function onTransposeCellContext(rowIndex: number, actualColIdx: number, event: MouseEvent) { @@ -6783,10 +7028,10 @@ function generateSelectionMenuItems(disabled: boolean): ContextMenuItem[] { ]; } -function cutSelection() { +async function cutSelection() { if (!props.editable || !selectedRange.value) return; - void copyWithExtractor("tsv"); const range = selectedRange.value; + if (!(await copyWithExtractor("tsv"))) return; const allowDraftSelectionValue = selectedRangeTargetsOnlyDraftRow(); beginBatch(); try { @@ -7038,7 +7283,7 @@ function editSelectedCell(): boolean { const item = displayItemAt(position.rowIndex); const actualColIndex = actualColumnIndex(position.colIndex); if (!item || !canEditCellItem(item, actualColIndex)) return false; - startEdit(item.id, actualColIndex); + void startCellEdit(item.id, actualColIndex, false); return true; } @@ -7056,10 +7301,10 @@ function copyCurrentRow(): boolean { const rowIds = selectedOrCurrentRowIds().filter((rowId) => !getRowItem(rowId)?.isDraft); if (rowIds.length === 0) return false; if (rowIds.length === 1) { - cloneRow(rowIds[0]); + void cloneRow(rowIds[0]); return true; } - cloneRows(rowIds); + void cloneRows(rowIds); return true; } @@ -7101,7 +7346,7 @@ async function commitEditFromCellBlur() { await commitEditFromBlur({ promoteDraft: false }); nextTick(() => { const item = getRowItem(target.rowId); - if (item && canEditCellItem(item, target.col)) startEdit(target.rowId, target.col); + if (item && canEditCellItem(item, target.col)) void startCellEdit(target.rowId, target.col, false); }); return; } @@ -7303,7 +7548,7 @@ async function onGridKeydown(event: KeyboardEvent) { if (clipboardShortcut(event, "x")) { if (!props.editable || !selectedRange.value) return; event.preventDefault(); - cutSelection(); + await cutSelection(); return; } if (clipboardShortcut(event, "v")) { @@ -7316,24 +7561,28 @@ async function onGridKeydown(event: KeyboardEvent) { } } -function copyDetailValue() { +async function copyDetailValue() { + const initialDetail = activeCellDetail.value; + if (!initialDetail || !(await hydrateLargeValueCell(initialDetail.rowId, initialDetail.colIndex))) return; const detail = activeCellDetail.value; if (!detail) return; const text = detail.value === null ? "" : displayCellValue(detail.value); copyText(text); } -function copyDetailFormattedJson() { +async function copyDetailFormattedJson() { + const initialDetail = activeCellDetail.value; + if (!initialDetail || !(await hydrateLargeValueCell(initialDetail.rowId, initialDetail.colIndex))) return; const detail = activeCellDetail.value; if (!detail?.formattedJson) return; copyText(detail.formattedJson); } -function copyDetailCurrentValue() { +async function copyDetailCurrentValue() { if (sideDetailJsonView.value && activeCellDetail.value?.formattedJson) { - copyDetailFormattedJson(); + await copyDetailFormattedJson(); } else { - copyDetailValue(); + await copyDetailValue(); } } @@ -7343,7 +7592,9 @@ function copyDetailColumnName() { } function canDownloadDetailBinaryValue(detail: DataGridCellDetail | null): boolean { - return !!detail && canDownloadBinaryCellValue(detail.value, detail.type); + if (!detail) return false; + const item = getRowItem(detail.rowId); + return (isLargeValuePreview(item, detail.colIndex) && isBinaryCellColumnType(detail.type)) || canDownloadBinaryCellValue(detail.value, detail.type); } function canImportDetailBinaryValue(detail: DataGridCellDetail | null): boolean { @@ -7390,10 +7641,13 @@ function downloadCellBinaryValue(rowIndex: number, columnIndex: number, mode: Bi async function downloadDetailBinaryValue(detail: DataGridCellDetail | null, mode: BinaryCellDownloadMode) { if (!detail || !canDownloadDetailBinaryValue(detail)) return; try { - const payload = binaryCellDownloadPayload(detail.value, mode, detail.type); + if (!(await hydrateLargeValueCell(detail.rowId, detail.colIndex))) return; + const resolvedDetail = cellDetailFor(detail.rowNumber - 1, detail.colIndex); + if (!resolvedDetail) return; + const payload = binaryCellDownloadPayload(resolvedDetail.value, mode, resolvedDetail.type); const fileName = binaryCellDownloadFileName({ - column: detail.column, - rowNumber: detail.rowNumber, + column: resolvedDetail.column, + rowNumber: resolvedDetail.rowNumber, mode, extension: payload.extension, }); @@ -7444,35 +7698,102 @@ async function openDialogCellInSidePanel() { showCellDetails(detail.rowNumber - 1, detail.colIndex); cellDetailDialogOpen.value = false; await nextTick(); - if (detail.isEditable) startDetailEdit(); + if (detail.isEditable) await startDetailEdit(); } -function copyRowDetailJson() { - const detail = rowDetail.value; - if (!detail) return; - copyText(dataGridRowDetailJson(detail)); +async function resolvedRowDetailForCopy() { + if (rowDetailDialogRowId.value === null) return null; + const item = getRowItem(rowDetailDialogRowId.value); + if (!item) return null; + const resolved = await resolveLargeValueCells([item.id], visibleColumnIndexes.value); + const data = [...item.data]; + for (const [columnIndex, value] of resolved.get(item.id) ?? []) data[columnIndex] = value; + return buildDataGridRowDetail({ + rowIndex: item.displayIndex, + rowId: item.id, + row: data, + columns: props.result.columns, + columnIndexes: visibleColumnIndexes.value, + typeByColumn: columnTypeMap.value, + resultColumnTypes: props.result.column_types, + commentByColumn: columnCommentMap.value, + displayValue: (value, index) => formatCellCached(value, index), + isEditableColumn: (columnIndex) => canEditGridCellDetail({ canEditCell: canEditCellItem(item, columnIndex), isDraft: !!item.isDraft }), + }); } -function copyRowDetailTsv() { - const detail = rowDetail.value; - if (!detail) return; - copyText(dataGridRowDetailTsv(detail)); +async function resolvedColumnDetailForCopy() { + if (columnDetailDialogColumnIndex.value === null) return null; + const columnIndex = columnDetailDialogColumnIndex.value; + const items = displayItems.value.filter((item) => !item.isDraft); + const resolved = await resolveLargeValueCells( + items.map((item) => item.id), + [columnIndex], + ); + return buildDataGridColumnDetail({ + rows: items.map((item) => { + const row = [...item.data]; + if (resolved.get(item.id)?.has(columnIndex)) row[columnIndex] = resolved.get(item.id)!.get(columnIndex) ?? null; + return { + rowIndex: item.displayIndex, + rowId: item.id, + row, + isEditable: canEditGridCellDetail({ canEditCell: canEditCellItem(item, columnIndex), isDraft: !!item.isDraft }), + }; + }), + columns: props.result.columns, + columnIndex, + typeByColumn: columnTypeMap.value, + resultColumnTypes: props.result.column_types, + commentByColumn: columnCommentMap.value, + displayValue: (value, index) => formatCellCached(value, index), + }); } -function copyRowDetailFieldValue(field: DataGridCellDetail) { - copyText(field.value === null ? "" : displayCellValue(field.value)); +async function copyRowDetailJson() { + try { + const detail = await resolvedRowDetailForCopy(); + if (!detail) return; + copyText(dataGridRowDetailJson(detail)); + } catch (error) { + reportLargeValueLoadError(error); + } } -function copyColumnDetailJson() { - const detail = columnDetail.value; - if (!detail) return; - copyText(dataGridColumnDetailJson(detail)); +async function copyRowDetailTsv() { + try { + const detail = await resolvedRowDetailForCopy(); + if (!detail) return; + copyText(dataGridRowDetailTsv(detail)); + } catch (error) { + reportLargeValueLoadError(error); + } } -function copyColumnDetailTsv() { - const detail = columnDetail.value; - if (!detail) return; - copyText(dataGridColumnDetailTsv(detail)); +async function copyRowDetailFieldValue(field: DataGridCellDetail) { + if (!(await hydrateLargeValueCell(field.rowId, field.colIndex))) return; + const resolved = cellDetailFor(field.rowNumber - 1, field.colIndex); + if (resolved) copyText(resolved.value === null ? "" : displayCellValue(resolved.value)); +} + +async function copyColumnDetailJson() { + try { + const detail = await resolvedColumnDetailForCopy(); + if (!detail) return; + copyText(dataGridColumnDetailJson(detail)); + } catch (error) { + reportLargeValueLoadError(error); + } +} + +async function copyColumnDetailTsv() { + try { + const detail = await resolvedColumnDetailForCopy(); + if (!detail) return; + copyText(dataGridColumnDetailTsv(detail)); + } catch (error) { + reportLargeValueLoadError(error); + } } function copyColumnDetailColumnName() { @@ -7481,8 +7802,10 @@ function copyColumnDetailColumnName() { copyText(detail.column); } -function copyColumnDetailFieldValue(field: DataGridCellDetail) { - copyText(field.value === null ? "" : displayCellValue(field.value)); +async function copyColumnDetailFieldValue(field: DataGridCellDetail) { + if (!(await hydrateLargeValueCell(field.rowId, field.colIndex))) return; + const resolved = cellDetailFor(field.rowNumber - 1, field.colIndex); + if (resolved) copyText(resolved.value === null ? "" : displayCellValue(resolved.value)); } const transposeRecordWidths = ref([]); @@ -8705,7 +9028,7 @@ async function navigateToForeignKeyCell(rowIndex: number, actualColIdx: number) const association = cellForeignKeyAssociation(actualColIdx); const item = displayItems.value[rowIndex]; if (!association || !item || !props.connectionId) return; - const cells = foreignKeyAssociationCells({ + let cells = foreignKeyAssociationCells({ association, context: props.context, resultColumns: props.result.columns, @@ -8714,6 +9037,15 @@ async function navigateToForeignKeyCell(rowIndex: number, actualColIdx: number) }); if (!cells) return; try { + const resolved = await resolveLargeValueCells( + [item.id], + cells.map((cell) => cell.columnIndex), + ); + const resolvedRow = resolved.get(item.id); + if (resolvedRow) { + cells = cells.map((cell) => (resolvedRow.has(cell.columnIndex) ? { ...cell, value: resolvedRow.get(cell.columnIndex) ?? null } : cell)); + if (cells.some((cell) => cell.value === null)) throw new Error(t("grid.largeValueRowUnavailable")); + } const conditions = await Promise.all( cells.map(({ foreignKey, value }) => buildColumnValueFilterCondition({ @@ -9461,7 +9793,7 @@ const gridContextMenuItems = computed(() => { labels: rowLabels, icons: { clone: CopyPlus, restore: Undo2, delete: Trash2 }, actions: { - clone: () => (isMultiRow.value ? cloneRows(affectedRowIds()) : row && cloneRow(row.id)), + clone: () => void (isMultiRow.value ? cloneRows(affectedRowIds()) : row && cloneRow(row.id)), restore: () => (isMultiRow.value ? restoreRows(affectedRowIds()) : row && restoreRow(row.id)), delete: () => (isMultiRow.value ? requestDeleteRows(deletableRowIds(affectedRowIds())) : row && requestDeleteRow(row.id)), }, diff --git a/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts b/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts index 56282cbf02..d26d4eb8b9 100644 --- a/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts +++ b/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts @@ -15,6 +15,9 @@ const mocks = vi.hoisted(() => ({ listIndexes: vi.fn(), ensureConnected: vi.fn(), tableOpenPageSize: 100, + infiniteScroll: true, + queryResultMaxRowsEnabled: true, + queryResultMaxRows: 10_000, tabs: [] as QueryTab[], setTableMeta: vi.fn(), clearInvalidDataTabSort: vi.fn(), @@ -80,7 +83,14 @@ vi.mock("@/stores/queryStore", () => ({ })); vi.mock("@/stores/settingsStore", () => ({ - useSettingsStore: () => ({ editorSettings: { tableOpenPageSize: mocks.tableOpenPageSize } }), + useSettingsStore: () => ({ + editorSettings: { + tableOpenPageSize: mocks.tableOpenPageSize, + infiniteScroll: mocks.infiniteScroll, + queryResultMaxRowsEnabled: mocks.queryResultMaxRowsEnabled, + queryResultMaxRows: mocks.queryResultMaxRows, + }, + }), })); vi.mock("@/composables/useToast", () => ({ @@ -118,6 +128,9 @@ describe("useDataGridActions", () => { vi.clearAllMocks(); mocks.tabs.length = 0; mocks.tableOpenPageSize = 100; + mocks.infiniteScroll = true; + mocks.queryResultMaxRowsEnabled = true; + mocks.queryResultMaxRows = 10_000; mocks.getConfig.mockReturnValue({ id: "postgres-1", db_type: "postgres" }); mocks.buildTableSelectSql.mockResolvedValue("SELECT * FROM public.users LIMIT 100 OFFSET 0"); mocks.buildSortedQuerySql.mockResolvedValue({ ok: true, sql: "SELECT sorted" }); @@ -160,6 +173,23 @@ describe("useDataGridActions", () => { expect(mocks.executeTabSql.mock.calls[0]?.[2]).not.toHaveProperty("preserveTotalRowCountDuringExecution"); }); + it("keeps infinite-scroll appends for ordinary table-data pages", async () => { + const tab = tableDataTab({ + result: { + columns: ["id"], + rows: Array.from({ length: 100 }, (_, index) => [index + 1]), + affected_rows: 0, + execution_time_ms: 1, + }, + }); + mocks.buildTableSelectSql.mockResolvedValueOnce("SELECT * FROM public.users LIMIT 100 OFFSET 100"); + const actions = useDataGridActions(computed(() => tab)); + + await actions.onPaginate(100, 100); + + expect(mocks.executeTabSql).toHaveBeenCalledWith("tab-1", "SELECT * FROM public.users LIMIT 100 OFFSET 100", expect.objectContaining({ appendResult: { maxRows: 10_000 } })); + }); + it("ignores a stale structured order when its column was renamed", async () => { const tab = tableDataTab({ resultSortColumn: "old_name", diff --git a/apps/desktop/src/composables/__tests__/useDataGridEditor.spec.ts b/apps/desktop/src/composables/__tests__/useDataGridEditor.spec.ts index a5db1546c4..fd5e452506 100644 --- a/apps/desktop/src/composables/__tests__/useDataGridEditor.spec.ts +++ b/apps/desktop/src/composables/__tests__/useDataGridEditor.spec.ts @@ -243,4 +243,13 @@ describe("useDataGridEditor appendPastedRowsToNewRow", () => { ["Grace", null, null], ]); }); + + it("uses resolved full values instead of preview values when cloning", () => { + const editor = createEditor(); + editor.newRows.value = [["Ada", "preview...", "Lovelace"]]; + + editor.cloneRow(-1, new Map([[1, "full payload"]])); + + expect(editor.newRows.value[1]).toEqual(["Ada", "full payload", "Lovelace"]); + }); }); diff --git a/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts b/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts index e73c78b5de..30adb431b2 100644 --- a/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts +++ b/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts @@ -763,6 +763,103 @@ describe("useDataGridExport prepared row statements", () => { expect(extractDataGridSelection).toHaveBeenNthCalledWith(2, expect.objectContaining({ rows: [[7, { name: "Ada", tags: ["admin"] }]] })); }); + it("resolves large-value previews before building an extractor request", async () => { + const matrix: CellSelectionMatrix = { + rowIndexes: [0], + columnIndexes: [0, 1], + columns: ["id", "name"], + rows: [[7, "preview..."]], + }; + const item = row([7, "preview..."]); + const resolveSourceValues = vi.fn(async () => new Map([[item.id, new Map([[1, "full payload"]])]])); + const options: UseDataGridExportOptions = { + columns: computed(() => ["id", "name"]), + displayItems: computed(() => [item]), + allColumns: computed(() => ["id", "name"]), + allDisplayItems: computed(() => [item]), + visibleColumnIndexes: computed(() => [0, 1]), + sql: computed(() => undefined), + tableMeta: computed(() => editableTable), + databaseType: computed(() => "mysql"), + connectionId: computed(() => "connection-1"), + database: computed(() => "dbx"), + context: computed(() => "table-data"), + sourceColumns: computed(() => ["id", "name"]), + columnTypes: computed(() => ["int", "varchar"]), + whereInput: computed(() => undefined), + orderBy: computed(() => undefined), + exportBatchSize: computed(() => 1000), + hasCellSelection: computed(() => true), + selectedCells: computed(() => matrix), + selectedCellMatrix: computed(() => matrix), + selectedRange: computed(() => ({ startRow: 0, endRow: 0, startCol: 0, endCol: 1 })), + contextCell: ref({ rowId: item.id, rowIndex: 0, col: 1 }), + contextSelectionIsSynthetic: ref(false), + getRowItem: () => item, + selectedRowIds: ref(new Set()), + hasRowSelection: computed(() => false), + resolveSourceValues, + }; + vi.mocked(extractDataGridSelection).mockResolvedValueOnce({ + text: "copied", + mimeType: "text/tab-separated-values", + fileExtension: "tsv", + rowCount: 1, + columnCount: 2, + }); + + await expect(useDataGridExport(options).copyWithExtractor("tsv")).resolves.toBe(true); + + expect(resolveSourceValues).toHaveBeenCalledWith([item.id], [0, 1]); + expect(extractDataGridSelection).toHaveBeenCalledWith(expect.objectContaining({ rows: [[7, "full payload"]] })); + }); + + it("does not copy a preview when full-value resolution fails", async () => { + const matrix: CellSelectionMatrix = { + rowIndexes: [0], + columnIndexes: [0], + columns: ["name"], + rows: [["preview..."]], + }; + const item = row(["preview..."]); + const state = useDataGridExport({ + columns: computed(() => ["name"]), + displayItems: computed(() => [item]), + allColumns: computed(() => ["name"]), + allDisplayItems: computed(() => [item]), + visibleColumnIndexes: computed(() => [0]), + sql: computed(() => undefined), + tableMeta: computed(() => editableTable), + databaseType: computed(() => "mysql"), + connectionId: computed(() => "connection-1"), + database: computed(() => "dbx"), + context: computed(() => "table-data"), + sourceColumns: computed(() => ["name"]), + columnTypes: computed(() => ["varchar"]), + whereInput: computed(() => undefined), + orderBy: computed(() => undefined), + exportBatchSize: computed(() => 1000), + hasCellSelection: computed(() => true), + selectedCells: computed(() => matrix), + selectedCellMatrix: computed(() => matrix), + selectedRange: computed(() => ({ startRow: 0, endRow: 0, startCol: 0, endCol: 0 })), + contextCell: ref({ rowId: item.id, rowIndex: 0, col: 0 }), + contextSelectionIsSynthetic: ref(false), + getRowItem: () => item, + selectedRowIds: ref(new Set()), + hasRowSelection: computed(() => false), + resolveSourceValues: async () => { + throw new Error("stable key required"); + }, + }); + + await expect(state.copyWithExtractor("tsv")).resolves.toBe(false); + + expect(extractDataGridSelection).not.toHaveBeenCalled(); + expect(copyToClipboard).not.toHaveBeenCalled(); + expect(toast).toHaveBeenCalledWith("grid.copyFailed: stable key required", 5000); + }); + it("rejects SQL UPDATE instead of silently skipping a row with a null primary key", async () => { const matrix: CellSelectionMatrix = { rowIndexes: [0], diff --git a/apps/desktop/src/composables/__tests__/useSidebarDataOpenRuntime.spec.ts b/apps/desktop/src/composables/__tests__/useSidebarDataOpenRuntime.spec.ts index b4f18dfeb9..bd294a5023 100644 --- a/apps/desktop/src/composables/__tests__/useSidebarDataOpenRuntime.spec.ts +++ b/apps/desktop/src/composables/__tests__/useSidebarDataOpenRuntime.spec.ts @@ -212,11 +212,13 @@ describe("useSidebarDataOpenRuntime", () => { await useSidebarDataOpenRuntime().openData(tableNode); mocks.tabs[0]!.isExecuting = false; mocks.tabs[0]!.executionId = undefined; + mocks.tabs[0]!.resultLocalSortOriginalLargeValueCells = [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]; await useSidebarDataOpenRuntime().openData(ordersNode); expect(mocks.tabs).toHaveLength(1); expect(mocks.tabs[0]?.title).toBe("orders"); expect(mocks.tabs[0]?.tableMeta?.tableName).toBe("orders"); + expect(mocks.tabs[0]?.resultLocalSortOriginalLargeValueCells).toBeUndefined(); }); it.each([ diff --git a/apps/desktop/src/composables/useDataGridEditor.ts b/apps/desktop/src/composables/useDataGridEditor.ts index a7ad84e384..210ae2d6fe 100644 --- a/apps/desktop/src/composables/useDataGridEditor.ts +++ b/apps/desktop/src/composables/useDataGridEditor.ts @@ -1120,12 +1120,13 @@ export function useDataGridEditor(options: UseDataGridEditorOptions) { return { ok: true, rowCount: mappedRows.length }; } - function clonedRowData(item: RowItem): CellValue[] { + function clonedRowData(item: RowItem, resolvedValues?: ReadonlyMap): CellValue[] { const columnInfoByName = new Map((tableMeta.value?.columns ?? []).map((column) => [column.name.toLowerCase(), column])); return item.data.map((val, i) => { const columnName = sourceColumns.value?.[i] ?? result.value.columns[i]; const columnInfo = columnInfoByName.get(columnName.toLowerCase()); - return shouldClearClonedColumn(columnName, columnInfo) ? null : val; + if (shouldClearClonedColumn(columnName, columnInfo)) return null; + return resolvedValues?.has(i) ? (resolvedValues.get(i) ?? null) : val; }); } @@ -1137,10 +1138,10 @@ export function useDataGridEditor(options: UseDataGridEditorOptions) { return /\b(auto_increment|autoincrement|identity|generated)\b/i.test(extra) || /\bnextval\s*\(/i.test(columnDefault); } - function cloneRow(rowId: number) { + function cloneRow(rowId: number, resolvedValues?: ReadonlyMap) { const item = getRowItem(rowId); if (!item) return; - const clonedData = clonedRowData(item); + const clonedData = clonedRowData(item, resolvedValues); pushUndoSnapshot(); rowStatusFilter.value = rowStatusFilterAfterAddingRow(rowStatusFilter.value); newRows.value.push(clonedData); @@ -1159,13 +1160,13 @@ export function useDataGridEditor(options: UseDataGridEditorOptions) { }); } - function cloneRows(rowIds: number[]) { + function cloneRows(rowIds: number[], resolvedValues?: ReadonlyMap>) { const rowsToClone = rowIds.map((rowId) => getRowItem(rowId)).filter(Boolean) as RowItem[]; if (rowsToClone.length === 0) return; pushUndoSnapshot(); rowStatusFilter.value = rowStatusFilterAfterAddingRow(rowStatusFilter.value); for (const item of rowsToClone) { - const clonedData = clonedRowData(item); + const clonedData = clonedRowData(item, resolvedValues?.get(item.id)); newRows.value.push(clonedData); newRowMeta.value.push(clonedRowMeta(item, clonedData)); } diff --git a/apps/desktop/src/composables/useDataGridExport.ts b/apps/desktop/src/composables/useDataGridExport.ts index a1ff44be2f..0708108237 100644 --- a/apps/desktop/src/composables/useDataGridExport.ts +++ b/apps/desktop/src/composables/useDataGridExport.ts @@ -98,6 +98,7 @@ export interface UseDataGridExportOptions { getRowItem: (rowId: number) => RowItem | undefined; selectedRowIds: Ref> | ComputedRef>; hasRowSelection: ComputedRef; + resolveSourceValues?: (rowIds: number[], sourceColumnIndexes: number[]) => Promise>>; fullExportResult?: (onProgress?: (info: { rowsExported: number; totalRows: number | null }) => void) => Promise; queryResultExportRequest?: (options: { exportId: string; filePath: string; format: "csv" | "xlsx" | "txt" | "sql"; includeSqlSheet?: boolean; exportTableName?: string; exportColumnTypes?: Array }) => Promise; /** @@ -182,6 +183,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { getRowItem, selectedRowIds, hasRowSelection, + resolveSourceValues, fullExportResult, queryResultExportRequest, hasCompleteLocalResult, @@ -223,6 +225,25 @@ export function useDataGridExport(options: UseDataGridExportOptions) { return displayItems.value.filter((item) => rowIdSet.has(item.id) && !item.isDraft); } + async function resolveVisibleRowValues(items: RowItem[], visibleIndexes = visibleColumnIndexes.value): Promise { + if (!resolveSourceValues || items.length === 0 || visibleIndexes.length === 0) return items; + const resolved = await resolveSourceValues( + items.map((item) => item.id), + visibleIndexes, + ); + if (resolved.size === 0) return items; + return items.map((item) => { + const values = resolved.get(item.id); + if (!values) return item; + const data = [...item.data]; + visibleIndexes.forEach((sourceIndex) => { + const visibleIndex = visibleColumnIndexes.value.indexOf(sourceIndex); + if (visibleIndex >= 0 && values.has(sourceIndex)) data[visibleIndex] = values.get(sourceIndex) ?? null; + }); + return { ...item, data }; + }); + } + function applyGlobalDateTimeExportFormat(result: { columns: string[]; columnTypes: string[]; rows: CellValue[][] }, enabled: boolean) { const pattern = enabled ? useSettingsStore().editorSettings.globalDateTimeExportFormat : ""; return pattern ? { ...result, rows: formatTemporalRowsForExport(result.rows, result.columnTypes, pattern) } : result; @@ -315,7 +336,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { { columns: columns.value, columnTypes: (columnTypes.value ?? []).map((type) => type ?? ""), - rows: rowsToExport(rowIds).map((item) => item.data), + rows: (await resolveVisibleRowValues(rowsToExport(rowIds))).map((item) => item.data), }, formatDateTime, ), @@ -436,7 +457,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) { async function copyRowsAsJson(items: RowItem[]) { if (items.length === 0) return; - const value = items.length === 1 ? rowToJsonObject(items[0]) : items.map(rowToJsonObject); + const resolvedItems = await resolveVisibleRowValues(items); + const value = resolvedItems.length === 1 ? rowToJsonObject(resolvedItems[0]) : resolvedItems.map(rowToJsonObject); const hasOriginalMongoDocuments = options.databaseType.value === "mongodb" && items.every((item) => item.sourceIndex !== undefined && options.mongoDocuments?.value?.[item.sourceIndex] !== undefined); const copyValue = options.databaseType.value === "mongodb" && !hasOriginalMongoDocuments ? expandNestedJsonStringsForCopy(value) : value; await copyText(JSON.stringify(copyValue, null, 2)); @@ -447,7 +469,9 @@ export function useDataGridExport(options: UseDataGridExportOptions) { if (!contextCell.value || contextCell.value.col < 0) return; const item = getRowItem(contextCell.value.rowId); if (!item || item.isDraft) return; - const val = item?.data[contextCell.value.col] ?? null; + const sourceIndex = visibleColumnIndexes.value[contextCell.value.col] ?? contextCell.value.col; + const [resolvedItem] = await resolveVisibleRowValues([item], [sourceIndex]); + const val = resolvedItem?.data[contextCell.value.col] ?? null; await copyText(displayCellValue(val), { rows: [[val]] }); } @@ -575,6 +599,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { hasRowSelection, hasColumnSelection, selectedRowIds, + resolveSourceValues, copyText, canCopySqlInsert: (request) => { const selectedColumns = request.selectedColumnIndexes.map((index) => request.columns[index]?.sourceName ?? request.columns[index]?.displayName).filter((column): column is string => !!column); @@ -589,7 +614,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { async function copyAll() { const header = columns.value.join("\t"); - const rows = displayItems.value.filter((item) => !item.isDraft).map((item) => item.data); + const rows = (await resolveVisibleRowValues(displayItems.value.filter((item) => !item.isDraft))).map((item) => item.data); const body = rows.map((row) => row.map((cell) => displayCellValue(cell)).join("\t")).join("\n"); await copyText(`${header}\n${body}`, { rows, header: columns.value }); } diff --git a/apps/desktop/src/composables/useDataGridExtractor.ts b/apps/desktop/src/composables/useDataGridExtractor.ts index abd4d89460..bb1aa2f986 100644 --- a/apps/desktop/src/composables/useDataGridExtractor.ts +++ b/apps/desktop/src/composables/useDataGridExtractor.ts @@ -45,6 +45,7 @@ interface UseDataGridExtractorOptions { hasRowSelection: ComputedRef; hasColumnSelection: ComputedRef; selectedRowIds: Ref> | ComputedRef>; + resolveSourceValues?: (rowIds: number[], sourceColumnIndexes: number[]) => Promise>>; contextCell: ComputedRef<{ rowId: number; rowIndex: number; col: number } | null> | Ref<{ rowId: number; rowIndex: number; col: number } | null>; contextSelectionIsSynthetic: ComputedRef | Ref; copyText: (text: string, gridCopy?: { rows: readonly (readonly unknown[])[]; header?: readonly unknown[] }) => Promise; @@ -58,6 +59,7 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { const { t } = useI18n(); const { toast } = useToast(); const hasUnsupportedDiscreteSelection = computed(() => options.hasCellSelection.value && options.selectedCellMatrix.value === null); + const requestSources = new WeakMap; normalizeValues: boolean }>(); function normalizeCellValue(value: unknown, columnType: string | undefined): unknown { if (typeof value !== "string" || columnType?.trim().toLowerCase() !== "json") return value; @@ -82,6 +84,7 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { const fullColumns = options.allColumns.value; const fullItemsById = new Map(options.allDisplayItems.value.map((item) => [item.id, item])); let sourceRows: unknown[][] = []; + let sourceRowIds: number[] = []; let selectedSourceIndexes: number[] = []; let selectionKind: DataGridExtractRequest["selectionKind"] = "cells"; @@ -93,14 +96,15 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { const hasRightClickContext = !!options.contextCell.value && options.contextSelectionIsSynthetic.value; if (options.hasRowSelection.value && options.selectedRowIds.value.size > 0) { - sourceRows = options.displayItems.value.filter((item) => options.selectedRowIds.value.has(item.id) && !item.isDraft).map((item) => (fullItemsById.get(item.id) ?? item).data); + const items = options.displayItems.value.filter((item) => options.selectedRowIds.value.has(item.id) && !item.isDraft); + sourceRows = items.map((item) => (fullItemsById.get(item.id) ?? item).data); + sourceRowIds = items.map((item) => item.id); selectedSourceIndexes = dedupeColumnIndexes(visibleIndexes).filter((index) => index < fullColumns.length); selectionKind = "rows"; } else if (isMultiCellSelection && matrix) { - sourceRows = matrix.rowIndexes - .map((rowIndex) => options.displayItems.value[rowIndex]) - .filter((item): item is ExtractorRowItem => !!item && !item.isDraft) - .map((item) => (fullItemsById.get(item.id) ?? item).data); + const items = matrix.rowIndexes.map((rowIndex) => options.displayItems.value[rowIndex]).filter((item): item is ExtractorRowItem => !!item && !item.isDraft); + sourceRows = items.map((item) => (fullItemsById.get(item.id) ?? item).data); + sourceRowIds = items.map((item) => item.id); selectedSourceIndexes = dedupeColumnIndexes(matrix.columnIndexes.map((index) => visibleIndexes[index] ?? index)).filter((index) => index < fullColumns.length); if (options.hasColumnSelection.value) selectionKind = "columns"; } else if (hasRightClickContext && options.contextCell.value) { @@ -108,15 +112,15 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { const item = fullItemsById.get(options.contextCell.value.rowId); if (item && !item.isDraft) { sourceRows = [item.data]; + sourceRowIds = [item.id]; selectedSourceIndexes = dedupeColumnIndexes(visibleIndexes).filter((index) => index < fullColumns.length); selectionKind = "rows"; } } else if (matrix) { // Single-cell matrix without a context cell — still honor the explicit selection. - sourceRows = matrix.rowIndexes - .map((rowIndex) => options.displayItems.value[rowIndex]) - .filter((item): item is ExtractorRowItem => !!item && !item.isDraft) - .map((item) => (fullItemsById.get(item.id) ?? item).data); + const items = matrix.rowIndexes.map((rowIndex) => options.displayItems.value[rowIndex]).filter((item): item is ExtractorRowItem => !!item && !item.isDraft); + sourceRows = items.map((item) => (fullItemsById.get(item.id) ?? item).data); + sourceRowIds = items.map((item) => item.id); selectedSourceIndexes = dedupeColumnIndexes(matrix.columnIndexes.map((index) => visibleIndexes[index] ?? index)).filter((index) => index < fullColumns.length); if (options.hasColumnSelection.value) selectionKind = "columns"; } @@ -146,7 +150,7 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { columns.map((column) => column.sourceName ?? column.displayName), ) : undefined; - return { + const request: DataGridExtractRequest = { version: DATA_GRID_EXTRACTOR_CONTRACT_VERSION, extractor, databaseType: descriptor.category === "sql" ? options.databaseType.value : undefined, @@ -157,6 +161,31 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { selectionKind, options: normalizeDataGridExtractorOptions(extractorOptions), }; + requestSources.set(request, { + rowIds: sourceRowIds, + sourceColumnIndexes: requiredSourceIndexes, + columnTypes: requiredSourceIndexes.map((sourceIndex) => columnTypesBySource.get(sourceIndex)), + normalizeValues: descriptor.category === "json" || descriptor.category === "sql", + }); + return request; + } + + async function resolveRequestSourceValues(request: DataGridExtractRequest, rowLimit?: number): Promise { + const source = requestSources.get(request); + if (!source || !options.resolveSourceValues) return rowLimit === undefined ? request : { ...request, rows: request.rows.slice(0, rowLimit) }; + const rowIds = rowLimit === undefined ? source.rowIds : source.rowIds.slice(0, rowLimit); + const overlays = await options.resolveSourceValues(rowIds, source.sourceColumnIndexes); + const rows = request.rows.slice(0, rowIds.length).map((row, rowIndex) => { + const values = overlays.get(rowIds[rowIndex]!); + if (!values) return row; + return row.map((value, compactIndex) => { + const sourceIndex = source.sourceColumnIndexes[compactIndex]; + if (sourceIndex === undefined || !values.has(sourceIndex)) return value; + const resolved = values.get(sourceIndex); + return source.normalizeValues ? normalizeCellValue(resolved, source.columnTypes[compactIndex]) : resolved; + }); + }); + return { ...request, rows }; } function canBuildSqlUpdateRequest(): boolean { @@ -219,9 +248,10 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { return false; } if (!canCopyWithExtractor(extractor, extractorOptions)) return false; - const request = buildRequest(extractor, extractorOptions); - if (!request) return false; + const initialRequest = buildRequest(extractor, extractorOptions); + if (!initialRequest) return false; try { + const request = await resolveRequestSourceValues(initialRequest); const mongoResult = await resolveMongoExtractorResult(extractor, request); const result = mongoResult ?? (await api.extractDataGridSelection(request)); if (!result.text) return false; @@ -253,12 +283,13 @@ export function useDataGridExtractor(options: UseDataGridExtractorOptions) { async function previewWithExtractor(extractor: DataGridCopyExtractorId, extractorOptions: DataGridExtractorOptions): Promise { if (hasUnsupportedDiscreteSelection.value) throw new Error(t("grid.copyExtractorUnsupportedSelection")); - const request = buildRequest(extractor, extractorOptions); - if (!request) throw new Error(t("grid.copyExtractorEmptySelection")); - const sourceRowCount = request.rows.length; + const initialRequest = buildRequest(extractor, extractorOptions); + if (!initialRequest) throw new Error(t("grid.copyExtractorEmptySelection")); + const sourceRowCount = initialRequest.rows.length; const previewRowCount = Math.min(sourceRowCount, DATA_GRID_EXTRACTOR_PREVIEW_MAX_ROWS); + const request = await resolveRequestSourceValues(initialRequest, previewRowCount); const mongoResult = await resolveMongoExtractorResult(extractor, request, previewRowCount); - const result = mongoResult ?? (await api.extractDataGridSelection({ ...request, rows: request.rows.slice(0, DATA_GRID_EXTRACTOR_PREVIEW_MAX_ROWS) })); + const result = mongoResult ?? (await api.extractDataGridSelection(request)); if (!result.text) throw new Error(t("grid.copyExtractorEmptySelection")); return { ...result, sourceRowCount, truncated: sourceRowCount > result.rowCount }; } diff --git a/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts b/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts index 1853e7ea64..6df69b4587 100644 --- a/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts +++ b/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts @@ -140,6 +140,7 @@ export function useSidebarDataOpenRuntime() { tab.resultSortDirection = undefined; tab.resultSortMode = undefined; tab.resultLocalSortOriginalRows = undefined; + tab.resultLocalSortOriginalLargeValueCells = undefined; tab.resultLocalSortOriginalMongoDocuments = undefined; tab.resultLocalSortOriginalMongoCopyDocuments = undefined; tab.resultSortedSql = undefined; diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index f9e266e201..fa4fe5c408 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -1818,6 +1818,12 @@ export default { detailSearchNoMatch: "No matches", rawValue: "Raw Value", largeValuePreviewHint: "Previewing first {count} characters. Copy still uses the full value.", + largeValueRowUnavailable: "The row could not be located in the database. The operation was canceled without using the preview value.", + largeValueColumnUnavailable: "The source column for this large value could not be located.", + largeValueLoadFailed: "Failed to load the full large-field value.", + largeValueNeedsStableKey: "This table has no usable primary key, so the full large-field value cannot be loaded safely. The operation was canceled.", + largeValueLoadFailedWithMessage: "Failed to load the full large-field value: {message}", + largeValueLocalSortUnavailable: "This column contains large-value previews and cannot be sorted locally by preview content. Use database sorting instead.", copyValue: "Copy Value", importBinaryValue: "Import from File", binaryImportApplied: "Imported {count} bytes. Save changes to write them to the database.", diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index aa4b783c2c..11bdc30f0b 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -1686,6 +1686,12 @@ export default withEnglishFallback({ detailSearchNoMatch: "Sin coincidencias", rawValue: "Valor sin procesar", largeValuePreviewHint: "Mostrando los primeros {count} caracteres. Copiar sigue usando el valor completo.", + largeValueRowUnavailable: "No se pudo localizar la fila en la base de datos. La operación se canceló sin usar el valor de vista previa.", + largeValueColumnUnavailable: "No se pudo localizar la columna de origen de este valor grande.", + largeValueLoadFailed: "No se pudo cargar el valor completo del campo grande.", + largeValueNeedsStableKey: "Esta tabla no tiene una clave primaria utilizable, por lo que el valor completo del campo grande no se puede cargar de forma segura. La operación se canceló.", + largeValueLoadFailedWithMessage: "No se pudo cargar el valor completo del campo grande: {message}", + largeValueLocalSortUnavailable: "Esta columna contiene vistas previas de valores grandes y no se puede ordenar localmente por su contenido. Use la ordenación de la base de datos.", copyValue: "Copiar valor", downloadBinaryValue: "Descargar valor", downloadSaved: "Guardado en {path}", diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index 563b4aa4af..a0bcd75175 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -1684,6 +1684,12 @@ export default withEnglishFallback({ detailSearchNoMatch: "Nessun risultato", rawValue: "Valore Grezzo", largeValuePreviewHint: "Anteprima dei primi {count} caratteri. La copia utilizzerà comunque il valore completo.", + largeValueRowUnavailable: "Impossibile individuare la riga nel database. L'operazione è stata annullata senza usare il valore di anteprima.", + largeValueColumnUnavailable: "Impossibile individuare la colonna di origine per questo valore di grandi dimensioni.", + largeValueLoadFailed: "Impossibile caricare il valore completo del campo di grandi dimensioni.", + largeValueNeedsStableKey: "Questa tabella non dispone di una chiave primaria utilizzabile, quindi il valore completo del campo non può essere caricato in modo sicuro. L'operazione è stata annullata.", + largeValueLoadFailedWithMessage: "Impossibile caricare il valore completo del campo di grandi dimensioni: {message}", + largeValueLocalSortUnavailable: "Questa colonna contiene anteprime di valori di grandi dimensioni e non può essere ordinata localmente in base all'anteprima. Usa l'ordinamento del database.", copyValue: "Copia Valore", downloadBinaryValue: "Scarica Valore", downloadSaved: "Salvato in {path}", diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index 4450449462..2698720455 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -1702,6 +1702,12 @@ export default withEnglishFallback({ detailSearchNoMatch: "一致なし", rawValue: "生の値", largeValuePreviewHint: "先頭{count}文字をプレビュー表示中です。コピー時は完全な値が使用されます。", + largeValueRowUnavailable: "データベース内で行を特定できません。プレビュー値を使用せず、操作をキャンセルしました。", + largeValueColumnUnavailable: "大きな値に対応するソース列を特定できません。", + largeValueLoadFailed: "大きなフィールドの完全な値を読み込めませんでした。", + largeValueNeedsStableKey: "このテーブルには使用可能な主キーがないため、大きなフィールドの完全な値を安全に読み込めません。操作をキャンセルしました。", + largeValueLoadFailedWithMessage: "大きなフィールドの完全な値を読み込めませんでした: {message}", + largeValueLocalSortUnavailable: "この列には大きな値のプレビューが含まれるため、プレビュー内容による現在のページの並べ替えはできません。データベース並べ替えを使用してください。", copyValue: "値をコピー", downloadBinaryValue: "値をダウンロード", downloadSaved: "{path} に保存しました", diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts index 9158ecc338..63c16b8a4a 100644 --- a/apps/desktop/src/i18n/locales/ko.ts +++ b/apps/desktop/src/i18n/locales/ko.ts @@ -1688,6 +1688,12 @@ export default withEnglishFallback({ detailSearchNoMatch: "일치 항목 없음", rawValue: "원시 값", largeValuePreviewHint: "처음 {count}자 미리보는 중. 복사에는 여전히 전체 값이 사용됩니다.", + largeValueRowUnavailable: "데이터베이스에서 행을 찾을 수 없습니다. 미리보기 값을 사용하지 않고 작업을 취소했습니다.", + largeValueColumnUnavailable: "큰 값에 해당하는 원본 열을 찾을 수 없습니다.", + largeValueLoadFailed: "큰 필드의 전체 값을 불러오지 못했습니다.", + largeValueNeedsStableKey: "이 테이블에는 사용할 수 있는 기본 키가 없어 큰 필드의 전체 값을 안전하게 불러올 수 없습니다. 작업을 취소했습니다.", + largeValueLoadFailedWithMessage: "큰 필드의 전체 값을 불러오지 못했습니다: {message}", + largeValueLocalSortUnavailable: "이 열에는 큰 값 미리보기가 포함되어 있어 미리보기 내용으로 현재 페이지를 정렬할 수 없습니다. 데이터베이스 정렬을 사용하세요.", copyValue: "값 복사", downloadBinaryValue: "값 다운로드", binaryImportTooLarge: "파일이 너무 큽니다({size}, 상한 {limit}). 큰 파일을 단일 셀에 가져오는 것은 지원되지 않습니다.", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index 687dd95f8a..373193b7a1 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -1686,6 +1686,12 @@ export default withEnglishFallback({ detailSearchNoMatch: "Nenhuma correspondência", rawValue: "Valor Bruto", largeValuePreviewHint: "Visualizando os primeiros {count} caracteres. A cópia ainda usa o valor completo.", + largeValueRowUnavailable: "Não foi possível localizar a linha no banco de dados. A operação foi cancelada sem usar o valor de visualização.", + largeValueColumnUnavailable: "Não foi possível localizar a coluna de origem deste valor grande.", + largeValueLoadFailed: "Falha ao carregar o valor completo do campo grande.", + largeValueNeedsStableKey: "Esta tabela não tem uma chave primária utilizável, portanto o valor completo do campo grande não pode ser carregado com segurança. A operação foi cancelada.", + largeValueLoadFailedWithMessage: "Falha ao carregar o valor completo do campo grande: {message}", + largeValueLocalSortUnavailable: "Esta coluna contém visualizações de valores grandes e não pode ser ordenada localmente pelo conteúdo da visualização. Use a ordenação do banco de dados.", copyValue: "Copiar Valor", downloadBinaryValue: "Baixar Valor", downloadSaved: "Salvo em {path}", diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index 5a201c9035..320e9e6802 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -1744,6 +1744,12 @@ export default withEnglishFallback({ detailSearchNoMatch: "无匹配项", rawValue: "原始值", largeValuePreviewHint: "仅预览前 {count} 个字符，复制仍会使用完整值。", + largeValueRowUnavailable: "无法在数据库中定位该行，操作已取消，未使用预览值。", + largeValueColumnUnavailable: "无法定位大字段对应的源列。", + largeValueLoadFailed: "读取大字段完整值失败。", + largeValueNeedsStableKey: "该表没有可用的主键，无法安全读取大字段完整值，操作已取消。", + largeValueLoadFailedWithMessage: "读取大字段完整值失败：{message}", + largeValueLocalSortUnavailable: "该列包含大字段预览，无法按预览内容进行当前页排序，请使用数据库排序。", copyValue: "复制值", importBinaryValue: "从文件导入", binaryImportApplied: "已导入 {count} 字节，保存更改后写入数据库。", diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index a37c385151..53e4a99d56 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -1686,6 +1686,12 @@ export default withEnglishFallback({ detailSearchNoMatch: "無相符項目", rawValue: "原始值", largeValuePreviewHint: "僅預覽前 {count} 個字元，複製仍會使用完整值。", + largeValueRowUnavailable: "無法在資料庫中定位該列，操作已取消，未使用預覽值。", + largeValueColumnUnavailable: "無法定位大欄位對應的來源欄。", + largeValueLoadFailed: "讀取大欄位完整值失敗。", + largeValueNeedsStableKey: "該資料表沒有可用的主鍵，無法安全讀取大欄位完整值，操作已取消。", + largeValueLoadFailedWithMessage: "讀取大欄位完整值失敗：{message}", + largeValueLocalSortUnavailable: "該欄包含大欄位預覽，無法依預覽內容進行目前頁排序，請使用資料庫排序。", copyValue: "複製值", importBinaryValue: "從檔案匯入", binaryImportApplied: "已匯入 {count} 位元組，儲存變更後寫入資料庫。", diff --git a/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts new file mode 100644 index 0000000000..d7391131bd --- /dev/null +++ b/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues"; +import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail"; + +describe("data grid large-value metadata", () => { + it("offsets appended segment rows and discards metadata beyond the append cap", () => { + expect( + appendLargeValueCells( + [{ row_index: 0, column_index: 1, original_bytes: 10_000 }], + [ + { row_index: 0, column_index: 1, original_bytes: 20_000 }, + { row_index: 1, column_index: 2, original_bytes: 30_000 }, + { row_index: 2, column_index: 1, original_bytes: 40_000 }, + ], + 2, + 2, + ), + ).toEqual([ + { row_index: 0, column_index: 1, original_bytes: 10_000 }, + { row_index: 2, column_index: 1, original_bytes: 20_000 }, + { row_index: 3, column_index: 2, original_bytes: 30_000 }, + ]); + }); + + it("remaps metadata to locally sorted row positions", () => { + expect( + remapLargeValueCells( + [ + { row_index: 0, column_index: 1, original_bytes: 10_000 }, + { row_index: 2, column_index: 1, original_bytes: 30_000 }, + ], + [2, 0, 1], + ), + ).toEqual([ + { row_index: 1, column_index: 1, original_bytes: 10_000 }, + { row_index: 0, column_index: 1, original_bytes: 30_000 }, + ]); + }); + + it("indexes metadata by source row and column", () => { + const result = { large_value_cells: [{ row_index: 4, column_index: 2, original_bytes: 65_536 }] }; + expect(largeValueCellMap(result).get("4:2")).toEqual(result.large_value_cells[0]); + }); + + it("marks a backend-bounded value as truncated even when the preview is shorter than the UI limit", () => { + const detail = buildDataGridCellDetail({ + rowIndex: 0, + rowId: 1, + row: ["preview..."], + columns: ["payload"], + columnIndex: 0, + displayValue: String, + isEditable: true, + isValuePreviewTruncated: true, + }); + + expect(detail?.isValuePreviewTruncated).toBe(true); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/tabs/tabResultCache.spec.ts b/apps/desktop/src/lib/__tests__/tabs/tabResultCache.spec.ts index 9a77c978c5..6a12e7ee0d 100644 --- a/apps/desktop/src/lib/__tests__/tabs/tabResultCache.spec.ts +++ b/apps/desktop/src/lib/__tests__/tabs/tabResultCache.spec.ts @@ -163,6 +163,25 @@ describe("tab result cache statement execution metadata", () => { expect(result?.spatial_values).toEqual([[4326], [3857], [null]]); }); + it("preserves large-value preview metadata", () => { + const encoded = encodeTabResultSnapshot({ + result: { + columns: ["id", "payload"], + rows: [[1, "preview..."]], + large_value_cells: [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }], + affected_rows: 0, + execution_time_ms: 1, + }, + resultLocalSortOriginalRows: [[1, "preview..."]], + resultLocalSortOriginalLargeValueCells: [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }], + cachedAt: 1, + }); + + const restored = decodeTabResultSnapshot(encoded); + expect(restored?.result?.large_value_cells).toEqual([{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]); + expect(restored?.resultLocalSortOriginalLargeValueCells).toEqual([{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]); + }); + it("restores multi-result, pagination, local-sort, and editable metadata fixtures", () => { const restored = decodeTabResultSnapshot(encodeTabResultSnapshot(queryResultLifecycleSnapshot())); diff --git a/apps/desktop/src/lib/backend/http.ts b/apps/desktop/src/lib/backend/http.ts index 083bb07fe1..c9e78010a3 100644 --- a/apps/desktop/src/lib/backend/http.ts +++ b/apps/desktop/src/lib/backend/http.ts @@ -927,6 +927,8 @@ export async function executeMulti( catalog?: string; fetchSize?: number; pageSize?: number; + maxResultBytes?: number; + resultKeyColumns?: string[]; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; @@ -967,6 +969,8 @@ export async function executeMultiWithProgress( catalog?: string; fetchSize?: number; pageSize?: number; + maxResultBytes?: number; + resultKeyColumns?: string[]; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 1de6a44336..ea7e6d1712 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -1145,6 +1145,8 @@ export async function executeMulti( catalog?: string; fetchSize?: number; pageSize?: number; + maxResultBytes?: number; + resultKeyColumns?: string[]; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; @@ -1189,6 +1191,8 @@ export async function executeMultiWithProgress( catalog?: string; fetchSize?: number; pageSize?: number; + maxResultBytes?: number; + resultKeyColumns?: string[]; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; diff --git a/apps/desktop/src/lib/dataGrid/dataGridDetail.ts b/apps/desktop/src/lib/dataGrid/dataGridDetail.ts index 427173fdd0..595a9bbdcb 100644 --- a/apps/desktop/src/lib/dataGrid/dataGridDetail.ts +++ b/apps/desktop/src/lib/dataGrid/dataGridDetail.ts @@ -49,6 +49,7 @@ export interface BuildDataGridCellDetailOptions { displayValue: (value: CellValue, columnIndex: number) => string; isEditable: boolean; includeBinaryImagePreview?: boolean; + isValuePreviewTruncated?: boolean; } export interface BuildDataGridRowDetailOptions { @@ -62,6 +63,7 @@ export interface BuildDataGridRowDetailOptions { commentByColumn?: ReadonlyMap; displayValue: (value: CellValue, columnIndex: number) => string; isEditableColumn?: (columnIndex: number) => boolean; + isValuePreviewTruncated?: (columnIndex: number) => boolean; } export interface BuildDataGridColumnDetailRow { @@ -69,6 +71,7 @@ export interface BuildDataGridColumnDetailRow { rowId: number; row: readonly CellValue[]; isEditable?: boolean; + isValuePreviewTruncated?: boolean; } export interface BuildDataGridColumnDetailOptions { @@ -105,7 +108,7 @@ export function buildDataGridCellDetail(options: BuildDataGridCellDetailOptions) rawValuePreview, displayValue, displayValuePreview, - isValuePreviewTruncated: rawValuePreview.length < rawValue.length || displayValuePreview.length < displayValue.length, + isValuePreviewTruncated: options.isValuePreviewTruncated === true || rawValuePreview.length < rawValue.length || displayValuePreview.length < displayValue.length, imagePreviewUrl: cellImagePreviewUrl(value, type, { binary: options.includeBinaryImagePreview !== false }), length: value === null ? 0 : String(value).length, formattedJson, @@ -131,6 +134,7 @@ export function buildDataGridColumnDetail(options: BuildDataGridColumnDetailOpti displayValue: options.displayValue, isEditable: row.isEditable ?? false, includeBinaryImagePreview: false, + isValuePreviewTruncated: row.isValuePreviewTruncated, }), ) .filter((field): field is DataGridCellDetail => field !== null); @@ -159,6 +163,7 @@ export function buildDataGridRowDetail(options: BuildDataGridRowDetailOptions): displayValue: options.displayValue, isEditable: options.isEditableColumn?.(columnIndex) ?? false, includeBinaryImagePreview: false, + isValuePreviewTruncated: options.isValuePreviewTruncated?.(columnIndex), }), ) .filter((field): field is DataGridCellDetail => field !== null); diff --git a/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts b/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts new file mode 100644 index 0000000000..f2e3739709 --- /dev/null +++ b/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts @@ -0,0 +1,29 @@ +import type { QueryResult } from "@/types/database"; + +export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024; + +export function largeValueCellKey(rowIndex: number, columnIndex: number): string { + return `${rowIndex}:${columnIndex}`; +} + +export function largeValueCellMap(result: Pick): Map[number]> { + return new Map((result.large_value_cells ?? []).map((cell) => [largeValueCellKey(cell.row_index, cell.column_index), cell])); +} + +export function appendLargeValueCells(previous: QueryResult["large_value_cells"], segment: QueryResult["large_value_cells"], rowOffset: number, appendedRowCount: number): QueryResult["large_value_cells"] { + const appended = (segment ?? []).filter((cell) => cell.row_index < appendedRowCount).map((cell) => ({ ...cell, row_index: cell.row_index + rowOffset })); + const combined = [...(previous ?? []), ...appended]; + return combined.length > 0 ? combined : undefined; +} + +export function remapLargeValueCells(cells: QueryResult["large_value_cells"], rowIndexes: number[]): QueryResult["large_value_cells"] { + if (!cells?.length) return undefined; + const targetBySource = new Map(rowIndexes.map((sourceIndex, targetIndex) => [sourceIndex, targetIndex])); + const remapped = cells + .map((cell) => { + const rowIndex = targetBySource.get(cell.row_index); + return rowIndex === undefined ? undefined : { ...cell, row_index: rowIndex }; + }) + .filter((cell): cell is NonNullable => !!cell); + return remapped.length > 0 ? remapped : undefined; +} diff --git a/apps/desktop/src/lib/tabs/tabResultCache.ts b/apps/desktop/src/lib/tabs/tabResultCache.ts index e2d3f93eaf..056e6bd62c 100644 --- a/apps/desktop/src/lib/tabs/tabResultCache.ts +++ b/apps/desktop/src/lib/tabs/tabResultCache.ts @@ -27,6 +27,7 @@ export interface TabResultSnapshot { * still return to the original result order. */ resultLocalSortOriginalRows?: QueryResult["rows"]; + resultLocalSortOriginalLargeValueCells?: QueryResult["large_value_cells"]; resultLocalSortOriginalMongoDocuments?: QueryResult["mongo_documents"]; resultLocalSortOriginalMongoCopyDocuments?: QueryResult["mongo_copy_documents"]; resultRuns?: QueryTab["resultRuns"]; @@ -48,6 +49,7 @@ interface ColumnarQueryResult { columns: string[]; spatial_columns?: QueryResult["spatial_columns"]; spatial_values?: QueryResult["spatial_values"]; + large_value_cells?: QueryResult["large_value_cells"]; execution_error?: true; statement_index?: number; column_types?: string[]; @@ -328,6 +330,7 @@ function stripSessionIds(result: QueryResult | undefined): QueryResult | undefin column_types: result.column_types ? [...result.column_types] : undefined, spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })), spatial_values: result.spatial_values?.map((row) => [...row]), + large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })), rows: result.rows.map((row) => [...row]), mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined, mongo_copy_documents: result.mongo_copy_documents ? clonePlain(result.mongo_copy_documents) : undefined, @@ -353,6 +356,7 @@ function stripResultRunSessionIds(resultRuns: QueryTab["resultRuns"]): QueryTab[ result: stripSessionIds(run.result), results: stripResultSessionIds(run.results), resultLocalSortOriginalRows: run.resultLocalSortOriginalRows?.map((row) => [...row]), + resultLocalSortOriginalLargeValueCells: run.resultLocalSortOriginalLargeValueCells?.map((cell) => ({ ...cell })), resultLocalSortOriginalMongoDocuments: run.resultLocalSortOriginalMongoDocuments ? clonePlain(run.resultLocalSortOriginalMongoDocuments) : undefined, resultLocalSortOriginalMongoCopyDocuments: run.resultLocalSortOriginalMongoCopyDocuments ? clonePlain(run.resultLocalSortOriginalMongoCopyDocuments) : undefined, resultSessionId: undefined, @@ -369,6 +373,7 @@ function toColumnarResult(result: QueryResult | undefined): ColumnarQueryResult column_types: result.column_types ? [...result.column_types] : undefined, spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })), spatial_values: result.spatial_values?.map((row) => [...row]), + large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })), columnValues, rowCount: result.rows.length, mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined, @@ -394,6 +399,7 @@ function fromColumnarResult(result: ColumnarQueryResult | undefined): QueryResul column_types: result.column_types ? [...result.column_types] : undefined, spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })), spatial_values: result.spatial_values?.map((row) => [...row]), + large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })), rows, mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined, mongo_copy_documents: result.mongo_copy_documents ? clonePlain(result.mongo_copy_documents) : undefined, diff --git a/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts b/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts index 407b4854e5..3a2df44263 100644 --- a/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts +++ b/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts @@ -166,6 +166,58 @@ describe("queryStore table data refresh", () => { expect(store.tabs.find((tab) => tab.id === secondTabId)?.result).toBeUndefined(); }); + it("keeps the configured MySQL page size when results contain large-value previews", async () => { + mocks.getConnectionConfig.mockReturnValue({ + id: "mysql-1", + name: "MySQL", + db_type: "mysql", + database: "app", + query_timeout_secs: 30, + }); + const { useQueryStore } = await import("@/stores/queryStore"); + const store = useQueryStore(); + const tabId = store.createTab("mysql-1", "app", "users", "data"); + store.setTableMeta(tabId, { + tableName: "users", + tableType: "TABLE", + columns: [ + { name: "id", data_type: "bigint", is_nullable: false, column_default: null, is_primary_key: true, extra: null }, + { name: "payload", data_type: "longtext", is_nullable: true, column_default: null, is_primary_key: false, extra: null }, + ], + primaryKeys: ["id"], + }); + const tab = store.tabs.find((candidate) => candidate.id === tabId)!; + tab.resultPageLimit = 100; + tab.resultPageOffset = 0; + mocks.executeMulti.mockResolvedValueOnce([ + { + columns: ["id", "payload"], + rows: Array.from({ length: 100 }, (_, index) => [String(index + 1), "preview..."]), + large_value_cells: Array.from({ length: 100 }, (_, index) => ({ row_index: index, column_index: 1, original_bytes: 1_000_000 })), + affected_rows: 100, + execution_time_ms: 1, + }, + ]); + + await expect(store.refreshDataTab(tabId)).resolves.toBe(true); + + expect(tab.result?.rows).toHaveLength(100); + expect(tab.resultPageLimit).toBe(100); + expect(mocks.executeMulti).toHaveBeenCalledWith( + "mysql-1", + "app", + expect.any(String), + undefined, + expect.any(String), + expect.objectContaining({ + maxRows: 100, + fetchSize: 100, + maxResultBytes: 32 * 1024 * 1024, + resultKeyColumns: ["id"], + }), + ); + }); + it("keeps a MySQL table refresh unqualified in the selected database context", async () => { mocks.getConnectionConfig.mockReturnValue({ id: "mysql-1", @@ -216,6 +268,7 @@ describe("queryStore table data refresh", () => { tab.resultSortMode = "database"; tab.resultSortedSql = 'SELECT * FROM public.users ORDER BY "old_name" ASC'; tab.resultLocalSortOriginalRows = [[1, "alpha"]]; + tab.resultLocalSortOriginalLargeValueCells = [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]; tab.resultLocalSortOriginalMongoDocuments = [{ id: 1, old_name: "alpha" }]; tab.resultLocalSortOriginalMongoCopyDocuments = [{ id: 1, old_name: "alpha" }]; tab.resultPageLimit = 25; @@ -238,6 +291,7 @@ describe("queryStore table data refresh", () => { expect(tab.resultSortMode).toBeUndefined(); expect(tab.resultSortedSql).toBeUndefined(); expect(tab.resultLocalSortOriginalRows).toBeUndefined(); + expect(tab.resultLocalSortOriginalLargeValueCells).toBeUndefined(); expect(tab.resultLocalSortOriginalMongoDocuments).toBeUndefined(); expect(tab.resultLocalSortOriginalMongoCopyDocuments).toBeUndefined(); expect(tab.orderByInput).toBeUndefined(); diff --git a/apps/desktop/src/stores/queryStore.ts b/apps/desktop/src/stores/queryStore.ts index 4872326375..14f7b5412d 100644 --- a/apps/desktop/src/stores/queryStore.ts +++ b/apps/desktop/src/stores/queryStore.ts @@ -46,6 +46,7 @@ import { connectionObjectTreeNodeSchema, connectionQueryExecutionSchema, connect import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout"; import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource"; import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection"; +import { appendLargeValueCells, remapLargeValueCells, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues"; import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort"; import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize"; import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit"; @@ -150,6 +151,7 @@ function droppedTableObjectSchemaCandidates(target: DroppedTableObjectTarget): S function markQueryResultRowsRaw(result: QueryResult): QueryResult { markRaw(result.rows); + if (result.large_value_cells) markRaw(result.large_value_cells); if (result.mongo_documents) markRaw(result.mongo_documents); if (result.mongo_copy_documents) markRaw(result.mongo_copy_documents); return result; @@ -201,6 +203,7 @@ export function appendQueryResultSegment(previous: QueryResult, segment: QueryRe rows: [...previous.rows, ...segment.rows.slice(0, appendedRowCount)], spatial_columns: spatial_columns.length > 0 ? spatial_columns : undefined, spatial_values: appendParallelValues(previous.spatial_values, segment.spatial_values), + large_value_cells: appendLargeValueCells(previous.large_value_cells, segment.large_value_cells, previous.rows.length, appendedRowCount), mongo_documents: appendParallelValues(previous.mongo_documents, segment.mongo_documents), mongo_copy_documents: appendParallelValues(previous.mongo_copy_documents, segment.mongo_copy_documents), execution_time_ms: (previous.execution_time_ms ?? 0) + (segment.execution_time_ms ?? 0), @@ -213,6 +216,7 @@ function markQueryResultRunsRowsRaw(resultRuns: NonNullable { tab.batchSqlExecution = undefined; tab.resultEditorFingerprint = undefined; tab.resultLocalSortOriginalRows = undefined; + tab.resultLocalSortOriginalLargeValueCells = undefined; tab.resultLocalSortOriginalMongoDocuments = undefined; tab.resultLocalSortOriginalMongoCopyDocuments = undefined; tab.resultSortMode = undefined; @@ -954,6 +959,7 @@ export const useQueryStore = defineStore("query", () => { tab.resultSortDirection = undefined; tab.resultSortMode = undefined; tab.resultLocalSortOriginalRows = undefined; + tab.resultLocalSortOriginalLargeValueCells = undefined; tab.resultLocalSortOriginalMongoDocuments = undefined; tab.resultLocalSortOriginalMongoCopyDocuments = undefined; tab.orderByInput = undefined; @@ -978,6 +984,7 @@ export const useQueryStore = defineStore("query", () => { run.result = undefined; run.results = undefined; run.resultLocalSortOriginalRows = undefined; + run.resultLocalSortOriginalLargeValueCells = undefined; run.resultLocalSortOriginalMongoDocuments = undefined; run.resultLocalSortOriginalMongoCopyDocuments = undefined; run.resultSessionId = undefined; @@ -1007,6 +1014,7 @@ export const useQueryStore = defineStore("query", () => { tab.resultSortDirection = run.resultSortDirection; tab.resultSortMode = run.resultSortMode; tab.resultLocalSortOriginalRows = run.resultLocalSortOriginalRows; + tab.resultLocalSortOriginalLargeValueCells = run.resultLocalSortOriginalLargeValueCells; tab.resultLocalSortOriginalMongoDocuments = run.resultLocalSortOriginalMongoDocuments; tab.resultLocalSortOriginalMongoCopyDocuments = run.resultLocalSortOriginalMongoCopyDocuments; tab.orderByInput = run.orderByInput; @@ -1243,6 +1251,7 @@ export const useQueryStore = defineStore("query", () => { resultSortDirection: tab.resultSortDirection, resultSortMode: tab.resultSortMode, resultLocalSortOriginalRows: tab.resultLocalSortOriginalRows, + resultLocalSortOriginalLargeValueCells: tab.resultLocalSortOriginalLargeValueCells, resultLocalSortOriginalMongoDocuments: tab.resultLocalSortOriginalMongoDocuments, resultLocalSortOriginalMongoCopyDocuments: tab.resultLocalSortOriginalMongoCopyDocuments, orderByInput: tab.orderByInput, @@ -1339,6 +1348,7 @@ export const useQueryStore = defineStore("query", () => { resultSortDirection: tab.resultSortDirection, resultSortMode: tab.resultSortMode, resultLocalSortOriginalRows: tab.resultLocalSortOriginalRows, + resultLocalSortOriginalLargeValueCells: tab.resultLocalSortOriginalLargeValueCells, resultLocalSortOriginalMongoDocuments: tab.resultLocalSortOriginalMongoDocuments, resultLocalSortOriginalMongoCopyDocuments: tab.resultLocalSortOriginalMongoCopyDocuments, orderByInput: tab.orderByInput, @@ -1392,6 +1402,7 @@ export const useQueryStore = defineStore("query", () => { if (!tab.resultLocalSortOriginalRows) { tab.resultLocalSortOriginalRows = tab.result.rows.slice(); + tab.resultLocalSortOriginalLargeValueCells = tab.result.large_value_cells?.map((cell) => ({ ...cell })); tab.resultLocalSortOriginalMongoDocuments = tab.result.mongo_documents?.slice(); tab.resultLocalSortOriginalMongoCopyDocuments = tab.result.mongo_copy_documents?.slice(); } @@ -1404,7 +1415,8 @@ export const useQueryStore = defineStore("query", () => { const mongo_documents = originalMongoDocuments ? rowIndexes.map((index) => originalMongoDocuments[index]) : undefined; const originalMongoCopyDocuments = tab.resultLocalSortOriginalMongoCopyDocuments; const mongo_copy_documents = originalMongoCopyDocuments ? rowIndexes.map((index) => originalMongoCopyDocuments[index]) : undefined; - assignDisplayedResult(tab, { ...tab.result, rows, mongo_documents, mongo_copy_documents }); + const large_value_cells = remapLargeValueCells(tab.resultLocalSortOriginalLargeValueCells, rowIndexes); + assignDisplayedResult(tab, { ...tab.result, rows, large_value_cells, mongo_documents, mongo_copy_documents }); tab.resultSortColumn = direction ? column : undefined; tab.resultSortColumnIndex = direction ? columnIndex : undefined; @@ -1413,6 +1425,7 @@ export const useQueryStore = defineStore("query", () => { tab.resultSortedSql = undefined; if (!direction) { tab.resultLocalSortOriginalRows = undefined; + tab.resultLocalSortOriginalLargeValueCells = undefined; tab.resultLocalSortOriginalMongoDocuments = undefined; tab.resultLocalSortOriginalMongoCopyDocuments = undefined; } @@ -2544,6 +2557,7 @@ export const useQueryStore = defineStore("query", () => { resultSortDirection: undefined, resultSortMode: undefined, resultLocalSortOriginalRows: undefined, + resultLocalSortOriginalLargeValueCells: undefined, resultLocalSortOriginalMongoDocuments: undefined, resultLocalSortOriginalMongoCopyDocuments: undefined, orderByInput: undefined, @@ -3124,6 +3138,7 @@ export const useQueryStore = defineStore("query", () => { tab.resultSortMode = undefined; tab.resultSortedSql = undefined; tab.resultLocalSortOriginalRows = undefined; + tab.resultLocalSortOriginalLargeValueCells = undefined; tab.resultLocalSortOriginalMongoDocuments = undefined; tab.resultLocalSortOriginalMongoCopyDocuments = undefined; } @@ -3808,6 +3823,7 @@ export const useQueryStore = defineStore("query", () => { const previousDisplayedSql = tab.resultBaseSql ?? tab.lastExecutedSql ?? tab.sql; tab.lastExecutedSql = sql; tab.resultLocalSortOriginalRows = undefined; + tab.resultLocalSortOriginalLargeValueCells = undefined; tab.resultLocalSortOriginalMongoDocuments = undefined; tab.resultLocalSortOriginalMongoCopyDocuments = undefined; if (captureResultRun && tab.result && !tab.activeResultRunId) { @@ -4465,6 +4481,12 @@ export const useQueryStore = defineStore("query", () => { : { maxRows: pageLimit, fetchSize: pageLimit } : { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }), ...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}), + ...(tab.mode === "data" && effectiveDbType === "mysql" + ? { + maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES, + resultKeyColumns: tableMetaForDataTab(tab)?.primaryKeys ?? [], + } + : {}), timeoutSecs: queryTimeoutSecs, catalog: executionCatalog, continueOnError: settingsStore.editorSettings.continueOnErrorOnBatch, @@ -5177,6 +5199,7 @@ export const useQueryStore = defineStore("query", () => { tab.activeResultIndex = index; tab.result = tab.results[index]; tab.resultLocalSortOriginalRows = undefined; + tab.resultLocalSortOriginalLargeValueCells = undefined; tab.resultLocalSortOriginalMongoDocuments = undefined; tab.resultLocalSortOriginalMongoCopyDocuments = undefined; tab.resultSortColumn = undefined; @@ -5300,6 +5323,7 @@ export const useQueryStore = defineStore("query", () => { tab.resultEditorFingerprint = snapshot.resultEditorFingerprint; tab.result = snapshot.result ? markQueryResultRowsRaw(snapshot.result) : results?.[activeIndex] ? markQueryResultRowsRaw(results[activeIndex]) : undefined; tab.resultLocalSortOriginalRows = snapshot.resultLocalSortOriginalRows ? markRaw(snapshot.resultLocalSortOriginalRows) : undefined; + tab.resultLocalSortOriginalLargeValueCells = snapshot.resultLocalSortOriginalLargeValueCells?.map((cell) => ({ ...cell })); tab.resultLocalSortOriginalMongoDocuments = snapshot.resultLocalSortOriginalMongoDocuments ? markRaw(snapshot.resultLocalSortOriginalMongoDocuments) : undefined; tab.resultLocalSortOriginalMongoCopyDocuments = snapshot.resultLocalSortOriginalMongoCopyDocuments ? markRaw(snapshot.resultLocalSortOriginalMongoCopyDocuments) : undefined; // 快照编解码会重建负载，落盘前的各 run 估算值不再对应恢复后的对象， diff --git a/apps/desktop/src/types/database.ts b/apps/desktop/src/types/database.ts index 667ac9e8a0..cad989635d 100644 --- a/apps/desktop/src/types/database.ts +++ b/apps/desktop/src/types/database.ts @@ -713,6 +713,8 @@ export interface QueryResult { /** Whether a backend-reported result total is exact. */ total_is_exact?: boolean; truncated?: boolean; + /** Variable-length cells represented by bounded previews in `rows`. */ + large_value_cells?: Array<{ row_index: number; column_index: number; original_bytes: number }>; session_id?: string | null; has_more?: boolean; /** For Elasticsearch REST search results parsed into a _source table, @@ -778,6 +780,7 @@ export interface QueryResultRun { resultSortDirection?: "asc" | "desc"; resultSortMode?: "database" | "local"; resultLocalSortOriginalRows?: QueryResult["rows"]; + resultLocalSortOriginalLargeValueCells?: QueryResult["large_value_cells"]; resultLocalSortOriginalMongoDocuments?: QueryResult["mongo_documents"]; resultLocalSortOriginalMongoCopyDocuments?: QueryResult["mongo_copy_documents"]; orderByInput?: string; @@ -1085,6 +1088,7 @@ export interface QueryTab { resultSortDirection?: "asc" | "desc"; resultSortMode?: "database" | "local"; resultLocalSortOriginalRows?: QueryResult["rows"]; + resultLocalSortOriginalLargeValueCells?: QueryResult["large_value_cells"]; resultLocalSortOriginalMongoDocuments?: QueryResult["mongo_documents"]; resultLocalSortOriginalMongoCopyDocuments?: QueryResult["mongo_copy_documents"]; orderByInput?: string; diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-core/src/db/mysql.rs index 9866837afb..5c57f9276f 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-core/src/db/mysql.rs @@ -1,6 +1,6 @@ use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use futures::StreamExt; -use mysql_async::consts::ColumnType; +use mysql_async::consts::{ColumnFlags, ColumnType}; use mysql_async::prelude::*; use percent_encoding::percent_decode_str; use rust_decimal::Decimal; @@ -21,8 +21,8 @@ use crate::sql::{starts_with_executable_sql_keyword, starts_with_executable_sql_ use crate::types::{ ColumnInfo, CompletionAssistantCandidate, CompletionAssistantCandidateKind, CompletionAssistantMatchMode, CompletionAssistantObjectKind, CompletionAssistantRequest, CompletionAssistantResponse, DatabaseInfo, - ForeignKeyInfo, IndexInfo, ObjectInfo, ObjectStatistics, QueryMessage, QueryResult, SpatialColumnBuilder, - TableInfo, TriggerInfo, + ForeignKeyInfo, IndexInfo, LargeValueCell, ObjectInfo, ObjectStatistics, QueryMessage, QueryResult, + SpatialColumnBuilder, TableInfo, TriggerInfo, }; use super::file_validator::validate_file_path; @@ -57,6 +57,21 @@ pub struct MySqlQueryDialect { supports_admin_show_results: bool, } +#[derive(Debug)] +pub struct MySqlQueryResult { + pub result: QueryResult, + pub large_value_cells: Vec, +} + +impl MySqlQueryResult { + fn exact(result: QueryResult) -> Self { + Self { result, large_value_cells: Vec::new() } + } +} + +const MYSQL_RESULT_CELL_PREVIEW_MIN_BYTES: usize = 256; +const MYSQL_RESULT_CELL_PREVIEW_MAX_BYTES: usize = 8 * 1024; + impl MySqlQueryDialect { pub fn for_connection(db_type: DatabaseType, driver_profile: Option<&str>) -> Self { let profile = driver_profile.map(str::to_ascii_lowercase); @@ -569,6 +584,102 @@ fn mysql_spatial_column_builder(columns: &[mysql_async::Column]) -> SpatialColum ) } +fn mysql_result_cell_preview_bytes( + max_result_bytes: usize, + row_limit: usize, + columns: &[mysql_async::Column], +) -> usize { + let serialized_column_weight = columns + .iter() + .map( + |column| { + if is_mysql_blob_column(column) || is_mysql_binary_string_column(column) { + 2usize + } else { + 1usize + } + }, + ) + .sum::() + .max(1); + max_result_bytes + .checked_div(row_limit.max(1)) + .and_then(|per_row| per_row.checked_div(serialized_column_weight)) + .unwrap_or(MYSQL_RESULT_CELL_PREVIEW_MIN_BYTES) + .clamp(MYSQL_RESULT_CELL_PREVIEW_MIN_BYTES, MYSQL_RESULT_CELL_PREVIEW_MAX_BYTES) +} + +fn mysql_column_can_use_bounded_preview(column: &mysql_async::Column) -> bool { + matches!( + column.column_type(), + ColumnType::MYSQL_TYPE_JSON + | ColumnType::MYSQL_TYPE_BLOB + | ColumnType::MYSQL_TYPE_LONG_BLOB + | ColumnType::MYSQL_TYPE_MEDIUM_BLOB + | ColumnType::MYSQL_TYPE_TINY_BLOB + | ColumnType::MYSQL_TYPE_STRING + | ColumnType::MYSQL_TYPE_VAR_STRING + | ColumnType::MYSQL_TYPE_VARCHAR + ) +} + +fn mysql_bounded_value_preview( + row: &mysql_async::Row, + index: usize, + preview_bytes: usize, + protected_indexes: &HashSet, +) -> Option<(serde_json::Value, usize)> { + if protected_indexes.contains(&index) { + return None; + } + let column = row.columns_ref().get(index)?; + if !mysql_column_can_use_bounded_preview(column) { + return None; + } + let mysql_async::Value::Bytes(bytes) = row.as_ref(index)? else { + return None; + }; + if bytes.len() <= preview_bytes { + return None; + } + + let end = if is_mysql_binary_charset(column) { + preview_bytes.min(bytes.len()) + } else { + let mut end = preview_bytes.min(bytes.len()); + while end > 0 && std::str::from_utf8(&bytes[..end]).is_err() { + end -= 1; + } + end + }; + let mut value = if is_mysql_binary_charset(column) { + mysql_bytes_to_json(bytes[..end].to_vec(), column) + } else { + let text = String::from_utf8_lossy(&bytes[..end]); + serde_json::Value::String(fix_potential_double_encoding(&text)) + }; + if let serde_json::Value::String(text) = &mut value { + text.push_str("..."); + } + Some((value, bytes.len())) +} + +fn mysql_result_protected_column_indexes( + columns: &[mysql_async::Column], + protected_columns: &[String], +) -> HashSet { + let protected: HashSet = protected_columns.iter().map(|column| column.to_ascii_lowercase()).collect(); + columns + .iter() + .enumerate() + .filter_map(|(index, column)| { + (column.flags().contains(ColumnFlags::PRI_KEY_FLAG) + || protected.contains(&column.name_str().to_ascii_lowercase())) + .then_some(index) + }) + .collect() +} + fn mysql_row_to_json_with_srids( row: &mysql_async::Row, spatial_columns: &mut SpatialColumnBuilder, @@ -603,6 +714,51 @@ fn mysql_row_to_json_with_srids( (values, srids) } +fn mysql_row_to_json_with_srids_and_previews( + row: &mysql_async::Row, + spatial_columns: &mut SpatialColumnBuilder, + row_index: usize, + preview_bytes: Option, + protected_indexes: &HashSet, +) -> (Vec, Vec>, Vec) { + let mut srids = vec![None; row.len()]; + let mut large_value_cells = Vec::new(); + let values = (0..row.len()) + .map(|index| { + let is_geometry = row + .columns_ref() + .get(index) + .is_some_and(|column| column.column_type() == ColumnType::MYSQL_TYPE_GEOMETRY); + if is_geometry { + let Some(bytes) = row_get::, _>(row, index) else { + spatial_columns.observe(index, None); + return serde_json::Value::Null; + }; + return match decode_mysql_geometry(&bytes) { + Some(geometry) => { + spatial_columns.observe(index, geometry.srid); + srids[index] = geometry.srid; + serde_json::Value::String(geometry.wkt) + } + None => { + spatial_columns.observe(index, None); + super::binary_value_to_json(&bytes) + } + }; + } + + if let Some((preview, original_bytes)) = + preview_bytes.and_then(|limit| mysql_bounded_value_preview(row, index, limit, protected_indexes)) + { + large_value_cells.push(LargeValueCell { row_index, column_index: index, original_bytes }); + return preview; + } + mysql_value_to_json(row, index) + }) + .collect(); + (values, srids, large_value_cells) +} + fn mysql_temporal_value_to_json( column_type: ColumnType, datetime: Option, @@ -3923,8 +4079,10 @@ async fn execute_result_set_with_text_protocol_on_conn( sql: &str, row_limit: usize, max_rows: Option, + max_result_bytes: Option, + result_key_columns: &[String], start: Instant, -) -> Result { +) -> Result { let mut result = conn.query_iter(sql).await.map_err(|e| e.to_string())?; if !advance_to_result_set_with_columns(&mut result).await? { let affected_rows = result.affected_rows(); @@ -3932,7 +4090,7 @@ async fn execute_result_set_with_text_protocol_on_conn( let info = result.info().into_owned(); drop(result); let messages = collect_mysql_server_messages(conn, warnings, &info).await; - return Ok(QueryResult { + return Ok(MySqlQueryResult::exact(QueryResult { columns: vec![], column_types: Vec::new(), column_sortables: vec![], @@ -3946,13 +4104,16 @@ async fn execute_result_set_with_text_protocol_on_conn( has_more: false, elasticsearch_raw_body: None, messages, - }); + })); } let columns: Vec = result.columns_ref().iter().map(|c| c.name_str().to_string()).collect(); let column_types: Vec = result.columns_ref().iter().map(mysql_column_type_name).collect(); let mut spatial_columns = mysql_spatial_column_builder(result.columns_ref()); + let preview_bytes = + max_result_bytes.map(|max_bytes| mysql_result_cell_preview_bytes(max_bytes, row_limit, result.columns_ref())); + let protected_indexes = mysql_result_protected_column_indexes(result.columns_ref(), result_key_columns); - if should_collect_text_result_set(sql, row_limit, max_rows) { + if max_result_bytes.is_none() && should_collect_text_result_set(sql, row_limit, max_rows) { let rows: Vec = result.collect_and_drop().await.map_err(|e| e.to_string())?; // collect_and_drop consumed the result; warnings/info now reflect the // trailing EOF/OK packet of the finished result set. @@ -3971,7 +4132,7 @@ async fn execute_result_set_with_text_protocol_on_conn( }) .collect(); - return Ok(QueryResult { + return Ok(MySqlQueryResult::exact(QueryResult { columns, column_types, column_sortables: vec![], @@ -3985,11 +4146,12 @@ async fn execute_result_set_with_text_protocol_on_conn( has_more: false, elasticsearch_raw_body: None, messages, - }); + })); } let mut result_rows: Vec> = Vec::new(); let mut spatial_values: Vec>> = Vec::new(); + let mut large_value_cells = Vec::new(); let mut truncated = false; let mut stream = result .stream::() @@ -4003,9 +4165,17 @@ async fn execute_result_set_with_text_protocol_on_conn( truncated = true; break; } - let (values, srids) = mysql_row_to_json_with_srids(&row, &mut spatial_columns); + let row_index = result_rows.len(); + let (values, srids, mut row_large_values) = mysql_row_to_json_with_srids_and_previews( + &row, + &mut spatial_columns, + row_index, + preview_bytes, + &protected_indexes, + ); result_rows.push(values); spatial_values.push(srids); + large_value_cells.append(&mut row_large_values); } drop(stream); @@ -4023,20 +4193,23 @@ async fn execute_result_set_with_text_protocol_on_conn( collect_mysql_server_messages(conn, warnings, &info).await }; - Ok(QueryResult { - columns, - column_types, - column_sortables: vec![], - spatial_columns: spatial_columns.finish(), - spatial_values, - rows: result_rows, - affected_rows: 0, - execution_time_ms: start.elapsed().as_millis(), - truncated, - session_id: None, - has_more: false, - elasticsearch_raw_body: None, - messages, + Ok(MySqlQueryResult { + result: QueryResult { + columns, + column_types, + column_sortables: vec![], + spatial_columns: spatial_columns.finish(), + spatial_values, + rows: result_rows, + affected_rows: 0, + execution_time_ms: start.elapsed().as_millis(), + truncated, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages, + }, + large_value_cells, }) } @@ -4045,8 +4218,10 @@ async fn execute_result_sets_with_text_protocol_on_conn( sql: &str, row_limit: usize, max_rows: Option, + max_result_bytes: Option, + result_key_columns: &[String], start: Instant, -) -> Result, String> { +) -> Result, String> { // Per-set warnings/info read after each `collect()` are only accurate when // the connection negotiated CLIENT_DEPRECATE_EOF: with legacy EOF packets // the next result set's column-definition EOF clobbers the previous OK @@ -4059,7 +4234,7 @@ async fn execute_result_sets_with_text_protocol_on_conn( // on the last result set reports warnings. let capture_per_set_messages = conn.opts().deprecate_eof(); let mut result = conn.query_iter(sql).await.map_err(|e| e.to_string())?; - let mut results: Vec = Vec::new(); + let mut results: Vec = Vec::new(); let mut result_set_warnings: Vec = Vec::new(); while advance_to_result_set_with_columns(&mut result).await? { @@ -4067,9 +4242,13 @@ async fn execute_result_sets_with_text_protocol_on_conn( let column_types: Vec = result.columns_ref().iter().map(mysql_column_type_name).collect(); let mut spatial_columns = mysql_spatial_column_builder(result.columns_ref()); let mut spatial_values = Vec::new(); + let mut large_value_cells = Vec::new(); let mut truncated = false; + let preview_bytes = max_result_bytes + .map(|max_bytes| mysql_result_cell_preview_bytes(max_bytes, row_limit, result.columns_ref())); + let protected_indexes = mysql_result_protected_column_indexes(result.columns_ref(), result_key_columns); - let rows = if should_collect_text_result_set(sql, row_limit, max_rows) { + let rows = if max_result_bytes.is_none() && should_collect_text_result_set(sql, row_limit, max_rows) { let rows: Vec = result.collect().await.map_err(|e| e.to_string())?; truncated = rows.len() > row_limit; rows.iter() @@ -4091,9 +4270,17 @@ async fn execute_result_sets_with_text_protocol_on_conn( while let Some(row) = stream.next().await { let row = row.map_err(|e| e.to_string())?; if rows.len() < row_limit { - let (values, srids) = mysql_row_to_json_with_srids(&row, &mut spatial_columns); + let row_index = rows.len(); + let (values, srids, mut row_large_values) = mysql_row_to_json_with_srids_and_previews( + &row, + &mut spatial_columns, + row_index, + preview_bytes, + &protected_indexes, + ); rows.push(values); spatial_values.push(srids); + large_value_cells.append(&mut row_large_values); } else { truncated = true; } @@ -4108,20 +4295,23 @@ async fn execute_result_sets_with_text_protocol_on_conn( Vec::new() }; result_set_warnings.push(warnings); - results.push(QueryResult { - columns, - column_types, - column_sortables: vec![], - spatial_columns: spatial_columns.finish(), - spatial_values, - rows, - affected_rows: 0, - execution_time_ms: start.elapsed().as_millis(), - truncated, - session_id: None, - has_more: false, - elasticsearch_raw_body: None, - messages, + results.push(MySqlQueryResult { + result: QueryResult { + columns, + column_types, + column_sortables: vec![], + spatial_columns: spatial_columns.finish(), + spatial_values, + rows, + affected_rows: 0, + execution_time_ms: start.elapsed().as_millis(), + truncated, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages, + }, + large_value_cells, }); } @@ -4131,7 +4321,7 @@ async fn execute_result_sets_with_text_protocol_on_conn( let info = result.info().into_owned(); drop(result); let messages = collect_mysql_server_messages(conn, warnings, &info).await; - results.push(QueryResult { + results.push(MySqlQueryResult::exact(QueryResult { columns: vec![], column_types: Vec::new(), column_sortables: vec![], @@ -4145,7 +4335,7 @@ async fn execute_result_sets_with_text_protocol_on_conn( has_more: false, elasticsearch_raw_body: None, messages, - }); + })); return Ok(results); } // Without per-set capture (no CLIENT_DEPRECATE_EOF) the per-iteration @@ -4168,9 +4358,9 @@ async fn execute_result_sets_with_text_protocol_on_conn( } if index == last_index { let mut messages = collect_mysql_server_messages(conn, warnings, "").await; - results[index].messages.append(&mut messages); + results[index].result.messages.append(&mut messages); } else { - results[index].messages.push(mysql_warnings_fallback_message(warnings)); + results[index].result.messages.push(mysql_warnings_fallback_message(warnings)); } } @@ -4193,15 +4383,21 @@ async fn execute_result_set_with_prepared_protocol_on_conn( conn: &mut mysql_async::Conn, sql: &str, row_limit: usize, + max_result_bytes: Option, + result_key_columns: &[String], start: Instant, -) -> Result { +) -> Result { let mut result = conn.exec_iter(sql, ()).await.map_err(|e| e.to_string())?; let columns: Vec = result.columns_ref().iter().map(|c| c.name_str().to_string()).collect(); let column_types: Vec = result.columns_ref().iter().map(mysql_column_type_name).collect(); let mut spatial_columns = mysql_spatial_column_builder(result.columns_ref()); + let preview_bytes = + max_result_bytes.map(|max_bytes| mysql_result_cell_preview_bytes(max_bytes, row_limit, result.columns_ref())); + let protected_indexes = mysql_result_protected_column_indexes(result.columns_ref(), result_key_columns); let mut result_rows: Vec> = Vec::new(); let mut spatial_values: Vec>> = Vec::new(); + let mut large_value_cells = Vec::new(); let mut truncated = false; let mut stream = result .stream::() @@ -4215,9 +4411,17 @@ async fn execute_result_set_with_prepared_protocol_on_conn( truncated = true; break; } - let (values, srids) = mysql_row_to_json_with_srids(&row, &mut spatial_columns); + let row_index = result_rows.len(); + let (values, srids, mut row_large_values) = mysql_row_to_json_with_srids_and_previews( + &row, + &mut spatial_columns, + row_index, + preview_bytes, + &protected_indexes, + ); result_rows.push(values); spatial_values.push(srids); + large_value_cells.append(&mut row_large_values); } drop(stream); @@ -4235,20 +4439,23 @@ async fn execute_result_set_with_prepared_protocol_on_conn( collect_mysql_server_messages(conn, warnings, &info).await }; - Ok(QueryResult { - columns, - column_types, - column_sortables: vec![], - spatial_columns: spatial_columns.finish(), - spatial_values, - rows: result_rows, - affected_rows: 0, - execution_time_ms: start.elapsed().as_millis(), - truncated, - session_id: None, - has_more: false, - elasticsearch_raw_body: None, - messages, + Ok(MySqlQueryResult { + result: QueryResult { + columns, + column_types, + column_sortables: vec![], + spatial_columns: spatial_columns.finish(), + spatial_values, + rows: result_rows, + affected_rows: 0, + execution_time_ms: start.elapsed().as_millis(), + truncated, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages, + }, + large_value_cells, }) } @@ -4479,17 +4686,56 @@ pub async fn execute_query_on_conn_with_max_rows( max_rows: Option, dialect: MySqlQueryDialect, ) -> Result { + execute_query_on_conn_with_limits(conn, sql, bare, max_rows, None, &[], dialect).await.map(|result| result.result) +} + +pub async fn execute_query_on_conn_with_limits( + conn: &mut mysql_async::Conn, + sql: &str, + bare: bool, + max_rows: Option, + max_result_bytes: Option, + result_key_columns: &[String], + dialect: MySqlQueryDialect, +) -> Result { let start = Instant::now(); let row_limit = query_result_row_limit(max_rows); if is_result_set_query(sql, dialect) { if bare || prefers_text_protocol_query(sql, dialect) { - execute_result_set_with_text_protocol_on_conn(conn, sql, row_limit, max_rows, start).await + execute_result_set_with_text_protocol_on_conn( + conn, + sql, + row_limit, + max_rows, + max_result_bytes, + result_key_columns, + start, + ) + .await } else { - match execute_result_set_with_prepared_protocol_on_conn(conn, sql, row_limit, start).await { + match execute_result_set_with_prepared_protocol_on_conn( + conn, + sql, + row_limit, + max_result_bytes, + result_key_columns, + start, + ) + .await + { Ok(result) => Ok(result), Err(err) if mysql_error_should_retry_with_text_protocol(&err) => { - execute_result_set_with_text_protocol_on_conn(conn, sql, row_limit, max_rows, start).await + execute_result_set_with_text_protocol_on_conn( + conn, + sql, + row_limit, + max_rows, + max_result_bytes, + result_key_columns, + start, + ) + .await } Err(err) => Err(err), } @@ -4514,7 +4760,7 @@ pub async fn execute_query_on_conn_with_max_rows( restore_explicit_timestamp_defaults_for_query(conn, previous_explicit_timestamp_defaults).await; drop_result.map_err(|e| e.to_string())?; - Ok(QueryResult { + Ok(MySqlQueryResult::exact(QueryResult { columns: vec![], column_types: Vec::new(), column_sortables: vec![], @@ -4528,7 +4774,7 @@ pub async fn execute_query_on_conn_with_max_rows( has_more: false, elasticsearch_raw_body: None, messages, - }) + })) } } @@ -4539,12 +4785,36 @@ pub async fn execute_query_results_on_conn_with_max_rows( max_rows: Option, dialect: MySqlQueryDialect, ) -> Result, String> { + execute_query_results_on_conn_with_limits(conn, sql, bare, max_rows, None, &[], dialect) + .await + .map(|results| results.into_iter().map(|result| result.result).collect()) +} + +pub async fn execute_query_results_on_conn_with_limits( + conn: &mut mysql_async::Conn, + sql: &str, + bare: bool, + max_rows: Option, + max_result_bytes: Option, + result_key_columns: &[String], + dialect: MySqlQueryDialect, +) -> Result, String> { if is_result_set_query(sql, dialect) && (bare || prefers_text_protocol_query(sql, dialect)) { let start = Instant::now(); - execute_result_sets_with_text_protocol_on_conn(conn, sql, query_result_row_limit(max_rows), max_rows, start) - .await + execute_result_sets_with_text_protocol_on_conn( + conn, + sql, + query_result_row_limit(max_rows), + max_rows, + max_result_bytes, + result_key_columns, + start, + ) + .await } else { - execute_query_on_conn_with_max_rows(conn, sql, bare, max_rows, dialect).await.map(|result| vec![result]) + execute_query_on_conn_with_limits(conn, sql, bare, max_rows, max_result_bytes, result_key_columns, dialect) + .await + .map(|result| vec![result]) } } @@ -5391,6 +5661,10 @@ mod tests { new_row(values, columns) } + fn mysql_test_row_with_columns(values: Vec, columns: Vec) -> mysql_async::Row { + new_row(values, columns.into()) + } + #[test] fn mysql_first_column_value_ignores_extra_result_columns() { let row = mysql_async::from_row::(mysql_test_row(vec![ @@ -6493,6 +6767,88 @@ mod tests { ); } + #[test] + fn mysql_large_text_preview_preserves_key_and_reports_cell_coordinates() { + let id_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING) + .with_name(b"id") + .with_character_set(45) + .with_column_length(64); + let payload_column = Column::new(ColumnType::MYSQL_TYPE_LONG_BLOB) + .with_name(b"payload") + .with_character_set(45) + .with_column_length(u32::MAX); + let row = mysql_test_row_with_columns( + vec![Value::Bytes(vec![b'k'; 2048]), Value::Bytes(vec![b'x'; 32 * 1024])], + vec![id_column, payload_column], + ); + let protected = mysql_result_protected_column_indexes(row.columns_ref(), &["id".to_string()]); + let mut spatial_columns = mysql_spatial_column_builder(row.columns_ref()); + + let (values, _srids, cells) = + mysql_row_to_json_with_srids_and_previews(&row, &mut spatial_columns, 7, Some(1024), &protected); + + assert_eq!(values[0], serde_json::json!("k".repeat(2048))); + assert_eq!(values[1].as_str().map(str::len), Some(1027)); + assert!(values[1].as_str().is_some_and(|value| value.ends_with("..."))); + assert_eq!(cells, vec![LargeValueCell { row_index: 7, column_index: 1, original_bytes: 32 * 1024 }]); + } + + #[test] + fn mysql_large_value_preview_protects_protocol_primary_keys_without_frontend_metadata() { + let id_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING) + .with_name(b"id") + .with_character_set(45) + .with_flags(ColumnFlags::PRI_KEY_FLAG) + .with_column_length(3072); + let payload_column = Column::new(ColumnType::MYSQL_TYPE_JSON) + .with_name(b"payload") + .with_character_set(45) + .with_column_length(u32::MAX); + let row = mysql_test_row_with_columns( + vec![Value::Bytes(vec![b'k'; 2048]), Value::Bytes(vec![b'x'; 4096])], + vec![id_column, payload_column], + ); + let protected = mysql_result_protected_column_indexes(row.columns_ref(), &[]); + let mut spatial_columns = mysql_spatial_column_builder(row.columns_ref()); + + let (values, _srids, cells) = + mysql_row_to_json_with_srids_and_previews(&row, &mut spatial_columns, 0, Some(512), &protected); + + assert_eq!(values[0], serde_json::json!("k".repeat(2048))); + assert_eq!(cells, vec![LargeValueCell { row_index: 0, column_index: 1, original_bytes: 4096 }]); + } + + #[test] + fn mysql_large_binary_preview_is_bounded_hex() { + let payload_column = Column::new(ColumnType::MYSQL_TYPE_LONG_BLOB) + .with_name(b"payload") + .with_character_set(63) + .with_flags(ColumnFlags::BLOB_FLAG) + .with_column_length(u32::MAX); + let row = mysql_test_row_with_columns(vec![Value::Bytes(vec![0xAB; 4096])], vec![payload_column]); + let mut spatial_columns = mysql_spatial_column_builder(row.columns_ref()); + + let (values, _srids, cells) = + mysql_row_to_json_with_srids_and_previews(&row, &mut spatial_columns, 0, Some(512), &HashSet::new()); + + let preview = values[0].as_str().unwrap(); + assert!(preview.starts_with("0xabab")); + assert!(preview.ends_with("...")); + assert_eq!(preview.len(), 2 + 512 * 2 + 3); + assert_eq!(cells[0].original_bytes, 4096); + } + + #[test] + fn mysql_result_preview_budget_scales_with_page_and_column_count() { + let text_column = mysql_test_column(ColumnType::MYSQL_TYPE_LONG_BLOB, 45, ColumnFlags::empty(), u32::MAX); + let binary_column = mysql_test_column(ColumnType::MYSQL_TYPE_LONG_BLOB, 63, ColumnFlags::BLOB_FLAG, u32::MAX); + assert_eq!(mysql_result_cell_preview_bytes(32 * 1024 * 1024, 1, std::slice::from_ref(&text_column)), 8 * 1024); + assert_eq!(mysql_result_cell_preview_bytes(32 * 1024 * 1024, 1000, &vec![text_column.clone(); 4]), 8 * 1024); + assert_eq!(mysql_result_cell_preview_bytes(32 * 1024 * 1024, 1000, &vec![text_column.clone(); 16]), 2097); + assert_eq!(mysql_result_cell_preview_bytes(1, 1000, &vec![text_column; 100]), 256); + assert_eq!(mysql_result_cell_preview_bytes(32 * 1024 * 1024, 1000, &vec![binary_column; 4]), 4194); + } + #[test] fn mysql_binary_preview_uses_charset_to_separate_blob_from_text() { let text_column = mysql_test_column(ColumnType::MYSQL_TYPE_BLOB, 45, ColumnFlags::empty(), 65_535); diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query.rs index dac54feb6d..0ff4f6c71b 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query.rs @@ -168,6 +168,8 @@ fn append_typed_sql_error_context(error: &str, _sql: &str) -> String { pub struct ExecuteMultiResult { #[serde(flatten)] pub result: db::QueryResult, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub large_value_cells: Vec, #[serde(skip_serializing_if = "is_false")] pub execution_error: bool, #[serde(skip_serializing_if = "Option::is_none")] @@ -215,12 +217,26 @@ fn report_execute_multi_progress( impl ExecuteMultiResult { fn execution_error(result: db::QueryResult) -> Self { let error = error_from_query_result(&result); - Self { result, execution_error: true, statement_index: None, error, server_message: false } + Self { + result, + large_value_cells: Vec::new(), + execution_error: true, + statement_index: None, + error, + server_message: false, + } } fn execution_error_with_index(result: db::QueryResult, statement_index: usize) -> Self { let error = error_from_query_result(&result); - Self { result, execution_error: true, statement_index: Some(statement_index), error, server_message: false } + Self { + result, + large_value_cells: Vec::new(), + execution_error: true, + statement_index: Some(statement_index), + error, + server_message: false, + } } fn execution_error_with_backend( @@ -228,12 +244,35 @@ impl ExecuteMultiResult { statement_index: Option, error: crate::backend_error::BackendError, ) -> Self { - Self { result, execution_error: true, statement_index, error: Some(error), server_message: false } + Self { + result, + large_value_cells: Vec::new(), + execution_error: true, + statement_index, + error: Some(error), + server_message: false, + } } fn success_with_index(result: db::QueryResult, statement_index: usize) -> Self { Self { result, + large_value_cells: Vec::new(), + execution_error: false, + statement_index: Some(statement_index), + error: None, + server_message: false, + } + } + + fn success_with_index_and_large_values( + result: db::QueryResult, + statement_index: usize, + large_value_cells: Vec, + ) -> Self { + Self { + result, + large_value_cells, execution_error: false, statement_index: Some(statement_index), error: None, @@ -253,7 +292,14 @@ impl ExecuteMultiResult { impl From for ExecuteMultiResult { fn from(result: db::QueryResult) -> Self { - Self { result, execution_error: false, statement_index: None, error: None, server_message: false } + Self { + result, + large_value_cells: Vec::new(), + execution_error: false, + statement_index: None, + error: None, + server_message: false, + } } } @@ -261,6 +307,7 @@ impl From for ExecuteMultiResult { fn from(result: db::sqlserver::SqlServerBatchResult) -> Self { Self { result: result.result, + large_value_cells: Vec::new(), execution_error: false, statement_index: None, error: None, @@ -588,6 +635,10 @@ pub struct QueryExecutionOptions { pub max_rows: Option, pub fetch_size: Option, pub page_size: Option, + pub max_result_bytes: Option, + /// Result columns that must stay exact because clients use them as stable + /// row identifiers when fetching full large-cell values on demand. + pub result_key_columns: Vec, /// Doris / StarRocks catalog selected for this query tab. pub catalog: Option, pub result_session_id: Option, @@ -1318,6 +1369,7 @@ async fn do_execute_typed( let p = p.clone(); let bare = *mode == crate::connection::MysqlMode::Bare; let max_rows = options.max_rows; + let max_result_bytes = options.max_result_bytes.filter(|value| *value > 0); drop(connections); let mut conn = match db::mysql::get_conn_with_health_check_with_cancel( &p, @@ -1358,12 +1410,21 @@ async fn do_execute_typed( ), ) .await?; - wait_for_query_opt( + wait_for_result_opt( cancel_token, query_timeout, - db::mysql::execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, mysql_dialect), + db::mysql::execute_query_on_conn_with_limits( + &mut conn, + sql, + bare, + max_rows, + max_result_bytes, + &options.result_key_columns, + mysql_dialect, + ), ) .await + .map(|result| result.result) } PoolKind::Postgres(p) => { let p = p.clone(); @@ -2219,7 +2280,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed( } }; - if statements.len() <= 1 { + if statements.len() <= 1 && !(mysql_pool.is_some() && options.max_result_bytes.is_some_and(|value| value > 0)) { let single_sql = statements.into_iter().next().unwrap_or_default(); return single_statement_multi_result( execute_sql_statement_with_options_typed( @@ -2323,13 +2384,15 @@ fn single_statement_multi_result( } trait MysqlBatchStatementExecutor { - async fn execute_statement(&mut self, statement: &str) -> Result, String>; + async fn execute_statement(&mut self, statement: &str) -> Result, String>; async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome { let mut results = Vec::with_capacity(statements.len()); for statement in statements { match self.execute_statement(statement).await { - Ok(mut statement_results) if statement_results.len() == 1 => results.append(&mut statement_results), + Ok(statement_results) if statement_results.len() == 1 => { + results.push(statement_results.into_iter().next().expect("single MySQL batch result").result); + } Ok(_) => { return db::mysql::MySqlNonResultBatchOutcome { results, @@ -2349,19 +2412,23 @@ struct MysqlBatchConnection<'a> { query_timeout: Option, bare: bool, max_rows: Option, + max_result_bytes: Option, + result_key_columns: &'a [String], dialect: db::mysql::MySqlQueryDialect, } impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> { - async fn execute_statement(&mut self, statement: &str) -> Result, String> { + async fn execute_statement(&mut self, statement: &str) -> Result, String> { wait_for_result_opt( self.cancel_token.clone(), self.query_timeout, - db::mysql::execute_query_results_on_conn_with_max_rows( + db::mysql::execute_query_results_on_conn_with_limits( &mut *self.conn, statement, self.bare, self.max_rows, + self.max_result_bytes, + self.result_key_columns, self.dialect, ), ) @@ -2493,13 +2560,22 @@ where match executor.execute_statement(statement).await { Ok(statement_results) => { if let Some(result) = statement_results.last() { - report_execute_multi_progress(progress, statement_index, statements.len(), result, true, None); + report_execute_multi_progress( + progress, + statement_index, + statements.len(), + &result.result, + true, + None, + ); } - results.extend( - statement_results - .into_iter() - .map(|result| ExecuteMultiResult::success_with_index(result, statement_index)), - ); + results.extend(statement_results.into_iter().map(|result| { + ExecuteMultiResult::success_with_index_and_large_values( + result.result, + statement_index, + result.large_value_cells, + ) + })); } Err(err) => { let action = mysql_batch_pool_error_action(db_type, &err); @@ -2545,6 +2621,7 @@ async fn execute_multi_mysql( let operation_budget = operation_budget_for_pool_key(state, pool_key, query_timeout).await; let bare = mode == crate::connection::MysqlMode::Bare; let max_rows = options.max_rows; + let max_result_bytes = options.max_result_bytes.filter(|value| *value > 0); let pipeline_non_result_statements = !options.continue_on_error && mode == crate::connection::MysqlMode::Normal; let mut conn = match db::mysql::get_conn_with_health_check_with_cancel( pool, @@ -2587,6 +2664,8 @@ async fn execute_multi_mysql( query_timeout, bare, max_rows, + max_result_bytes, + result_key_columns: &options.result_key_columns, dialect, }; let (results, error_action) = execute_mysql_batch_statements( @@ -4922,26 +5001,34 @@ for line in sys.stdin: } struct FakeMysqlBatchExecutor { - outcomes: std::collections::VecDeque, String>>, + outcomes: std::collections::VecDeque, String>>, executed: Vec, } impl MysqlBatchStatementExecutor for FakeMysqlBatchExecutor { - async fn execute_statement(&mut self, statement: &str) -> Result, String> { + async fn execute_statement(&mut self, statement: &str) -> Result, String> { self.executed.push(statement.to_string()); self.outcomes.pop_front().expect("test outcome for statement") } } + fn mysql_query_result(result: db::QueryResult) -> db::mysql::MySqlQueryResult { + db::mysql::MySqlQueryResult { result, large_value_cells: Vec::new() } + } + + fn mysql_batch_result(result: db::QueryResult) -> Result, String> { + Ok(vec![mysql_query_result(result)]) + } + struct FakePipelinedMysqlBatchExecutor { batch_outcomes: std::collections::VecDeque, - statement_outcomes: std::collections::VecDeque, String>>, + statement_outcomes: std::collections::VecDeque, String>>, batches: Vec>, statements: Vec, } impl MysqlBatchStatementExecutor for FakePipelinedMysqlBatchExecutor { - async fn execute_statement(&mut self, statement: &str) -> Result, String> { + async fn execute_statement(&mut self, statement: &str) -> Result, String> { self.statements.push(statement.to_string()); self.statement_outcomes.pop_front().expect("test outcome for single statement") } @@ -4952,10 +5039,6 @@ for line in sys.stdin: } } - fn mysql_batch_result(result: db::QueryResult) -> Result, String> { - Ok(vec![result]) - } - async fn assert_sqlite_batch_error_behavior(failure_first: bool, continue_on_error: bool) { let dir = std::env::temp_dir().join(format!("dbx-query-batch-error-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); @@ -5337,7 +5420,11 @@ for line in sys.stdin: }; let mut executor = FakeMysqlBatchExecutor { outcomes: std::collections::VecDeque::from([ - Ok(vec![result_set(1), result_set(2), result_set(3)]), + Ok(vec![ + mysql_query_result(result_set(1)), + mysql_query_result(result_set(2)), + mysql_query_result(result_set(3)), + ]), mysql_batch_result(empty_query_result(1)), ]), executed: Vec::new(), @@ -5497,6 +5584,7 @@ for line in sys.stdin: assert!(success.get("execution_error").is_none()); assert!(success.get("statement_index").is_none()); assert!(success.get("server_message").is_none()); + assert!(success.get("large_value_cells").is_none()); let mut error_column = empty_query_result(0); error_column.columns = vec!["Error".to_string()]; @@ -5526,6 +5614,21 @@ for line in sys.stdin: assert!(redacted.get("error").and_then(|value| value.get("detail")).is_none()); } + #[test] + fn execute_multi_result_serializes_large_value_metadata_only_when_present() { + let result = ExecuteMultiResult::success_with_index_and_large_values( + empty_query_result(1), + 0, + vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }], + ); + + let serialized = serde_json::to_value(result).unwrap(); + assert_eq!( + serialized.get("large_value_cells"), + Some(&serde_json::json!([{"row_index": 2, "column_index": 3, "original_bytes": 65_536}])) + ); + } + #[test] fn sqlserver_batch_results_do_not_claim_statement_indexes() { assert_eq!(split_sql_batches("SELECT 1; SELECT 2;").len(), 1); diff --git a/crates/dbx-core/src/types.rs b/crates/dbx-core/src/types.rs index 3dd440826c..972404d7ac 100644 --- a/crates/dbx-core/src/types.rs +++ b/crates/dbx-core/src/types.rs @@ -341,6 +341,15 @@ impl QueryMessage { } } +/// A result cell whose full variable-length value was replaced by a bounded +/// preview before the result crossed the desktop/web transport boundary. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LargeValueCell { + pub row_index: usize, + pub column_index: usize, + pub original_bytes: usize, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QueryResult { pub columns: Vec, diff --git a/crates/dbx-web/src/routes/query.rs b/crates/dbx-web/src/routes/query.rs index 5381ac65d6..dd7ee00123 100644 --- a/crates/dbx-web/src/routes/query.rs +++ b/crates/dbx-web/src/routes/query.rs @@ -21,6 +21,9 @@ pub struct ExecuteQueryRequest { pub max_rows: Option, pub fetch_size: Option, pub page_size: Option, + pub max_result_bytes: Option, + #[serde(default)] + pub result_key_columns: Vec, pub result_session_id: Option, pub client_session_id: Option, pub timeout_secs: Option, @@ -357,6 +360,8 @@ pub async fn execute_query( max_rows: req.max_rows, fetch_size: req.fetch_size, page_size: req.page_size, + max_result_bytes: req.max_result_bytes, + result_key_columns: req.result_key_columns, catalog: req.catalog, result_session_id: req.result_session_id, client_session_id: req.client_session_id, @@ -403,6 +408,8 @@ pub async fn execute_multi( max_rows: req.max_rows, fetch_size: req.fetch_size, page_size: req.page_size, + max_result_bytes: req.max_result_bytes, + result_key_columns: req.result_key_columns, catalog: req.catalog, result_session_id: req.result_session_id, client_session_id: req.client_session_id, @@ -1031,6 +1038,7 @@ mod tests { elasticsearch_raw_body: None, messages: Vec::new(), }, + large_value_cells: Vec::new(), execution_error: true, statement_index: Some(1), error: Some(dbx_core::backend_error::BackendError::from_sql_detail( diff --git a/src-tauri/src/commands/query.rs b/src-tauri/src/commands/query.rs index a9700c6544..5ca0ac110c 100644 --- a/src-tauri/src/commands/query.rs +++ b/src-tauri/src/commands/query.rs @@ -88,6 +88,8 @@ pub async fn execute_multi( max_rows: Option, fetch_size: Option, page_size: Option, + max_result_bytes: Option, + result_key_columns: Option>, result_session_id: Option, client_session_id: Option, timeout_secs: Option, @@ -144,6 +146,8 @@ pub async fn execute_multi( max_rows, fetch_size, page_size, + max_result_bytes, + result_key_columns: result_key_columns.unwrap_or_default(), catalog, result_session_id, client_session_id,