From d58a79d6c8cef9e15da90ae89c05c516ebfd55ce Mon Sep 17 00:00:00 2001 From: lxk955 <164191365+lxk955@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:58:49 +0800 Subject: [PATCH] feat(grid): allow load all to exceed the per-request row cap Closes #10752 --- apps/desktop/src/components/grid/DataGrid.vue | 24 +++++--- .../__tests__/useDataGridActions.spec.ts | 32 ++++++++++ .../src/composables/useDataGridActions.ts | 4 +- .../dataGrid/dataGridInfiniteScroll.spec.ts | 21 ++++++- .../src/lib/dataGrid/dataGridPagination.ts | 15 +++++ apps/desktop/src/stores/queryStore.ts | 2 +- packages/app-tests/dataGridPagination.test.ts | 21 ++++++- packages/app-tests/useDataGridActions.test.ts | 60 +++++++++++++++++++ 8 files changed, 167 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index 34e3b79c9c..5225c55828 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -222,6 +222,7 @@ import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid import { canFetchNextDataGridSegment, canGoNextDataGridPage, + dataGridLoadAllInitialTarget, dataGridLoadAllNextSegment, dataGridLoadAllSegment, dataGridTotalRowCountLabelKey, @@ -3817,14 +3818,17 @@ function loadAllRowsAndGoToLast() { // against those cursors is untested, so ES/Easysearch grids keep the // reveal-only shortcut instead of loading everything. if (isResultsContext.value && (resolvedDatabaseType.value === "elasticsearch" || resolvedDatabaseType.value === "easysearch")) return; - const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value); + const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value; + const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal); + const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined); + const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore); if (!segment) { loadAllRowsActive.value = true; infiniteScrollAllLoaded = true; selectAndRevealLastLoadedRow(); return; } - const knownTotal = displayedTotalRowCount.value; + const knownTotal = effectiveTotal; const remaining = typeof knownTotal === "number" && Number.isFinite(knownTotal) && knownTotal >= props.result.rows.length ? knownTotal - props.result.rows.length : segment.limit; if (remaining > LOAD_ALL_ROWS_CONFIRM_ROW_THRESHOLD) { pendingLoadAllRows.value = { remaining }; @@ -3837,6 +3841,7 @@ function loadAllRowsAndGoToLast() { function startLoadAllRows(segment: { offset: number; limit: number }) { loadAllRowsActive.value = true; loadAllRowsLoopActive = true; + infiniteScrollAllLoaded = false; infiniteScrollLoadAllPending = true; infiniteScrollLoading.value = true; isInfiniteScrollPaginating.value = true; @@ -3852,12 +3857,14 @@ function startLoadAllRows(segment: { offset: number; limit: number }) { // The per-request result-row cap bounds each chunk, never the run (#10752). function finishOrContinueLoadAllRun(requestedOffset: number | undefined, requestedLimit: number | undefined): boolean { if (!loadAllRowsLoopActive) return false; - const nextSegment = canFetchNextInfiniteScrollSegment.value + const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value; + const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal); + const nextSegment = canFetchMore ? dataGridLoadAllNextSegment({ loadedRowCount: props.result.rows.length, requestedOffset: requestedOffset ?? props.result.rows.length, requestedLimit: requestedLimit ?? pageSize.value, - totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined, + totalRowCount: totalRowCountIsExact.value ? effectiveTotal : undefined, }) : null; if (!nextSegment) { @@ -3875,7 +3882,10 @@ function confirmLoadAllRows() { if (!pending) return; pendingLoadAllRows.value = undefined; loadAllRowsConfirmOpen.value = false; - const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value); + const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value; + const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal); + const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined); + const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore); if (segment) startLoadAllRows(segment); } function checkInfiniteScroll(scroller: HTMLElement) { @@ -10740,10 +10750,10 @@ watch( // While a "load all" run is active the per-request row cap must not be // read as the end of data — only the requested-vs-appended count and an // exact known total end the run (#10752). - ...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined } } : {}), + ...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? (paginationTotalRowCount.value ?? displayedTotalRowCount.value) : undefined } } : {}), }); if (appendCompletion) { - if (infiniteScrollEnabled.value) { + if (infiniteScrollEnabled.value || loadAllRowsLoopActive) { currentPage.value = appendCompletion.loadedPage; lastInfiniteScrollPage = Math.max(0, appendCompletion.loadedPage - 1); infiniteScrollAllLoaded = appendCompletion.allLoaded; diff --git a/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts b/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts index 8181fe92a6..433bdd5fd0 100644 --- a/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts +++ b/apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts @@ -2,6 +2,7 @@ import { computed, reactive } from "vue"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { useDataGridActions } from "@/composables/useDataGridActions"; import { clearTableMetadataCache } from "@/lib/metadata/tableMetadataCache"; +import { MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit"; import { restoredDataTabReloadFilters } from "@/lib/table/tableDataRefresh"; import type { IndexInfo, QueryTab } from "@/types/database"; @@ -618,6 +619,37 @@ describe("useDataGridActions", () => { expect(mocks.executeTabSql).toHaveBeenCalledWith("tab-1", expect.any(String), expect.objectContaining({ pagination: { offset: 100, limit: 99_900, sessionId: undefined, clientSessionId: undefined } })); }); + it("allows load-all to append rows past the continuous query result cap", async () => { + mocks.queryResultMaxRows = 100_000; + const tab = tableDataTab({ + id: "tab-large", + resultTotalRowCount: 260_000, + resultPageLimit: 100, + resultPageOffset: 0, + result: { + columns: ["id"], + rows: Array.from({ length: 100_000 }, (_, index) => [index + 1]), + affected_rows: 0, + execution_time_ms: 1, + }, + }); + mocks.buildTableSelectSql.mockResolvedValueOnce('SELECT * FROM "public"."users" LIMIT 100000 OFFSET 100000;'); + mocks.tabs.push(tab); + const actions = useDataGridActions(computed(() => tab)); + + await actions.onPaginate(tab.id, 100_000, 100_000, "", undefined, true); + + expect(mocks.executeTabSql).toHaveBeenCalledWith( + "tab-large", + expect.any(String), + expect.objectContaining({ + appendResult: { + maxRows: MAX_QUERY_RESULT_MAX_ROWS, + }, + }), + ); + }); + it("continues the matching Cassandra table cursor and client session", async () => { mocks.infiniteScroll = false; mocks.getConfig.mockReturnValue({ id: "cassandra-1", db_type: "cassandra" }); diff --git a/apps/desktop/src/composables/useDataGridActions.ts b/apps/desktop/src/composables/useDataGridActions.ts index c881879a19..5da614ee6e 100644 --- a/apps/desktop/src/composables/useDataGridActions.ts +++ b/apps/desktop/src/composables/useDataGridActions.ts @@ -21,7 +21,7 @@ import { applyMongoFindSort } from "@/lib/mongo/mongoShellCommand"; import { uuid } from "@/lib/common/utils"; import { simpleDataGridOrderByReferencesMissingColumn, type DataGridSortMode } from "@/lib/dataGrid/dataGridSort"; import type { DataGridReloadIntent } from "@/lib/dataGrid/dataGridToolbar"; -import { continuousQueryResultMaxRows } from "@/lib/dataGrid/queryResultRowLimit"; +import { continuousQueryResultMaxRows, MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit"; import { queryResultBaseSql, queryResultExecutionSql } from "@/lib/tabs/tabPresentation"; import { sqlExecutionTargetCapabilities } from "@/lib/database/sqlExecutionTargetCapabilities"; import { usesAgentCursorForTableData } from "@/lib/database/databaseDriverManifest"; @@ -503,7 +503,7 @@ export function useDataGridActions(activeTab: ComputedRef) const appendOptions = appendResult ? { appendResult: { - maxRows: continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows), + maxRows: appendRequested ? MAX_QUERY_RESULT_MAX_ROWS : continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows), }, } : {}; diff --git a/apps/desktop/src/lib/__tests__/dataGrid/dataGridInfiniteScroll.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/dataGridInfiniteScroll.spec.ts index a1116f9c89..375943e876 100644 --- a/apps/desktop/src/lib/__tests__/dataGrid/dataGridInfiniteScroll.spec.ts +++ b/apps/desktop/src/lib/__tests__/dataGrid/dataGridInfiniteScroll.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { dataGridBottomScrollTop, dataGridInfiniteScrollAppendCompletion, didDataGridInfiniteScrollContextChange, isDataGridAtScrollBottom, restoredDataGridScrollLeft } from "@/lib/dataGrid/dataGridInfiniteScroll"; -import { dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination"; +import { dataGridLoadAllInitialTarget, dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination"; describe("data grid bottom anchoring", () => { it("keeps DOM rows anchored when scrollbar padding increases the scroll height", () => { @@ -145,3 +145,22 @@ describe("dataGridLoadAllNextSegment", () => { expect(dataGridLoadAllNextSegment({ loadedRowCount: 200_000, requestedOffset: 200_000, requestedLimit: 100_000 })).toBeNull(); }); }); + +describe("dataGridLoadAllInitialTarget", () => { + it("targets the cap or total when below the chunk limit", () => { + expect(dataGridLoadAllInitialTarget(100, 100_000, 260_000)).toBe(100_000); + expect(dataGridLoadAllInitialTarget(100, 100_000, 50_000)).toBe(50_000); + expect(dataGridLoadAllInitialTarget(100, 100_000)).toBe(100_000); + }); + + it("targets the next chunk increment when already at or past the chunk limit", () => { + expect(dataGridLoadAllInitialTarget(100_000, 100_000, 260_000)).toBe(200_000); + expect(dataGridLoadAllInitialTarget(200_000, 100_000, 260_000)).toBe(260_000); + expect(dataGridLoadAllInitialTarget(260_000, 100_000, 260_000)).toBe(260_000); + }); + + it("advances by chunk increment when total is unknown", () => { + expect(dataGridLoadAllInitialTarget(100_000, 100_000)).toBe(200_000); + expect(dataGridLoadAllInitialTarget(200_000, 100_000)).toBe(300_000); + }); +}); diff --git a/apps/desktop/src/lib/dataGrid/dataGridPagination.ts b/apps/desktop/src/lib/dataGrid/dataGridPagination.ts index 49bfa27b0e..523407c139 100644 --- a/apps/desktop/src/lib/dataGrid/dataGridPagination.ts +++ b/apps/desktop/src/lib/dataGrid/dataGridPagination.ts @@ -163,6 +163,21 @@ export function dataGridLoadAllSegment(loadedRowCount: number, maxRows: number, return { offset, limit: boundedMaxRows - offset }; } +/** + * Target cumulative row count for the initial chunk of an explicit "load all" + * run. Each chunk is bounded by `chunkLimit` (the per-request result-row cap, + * e.g. 100,000), but an existing dataset that has already reached or exceeded + * `chunkLimit` must still be able to start loading subsequent chunks up to + * `totalRowCount` (if known) or by another chunk increment (#10752). + */ +export function dataGridLoadAllInitialTarget(loadedRowCount: number, chunkLimit: number, totalRowCount?: number): number { + const loaded = Number.isFinite(loadedRowCount) ? Math.max(0, Math.trunc(loadedRowCount)) : 0; + const chunk = Number.isFinite(chunkLimit) ? Math.max(1, Math.trunc(chunkLimit)) : 1; + const chunkCap = loaded < chunk ? chunk : loaded + chunk; + const total = typeof totalRowCount === "number" && Number.isFinite(totalRowCount) && totalRowCount >= 0 ? Math.trunc(totalRowCount) : undefined; + return total !== undefined ? Math.min(chunkCap, total) : chunkCap; +} + /** * Next chunk of an explicit "load all" run, issued after the previous chunk * completed. The per-request result-row cap bounds each request, not the run: diff --git a/apps/desktop/src/stores/queryStore.ts b/apps/desktop/src/stores/queryStore.ts index e213137537..fa631e754f 100644 --- a/apps/desktop/src/stores/queryStore.ts +++ b/apps/desktop/src/stores/queryStore.ts @@ -7724,7 +7724,7 @@ export const useQueryStore = defineStore("query", () => { return producedResult; } - const queryResultMaxRows = tab.mode === "query" ? effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows) : undefined; + const queryResultMaxRows = tab.mode === "query" ? (options?.appendResult ? options.appendResult.maxRows : effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows)) : undefined; if (tab.mode === "query") { const prepared = await prepareEditableQueryExecution(tab, sqlToExecute, conn, effectiveDbType, executionDatabase, traceId, elapsed); diff --git a/packages/app-tests/dataGridPagination.test.ts b/packages/app-tests/dataGridPagination.test.ts index 5d7fcaf804..7ad071171d 100644 --- a/packages/app-tests/dataGridPagination.test.ts +++ b/packages/app-tests/dataGridPagination.test.ts @@ -1,6 +1,6 @@ import { strict as assert } from "node:assert"; import { test } from "vitest"; -import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridLoadAllSegment, dataGridUserFacingPage, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal } from "../../apps/desktop/src/lib/dataGrid/dataGridPagination.ts"; +import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridLoadAllInitialTarget, dataGridLoadAllSegment, dataGridUserFacingPage, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal } from "../../apps/desktop/src/lib/dataGrid/dataGridPagination.ts"; test("estimated display totals do not become pagination bounds", () => { assert.equal( @@ -197,6 +197,25 @@ test("load-all does not request past the cap or after the result is complete", ( assert.equal(dataGridLoadAllSegment(100, 5_000, false), null); }); +test("load-all initial target calculates chunk target across boundaries and known totals", () => { + // Below cap: targets cap or total + assert.equal(dataGridLoadAllInitialTarget(100, 100_000, 260_000), 100_000); + assert.equal(dataGridLoadAllInitialTarget(100, 100_000, 50_000), 50_000); + assert.equal(dataGridLoadAllInitialTarget(100, 100_000), 100_000); + + // At or past cap: targets next chunk increment up to total + assert.equal(dataGridLoadAllInitialTarget(100_000, 100_000, 260_000), 200_000); + assert.equal(dataGridLoadAllInitialTarget(200_000, 100_000, 260_000), 260_000); + assert.equal(dataGridLoadAllInitialTarget(260_000, 100_000, 260_000), 260_000); + + // When total is reached, dataGridLoadAllSegment returns null + const target = dataGridLoadAllInitialTarget(260_000, 100_000, 260_000); + assert.equal(dataGridLoadAllSegment(260_000, target, true), null); + + // Without known total, continues by chunk increment + assert.equal(dataGridLoadAllInitialTarget(100_000, 100_000), 200_000); +}); + // --- auto-redirect page calculation after refresh --- // These tests document the math used in DataGrid.vue's loading watcher: // lastPageNum = Math.max(1, Math.ceil(total / pageSize)) diff --git a/packages/app-tests/useDataGridActions.test.ts b/packages/app-tests/useDataGridActions.test.ts index 4f7509af03..7ffac7e20c 100644 --- a/packages/app-tests/useDataGridActions.test.ts +++ b/packages/app-tests/useDataGridActions.test.ts @@ -506,4 +506,64 @@ describe("multi-result preservation after submit", () => { restoreStorage(); } }); + + test("onPaginate uses MAX_QUERY_RESULT_MAX_ROWS when appendRequested is true", async () => { + const restoreStorage = installMemoryStorage(); + const originalFetch = globalThis.fetch; + const { useDataGridActions } = await import("../../apps/desktop/src/composables/useDataGridActions.ts"); + const { MAX_QUERY_RESULT_MAX_ROWS } = await import("../../apps/desktop/src/lib/dataGrid/queryResultRowLimit.ts"); + + globalThis.fetch = (async (input) => { + const url = new URL(String(input), "http://localhost"); + if (url.pathname === "/api/connection/check-health") return Response.json(null); + if (url.pathname === "/api/query/build-table-select-sql") { + return Response.json("SELECT * FROM `orders` LIMIT 100 OFFSET 100"); + } + return new Response(`unexpected ${url.pathname}`, { status: 500 }); + }) as typeof fetch; + + try { + setActivePinia(createPinia()); + const connectionStore = useConnectionStore(); + const queryStore = useQueryStore(); + const settingsStore = useSettingsStore(); + settingsStore.updateEditorSettings({ + queryResultMaxRows: 100_000, + queryResultMaxRowsEnabled: true, + }); + + connectionStore.addEphemeralConnection(conn("mysql-1")); + const tabId = queryStore.createTab("mysql-1", "app", "orders", "data"); + queryStore.setTableMeta(tabId, { tableName: "orders", tableType: "TABLE", columns: [], primaryKeys: [] }); + const tab = queryStore.tabs.find((item) => item.id === tabId); + assert.ok(tab); + tab.result = { + columns: ["id"], + rows: Array.from({ length: 100 }, (_, i) => [i + 1]), + affected_rows: 0, + execution_time_ms: 1, + }; + + let capturedOptions: any; + vi.spyOn(queryStore, "executeTabSql").mockImplementation(async (_tabId, _sql, options) => { + capturedOptions = options; + return true; + }); + + const actions = useDataGridActions(computed(() => tab)); + + // Normal infinite scroll append + settingsStore.updateEditorSettings({ infiniteScroll: true }); + await actions.onPaginate(tabId, 100, 100, undefined, undefined, false); + assert.equal(capturedOptions?.appendResult?.maxRows, 100_000); + + // Explicit load-all append + await actions.onPaginate(tabId, 100, 99_900, undefined, undefined, true); + assert.equal(capturedOptions?.appendResult?.maxRows, MAX_QUERY_RESULT_MAX_ROWS); + } finally { + globalThis.fetch = originalFetch; + restoreStorage(); + } + }); }); +