feat(grid): allow load all to exceed the per-request row cap

Closes #10752

apps/desktop/src/components/grid/DataGrid.vue

@@ -222,6 +222,7 @@ import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid
 import {
   canFetchNextDataGridSegment,
   canGoNextDataGridPage,
+  dataGridLoadAllInitialTarget,
   dataGridLoadAllNextSegment,
   dataGridLoadAllSegment,
   dataGridTotalRowCountLabelKey,
@@ -3817,14 +3818,17 @@ function loadAllRowsAndGoToLast() {
   // against those cursors is untested, so ES/Easysearch grids keep the
   // reveal-only shortcut instead of loading everything.
   if (isResultsContext.value && (resolvedDatabaseType.value === "elasticsearch" || resolvedDatabaseType.value === "easysearch")) return;
-  const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value);
+  const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
+  const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
+  const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined);
+  const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore);
   if (!segment) {
     loadAllRowsActive.value = true;
     infiniteScrollAllLoaded = true;
     selectAndRevealLastLoadedRow();
     return;
   }
-  const knownTotal = displayedTotalRowCount.value;
+  const knownTotal = effectiveTotal;
   const remaining = typeof knownTotal === "number" && Number.isFinite(knownTotal) && knownTotal >= props.result.rows.length ? knownTotal - props.result.rows.length : segment.limit;
   if (remaining > LOAD_ALL_ROWS_CONFIRM_ROW_THRESHOLD) {
     pendingLoadAllRows.value = { remaining };
@@ -3837,6 +3841,7 @@ function loadAllRowsAndGoToLast() {
 function startLoadAllRows(segment: { offset: number; limit: number }) {
   loadAllRowsActive.value = true;
   loadAllRowsLoopActive = true;
+  infiniteScrollAllLoaded = false;
   infiniteScrollLoadAllPending = true;
   infiniteScrollLoading.value = true;
   isInfiniteScrollPaginating.value = true;
@@ -3852,12 +3857,14 @@ function startLoadAllRows(segment: { offset: number; limit: number }) {
 // The per-request result-row cap bounds each chunk, never the run (#10752).
 function finishOrContinueLoadAllRun(requestedOffset: number | undefined, requestedLimit: number | undefined): boolean {
   if (!loadAllRowsLoopActive) return false;
-  const nextSegment = canFetchNextInfiniteScrollSegment.value
+  const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
+  const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
+  const nextSegment = canFetchMore
     ? dataGridLoadAllNextSegment({
         loadedRowCount: props.result.rows.length,
         requestedOffset: requestedOffset ?? props.result.rows.length,
         requestedLimit: requestedLimit ?? pageSize.value,
-        totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined,
+        totalRowCount: totalRowCountIsExact.value ? effectiveTotal : undefined,
       })
     : null;
   if (!nextSegment) {
@@ -3875,7 +3882,10 @@ function confirmLoadAllRows() {
   if (!pending) return;
   pendingLoadAllRows.value = undefined;
   loadAllRowsConfirmOpen.value = false;
-  const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value);
+  const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
+  const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
+  const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined);
+  const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore);
   if (segment) startLoadAllRows(segment);
 }
 function checkInfiniteScroll(scroller: HTMLElement) {
@@ -10740,10 +10750,10 @@ watch(
       // While a "load all" run is active the per-request row cap must not be
       // read as the end of data — only the requested-vs-appended count and an
       // exact known total end the run (#10752).
-      ...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined } } : {}),
+      ...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? (paginationTotalRowCount.value ?? displayedTotalRowCount.value) : undefined } } : {}),
     });
     if (appendCompletion) {
-      if (infiniteScrollEnabled.value) {
+      if (infiniteScrollEnabled.value || loadAllRowsLoopActive) {
         currentPage.value = appendCompletion.loadedPage;
         lastInfiniteScrollPage = Math.max(0, appendCompletion.loadedPage - 1);
         infiniteScrollAllLoaded = appendCompletion.allLoaded;

apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts

@@ -2,6 +2,7 @@ import { computed, reactive } from "vue";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 import { useDataGridActions } from "@/composables/useDataGridActions";
 import { clearTableMetadataCache } from "@/lib/metadata/tableMetadataCache";
+import { MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit";
 import { restoredDataTabReloadFilters } from "@/lib/table/tableDataRefresh";
 import type { IndexInfo, QueryTab } from "@/types/database";
 
@@ -618,6 +619,37 @@ describe("useDataGridActions", () => {
     expect(mocks.executeTabSql).toHaveBeenCalledWith("tab-1", expect.any(String), expect.objectContaining({ pagination: { offset: 100, limit: 99_900, sessionId: undefined, clientSessionId: undefined } }));
   });
 
+  it("allows load-all to append rows past the continuous query result cap", async () => {
+    mocks.queryResultMaxRows = 100_000;
+    const tab = tableDataTab({
+      id: "tab-large",
+      resultTotalRowCount: 260_000,
+      resultPageLimit: 100,
+      resultPageOffset: 0,
+      result: {
+        columns: ["id"],
+        rows: Array.from({ length: 100_000 }, (_, index) => [index + 1]),
+        affected_rows: 0,
+        execution_time_ms: 1,
+      },
+    });
+    mocks.buildTableSelectSql.mockResolvedValueOnce('SELECT * FROM "public"."users" LIMIT 100000 OFFSET 100000;');
+    mocks.tabs.push(tab);
+    const actions = useDataGridActions(computed(() => tab));
+
+    await actions.onPaginate(tab.id, 100_000, 100_000, "", undefined, true);
+
+    expect(mocks.executeTabSql).toHaveBeenCalledWith(
+      "tab-large",
+      expect.any(String),
+      expect.objectContaining({
+        appendResult: {
+          maxRows: MAX_QUERY_RESULT_MAX_ROWS,
+        },
+      }),
+    );
+  });
+
   it("continues the matching Cassandra table cursor and client session", async () => {
     mocks.infiniteScroll = false;
     mocks.getConfig.mockReturnValue({ id: "cassandra-1", db_type: "cassandra" });

apps/desktop/src/composables/useDataGridActions.ts

@@ -21,7 +21,7 @@ import { applyMongoFindSort } from "@/lib/mongo/mongoShellCommand";
 import { uuid } from "@/lib/common/utils";
 import { simpleDataGridOrderByReferencesMissingColumn, type DataGridSortMode } from "@/lib/dataGrid/dataGridSort";
 import type { DataGridReloadIntent } from "@/lib/dataGrid/dataGridToolbar";
-import { continuousQueryResultMaxRows } from "@/lib/dataGrid/queryResultRowLimit";
+import { continuousQueryResultMaxRows, MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit";
 import { queryResultBaseSql, queryResultExecutionSql } from "@/lib/tabs/tabPresentation";
 import { sqlExecutionTargetCapabilities } from "@/lib/database/sqlExecutionTargetCapabilities";
 import { usesAgentCursorForTableData } from "@/lib/database/databaseDriverManifest";
@@ -503,7 +503,7 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
     const appendOptions = appendResult
       ? {
           appendResult: {
-            maxRows: continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows),
+            maxRows: appendRequested ? MAX_QUERY_RESULT_MAX_ROWS : continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows),
           },
         }
       : {};

apps/desktop/src/lib/__tests__/dataGrid/dataGridInfiniteScroll.spec.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { dataGridBottomScrollTop, dataGridInfiniteScrollAppendCompletion, didDataGridInfiniteScrollContextChange, isDataGridAtScrollBottom, restoredDataGridScrollLeft } from "@/lib/dataGrid/dataGridInfiniteScroll";
-import { dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination";
+import { dataGridLoadAllInitialTarget, dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination";
 
 describe("data grid bottom anchoring", () => {
   it("keeps DOM rows anchored when scrollbar padding increases the scroll height", () => {
@@ -145,3 +145,22 @@ describe("dataGridLoadAllNextSegment", () => {
     expect(dataGridLoadAllNextSegment({ loadedRowCount: 200_000, requestedOffset: 200_000, requestedLimit: 100_000 })).toBeNull();
   });
 });
+
+describe("dataGridLoadAllInitialTarget", () => {
+  it("targets the cap or total when below the chunk limit", () => {
+    expect(dataGridLoadAllInitialTarget(100, 100_000, 260_000)).toBe(100_000);
+    expect(dataGridLoadAllInitialTarget(100, 100_000, 50_000)).toBe(50_000);
+    expect(dataGridLoadAllInitialTarget(100, 100_000)).toBe(100_000);
+  });
+
+  it("targets the next chunk increment when already at or past the chunk limit", () => {
+    expect(dataGridLoadAllInitialTarget(100_000, 100_000, 260_000)).toBe(200_000);
+    expect(dataGridLoadAllInitialTarget(200_000, 100_000, 260_000)).toBe(260_000);
+    expect(dataGridLoadAllInitialTarget(260_000, 100_000, 260_000)).toBe(260_000);
+  });
+
+  it("advances by chunk increment when total is unknown", () => {
+    expect(dataGridLoadAllInitialTarget(100_000, 100_000)).toBe(200_000);
+    expect(dataGridLoadAllInitialTarget(200_000, 100_000)).toBe(300_000);
+  });
+});

apps/desktop/src/lib/dataGrid/dataGridPagination.ts

@@ -163,6 +163,21 @@ export function dataGridLoadAllSegment(loadedRowCount: number, maxRows: number,
   return { offset, limit: boundedMaxRows - offset };
 }
 
+/**
+ * Target cumulative row count for the initial chunk of an explicit "load all"
+ * run. Each chunk is bounded by `chunkLimit` (the per-request result-row cap,
+ * e.g. 100,000), but an existing dataset that has already reached or exceeded
+ * `chunkLimit` must still be able to start loading subsequent chunks up to
+ * `totalRowCount` (if known) or by another chunk increment (#10752).
+ */
+export function dataGridLoadAllInitialTarget(loadedRowCount: number, chunkLimit: number, totalRowCount?: number): number {
+  const loaded = Number.isFinite(loadedRowCount) ? Math.max(0, Math.trunc(loadedRowCount)) : 0;
+  const chunk = Number.isFinite(chunkLimit) ? Math.max(1, Math.trunc(chunkLimit)) : 1;
+  const chunkCap = loaded < chunk ? chunk : loaded + chunk;
+  const total = typeof totalRowCount === "number" && Number.isFinite(totalRowCount) && totalRowCount >= 0 ? Math.trunc(totalRowCount) : undefined;
+  return total !== undefined ? Math.min(chunkCap, total) : chunkCap;
+}
+
 /**
  * Next chunk of an explicit "load all" run, issued after the previous chunk
  * completed. The per-request result-row cap bounds each request, not the run:

apps/desktop/src/stores/queryStore.ts

@@ -7724,7 +7724,7 @@ export const useQueryStore = defineStore("query", () => {
         return producedResult;
       }
 
-      const queryResultMaxRows = tab.mode === "query" ? effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows) : undefined;
+      const queryResultMaxRows = tab.mode === "query" ? (options?.appendResult ? options.appendResult.maxRows : effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows)) : undefined;
 
       if (tab.mode === "query") {
         const prepared = await prepareEditableQueryExecution(tab, sqlToExecute, conn, effectiveDbType, executionDatabase, traceId, elapsed);

packages/app-tests/dataGridPagination.test.ts

@@ -1,6 +1,6 @@
 import { strict as assert } from "node:assert";
 import { test } from "vitest";
-import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridLoadAllSegment, dataGridUserFacingPage, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal } from "../../apps/desktop/src/lib/dataGrid/dataGridPagination.ts";
+import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridLoadAllInitialTarget, dataGridLoadAllSegment, dataGridUserFacingPage, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal } from "../../apps/desktop/src/lib/dataGrid/dataGridPagination.ts";
 
 test("estimated display totals do not become pagination bounds", () => {
   assert.equal(
@@ -197,6 +197,25 @@ test("load-all does not request past the cap or after the result is complete", (
   assert.equal(dataGridLoadAllSegment(100, 5_000, false), null);
 });
 
+test("load-all initial target calculates chunk target across boundaries and known totals", () => {
+  // Below cap: targets cap or total
+  assert.equal(dataGridLoadAllInitialTarget(100, 100_000, 260_000), 100_000);
+  assert.equal(dataGridLoadAllInitialTarget(100, 100_000, 50_000), 50_000);
+  assert.equal(dataGridLoadAllInitialTarget(100, 100_000), 100_000);
+
+  // At or past cap: targets next chunk increment up to total
+  assert.equal(dataGridLoadAllInitialTarget(100_000, 100_000, 260_000), 200_000);
+  assert.equal(dataGridLoadAllInitialTarget(200_000, 100_000, 260_000), 260_000);
+  assert.equal(dataGridLoadAllInitialTarget(260_000, 100_000, 260_000), 260_000);
+
+  // When total is reached, dataGridLoadAllSegment returns null
+  const target = dataGridLoadAllInitialTarget(260_000, 100_000, 260_000);
+  assert.equal(dataGridLoadAllSegment(260_000, target, true), null);
+
+  // Without known total, continues by chunk increment
+  assert.equal(dataGridLoadAllInitialTarget(100_000, 100_000), 200_000);
+});
+
 // --- auto-redirect page calculation after refresh ---
 // These tests document the math used in DataGrid.vue's loading watcher:
 //   lastPageNum = Math.max(1, Math.ceil(total / pageSize))

packages/app-tests/useDataGridActions.test.ts

@@ -506,4 +506,64 @@ describe("multi-result preservation after submit", () => {
       restoreStorage();
     }
   });
+
+  test("onPaginate uses MAX_QUERY_RESULT_MAX_ROWS when appendRequested is true", async () => {
+    const restoreStorage = installMemoryStorage();
+    const originalFetch = globalThis.fetch;
+    const { useDataGridActions } = await import("../../apps/desktop/src/composables/useDataGridActions.ts");
+    const { MAX_QUERY_RESULT_MAX_ROWS } = await import("../../apps/desktop/src/lib/dataGrid/queryResultRowLimit.ts");
+
+    globalThis.fetch = (async (input) => {
+      const url = new URL(String(input), "http://localhost");
+      if (url.pathname === "/api/connection/check-health") return Response.json(null);
+      if (url.pathname === "/api/query/build-table-select-sql") {
+        return Response.json("SELECT * FROM `orders` LIMIT 100 OFFSET 100");
+      }
+      return new Response(`unexpected ${url.pathname}`, { status: 500 });
+    }) as typeof fetch;
+
+    try {
+      setActivePinia(createPinia());
+      const connectionStore = useConnectionStore();
+      const queryStore = useQueryStore();
+      const settingsStore = useSettingsStore();
+      settingsStore.updateEditorSettings({
+        queryResultMaxRows: 100_000,
+        queryResultMaxRowsEnabled: true,
+      });
+
+      connectionStore.addEphemeralConnection(conn("mysql-1"));
+      const tabId = queryStore.createTab("mysql-1", "app", "orders", "data");
+      queryStore.setTableMeta(tabId, { tableName: "orders", tableType: "TABLE", columns: [], primaryKeys: [] });
+      const tab = queryStore.tabs.find((item) => item.id === tabId);
+      assert.ok(tab);
+      tab.result = {
+        columns: ["id"],
+        rows: Array.from({ length: 100 }, (_, i) => [i + 1]),
+        affected_rows: 0,
+        execution_time_ms: 1,
+      };
+
+      let capturedOptions: any;
+      vi.spyOn(queryStore, "executeTabSql").mockImplementation(async (_tabId, _sql, options) => {
+        capturedOptions = options;
+        return true;
+      });
+
+      const actions = useDataGridActions(computed(() => tab));
+
+      // Normal infinite scroll append
+      settingsStore.updateEditorSettings({ infiniteScroll: true });
+      await actions.onPaginate(tabId, 100, 100, undefined, undefined, false);
+      assert.equal(capturedOptions?.appendResult?.maxRows, 100_000);
+
+      // Explicit load-all append
+      await actions.onPaginate(tabId, 100, 99_900, undefined, undefined, true);
+      assert.equal(capturedOptions?.appendResult?.maxRows, MAX_QUERY_RESULT_MAX_ROWS);
+    } finally {
+      globalThis.fetch = originalFetch;
+      restoreStorage();
+    }
+  });
 });
+