feat(es): paginate past 10k documents via pit and search_after

apps/desktop/src/composables/__tests__/useDataGridActions.spec.ts

@@ -1,4 +1,4 @@
-import { computed } from "vue";
+import { computed, reactive } from "vue";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 import { useDataGridActions } from "@/composables/useDataGridActions";
 import { clearTableMetadataCache } from "@/lib/metadata/tableMetadataCache";
@@ -130,6 +130,7 @@ describe("useDataGridActions", () => {
   beforeEach(() => {
     clearTableMetadataCache();
     vi.clearAllMocks();
+    mocks.executeTabSql.mockReset();
     mocks.tabs.length = 0;
     mocks.tableOpenPageSize = 100;
     mocks.infiniteScroll = true;
@@ -543,6 +544,71 @@ describe("useDataGridActions", () => {
     );
   });
 
+  it("advances Elasticsearch cursors when jumping across pages", async () => {
+    mocks.infiniteScroll = false;
+    mocks.getConfig.mockReturnValue({ id: "elasticsearch-1", db_type: "elasticsearch" });
+    const tab = reactive({
+      id: "tab-1",
+      connectionId: "elasticsearch-1",
+      database: "",
+      title: "Query",
+      sql: "SELECT * FROM `dbx-app-logs-v1` AS dalv",
+      resultBaseSql: "SELECT * FROM `dbx-app-logs-v1` AS dalv",
+      resultPageLimit: 100,
+      resultPageOffset: 0,
+      resultSessionId: "cursor-1",
+      resultClientSessionId: "client-1",
+      result: {
+        columns: ["message"],
+        rows: [["page-1"]],
+        affected_rows: 100_000,
+        execution_time_ms: 1,
+        session_id: "cursor-1",
+        has_more: true,
+      },
+      mode: "query",
+      isDirty: false,
+      isExecuting: false,
+      isCancelling: false,
+      isExplaining: false,
+    } as QueryTab);
+    mocks.executeTabSql.mockImplementation(async (_id, _sql, options) => {
+      const pagination = options.pagination as { offset: number };
+      expect(tab.resultPageJumpProgress).toEqual({
+        completedRequests: pagination.offset / 100 - 1,
+        totalRequests: 3,
+        targetPage: 4,
+      });
+      if (options.retainDisplayedResult) {
+        expect(tab.resultPageOffset).toBe(0);
+        tab.resultSessionId = `cursor-${pagination.offset / 100 + 1}`;
+        return;
+      }
+      tab.resultPageOffset = pagination.offset;
+      tab.resultSessionId = `cursor-${pagination.offset / 100 + 1}`;
+      tab.result = {
+        columns: ["message"],
+        rows: [[`page-${pagination.offset / 100 + 1}`]],
+        affected_rows: 100_000,
+        execution_time_ms: 1,
+        session_id: `cursor-${pagination.offset / 100 + 1}`,
+        has_more: true,
+      };
+    });
+    const actions = useDataGridActions(computed(() => tab));
+
+    await actions.onPaginate(300, 100);
+
+    expect(mocks.executeTabSql.mock.calls.map((call) => call[2].pagination)).toEqual([
+      { offset: 100, limit: 100, sessionId: "cursor-1", clientSessionId: "client-1" },
+      { offset: 200, limit: 100, sessionId: "cursor-2", clientSessionId: "client-1" },
+      { offset: 300, limit: 100, sessionId: "cursor-3", clientSessionId: "client-1" },
+    ]);
+    expect(mocks.executeTabSql.mock.calls.map((call) => call[2].retainDisplayedResult)).toEqual([true, true, undefined]);
+    expect(tab.resultPageOffset).toBe(300);
+    expect(tab.resultPageJumpProgress).toBeUndefined();
+  });
+
   it("uses the active multi-database result target for pagination", async () => {
     const tab = {
       id: "tab-1",

apps/desktop/src/composables/useDataGridActions.ts

@@ -6,6 +6,7 @@ import { useSettingsStore } from "@/stores/settingsStore";
 import { buildTableSelectSql, quoteTableDataIdentifier } from "@/lib/table/tableSelectSql";
 import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
 import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
+import { elasticsearchCursorPageJumpRequestCount } from "@/lib/dataGrid/dataGridPagination";
 import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
 import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
 import * as api from "@/lib/backend/api";
@@ -335,26 +336,100 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
       const continuesResultSession = appendResult ? offset === expectedNextOffset : offset === expectedNextOffset && limit === tab.resultPageLimit;
       const sessionId = tab.result?.has_more && tab.result?.session_id && continuesResultSession ? tab.result.session_id : undefined;
       const resultBaseSql = queryResultBaseSql(tab);
-      await queryStore.executeTabSql(tab.id, baseSql, {
-        ...activeQueryTargetOptions(tab),
-        resultBaseSql,
-        resultSortedSql: tab.resultSortedSql,
-        ...(hasDatabaseSort
-          ? {
-              querySort: {
-                resultColumns: sortColumns.resultColumns,
-                columnIndex: sortColumns.columnIndex,
-                column: tab.resultSortColumn!,
-                direction: tab.resultSortDirection!,
-              },
+      const executePage = (pageOffset: number, pageSessionId?: string, retainDisplayedResult = false) =>
+        queryStore.executeTabSql(tab.id, baseSql, {
+          ...activeQueryTargetOptions(tab),
+          resultBaseSql,
+          resultSortedSql: tab.resultSortedSql,
+          ...(hasDatabaseSort
+            ? {
+                querySort: {
+                  resultColumns: sortColumns.resultColumns,
+                  columnIndex: sortColumns.columnIndex,
+                  column: tab.resultSortColumn!,
+                  direction: tab.resultSortDirection!,
+                },
+              }
+            : {}),
+          pagination: { offset: pageOffset, limit, sessionId: pageSessionId, clientSessionId: pageSessionId ? tab.resultClientSessionId : undefined },
+          ...appendOptions,
+          preserveResultDuringExecution: true,
+          preserveTotalRowCountDuringExecution: true,
+          replaceActiveResultInGroup: true,
+          ...(retainDisplayedResult ? { retainDisplayedResult: true } : {}),
+        });
+      const executionTarget = queryStore.activeResultExecutionTarget(tab.id);
+      const executionConnection = connectionStore.getConfig(executionTarget?.connectionId ?? tab.connectionId);
+      const effectiveDbType = effectiveDatabaseTypeForConnection(executionConnection);
+      const usesElasticsearchCursor = effectiveDbType === "elasticsearch" || effectiveDbType === "easysearch";
+
+      // Elasticsearch search_after has no random page access, so every missing
+      // cursor must be fetched in order. This workaround still sends one ES
+      // request per skipped page (page 1 -> 101 sends 100 requests);
+      // retainDisplayedResult only hides those intermediate pages from the UI.
+      const currentResultSessionId = tab.resultSessionId ?? tab.result?.session_id;
+      if (usesElasticsearchCursor && !appendResult && typeof expectedNextOffset === "number" && limit === tab.resultPageLimit && currentResultSessionId) {
+        let nextOffset = expectedNextOffset;
+        let nextSessionId: string | undefined = currentResultSessionId;
+        const currentPage = Math.floor(nextOffset / limit);
+        const targetPage = Math.floor(offset / limit) + 1;
+        const totalRequests = elasticsearchCursorPageJumpRequestCount(currentPage, targetPage);
+        const jumpProgress =
+          totalRequests > 1
+            ? {
+                completedRequests: 0,
+                totalRequests,
+                targetPage,
+              }
+            : undefined;
+        if (jumpProgress) {
+          tab.resultPageJumpProgress = jumpProgress;
+        }
+        const activeJumpProgress = tab.resultPageJumpProgress;
+        const executeCursorPage = async (pageOffset: number, pageSessionId?: string, retainDisplayedResult = false) => {
+          await executePage(pageOffset, pageSessionId, retainDisplayedResult);
+          if (activeJumpProgress) {
+            activeJumpProgress.completedRequests += 1;
+          }
+        };
+
+        try {
+          if (offset < nextOffset) {
+            await executeCursorPage(0, undefined, offset !== 0);
+            if (offset === 0) {
+              return;
             }
-          : {}),
-        pagination: { offset, limit, sessionId, clientSessionId: sessionId ? tab.resultClientSessionId : undefined },
-        ...appendOptions,
-        preserveResultDuringExecution: true,
-        preserveTotalRowCountDuringExecution: true,
-        replaceActiveResultInGroup: true,
-      });
+            const restartedTab = activeTab.value;
+            if (restartedTab?.id !== tab.id) {
+              return;
+            }
+            nextOffset = limit;
+            nextSessionId = restartedTab.resultSessionId;
+          }
+          while (nextOffset < offset) {
+            if (!nextSessionId) {
+              return;
+            }
+            await executeCursorPage(nextOffset, nextSessionId, true);
+            const advancedTab = activeTab.value;
+            if (advancedTab?.id !== tab.id) {
+              return;
+            }
+            nextOffset += limit;
+            nextSessionId = advancedTab.resultSessionId;
+          }
+          if (nextOffset === offset && nextSessionId) {
+            await executeCursorPage(offset, nextSessionId);
+            return;
+          }
+        } finally {
+          if (tab.resultPageJumpProgress === activeJumpProgress) {
+            tab.resultPageJumpProgress = undefined;
+          }
+        }
+      }
+
+      await executePage(offset, sessionId);
       return;
     }