feat(es): paginate past 10k documents via pit and search_after

apps/desktop/src/lib/__tests__/document/elasticsearchDocumentTotals.spec.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { clampDocumentPage, clampElasticsearchRequestLimit, documentPageRequestLimit, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals";
+import { clampDocumentPage, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals";
 
 describe("Elasticsearch document totals", () => {
   it("keeps a lower-bound search total separate from an exact background count", () => {
@@ -39,21 +39,5 @@ describe("Elasticsearch document totals", () => {
   it("clamps the final request without exceeding the conservative page cap", () => {
     expect(clampDocumentPage(30, 333, 10_000)).toBe(30);
     expect(clampDocumentPage(31, 333, 10_000)).toBe(30);
-    expect(documentPageRequestLimit(30, 333, 10_000)).toBe(10);
-    expect(documentPageRequestLimit(0, 100, undefined)).toBe(100);
-  });
-
-  it("caps the initial page request to Elasticsearch's default max_result_window even without a known total (#7231)", () => {
-    // Opening an index for the first time has no paginationTotal yet; a "rows per page" preference
-    // set above 10,000 must not be forwarded to Elasticsearch as-is, or the from+size search fails outright.
-    expect(documentPageRequestLimit(0, 100_000, undefined)).toBe(10_000);
-    expect(documentPageRequestLimit(0, 5_000, undefined)).toBe(5_000);
-  });
-
-  it("clamps an explicit request limit against the remaining from+size window at a given skip", () => {
-    expect(clampElasticsearchRequestLimit(0, 100_000)).toBe(10_000);
-    expect(clampElasticsearchRequestLimit(9_000, 5_000)).toBe(1_000);
-    expect(clampElasticsearchRequestLimit(20_000, 5_000)).toBe(0);
-    expect(clampElasticsearchRequestLimit(0, 500)).toBe(500);
   });
 });

apps/desktop/src/lib/backend/http.ts

@@ -3797,7 +3797,20 @@ export async function mongoFindOne(connectionId: string, database: string, colle
   });
 }
 
-export async function documentFindDocuments(connectionId: string, database: string, collection: string, skip: number, limit: number, filter?: string, projection?: string, sort?: string, collation?: string, executionId?: string, cursor?: string): Promise<DocumentQueryResult> {
+export async function documentFindDocuments(
+  connectionId: string,
+  database: string,
+  collection: string,
+  skip: number,
+  limit: number,
+  filter?: string,
+  projection?: string,
+  sort?: string,
+  collation?: string,
+  executionId?: string,
+  cursor?: string,
+  cursorPagination?: boolean,
+): Promise<DocumentQueryResult> {
   return post("/api/document-store/find-documents", {
     connectionId,
     database,
@@ -3809,6 +3822,7 @@ export async function documentFindDocuments(connectionId: string, database: stri
     sort,
     collation,
     cursor,
+    cursorPagination,
     executionId,
   });
 }

apps/desktop/src/lib/backend/tauri.ts

@@ -3790,7 +3790,20 @@ export async function mongoParseShellCommand(source: string): Promise<MongoComma
   return normalizeRustMongoCommand(raw);
 }
 
-export async function documentFindDocuments(connectionId: string, database: string, collection: string, skip: number, limit: number, filter?: string, projection?: string, sort?: string, collation?: string, executionId?: string, cursor?: string): Promise<DocumentQueryResult> {
+export async function documentFindDocuments(
+  connectionId: string,
+  database: string,
+  collection: string,
+  skip: number,
+  limit: number,
+  filter?: string,
+  projection?: string,
+  sort?: string,
+  collation?: string,
+  executionId?: string,
+  cursor?: string,
+  cursorPagination?: boolean,
+): Promise<DocumentQueryResult> {
   return invoke("document_find_documents", {
     connectionId,
     database,
@@ -3802,6 +3815,7 @@ export async function documentFindDocuments(connectionId: string, database: stri
     sort,
     collation,
     cursor,
+    cursorPagination,
     executionId,
   });
 }

apps/desktop/src/lib/dataGrid/__tests__/dataGridPagination.spec.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+import { ELASTICSEARCH_PAGE_JUMP_WARNING_REQUESTS, elasticsearchCursorPageJumpRequestCount } from "@/lib/dataGrid/dataGridPagination";
+
+describe("Elasticsearch cursor page jumps", () => {
+  it("counts only missing forward cursors", () => {
+    expect(elasticsearchCursorPageJumpRequestCount(1, 101)).toBe(100);
+    expect(elasticsearchCursorPageJumpRequestCount(101, 102)).toBe(1);
+  });
+
+  it("counts a backward jump from the page-one restart", () => {
+    expect(elasticsearchCursorPageJumpRequestCount(104, 103)).toBe(103);
+    expect(elasticsearchCursorPageJumpRequestCount(104, 1)).toBe(1);
+  });
+
+  it("warns starting at one hundred sequential requests", () => {
+    expect(elasticsearchCursorPageJumpRequestCount(1, 100)).toBeLessThan(ELASTICSEARCH_PAGE_JUMP_WARNING_REQUESTS);
+    expect(elasticsearchCursorPageJumpRequestCount(1, 101)).toBe(ELASTICSEARCH_PAGE_JUMP_WARNING_REQUESTS);
+  });
+});

apps/desktop/src/lib/dataGrid/dataGridPagination.ts

@@ -33,6 +33,21 @@ export interface CanFetchNextDataGridSegmentOptions {
 
 export type DataGridInexactTotalRowCountMode = "at-least" | "estimated";
 
+export const ELASTICSEARCH_PAGE_JUMP_WARNING_REQUESTS = 100;
+
+export function elasticsearchCursorPageJumpRequestCount(currentPage: number, targetPage: number): number {
+  if (!Number.isSafeInteger(currentPage) || !Number.isSafeInteger(targetPage) || currentPage < 1 || targetPage < 1 || currentPage === targetPage) {
+    return 0;
+  }
+  if (targetPage > currentPage) {
+    return targetPage - currentPage;
+  }
+
+  // search_after only moves forward. The current workaround rebuilds the
+  // cursor from page 1 when navigating backward, including the target request.
+  return targetPage;
+}
+
 export function dataGridTruncationHintKey(databaseType?: DatabaseType): "grid.truncatedHint" | "grid.victoriaMetricsTruncatedHint" {
   return databaseType === "victoriametrics" ? "grid.victoriaMetricsTruncatedHint" : "grid.truncatedHint";
 }

apps/desktop/src/lib/document/elasticsearchDocumentTotals.ts

@@ -45,20 +45,3 @@ export function clampDocumentPage(page: number, pageSize: number, paginationTota
 // DBX can't know a target index's actual configured value up front, so it clamps requests to the
 // out-of-the-box default rather than letting an oversized "rows per page" preference reach the cluster as-is.
 export const ELASTICSEARCH_DEFAULT_MAX_RESULT_WINDOW = 10_000;
-
-// Clamps a requested page size against the from+size window still available at a given skip offset,
-// regardless of where the requested limit came from (a settings default or an explicit override).
-export function clampElasticsearchRequestLimit(skip: number, limit: number): number {
-  const normalizedSkip = Math.max(0, Math.floor(skip));
-  const normalizedLimit = Math.max(1, Math.floor(limit));
-  return Math.min(normalizedLimit, Math.max(0, ELASTICSEARCH_DEFAULT_MAX_RESULT_WINDOW - normalizedSkip));
-}
-
-export function documentPageRequestLimit(page: number, pageSize: number, paginationTotal?: number): number {
-  const normalizedPageSize = Math.max(1, Math.floor(pageSize));
-  const skip = Math.max(0, Math.floor(page)) * normalizedPageSize;
-  const cappedPageSize = clampElasticsearchRequestLimit(skip, normalizedPageSize);
-  if (paginationTotal === undefined) return cappedPageSize;
-  const remaining = paginationTotal - skip;
-  return remaining > 0 ? Math.min(cappedPageSize, remaining) : cappedPageSize;
-}