feat(es): paginate past 10k documents via pit and search_after

This commit is contained in:
Lan-zk authored and GitHub committed 2026-08-30 03:57:03 +08:00
1 parent 1d71a5fc23
commit 73a7b3f2c7
35 files changed
+1622 -153

No files matched your search

@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { clampDocumentPage, clampElasticsearchRequestLimit, documentPageRequestLimit, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals";
import { clampDocumentPage, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals";
describe("Elasticsearch document totals", () => {
it("keeps a lower-bound search total separate from an exact background count", () => {
@@ -39,21 +39,5 @@ describe("Elasticsearch document totals", () => {
it("clamps the final request without exceeding the conservative page cap", () => {
expect(clampDocumentPage(30, 333, 10_000)).toBe(30);
expect(clampDocumentPage(31, 333, 10_000)).toBe(30);
expect(documentPageRequestLimit(30, 333, 10_000)).toBe(10);
expect(documentPageRequestLimit(0, 100, undefined)).toBe(100);
});
it("caps the initial page request to Elasticsearch's default max_result_window even without a known total (#7231)", () => {
// Opening an index for the first time has no paginationTotal yet; a "rows per page" preference
// set above 10,000 must not be forwarded to Elasticsearch as-is, or the from+size search fails outright.
expect(documentPageRequestLimit(0, 100_000, undefined)).toBe(10_000);
expect(documentPageRequestLimit(0, 5_000, undefined)).toBe(5_000);
});
it("clamps an explicit request limit against the remaining from+size window at a given skip", () => {
expect(clampElasticsearchRequestLimit(0, 100_000)).toBe(10_000);
expect(clampElasticsearchRequestLimit(9_000, 5_000)).toBe(1_000);
expect(clampElasticsearchRequestLimit(20_000, 5_000)).toBe(0);
expect(clampElasticsearchRequestLimit(0, 500)).toBe(500);
});
});