feat(es): paginate past 10k documents via pit and search_after

This commit is contained in:
Lan-zk authored and GitHub committed 2026-08-30 03:57:03 +08:00
1 parent 1d71a5fc23
commit 73a7b3f2c7
35 files changed
+1622 -153

No files matched your search

@@ -14,6 +14,7 @@ const backend = vi.hoisted(() => ({
documentUpdateDocument: vi.fn(),
documentDeleteDocument: vi.fn(),
documentSaveMeilisearchBatch: vi.fn(),
closeQuerySession: vi.fn(),
}));
const documentJsonEditor = vi.hoisted(() => ({
@@ -79,6 +80,7 @@ vi.mock("@/lib/backend/api", () => ({
documentUpdateDocument: backend.documentUpdateDocument,
documentDeleteDocument: backend.documentDeleteDocument,
documentSaveMeilisearchBatch: backend.documentSaveMeilisearchBatch,
closeQuerySession: backend.closeQuerySession,
}));
vi.mock("@/stores/connectionStore", () => ({
@@ -315,6 +317,7 @@ beforeEach(async () => {
backend.documentUpdateDocument.mockReset();
backend.documentDeleteDocument.mockReset();
backend.documentSaveMeilisearchBatch.mockReset();
backend.closeQuerySession.mockReset();
dataGrid.fullExportResult = undefined;
dataGrid.customSaveHandler = undefined;
dataGrid.countTotalRows = undefined;
@@ -889,8 +892,39 @@ describe("DocumentBrowser MongoDB filter value types", () => {
expect(progress).toHaveBeenLastCalledWith({ rowsExported: 1, totalRows: null });
});
it("does not offer the MongoDB full export callback to Elasticsearch viewers", () => {
expect(dataGrid.fullExportResult).toBeUndefined();
it("exports Elasticsearch documents with cursor pagination", async () => {
settings.editorSettings.exportBatchSize = 2;
backend.documentFindDocuments.mockReset();
backend.documentFindDocuments
.mockResolvedValueOnce({
documents: [
{ _id: "one", title: "First" },
{ _id: "two", title: "Second" },
],
total: 3,
total_is_exact: true,
next_cursor: "cursor-1",
})
.mockResolvedValueOnce({
documents: [{ _id: "three", title: "Third" }],
total: 3,
total_is_exact: true,
});
expect(dataGrid.fullExportResult).toBeTypeOf("function");
const result = await dataGrid.fullExportResult!();
await flushUi();
expect(backend.documentFindDocuments.mock.calls.map((call) => [call[3], call[4], call[10], call[11]])).toEqual([
[0, 2, undefined, true],
[0, 2, "cursor-1", true],
]);
expect(result?.rows).toEqual([
["one", "First"],
["two", "Second"],
["three", "Third"],
]);
expect(backend.closeQuerySession).toHaveBeenCalledWith("connection-1", "", "cursor-1");
});
it("identifies consistently numeric MongoDB columns for shared grid alignment", async () => {
@@ -260,6 +260,10 @@ describe("DocumentBrowser infinite scroll (issue #6455)", () => {
await mountBrowser(databaseType);
expect(backend.documentFindDocuments.mock.calls[0]!.slice(0, 5)).toEqual(["mongo-1", "test", "docs", 0, 5]);
if (databaseType === "elasticsearch") {
expect(backend.documentFindDocuments.mock.calls[0]![10]).toBeUndefined();
expect(backend.documentFindDocuments.mock.calls[0]![11]).toBe(true);
}
expect(dataGrid.pageLimit).toBe(5);
expect(dataGrid.pageSizePreference).toBe("table-open");
});