From 891e60c2f958337a125a013bab0e24393449d95f Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Wed, 12 Aug 2026 04:55:53 +0000 Subject: [PATCH] fix(mongodb): export all documents from collection view Closes #4197 --- .../components/document/DocumentBrowser.vue | 50 +++++++ .../DocumentBrowserFieldSearch.spec.ts | 122 ++++++++++++++++++ 2 files changed, 172 insertions(+) diff --git a/apps/desktop/src/components/document/DocumentBrowser.vue b/apps/desktop/src/components/document/DocumentBrowser.vue index 6e5415afd1..1fc70f2d44 100644 --- a/apps/desktop/src/components/document/DocumentBrowser.vue +++ b/apps/desktop/src/components/document/DocumentBrowser.vue @@ -80,6 +80,7 @@ import { serializeMongoDocumentId, type MongoInputValue, } from "@/lib/mongo/mongoDocumentValues"; +import { mongoDocumentsToQueryResult } from "@/lib/mongo/mongoShellCommand"; import type { GridNewRowMeta } from "@/lib/dataGrid/gridNewRowPlacement"; import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize"; import { documentDataGridColumnLayoutScopeKey } from "@/lib/dataGrid/dataGridColumnLayoutStorage"; @@ -338,6 +339,54 @@ const gridResult = computed(() => { return { columns, column_types: columnTypes, rows, mongo_documents: docs, mongo_copy_documents: copyDocuments.value, affected_rows: 0, execution_time_ms: 0, truncated: false }; }); + +async function exportAllMongoDocuments(onProgress?: (info: { rowsExported: number; totalRows: number | null }) => void): Promise { + if (documentStoreProvider.value.kind !== "mongodb") return undefined; + + const connectionId = props.connectionId; + const database = props.database; + const collection = props.collection; + const filter = currentDocumentFilter(); + const sort = currentDocumentSortJson(sortInput.value); + const exportSettings = settingsStore.editorSettings; + const batchSize = Math.max(1, Math.trunc(exportSettings.exportBatchSize)); + const rowLimit = exportSettings.exportRowLimitEnabled ? Math.max(0, Math.trunc(exportSettings.exportRowLimit)) : Number.POSITIVE_INFINITY; + const exportExecutionId = uuid(); + const exportStartedAt = performance.now(); + const exportedDocuments: JsonRecord[] = []; + let exportedCopyDocuments: JsonRecord[] | undefined = []; + let totalRows: number | null = null; + + while (exportedDocuments.length < rowLimit) { + const requestLimit = Math.min(batchSize, rowLimit - exportedDocuments.length); + if (requestLimit <= 0) break; + const result = await api.documentFindDocuments(connectionId, database, collection, exportedDocuments.length, requestLimit, filter, undefined, sort, undefined, exportExecutionId); + const pageDocuments = result.documents.slice(0, requestLimit).map(asRecord); + exportedDocuments.push(...pageDocuments); + + if (exportedCopyDocuments) { + if (result.extended_documents?.length === result.documents.length) { + exportedCopyDocuments.push(...result.extended_documents.slice(0, pageDocuments.length).map(asRecord)); + } else { + exportedCopyDocuments = undefined; + } + } + + if (result.total_is_exact !== false) totalRows = Math.min(result.total, rowLimit); + onProgress?.({ rowsExported: exportedDocuments.length, totalRows }); + + const reachedExactTotal = result.total_is_exact !== false && exportedDocuments.length >= result.total; + if (pageDocuments.length === 0 || pageDocuments.length < requestLimit || reachedExactTotal) break; + } + + const result = mongoDocumentsToQueryResult(exportedDocuments, performance.now() - exportStartedAt, totalRows ?? exportedDocuments.length, exportedCopyDocuments, totalRows !== null); + if (result.columns.length === 0) result.columns = gridResult.value.columns; + result.column_types = mongoDocumentGridColumnTypes(exportedDocuments, result.columns); + result.affected_rows = exportedDocuments.length; + result.truncated = false; + result.has_more = false; + return result; +} const expandedDocumentFilterFieldPaths = ref>(new Set()); const elasticsearchFieldTypes = computed(() => new Map(elasticsearchMappingFields.value.map((field) => [field.name, field.data_type]))); const elasticsearchFilterFieldNames = computed(() => { @@ -1871,6 +1920,7 @@ defineExpose({ focusSearch }); :inexact-total-row-count-mode="documentStoreProvider.kind === 'mongodb' ? 'estimated' : 'at-least'" :pagination-total-row-count="pageTotal" :count-total-rows="countExactDocumentTotal" + :full-export-result="documentStoreProvider.kind === 'mongodb' ? exportAllMongoDocuments : undefined" @sort="onSort" @reload="refreshDocuments" @paginate="(offset: number, limit: number) => paginate(offset, limit)" diff --git a/apps/desktop/src/components/document/__tests__/DocumentBrowserFieldSearch.spec.ts b/apps/desktop/src/components/document/__tests__/DocumentBrowserFieldSearch.spec.ts index 9269868331..b5dcce2a6d 100644 --- a/apps/desktop/src/components/document/__tests__/DocumentBrowserFieldSearch.spec.ts +++ b/apps/desktop/src/components/document/__tests__/DocumentBrowserFieldSearch.spec.ts @@ -15,6 +15,17 @@ const documentJsonEditor = vi.hoisted(() => ({ openSearch: vi.fn().mockReturnValue(true), })); +const dataGrid = vi.hoisted(() => ({ + fullExportResult: undefined as + | ((onProgress?: (info: { rowsExported: number; totalRows: number | null }) => void) => Promise<{ + columns: string[]; + column_types?: string[]; + rows: Array>; + mongo_copy_documents?: unknown[]; + } | undefined>) + | undefined, +})); + const settings = vi.hoisted(() => ({ editorSettings: { pageSize: 100, @@ -25,6 +36,9 @@ const settings = vi.hoisted(() => ({ tableFontSize: 12, numericColumnRightAlign: true, confirmDangerousSqlExecution: true, + exportBatchSize: 2, + exportRowLimitEnabled: false, + exportRowLimit: 100_000, }, updateEditorSettings: vi.fn(), })); @@ -64,8 +78,10 @@ vi.mock("@/components/grid/DataGrid.vue", async () => { connectionId: { type: String, default: "" }, database: { type: String, default: "" }, columnLayoutScopeKey: { type: String, default: "" }, + fullExportResult: { type: Function, default: undefined }, }, setup(props, { expose, slots }) { + dataGrid.fullExportResult = props.fullExportResult as typeof dataGrid.fullExportResult; expose({ visibleColumnCount: 2, displayableColumnCount: 2, @@ -95,6 +111,7 @@ vi.mock("@/components/grid/DataGrid.vue", async () => { "data-column-layout-scope-key": props.columnLayoutScopeKey, "data-result-hidden-column-keys": JSON.stringify((props.result as { local_hidden_column_keys?: string[] }).local_hidden_column_keys ?? []), "data-result-column-types": JSON.stringify((props.result as { column_types?: string[] }).column_types ?? []), + "data-result-rows": JSON.stringify((props.result as { rows?: unknown[] }).rows ?? []), }, [ slots["search-bar"]?.({ @@ -258,6 +275,7 @@ beforeEach(async () => { backend.cancelQuery.mockReset(); backend.ensureConnected.mockReset(); backend.documentDeleteDocument.mockReset(); + dataGrid.fullExportResult = undefined; documentJsonEditor.openSearch.mockClear(); backend.documentDeleteDocument.mockResolvedValue(undefined); settings.editorSettings.mongoViewMode = "table"; @@ -267,6 +285,9 @@ beforeEach(async () => { settings.editorSettings.tableFontSize = 12; settings.editorSettings.numericColumnRightAlign = true; settings.editorSettings.confirmDangerousSqlExecution = true; + settings.editorSettings.exportBatchSize = 2; + settings.editorSettings.exportRowLimitEnabled = false; + settings.editorSettings.exportRowLimit = 100_000; settings.updateEditorSettings.mockReset(); settings.updateEditorSettings.mockImplementation((partial: Partial) => Object.assign(settings.editorSettings, partial)); backend.ensureConnected.mockResolvedValue(undefined); @@ -407,6 +428,107 @@ describe("DocumentBrowser Elasticsearch field search", () => { }); describe("DocumentBrowser MongoDB filter value types", () => { + it("exports all matching MongoDB documents without changing the visible page", async () => { + app?.unmount(); + settings.editorSettings.exportBatchSize = 2; + settings.editorSettings.exportRowLimitEnabled = true; + settings.editorSettings.exportRowLimit = 3; + backend.documentFindDocuments.mockReset(); + backend.documentFindDocuments + .mockResolvedValueOnce({ + documents: [{ _id: "visible", name: "Visible" }], + extended_documents: [{ _id: { $oid: "000000000000000000000001" }, name: "Visible" }], + total: 5, + total_is_exact: true, + }) + .mockResolvedValueOnce({ + documents: [ + { _id: "one", name: "First" }, + { _id: "two", name: "Second" }, + ], + extended_documents: [ + { _id: { $oid: "000000000000000000000002" }, name: "First" }, + { _id: { $oid: "000000000000000000000003" }, name: "Second" }, + ], + total: 5, + total_is_exact: true, + }) + .mockResolvedValueOnce({ + documents: [{ _id: "three", later: { $numberLong: "9007199254740993" } }], + extended_documents: [{ _id: { $oid: "000000000000000000000004" }, later: { $numberLong: "9007199254740993" } }], + total: 5, + total_is_exact: true, + }); + app = createApp(DocumentBrowser, { + connectionId: "mongo-1", + database: "test", + collection: "orders", + databaseType: "mongodb", + }); + app.mount(root!); + await flushUi(); + + const inputs = root!.querySelectorAll("textarea"); + inputs[0]!.value = '{"active":true}'; + inputs[0]!.dispatchEvent(new Event("input", { bubbles: true })); + inputs[1]!.value = '{"createdAt":-1}'; + inputs[1]!.dispatchEvent(new Event("input", { bubbles: true })); + await flushUi(); + + expect(dataGrid.fullExportResult).toBeTypeOf("function"); + const progress = vi.fn(); + const result = await dataGrid.fullExportResult!(progress); + + expect(backend.documentFindDocuments.mock.calls.slice(1).map((call) => call.slice(0, 9))).toEqual([ + ["mongo-1", "test", "orders", 0, 2, '{"active":true}', undefined, '{"createdAt":-1}', undefined], + ["mongo-1", "test", "orders", 2, 1, '{"active":true}', undefined, '{"createdAt":-1}', undefined], + ]); + expect(result?.columns).toEqual(["_id", "name", "later"]); + expect(result?.rows).toEqual([ + ["one", "First", null], + ["two", "Second", null], + ["three", null, "9007199254740993"], + ]); + expect(result?.mongo_copy_documents).toEqual([ + { _id: { $oid: "000000000000000000000002" }, name: "First" }, + { _id: { $oid: "000000000000000000000003" }, name: "Second" }, + { _id: { $oid: "000000000000000000000004" }, later: { $numberLong: "9007199254740993" } }, + ]); + expect(progress).toHaveBeenLastCalledWith({ rowsExported: 3, totalRows: 3 }); + const visibleGrid = root!.querySelector('[data-testid="data-grid"]')!; + expect(JSON.parse(visibleGrid.dataset.resultColumnTypes ?? "[]")).toEqual(["", ""]); + expect(JSON.parse(visibleGrid.dataset.resultRows ?? "[]")).toEqual([["visible", "Visible"]]); + expect(inputs[0]!.value).toBe('{"active":true}'); + expect(inputs[1]!.value).toBe('{"createdAt":-1}'); + }); + + it("stops MongoDB full export on a short page when the total is estimated", async () => { + app?.unmount(); + backend.documentFindDocuments.mockReset(); + backend.documentFindDocuments + .mockResolvedValueOnce({ documents: [{ _id: "visible" }], total: 500, total_is_exact: false }) + .mockResolvedValueOnce({ documents: [{ _id: "only" }], total: 500, total_is_exact: false }); + app = createApp(DocumentBrowser, { + connectionId: "mongo-1", + database: "test", + collection: "orders", + databaseType: "mongodb", + }); + app.mount(root!); + await flushUi(); + + const progress = vi.fn(); + const result = await dataGrid.fullExportResult!(progress); + + expect(backend.documentFindDocuments).toHaveBeenCalledTimes(2); + expect(result?.rows).toEqual([["only"]]); + expect(progress).toHaveBeenLastCalledWith({ rowsExported: 1, totalRows: null }); + }); + + it("does not offer the MongoDB full export callback to Elasticsearch viewers", () => { + expect(dataGrid.fullExportResult).toBeUndefined(); + }); + it("identifies consistently numeric MongoDB columns for shared grid alignment", async () => { app?.unmount(); backend.documentFindDocuments.mockReset();