From 9234c33e24a43cbb72074301fd41eb5ccfce85ac Mon Sep 17 00:00:00 2001 From: zipg Date: Wed, 15 Jul 2026 21:01:40 +0800 Subject: [PATCH] fix(mongodb): preserve BSON dates in copied inserts --- .../components/document/DocumentBrowser.vue | 5 ++- apps/desktop/src/components/grid/DataGrid.vue | 2 +- .../__tests__/useDataGridExport.spec.ts | 18 ++++++-- .../src/composables/useDataGridExport.ts | 30 ++++++++++---- apps/desktop/src/lib/backend/tauri.ts | 1 + apps/desktop/src/types/database.ts | 2 + .../dbx-core/src/db/elasticsearch_driver.rs | 7 +++- crates/dbx-core/src/db/mongo_driver.rs | 41 +++++++++++++++---- crates/dbx-core/src/db/vector_driver.rs | 8 +++- .../dbx-core/tests/live_mongodb_find_one.rs | 32 +++++++++++++++ packages/app-tests/documentBrowser.test.ts | 2 + packages/app-tests/useDataGridExport.test.ts | 38 ++++++++++++++++- 12 files changed, 160 insertions(+), 26 deletions(-) diff --git a/apps/desktop/src/components/document/DocumentBrowser.vue b/apps/desktop/src/components/document/DocumentBrowser.vue index 7b343c4147..a369a6f909 100644 --- a/apps/desktop/src/components/document/DocumentBrowser.vue +++ b/apps/desktop/src/components/document/DocumentBrowser.vue @@ -55,6 +55,7 @@ type JsonRecord = Record; type ViewMode = "document" | "table"; const documents = ref([]); +const copyDocuments = ref([]); const lastGridColumns = ref([]); const total = ref(0); const loading = ref(false); @@ -172,7 +173,7 @@ const gridResult = computed(() => { }), ); - return { columns, rows, mongo_documents: docs, affected_rows: 0, execution_time_ms: 0, truncated: false }; + return { columns, rows, mongo_documents: docs, mongo_copy_documents: copyDocuments.value, affected_rows: 0, execution_time_ms: 0, truncated: false }; }); const documentFilterFieldOptions = computed(() => gridResult.value.columns); const documentStructuredFilterCount = computed(() => (appliedDocumentFilter.value ? 1 : 0)); @@ -487,7 +488,9 @@ async function load() { } }) : result.documents.map(asRecord); + const nextCopyDocuments = result.extended_documents?.length === nextDocuments.length ? result.extended_documents.map(asRecord) : nextDocuments; documents.value = nextDocuments; + copyDocuments.value = nextCopyDocuments; if (nextDocuments.length > 0) { const keySet = new Set(); keySet.add("_id"); diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index 91796059ea..7b9ccd1de5 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -6651,7 +6651,7 @@ const { database: computed(() => props.executionDatabase ?? props.database), context: computed(() => props.context), sourceColumns: visibleSourceColumns, - mongoDocuments: computed(() => props.result.mongo_documents), + mongoDocuments: computed(() => props.result.mongo_copy_documents ?? props.result.mongo_documents), columnTypes: visibleColumnTypes, whereInput: computed(() => currentWhereInput()), orderBy: computed(() => currentOrderBy()), diff --git a/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts b/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts index e08b91649a..a482887dfd 100644 --- a/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts +++ b/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts @@ -212,10 +212,10 @@ describe("useDataGridExport prepared row statements", () => { }); it("preserves original Mongo string types in INSERT and applies explicit edits", async () => { - const item = { ...row(["123", "true", '{"kind":"literal"}', "2024-01-01 00:00:00", '{"role":"maintainer"}']), sourceIndex: 0 }; - item.isDirtyCol = [false, false, false, false, true]; + const item = { ...row(["123", "true", '{"kind":"literal"}', "2024-01-01 00:00:00", '{"role":"maintainer"}', 'ISODate("2025-05-06T08:35:32Z")']), sourceIndex: 0 }; + item.isDirtyCol = [false, false, false, false, true, false]; const state = createMongoExportState({ - columns: ["numericText", "booleanText", "jsonText", "dateText", "profile"], + columns: ["numericText", "booleanText", "jsonText", "dateText", "profile", "lastUpdatedDate"], item, mongoDocuments: [ { @@ -224,12 +224,22 @@ describe("useDataGridExport prepared row statements", () => { jsonText: '{"kind":"literal"}', dateText: "2024-01-01 00:00:00", profile: { role: "admin" }, + lastUpdatedDate: { $date: "2025-05-06T08:35:32Z" }, }, ], }); await state.copyRowAsInsert(); - expect(copyToClipboard).toHaveBeenCalledWith('db.getCollection("documents").insert({"numericText":"123","booleanText":"true","jsonText":"{\\"kind\\":\\"literal\\"}","dateText":"2024-01-01 00:00:00","profile":{"role":"maintainer"}});'); + expect(copyToClipboard).toHaveBeenCalledWith(`db.getCollection("documents").insert({ + "numericText": "123", + "booleanText": "true", + "jsonText": "{\\"kind\\":\\"literal\\"}", + "dateText": "2024-01-01 00:00:00", + "profile": { + "role": "maintainer" + }, + "lastUpdatedDate": ISODate("2025-05-06T08:35:32Z") +});`); }); }); diff --git a/apps/desktop/src/composables/useDataGridExport.ts b/apps/desktop/src/composables/useDataGridExport.ts index dc5773ce31..f6cc3d157a 100644 --- a/apps/desktop/src/composables/useDataGridExport.ts +++ b/apps/desktop/src/composables/useDataGridExport.ts @@ -13,6 +13,7 @@ import { uuid } from "@/lib/common/utils"; import { useSettingsStore } from "@/stores/settingsStore"; import { expandNestedJsonStringsForCopy } from "@/lib/common/jsonCopyValue"; import { buildMongoCopyDocumentFromOriginal, buildMongoCopyInsertDocument, formatMongoShellLiteral, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues"; +import { formatMongoShellText } from "@/lib/mongo/mongoFormatter"; import type { DatabaseType, QueryResult } from "@/types/database"; import type { QueryResultExportRequest } from "@/lib/backend/api"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; @@ -352,15 +353,17 @@ export function useDataGridExport(options: UseDataGridExportOptions) { const promise = Promise.resolve().then(async () => { const statement = databaseType.value === "mongodb" - ? buildMongoCopyInsertStatement({ - collection: copyInsertTargetLabel?.value || tableMeta.value?.tableName || "collection", - columns: columns.value, - sourceColumns: sourceColumns.value, - rows, - mongoDocuments: options.mongoDocuments?.value, - excludePrimaryKeys, - insertMode, - }) + ? formatMongoCopyInsertStatement( + buildMongoCopyInsertStatement({ + collection: copyInsertTargetLabel?.value || tableMeta.value?.tableName || "collection", + columns: columns.value, + sourceColumns: sourceColumns.value, + rows, + mongoDocuments: options.mongoDocuments?.value, + excludePrimaryKeys, + insertMode, + }), + ) : await buildDataGridCopyInsertStatement({ databaseType: databaseType.value, tableMeta: tableMeta.value, @@ -1375,6 +1378,15 @@ function buildMongoCopyInsertStatement(options: { collection: string; columns: s return `${collection}.insertMany(${formatMongoShellLiteral(documents)});`; } +function formatMongoCopyInsertStatement(statement: string | undefined): string | undefined { + if (!statement) return undefined; + try { + return formatMongoShellText(statement); + } catch { + return statement; + } +} + function compactLocalTimestamp(date = new Date()): string { const yy = String(date.getFullYear() % 100).padStart(2, "0"); const month = String(date.getMonth() + 1).padStart(2, "0"); diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 3f877a9992..18d92a2385 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -1752,6 +1752,7 @@ export async function zookeeperDelete(connectionId: string, key: string): Promis export interface MongoDocumentResult { documents: any[]; raw_documents?: string[]; + extended_documents?: any[]; total: number; } diff --git a/apps/desktop/src/types/database.ts b/apps/desktop/src/types/database.ts index b1c8046318..265584e811 100644 --- a/apps/desktop/src/types/database.ts +++ b/apps/desktop/src/types/database.ts @@ -515,6 +515,8 @@ export interface QueryResult { * preview. This is populated only for MongoDB document query results. */ mongo_documents?: unknown[]; + /** Type-preserving Extended JSON documents used when copying MongoDB values. */ + mongo_copy_documents?: unknown[]; affected_rows: number; execution_time_ms: number; truncated?: boolean; diff --git a/crates/dbx-core/src/db/elasticsearch_driver.rs b/crates/dbx-core/src/db/elasticsearch_driver.rs index b0f75894bf..4b211b8005 100644 --- a/crates/dbx-core/src/db/elasticsearch_driver.rs +++ b/crates/dbx-core/src/db/elasticsearch_driver.rs @@ -438,7 +438,12 @@ fn search_response_to_document_result(result: SearchResponse) -> Result, _>>() .map_err(|e| format!("Elasticsearch document serialization failed: {e}"))?; - Ok(MongoDocumentResult { documents, raw_documents: Some(raw_documents), total: result.hits.total.value() }) + Ok(MongoDocumentResult { + documents, + raw_documents: Some(raw_documents), + extended_documents: None, + total: result.hits.total.value(), + }) } fn build_find_documents_body( diff --git a/crates/dbx-core/src/db/mongo_driver.rs b/crates/dbx-core/src/db/mongo_driver.rs index 3825858bef..d4f4e471e1 100644 --- a/crates/dbx-core/src/db/mongo_driver.rs +++ b/crates/dbx-core/src/db/mongo_driver.rs @@ -17,6 +17,8 @@ pub struct MongoDocumentResult { pub documents: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub raw_documents: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extended_documents: Option>, pub total: u64, } @@ -586,12 +588,14 @@ pub async fn find_documents( let mut cursor = find.await.map_err(|e| e.to_string())?; let mut documents = Vec::new(); + let mut extended_documents = Vec::new(); while cursor.advance().await.map_err(|e| e.to_string())? { let doc = cursor.deserialize_current().map_err(|e| e.to_string())?; - documents.push(bson_to_json(&Bson::Document(doc))); + documents.push(bson_to_json(&Bson::Document(doc.clone()))); + extended_documents.push(Bson::Document(doc).into_relaxed_extjson()); } - Ok(MongoDocumentResult { documents, raw_documents: None, total }) + Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total }) } #[derive(Debug, Default, Deserialize)] @@ -727,7 +731,7 @@ pub async fn find_documents_extended_json( documents.push(Bson::Document(doc).into_relaxed_extjson()); } - Ok(MongoDocumentResult { documents, raw_documents: None, total }) + Ok(MongoDocumentResult { extended_documents: Some(documents.clone()), documents, raw_documents: None, total }) } pub async fn aggregate_documents( @@ -749,15 +753,18 @@ pub async fn aggregate_documents( let max_rows = max_rows.unwrap_or(100); let fetch_limit = max_rows.saturating_add(1); let mut documents = Vec::new(); + let mut extended_documents = Vec::new(); while documents.len() < fetch_limit && cursor.advance().await.map_err(|e| e.to_string())? { let doc = cursor.deserialize_current().map_err(|e| e.to_string())?; - documents.push(bson_to_json(&Bson::Document(doc))); + documents.push(bson_to_json(&Bson::Document(doc.clone()))); + extended_documents.push(Bson::Document(doc).into_relaxed_extjson()); } let total = documents.len() as u64; if documents.len() > max_rows { documents.truncate(max_rows); + extended_documents.truncate(max_rows); } - Ok(MongoDocumentResult { documents, raw_documents: None, total }) + Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total }) } pub async fn create_index( @@ -1119,11 +1126,17 @@ fn find_and_modify_array_filters( fn single_document_result(document: Option) -> MongoDocumentResult { match document { Some(document) => MongoDocumentResult { - documents: vec![bson_to_json(&Bson::Document(document))], + documents: vec![bson_to_json(&Bson::Document(document.clone()))], raw_documents: None, + extended_documents: Some(vec![Bson::Document(document).into_relaxed_extjson()]), total: 1, }, - None => MongoDocumentResult { documents: Vec::new(), raw_documents: None, total: 0 }, + None => MongoDocumentResult { + documents: Vec::new(), + raw_documents: None, + extended_documents: Some(Vec::new()), + total: 0, + }, } } @@ -1825,6 +1838,20 @@ mod tests { assert_eq!(value, serde_json::json!("ISODate(\"2026-06-10T13:59:31.287Z\")")); } + #[test] + fn mongo_document_result_keeps_extended_json_types_for_copying() { + let date = DateTime::parse_rfc3339_str("2025-05-06T08:35:32Z").unwrap(); + let result = single_document_result(Some(doc! { + "lastUpdatedDate": date, + "dateText": "ISODate(\"2025-05-06T08:35:32Z\")", + })); + + assert_eq!(result.documents[0]["lastUpdatedDate"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")")); + let extended = result.extended_documents.expect("extended documents"); + assert_eq!(extended[0]["lastUpdatedDate"], serde_json::json!({ "$date": "2025-05-06T08:35:32Z" })); + assert_eq!(extended[0]["dateText"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")")); + } + #[test] fn bson_to_json_preserves_unsafe_int64_for_js() { let value = bson_to_json(&Bson::Int64(2_326_645_729_978_441_729)); diff --git a/crates/dbx-core/src/db/vector_driver.rs b/crates/dbx-core/src/db/vector_driver.rs index 61b9d9416f..36510e8e7e 100644 --- a/crates/dbx-core/src/db/vector_driver.rs +++ b/crates/dbx-core/src/db/vector_driver.rs @@ -523,6 +523,7 @@ pub async fn find_documents( return Ok(crate::db::mongo_driver::MongoDocumentResult { documents, raw_documents: None, + extended_documents: None, total: result.affected_rows, }); } @@ -566,7 +567,12 @@ pub async fn find_documents( Value::Object(map) }) .collect(); - Ok(crate::db::mongo_driver::MongoDocumentResult { documents, raw_documents: None, total: result.affected_rows }) + Ok(crate::db::mongo_driver::MongoDocumentResult { + documents, + raw_documents: None, + extended_documents: None, + total: result.affected_rows, + }) } pub async fn execute_rest_query(client: &VectorClient, input: &str) -> Result { diff --git a/crates/dbx-core/tests/live_mongodb_find_one.rs b/crates/dbx-core/tests/live_mongodb_find_one.rs index 0b220467f6..dd9bb704a4 100644 --- a/crates/dbx-core/tests/live_mongodb_find_one.rs +++ b/crates/dbx-core/tests/live_mongodb_find_one.rs @@ -1,6 +1,7 @@ use std::time::Duration; use dbx_core::db::mongo_driver; +use mongodb::bson::{doc, Bson, DateTime}; #[tokio::test] #[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"] @@ -34,3 +35,34 @@ async fn find_one_returns_only_the_sorted_document() { assert_eq!(result.documents, vec![serde_json::json!({ "name": "new" })]); mongo_driver::drop_collection(&client, database, &collection).await.unwrap(); } + +#[tokio::test] +#[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"] +async fn find_documents_returns_type_preserving_copy_documents() { + let url = std::env::var("DBX_LIVE_MONGODB_URL").expect("DBX_LIVE_MONGODB_URL"); + let client = mongo_driver::connect(&url, Duration::from_secs(10), Duration::from_secs(60)).await.unwrap(); + let database = "dbx_live_find_one"; + let collection = format!("copy_types_{}", std::process::id()); + + client + .database(database) + .collection(&collection) + .insert_one(doc! { + "lastUpdatedDate": DateTime::parse_rfc3339_str("2025-05-06T08:35:32Z").unwrap(), + "dateText": Bson::String("ISODate(\"2025-05-06T08:35:32Z\")".to_string()), + }) + .await + .unwrap(); + + let result = + mongo_driver::find_documents(&client, database, &collection, 0, 10, Some("{}"), Some(r#"{"_id":0}"#), None) + .await + .unwrap(); + + assert_eq!(result.documents[0]["lastUpdatedDate"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")")); + let extended = result.extended_documents.expect("extended documents"); + assert_eq!(extended[0]["lastUpdatedDate"], serde_json::json!({ "$date": "2025-05-06T08:35:32Z" })); + assert_eq!(extended[0]["dateText"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")")); + + mongo_driver::drop_collection(&client, database, &collection).await.unwrap(); +} diff --git a/packages/app-tests/documentBrowser.test.ts b/packages/app-tests/documentBrowser.test.ts index 7e5b17f03b..05978eca24 100644 --- a/packages/app-tests/documentBrowser.test.ts +++ b/packages/app-tests/documentBrowser.test.ts @@ -26,4 +26,6 @@ test("mongo document table passes copy context to the data grid", () => { const source = documentBrowserSource(); assert.match(source, //); assert.match(source, /const customSaveHandler = computed\(\(\) => \(\{[\s\S]*?targetLabel: props\.collection,[\s\S]*?\}\)\);/); + assert.match(source, /mongo_copy_documents: copyDocuments\.value/); + assert.match(source, /result\.extended_documents\?\.length === nextDocuments\.length/); }); diff --git a/packages/app-tests/useDataGridExport.test.ts b/packages/app-tests/useDataGridExport.test.ts index a86466d7ea..f123aff8e0 100644 --- a/packages/app-tests/useDataGridExport.test.ts +++ b/packages/app-tests/useDataGridExport.test.ts @@ -314,6 +314,7 @@ test("copy MongoDB row as INSERT uses Mongo shell insert syntax", async () => { const jsonString = '{"endingBalance":{"beginningBalance":"0","endingBalance":"100","endingDate":"2024-11-25"},"Line":[]}'; const row = { id: 1, + sourceIndex: 0, data: ["6743e4bfa3f6f84bc3fff6c8", "577", "done", jsonString, 'ISODate("2024-11-25T02:45:36.184Z")'], isNew: false, isDeleted: false, @@ -331,6 +332,15 @@ test("copy MongoDB row as INSERT uses Mongo shell insert syntax", async () => { database: computed(() => "db"), context: computed(() => "results"), sourceColumns: computed(() => undefined), + mongoDocuments: computed(() => [ + { + _id: { $oid: "6743e4bfa3f6f84bc3fff6c8" }, + accountId: 577, + status: "done", + data: { endingBalance: { beginningBalance: "0", endingBalance: "100", endingDate: "2024-11-25" }, Line: [] }, + lastUpdatedDate: { $date: "2024-11-25T02:45:36.184Z" }, + }, + ]), columnTypes: computed(() => undefined), whereInput: computed(() => undefined), orderBy: computed(() => undefined), @@ -350,7 +360,20 @@ test("copy MongoDB row as INSERT uses Mongo shell insert syntax", async () => { assert.equal(apiMock.buildDataGridCopyInsertStatement.mock.calls.length, 0); assert.equal( clipboardMock.copyToClipboard.mock.calls[0][0], - 'db.getCollection("accounting_reconciliations").insert({"_id":ObjectId("6743e4bfa3f6f84bc3fff6c8"),"accountId":577,"status":"done","data":{"endingBalance":{"beginningBalance":"0","endingBalance":"100","endingDate":"2024-11-25"},"Line":[]},"lastUpdatedDate":ISODate("2024-11-25T02:45:36.184Z")});', + `db.getCollection("accounting_reconciliations").insert({ + "_id": ObjectId("6743e4bfa3f6f84bc3fff6c8"), + "accountId": 577, + "status": "done", + "data": { + "endingBalance": { + "beginningBalance": "0", + "endingBalance": "100", + "endingDate": "2024-11-25" + }, + "Line": [] + }, + "lastUpdatedDate": ISODate("2024-11-25T02:45:36.184Z") +});`, ); }); @@ -390,7 +413,18 @@ test("copy MongoDB rows as INSERT excludes _id for insert without primary keys", await composable.copyRowAsInsertWithoutPrimaryKeys(); assert.equal(apiMock.buildDataGridCopyInsertStatement.mock.calls.length, 0); - assert.equal(clipboardMock.copyToClipboard.mock.calls[0][0], 'db.getCollection("accounting_reconciliations").insertMany([{"status":"done"},{"status":"draft"}]);'); + assert.equal( + clipboardMock.copyToClipboard.mock.calls[0][0], + `db.getCollection("accounting_reconciliations") + .insertMany([ + { + "status": "done" + }, + { + "status": "draft" + } + ]);`, + ); }); test("copy row as INSERT refreshes prepared SQL after row data changes", async () => {