fix(mongodb): preserve BSON dates in copied inserts

This commit is contained in:
zipg authored and GitHub committed 2026-07-15 21:01:40 +08:00
1 parent 711d0ec57b
commit 9234c33e24
12 files changed
+160 -26

No files matched your search

@@ -55,6 +55,7 @@ type JsonRecord = Record<string, unknown>;
type ViewMode = "document" | "table"; type ViewMode = "document" | "table";
const documents = ref<JsonRecord[]>([]); const documents = ref<JsonRecord[]>([]);
const copyDocuments = ref<JsonRecord[]>([]);
const lastGridColumns = ref<string[]>([]); const lastGridColumns = ref<string[]>([]);
const total = ref(0); const total = ref(0);
const loading = ref(false); const loading = ref(false);
@@ -172,7 +173,7 @@ const gridResult = computed<QueryResult>(() => {
}), }),
); );
return { columns, rows, mongo_documents: docs, affected_rows: 0, execution_time_ms: 0, truncated: false }; return { columns, rows, mongo_documents: docs, mongo_copy_documents: copyDocuments.value, affected_rows: 0, execution_time_ms: 0, truncated: false };
}); });
const documentFilterFieldOptions = computed(() => gridResult.value.columns); const documentFilterFieldOptions = computed(() => gridResult.value.columns);
const documentStructuredFilterCount = computed(() => (appliedDocumentFilter.value ? 1 : 0)); const documentStructuredFilterCount = computed(() => (appliedDocumentFilter.value ? 1 : 0));
@@ -487,7 +488,9 @@ async function load() {
} }
}) })
: result.documents.map(asRecord); : result.documents.map(asRecord);
const nextCopyDocuments = result.extended_documents?.length === nextDocuments.length ? result.extended_documents.map(asRecord) : nextDocuments;
documents.value = nextDocuments; documents.value = nextDocuments;
copyDocuments.value = nextCopyDocuments;
if (nextDocuments.length > 0) { if (nextDocuments.length > 0) {
const keySet = new Set<string>(); const keySet = new Set<string>();
keySet.add("_id"); keySet.add("_id");
@@ -6651,7 +6651,7 @@ const {
database: computed(() => props.executionDatabase ?? props.database), database: computed(() => props.executionDatabase ?? props.database),
context: computed(() => props.context), context: computed(() => props.context),
sourceColumns: visibleSourceColumns, sourceColumns: visibleSourceColumns,
mongoDocuments: computed(() => props.result.mongo_documents), mongoDocuments: computed(() => props.result.mongo_copy_documents ?? props.result.mongo_documents),
columnTypes: visibleColumnTypes, columnTypes: visibleColumnTypes,
whereInput: computed(() => currentWhereInput()), whereInput: computed(() => currentWhereInput()),
orderBy: computed(() => currentOrderBy()), orderBy: computed(() => currentOrderBy()),
@@ -212,10 +212,10 @@ describe("useDataGridExport prepared row statements", () => {
}); });
it("preserves original Mongo string types in INSERT and applies explicit edits", async () => { it("preserves original Mongo string types in INSERT and applies explicit edits", async () => {
const item = { ...row(["123", "true", '{"kind":"literal"}', "2024-01-01 00:00:00", '{"role":"maintainer"}']), sourceIndex: 0 }; const item = { ...row(["123", "true", '{"kind":"literal"}', "2024-01-01 00:00:00", '{"role":"maintainer"}', 'ISODate("2025-05-06T08:35:32Z")']), sourceIndex: 0 };
item.isDirtyCol = [false, false, false, false, true]; item.isDirtyCol = [false, false, false, false, true, false];
const state = createMongoExportState({ const state = createMongoExportState({
columns: ["numericText", "booleanText", "jsonText", "dateText", "profile"], columns: ["numericText", "booleanText", "jsonText", "dateText", "profile", "lastUpdatedDate"],
item, item,
mongoDocuments: [ mongoDocuments: [
{ {
@@ -224,12 +224,22 @@ describe("useDataGridExport prepared row statements", () => {
jsonText: '{"kind":"literal"}', jsonText: '{"kind":"literal"}',
dateText: "2024-01-01 00:00:00", dateText: "2024-01-01 00:00:00",
profile: { role: "admin" }, profile: { role: "admin" },
lastUpdatedDate: { $date: "2025-05-06T08:35:32Z" },
}, },
], ],
}); });
await state.copyRowAsInsert(); await state.copyRowAsInsert();
expect(copyToClipboard).toHaveBeenCalledWith('db.getCollection("documents").insert({"numericText":"123","booleanText":"true","jsonText":"{\\"kind\\":\\"literal\\"}","dateText":"2024-01-01 00:00:00","profile":{"role":"maintainer"}});'); expect(copyToClipboard).toHaveBeenCalledWith(`db.getCollection("documents").insert({
"numericText": "123",
"booleanText": "true",
"jsonText": "{\\"kind\\":\\"literal\\"}",
"dateText": "2024-01-01 00:00:00",
"profile": {
"role": "maintainer"
},
"lastUpdatedDate": ISODate("2025-05-06T08:35:32Z")
});`);
}); });
}); });
@@ -13,6 +13,7 @@ import { uuid } from "@/lib/common/utils";
import { useSettingsStore } from "@/stores/settingsStore"; import { useSettingsStore } from "@/stores/settingsStore";
import { expandNestedJsonStringsForCopy } from "@/lib/common/jsonCopyValue"; import { expandNestedJsonStringsForCopy } from "@/lib/common/jsonCopyValue";
import { buildMongoCopyDocumentFromOriginal, buildMongoCopyInsertDocument, formatMongoShellLiteral, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues"; import { buildMongoCopyDocumentFromOriginal, buildMongoCopyInsertDocument, formatMongoShellLiteral, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
import { formatMongoShellText } from "@/lib/mongo/mongoFormatter";
import type { DatabaseType, QueryResult } from "@/types/database"; import type { DatabaseType, QueryResult } from "@/types/database";
import type { QueryResultExportRequest } from "@/lib/backend/api"; import type { QueryResultExportRequest } from "@/lib/backend/api";
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
@@ -352,15 +353,17 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
const promise = Promise.resolve().then(async () => { const promise = Promise.resolve().then(async () => {
const statement = const statement =
databaseType.value === "mongodb" databaseType.value === "mongodb"
? buildMongoCopyInsertStatement({ ? formatMongoCopyInsertStatement(
collection: copyInsertTargetLabel?.value || tableMeta.value?.tableName || "collection", buildMongoCopyInsertStatement({
columns: columns.value, collection: copyInsertTargetLabel?.value || tableMeta.value?.tableName || "collection",
sourceColumns: sourceColumns.value, columns: columns.value,
rows, sourceColumns: sourceColumns.value,
mongoDocuments: options.mongoDocuments?.value, rows,
excludePrimaryKeys, mongoDocuments: options.mongoDocuments?.value,
insertMode, excludePrimaryKeys,
}) insertMode,
}),
)
: await buildDataGridCopyInsertStatement({ : await buildDataGridCopyInsertStatement({
databaseType: databaseType.value, databaseType: databaseType.value,
tableMeta: tableMeta.value, tableMeta: tableMeta.value,
@@ -1375,6 +1378,15 @@ function buildMongoCopyInsertStatement(options: { collection: string; columns: s
return `${collection}.insertMany(${formatMongoShellLiteral(documents)});`; return `${collection}.insertMany(${formatMongoShellLiteral(documents)});`;
} }
function formatMongoCopyInsertStatement(statement: string | undefined): string | undefined {
if (!statement) return undefined;
try {
return formatMongoShellText(statement);
} catch {
return statement;
}
}
function compactLocalTimestamp(date = new Date()): string { function compactLocalTimestamp(date = new Date()): string {
const yy = String(date.getFullYear() % 100).padStart(2, "0"); const yy = String(date.getFullYear() % 100).padStart(2, "0");
const month = String(date.getMonth() + 1).padStart(2, "0"); const month = String(date.getMonth() + 1).padStart(2, "0");
+1
View File
@@ -1752,6 +1752,7 @@ export async function zookeeperDelete(connectionId: string, key: string): Promis
export interface MongoDocumentResult { export interface MongoDocumentResult {
documents: any[]; documents: any[];
raw_documents?: string[]; raw_documents?: string[];
extended_documents?: any[];
total: number; total: number;
} }
+2
View File
@@ -515,6 +515,8 @@ export interface QueryResult {
* preview. This is populated only for MongoDB document query results. * preview. This is populated only for MongoDB document query results.
*/ */
mongo_documents?: unknown[]; mongo_documents?: unknown[];
/** Type-preserving Extended JSON documents used when copying MongoDB values. */
mongo_copy_documents?: unknown[];
affected_rows: number; affected_rows: number;
execution_time_ms: number; execution_time_ms: number;
truncated?: boolean; truncated?: boolean;
@@ -438,7 +438,12 @@ fn search_response_to_document_result(result: SearchResponse) -> Result<MongoDoc
.collect::<Result<Vec<_>, _>>() .collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("Elasticsearch document serialization failed: {e}"))?; .map_err(|e| format!("Elasticsearch document serialization failed: {e}"))?;
Ok(MongoDocumentResult { documents, raw_documents: Some(raw_documents), total: result.hits.total.value() }) Ok(MongoDocumentResult {
documents,
raw_documents: Some(raw_documents),
extended_documents: None,
total: result.hits.total.value(),
})
} }
fn build_find_documents_body( fn build_find_documents_body(
+34 -7
View File
@@ -17,6 +17,8 @@ pub struct MongoDocumentResult {
pub documents: Vec<serde_json::Value>, pub documents: Vec<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub raw_documents: Option<Vec<String>>, pub raw_documents: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub extended_documents: Option<Vec<serde_json::Value>>,
pub total: u64, pub total: u64,
} }
@@ -586,12 +588,14 @@ pub async fn find_documents(
let mut cursor = find.await.map_err(|e| e.to_string())?; let mut cursor = find.await.map_err(|e| e.to_string())?;
let mut documents = Vec::new(); let mut documents = Vec::new();
let mut extended_documents = Vec::new();
while cursor.advance().await.map_err(|e| e.to_string())? { while cursor.advance().await.map_err(|e| e.to_string())? {
let doc = cursor.deserialize_current().map_err(|e| e.to_string())?; let doc = cursor.deserialize_current().map_err(|e| e.to_string())?;
documents.push(bson_to_json(&Bson::Document(doc))); documents.push(bson_to_json(&Bson::Document(doc.clone())));
extended_documents.push(Bson::Document(doc).into_relaxed_extjson());
} }
Ok(MongoDocumentResult { documents, raw_documents: None, total }) Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total })
} }
#[derive(Debug, Default, Deserialize)] #[derive(Debug, Default, Deserialize)]
@@ -727,7 +731,7 @@ pub async fn find_documents_extended_json(
documents.push(Bson::Document(doc).into_relaxed_extjson()); documents.push(Bson::Document(doc).into_relaxed_extjson());
} }
Ok(MongoDocumentResult { documents, raw_documents: None, total }) Ok(MongoDocumentResult { extended_documents: Some(documents.clone()), documents, raw_documents: None, total })
} }
pub async fn aggregate_documents( pub async fn aggregate_documents(
@@ -749,15 +753,18 @@ pub async fn aggregate_documents(
let max_rows = max_rows.unwrap_or(100); let max_rows = max_rows.unwrap_or(100);
let fetch_limit = max_rows.saturating_add(1); let fetch_limit = max_rows.saturating_add(1);
let mut documents = Vec::new(); let mut documents = Vec::new();
let mut extended_documents = Vec::new();
while documents.len() < fetch_limit && cursor.advance().await.map_err(|e| e.to_string())? { while documents.len() < fetch_limit && cursor.advance().await.map_err(|e| e.to_string())? {
let doc = cursor.deserialize_current().map_err(|e| e.to_string())?; let doc = cursor.deserialize_current().map_err(|e| e.to_string())?;
documents.push(bson_to_json(&Bson::Document(doc))); documents.push(bson_to_json(&Bson::Document(doc.clone())));
extended_documents.push(Bson::Document(doc).into_relaxed_extjson());
} }
let total = documents.len() as u64; let total = documents.len() as u64;
if documents.len() > max_rows { if documents.len() > max_rows {
documents.truncate(max_rows); documents.truncate(max_rows);
extended_documents.truncate(max_rows);
} }
Ok(MongoDocumentResult { documents, raw_documents: None, total }) Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total })
} }
pub async fn create_index( pub async fn create_index(
@@ -1119,11 +1126,17 @@ fn find_and_modify_array_filters(
fn single_document_result(document: Option<Document>) -> MongoDocumentResult { fn single_document_result(document: Option<Document>) -> MongoDocumentResult {
match document { match document {
Some(document) => MongoDocumentResult { Some(document) => MongoDocumentResult {
documents: vec![bson_to_json(&Bson::Document(document))], documents: vec![bson_to_json(&Bson::Document(document.clone()))],
raw_documents: None, raw_documents: None,
extended_documents: Some(vec![Bson::Document(document).into_relaxed_extjson()]),
total: 1, total: 1,
}, },
None => MongoDocumentResult { documents: Vec::new(), raw_documents: None, total: 0 }, None => MongoDocumentResult {
documents: Vec::new(),
raw_documents: None,
extended_documents: Some(Vec::new()),
total: 0,
},
} }
} }
@@ -1825,6 +1838,20 @@ mod tests {
assert_eq!(value, serde_json::json!("ISODate(\"2026-06-10T13:59:31.287Z\")")); assert_eq!(value, serde_json::json!("ISODate(\"2026-06-10T13:59:31.287Z\")"));
} }
#[test]
fn mongo_document_result_keeps_extended_json_types_for_copying() {
let date = DateTime::parse_rfc3339_str("2025-05-06T08:35:32Z").unwrap();
let result = single_document_result(Some(doc! {
"lastUpdatedDate": date,
"dateText": "ISODate(\"2025-05-06T08:35:32Z\")",
}));
assert_eq!(result.documents[0]["lastUpdatedDate"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")"));
let extended = result.extended_documents.expect("extended documents");
assert_eq!(extended[0]["lastUpdatedDate"], serde_json::json!({ "$date": "2025-05-06T08:35:32Z" }));
assert_eq!(extended[0]["dateText"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")"));
}
#[test] #[test]
fn bson_to_json_preserves_unsafe_int64_for_js() { fn bson_to_json_preserves_unsafe_int64_for_js() {
let value = bson_to_json(&Bson::Int64(2_326_645_729_978_441_729)); let value = bson_to_json(&Bson::Int64(2_326_645_729_978_441_729));
+7 -1
View File
@@ -523,6 +523,7 @@ pub async fn find_documents(
return Ok(crate::db::mongo_driver::MongoDocumentResult { return Ok(crate::db::mongo_driver::MongoDocumentResult {
documents, documents,
raw_documents: None, raw_documents: None,
extended_documents: None,
total: result.affected_rows, total: result.affected_rows,
}); });
} }
@@ -566,7 +567,12 @@ pub async fn find_documents(
Value::Object(map) Value::Object(map)
}) })
.collect(); .collect();
Ok(crate::db::mongo_driver::MongoDocumentResult { documents, raw_documents: None, total: result.affected_rows }) Ok(crate::db::mongo_driver::MongoDocumentResult {
documents,
raw_documents: None,
extended_documents: None,
total: result.affected_rows,
})
} }
pub async fn execute_rest_query(client: &VectorClient, input: &str) -> Result<QueryResult, String> { pub async fn execute_rest_query(client: &VectorClient, input: &str) -> Result<QueryResult, String> {
@@ -1,6 +1,7 @@
use std::time::Duration; use std::time::Duration;
use dbx_core::db::mongo_driver; use dbx_core::db::mongo_driver;
use mongodb::bson::{doc, Bson, DateTime};
#[tokio::test] #[tokio::test]
#[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"] #[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"]
@@ -34,3 +35,34 @@ async fn find_one_returns_only_the_sorted_document() {
assert_eq!(result.documents, vec![serde_json::json!({ "name": "new" })]); assert_eq!(result.documents, vec![serde_json::json!({ "name": "new" })]);
mongo_driver::drop_collection(&client, database, &collection).await.unwrap(); mongo_driver::drop_collection(&client, database, &collection).await.unwrap();
} }
#[tokio::test]
#[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"]
async fn find_documents_returns_type_preserving_copy_documents() {
let url = std::env::var("DBX_LIVE_MONGODB_URL").expect("DBX_LIVE_MONGODB_URL");
let client = mongo_driver::connect(&url, Duration::from_secs(10), Duration::from_secs(60)).await.unwrap();
let database = "dbx_live_find_one";
let collection = format!("copy_types_{}", std::process::id());
client
.database(database)
.collection(&collection)
.insert_one(doc! {
"lastUpdatedDate": DateTime::parse_rfc3339_str("2025-05-06T08:35:32Z").unwrap(),
"dateText": Bson::String("ISODate(\"2025-05-06T08:35:32Z\")".to_string()),
})
.await
.unwrap();
let result =
mongo_driver::find_documents(&client, database, &collection, 0, 10, Some("{}"), Some(r#"{"_id":0}"#), None)
.await
.unwrap();
assert_eq!(result.documents[0]["lastUpdatedDate"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")"));
let extended = result.extended_documents.expect("extended documents");
assert_eq!(extended[0]["lastUpdatedDate"], serde_json::json!({ "$date": "2025-05-06T08:35:32Z" }));
assert_eq!(extended[0]["dateText"], serde_json::json!("ISODate(\"2025-05-06T08:35:32Z\")"));
mongo_driver::drop_collection(&client, database, &collection).await.unwrap();
}
+2
View File
@@ -26,4 +26,6 @@ test("mongo document table passes copy context to the data grid", () => {
const source = documentBrowserSource(); const source = documentBrowserSource();
assert.match(source, /<DataGrid[\s\S]*?:database-type="props\.databaseType"[\s\S]*?<\/DataGrid>/); assert.match(source, /<DataGrid[\s\S]*?:database-type="props\.databaseType"[\s\S]*?<\/DataGrid>/);
assert.match(source, /const customSaveHandler = computed<CustomSaveHandler>\(\(\) => \(\{[\s\S]*?targetLabel: props\.collection,[\s\S]*?\}\)\);/); assert.match(source, /const customSaveHandler = computed<CustomSaveHandler>\(\(\) => \(\{[\s\S]*?targetLabel: props\.collection,[\s\S]*?\}\)\);/);
assert.match(source, /mongo_copy_documents: copyDocuments\.value/);
assert.match(source, /result\.extended_documents\?\.length === nextDocuments\.length/);
}); });
+36 -2
View File
@@ -314,6 +314,7 @@ test("copy MongoDB row as INSERT uses Mongo shell insert syntax", async () => {
const jsonString = '{"endingBalance":{"beginningBalance":"0","endingBalance":"100","endingDate":"2024-11-25"},"Line":[]}'; const jsonString = '{"endingBalance":{"beginningBalance":"0","endingBalance":"100","endingDate":"2024-11-25"},"Line":[]}';
const row = { const row = {
id: 1, id: 1,
sourceIndex: 0,
data: ["6743e4bfa3f6f84bc3fff6c8", "577", "done", jsonString, 'ISODate("2024-11-25T02:45:36.184Z")'], data: ["6743e4bfa3f6f84bc3fff6c8", "577", "done", jsonString, 'ISODate("2024-11-25T02:45:36.184Z")'],
isNew: false, isNew: false,
isDeleted: false, isDeleted: false,
@@ -331,6 +332,15 @@ test("copy MongoDB row as INSERT uses Mongo shell insert syntax", async () => {
database: computed(() => "db"), database: computed(() => "db"),
context: computed(() => "results"), context: computed(() => "results"),
sourceColumns: computed(() => undefined), sourceColumns: computed(() => undefined),
mongoDocuments: computed(() => [
{
_id: { $oid: "6743e4bfa3f6f84bc3fff6c8" },
accountId: 577,
status: "done",
data: { endingBalance: { beginningBalance: "0", endingBalance: "100", endingDate: "2024-11-25" }, Line: [] },
lastUpdatedDate: { $date: "2024-11-25T02:45:36.184Z" },
},
]),
columnTypes: computed(() => undefined), columnTypes: computed(() => undefined),
whereInput: computed(() => undefined), whereInput: computed(() => undefined),
orderBy: computed(() => undefined), orderBy: computed(() => undefined),
@@ -350,7 +360,20 @@ test("copy MongoDB row as INSERT uses Mongo shell insert syntax", async () => {
assert.equal(apiMock.buildDataGridCopyInsertStatement.mock.calls.length, 0); assert.equal(apiMock.buildDataGridCopyInsertStatement.mock.calls.length, 0);
assert.equal( assert.equal(
clipboardMock.copyToClipboard.mock.calls[0][0], clipboardMock.copyToClipboard.mock.calls[0][0],
'db.getCollection("accounting_reconciliations").insert({"_id":ObjectId("6743e4bfa3f6f84bc3fff6c8"),"accountId":577,"status":"done","data":{"endingBalance":{"beginningBalance":"0","endingBalance":"100","endingDate":"2024-11-25"},"Line":[]},"lastUpdatedDate":ISODate("2024-11-25T02:45:36.184Z")});', `db.getCollection("accounting_reconciliations").insert({
"_id": ObjectId("6743e4bfa3f6f84bc3fff6c8"),
"accountId": 577,
"status": "done",
"data": {
"endingBalance": {
"beginningBalance": "0",
"endingBalance": "100",
"endingDate": "2024-11-25"
},
"Line": []
},
"lastUpdatedDate": ISODate("2024-11-25T02:45:36.184Z")
});`,
); );
}); });
@@ -390,7 +413,18 @@ test("copy MongoDB rows as INSERT excludes _id for insert without primary keys",
await composable.copyRowAsInsertWithoutPrimaryKeys(); await composable.copyRowAsInsertWithoutPrimaryKeys();
assert.equal(apiMock.buildDataGridCopyInsertStatement.mock.calls.length, 0); assert.equal(apiMock.buildDataGridCopyInsertStatement.mock.calls.length, 0);
assert.equal(clipboardMock.copyToClipboard.mock.calls[0][0], 'db.getCollection("accounting_reconciliations").insertMany([{"status":"done"},{"status":"draft"}]);'); assert.equal(
clipboardMock.copyToClipboard.mock.calls[0][0],
`db.getCollection("accounting_reconciliations")
.insertMany([
{
"status": "done"
},
{
"status": "draft"
}
]);`,
);
}); });
test("copy row as INSERT refreshes prepared SQL after row data changes", async () => { test("copy row as INSERT refreshes prepared SQL after row data changes", async () => {