diff --git a/apps/desktop/src/components/document/DocumentBrowser.vue b/apps/desktop/src/components/document/DocumentBrowser.vue index 4d8837647e..7b343c4147 100644 --- a/apps/desktop/src/components/document/DocumentBrowser.vue +++ b/apps/desktop/src/components/document/DocumentBrowser.vue @@ -28,6 +28,8 @@ import { type DocumentFilterMode, type DocumentFilterRule, } from "@/lib/app/documentStoreProvider"; +import { documentStoreValueForGrid, parseDocumentStoreInputValue, serializeDocumentStoreId, stringifyDocumentStoreValue } from "@/lib/app/documentJsonValues"; +import { isLosslessJsonNumber, parseJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat"; import { buildMongoInsertDocument, buildMongoUpdateDocument, formatMongoShellLiteral, mongoDocumentIdForGrid, parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues"; import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize"; import { useSettingsStore } from "@/stores/settingsStore"; @@ -163,8 +165,8 @@ const gridResult = computed(() => { columns.map((col) => { const val = doc[col]; if (val === undefined || val === null) return null; - if (col === "_id") return mongoDocumentIdForGrid(val); - if (typeof val === "object") return JSON.stringify(val); + if (col === "_id") return documentStoreProvider.value.kind === "elasticsearch" ? documentStoreValueForGrid(val, "elasticsearch") : mongoDocumentIdForGrid(val); + if (typeof val === "object") return documentStoreValueForGrid(val, documentStoreProvider.value.kind); if (typeof val === "string" || typeof val === "number" || typeof val === "boolean") return val; return String(val); }), @@ -313,10 +315,10 @@ async function gridSave(changes: DocumentGridChanges) { if (newVal === null) { delete updated[col]; } else { - updated[col] = parseMongoDocumentInputValue(newVal); + updated[col] = parseDocumentStoreInputValue(newVal, "elasticsearch"); } } - await api.documentUpdateDocument(props.connectionId, props.database, props.collection, String(id), JSON.stringify(updated), routing); + await api.documentUpdateDocument(props.connectionId, props.database, props.collection, String(id), stringifyDocumentStoreValue(updated, "elasticsearch"), routing); continue; } @@ -337,13 +339,13 @@ async function gridSave(changes: DocumentGridChanges) { } for (const newRow of changes.newRows) { - const doc = buildMongoInsertDocument(newRow, cols); + const doc = isEs ? buildElasticsearchInsertDocument(newRow, cols) : buildMongoInsertDocument(newRow, cols); if (isEs) { const id = documentIdFromGridValue(newRow[idColIdx]); if (id) { - await api.documentUpdateDocument(props.connectionId, props.database, props.collection, id, JSON.stringify(doc), documentRoutingFromGridRow(newRow, cols)); + await api.documentUpdateDocument(props.connectionId, props.database, props.collection, id, stringifyDocumentStoreValue(doc, "elasticsearch"), documentRoutingFromGridRow(newRow, cols)); } else { - await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc)); + await api.documentInsertDocument(props.connectionId, props.database, props.collection, stringifyDocumentStoreValue(doc, "elasticsearch")); } continue; } @@ -353,6 +355,17 @@ async function gridSave(changes: DocumentGridChanges) { await load(); } +function buildElasticsearchInsertDocument(row: MongoInputValue[], columns: string[]): JsonRecord { + const doc: JsonRecord = {}; + for (let columnIndex = 0; columnIndex < columns.length; columnIndex += 1) { + const column = columns[columnIndex]; + if (!column || column === "_id" || column === "_routing") continue; + const value = row[columnIndex]; + if (value !== null) doc[column] = parseDocumentStoreInputValue(value, "elasticsearch"); + } + return doc; +} + function mongoIdPreview(val: unknown): string { if (val === null || val === undefined) return "null"; if (typeof val === "string" && /^[a-fA-F0-9]{24}$/.test(val)) return `ObjectId("${val}")`; @@ -368,13 +381,13 @@ function elasticsearchRoutingPreview(routing: string | undefined): string { } function buildElasticsearchPartialUpdateDocument(changes: Map, columns: string[]): Record { - const filtered = new Map(); + const document: Record = {}; for (const [colIdx, newVal] of changes) { const col = columns[colIdx]; if (col === "_id" || col === "_routing") continue; - filtered.set(colIdx, newVal); + if (col && newVal !== null) document[col] = parseDocumentStoreInputValue(newVal, "elasticsearch"); } - return buildMongoUpdateDocument(filtered, columns); + return document; } async function previewDocumentChanges(changes: DocumentGridChanges): Promise { @@ -391,7 +404,7 @@ async function previewDocumentChanges(changes: DocumentGridChanges): Promise= 0 ? documentIdFromGridValue(newRow[idColIdx]) : null; if (id) { - stmts.push(`PUT /${coll}/_doc/${elasticsearchPathIdPreview(id)}\n${JSON.stringify(doc, null, 2)}`); + stmts.push(`PUT /${coll}/_doc/${elasticsearchPathIdPreview(id)}\n${stringifyDocumentStoreValue(doc, "elasticsearch", 2)}`); } else { - stmts.push(`POST /${coll}/_doc\n${JSON.stringify(doc, null, 2)}`); + stmts.push(`POST /${coll}/_doc\n${stringifyDocumentStoreValue(doc, "elasticsearch", 2)}`); } } else { stmts.push(`db.${coll}.insertOne(${formatMongoShellLiteral(doc)})`); @@ -464,7 +477,16 @@ async function load() { const sort = currentDocumentSortJson(sortInput.value); const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, page.value * pageSize.value, pageSize.value, filter, undefined, sort, executionId); if (documentLoadExecutionId.value !== executionId) return; - const nextDocuments = result.documents.map(asRecord); + const nextDocuments = + documentStoreProvider.value.kind === "elasticsearch" && result.raw_documents?.length === result.documents.length + ? result.raw_documents.map((raw, index) => { + try { + return asRecord(parseJsonPreservingLargeNumbers(raw)); + } catch { + return asRecord(result.documents[index]); + } + }) + : result.documents.map(asRecord); documents.value = nextDocuments; if (nextDocuments.length > 0) { const keySet = new Set(); @@ -554,13 +576,13 @@ function syncSelectedDocumentAfterLoad(previousSelectedIdx: number | null, previ selectedIdx.value = nextIdx; if (!isEditing.value) { - editJson.value = JSON.stringify(documents.value[nextIdx], null, 2); + editJson.value = stringifyDocumentStoreValue(documents.value[nextIdx], documentStoreProvider.value.kind, 2); } } function selectDoc(idx: number) { selectedIdx.value = idx; - editJson.value = JSON.stringify(documents.value[idx], null, 2); + editJson.value = stringifyDocumentStoreValue(documents.value[idx], documentStoreProvider.value.kind, 2); isEditing.value = false; isNew.value = false; editFields.value = []; @@ -593,13 +615,24 @@ function cancelEdit() { return; } if (selectedDoc.value) { - editJson.value = JSON.stringify(selectedDoc.value, null, 2); + editJson.value = stringifyDocumentStoreValue(selectedDoc.value, documentStoreProvider.value.kind, 2); } editFields.value = []; error.value = ""; } function createEditNode(keyName: string, value: unknown, readonlyKey: boolean, readonlyValue: boolean): EditNode { + if (isLosslessJsonNumber(value)) { + return { + key: uuid(), + keyName, + kind: "value", + valueText: value.raw, + readonlyKey, + readonlyValue, + children: [], + }; + } if (Array.isArray(value)) { return { key: uuid(), @@ -655,12 +688,12 @@ function formatForEdit(value: unknown): string { if (value === undefined) return ""; if (value === null) return "null"; if (typeof value === "string") return JSON.stringify(value); - if (typeof value === "object") return JSON.stringify(value, null, 2); + if (typeof value === "object") return stringifyDocumentStoreValue(value, documentStoreProvider.value.kind, 2); return String(value); } function parseFieldValue(raw: string): unknown { - return parseMongoDocumentInputValue(raw); + return parseDocumentStoreInputValue(raw, documentStoreProvider.value.kind); } function buildObjectFromNodes(nodes: EditNode[], path: string): JsonRecord { @@ -695,7 +728,7 @@ async function saveDoc() { try { const doc = buildDocumentFromFields(); if (isNew.value) { - await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc)); + await api.documentInsertDocument(props.connectionId, props.database, props.collection, stringifyDocumentStoreValue(doc, documentStoreProvider.value.kind)); } else if (selectedIdx.value !== null) { const current = documents.value[selectedIdx.value]; const id = current?._id; @@ -703,14 +736,14 @@ async function saveDoc() { error.value = "No _id field"; return; } - await api.documentUpdateDocument(props.connectionId, props.database, props.collection, serializeMongoDocumentId(id), JSON.stringify(doc), documentRoutingFromDocument(current)); + await api.documentUpdateDocument(props.connectionId, props.database, props.collection, serializeDocumentStoreId(id, documentStoreProvider.value.kind), stringifyDocumentStoreValue(doc, documentStoreProvider.value.kind), documentRoutingFromDocument(current)); } isEditing.value = false; isNew.value = false; editFields.value = []; await load(); if (selectedIdx.value !== null && documents.value[selectedIdx.value]) { - editJson.value = JSON.stringify(documents.value[selectedIdx.value], null, 2); + editJson.value = stringifyDocumentStoreValue(documents.value[selectedIdx.value], documentStoreProvider.value.kind, 2); } } catch (e: unknown) { error.value = e instanceof Error ? e.message : String(e); @@ -723,7 +756,7 @@ async function applyDeleteDoc(idx: number) { if (!id) return; error.value = ""; try { - await api.documentDeleteDocument(props.connectionId, props.database, props.collection, serializeMongoDocumentId(id), documentRoutingFromDocument(doc)); + await api.documentDeleteDocument(props.connectionId, props.database, props.collection, serializeDocumentStoreId(id, documentStoreProvider.value.kind), documentRoutingFromDocument(doc)); if (selectedIdx.value === idx) { selectedIdx.value = null; editJson.value = ""; @@ -767,7 +800,7 @@ function docPreview(doc: JsonRecord): string { const keys = Object.keys(doc) .filter((k) => k !== "_id") .slice(0, 3); - const preview = keys.map((k) => `${k}: ${JSON.stringify(doc[k]).substring(0, 30)}`).join(", "); + const preview = keys.map((k) => `${k}: ${stringifyDocumentStoreValue(doc[k], documentStoreProvider.value.kind).substring(0, 30)}`).join(", "); return `${id} - ${preview}`; } diff --git a/apps/desktop/src/lib/app/documentJsonValues.ts b/apps/desktop/src/lib/app/documentJsonValues.ts new file mode 100644 index 0000000000..54dc47cffe --- /dev/null +++ b/apps/desktop/src/lib/app/documentJsonValues.ts @@ -0,0 +1,30 @@ +import { isLosslessJsonNumber, parseJsonPreservingLargeNumbers, stringifyJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat"; +import { parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues"; +import type { DocumentStoreKind } from "@/lib/app/documentStoreProvider"; + +export function parseDocumentStoreInputValue(raw: MongoInputValue, kind: DocumentStoreKind): unknown { + if (kind === "mongodb") return parseMongoDocumentInputValue(raw); + if (raw === null || typeof raw === "number" || typeof raw === "boolean") return raw; + + const trimmed = raw.trim(); + if (!trimmed) return ""; + try { + return parseJsonPreservingLargeNumbers(trimmed); + } catch { + return trimmed; + } +} + +export function stringifyDocumentStoreValue(value: unknown, kind: DocumentStoreKind, indent?: number): string { + return kind === "elasticsearch" ? stringifyJsonPreservingLargeNumbers(value, indent) : JSON.stringify(value, null, indent ?? undefined); +} + +export function documentStoreValueForGrid(value: unknown, kind: DocumentStoreKind): MongoInputValue { + if (kind === "elasticsearch" && isLosslessJsonNumber(value)) return value.raw; + if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value; + return stringifyDocumentStoreValue(value, kind); +} + +export function serializeDocumentStoreId(value: unknown, kind: DocumentStoreKind): string { + return kind === "elasticsearch" ? String(value) : serializeMongoDocumentId(value); +} diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 0233bf3947..969c6255a5 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -1729,6 +1729,7 @@ export async function zookeeperDelete(connectionId: string, key: string): Promis // --- MongoDB --- export interface MongoDocumentResult { documents: any[]; + raw_documents?: string[]; total: number; } diff --git a/apps/desktop/src/lib/common/__tests__/safeJsonFormat.spec.ts b/apps/desktop/src/lib/common/__tests__/safeJsonFormat.spec.ts index 92315c8138..967b17e8fb 100644 --- a/apps/desktop/src/lib/common/__tests__/safeJsonFormat.spec.ts +++ b/apps/desktop/src/lib/common/__tests__/safeJsonFormat.spec.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat } from "../safeJsonFormat"; +import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat, stringifyJsonPreservingLargeNumbers } from "../safeJsonFormat"; describe("safeJsonFormat", () => { it("preserves large integers exceeding MAX_SAFE_INTEGER", () => { @@ -85,6 +85,11 @@ describe("safeJsonFormat", () => { expect(safeJsonFormat(input, 2)).toContain("1.234567890123456789e20"); expect(safeJsonFormat(input, 2)).toContain("1e999"); }); + + it("stringifies parsed lossless numbers back to numeric JSON tokens", () => { + const parsed = parseJsonPreservingLargeNumbers('{"id":2018551659033767937,"stringId":"2018551659033767937"}'); + expect(stringifyJsonPreservingLargeNumbers(parsed)).toBe('{"id":2018551659033767937,"stringId":"2018551659033767937"}'); + }); }); describe("formatJsonSource", () => { diff --git a/apps/desktop/src/lib/common/safeJsonFormat.ts b/apps/desktop/src/lib/common/safeJsonFormat.ts index 5836bf9c1a..0f73ae1ed8 100644 --- a/apps/desktop/src/lib/common/safeJsonFormat.ts +++ b/apps/desktop/src/lib/common/safeJsonFormat.ts @@ -46,6 +46,35 @@ export function safeJsonFormat(text: string, indent?: number): string { return result; } +/** + * Stringifies values produced by {@link parseJsonPreservingLargeNumbers} + * without routing their numeric literals through JavaScript Number. + */ +export function stringifyJsonPreservingLargeNumbers(value: unknown, indent?: number): string { + let placeholderPrefix = "__DBX_LOSSLESS_NUMBER_"; + const ordinaryJson = JSON.stringify(value); + while (ordinaryJson?.includes(placeholderPrefix)) placeholderPrefix += "_"; + + const numbers = new Map(); + const result = JSON.stringify( + value, + (_key, item) => { + if (!isLosslessJsonNumber(item)) return item; + const placeholder = `${placeholderPrefix}${numbers.size}__`; + numbers.set(placeholder, item.raw); + return placeholder; + }, + indent ?? undefined, + ); + + if (result === undefined) throw new TypeError("Value is not JSON serializable"); + let restored = result; + for (const [placeholder, raw] of numbers) { + restored = restored.replaceAll(JSON.stringify(placeholder), raw); + } + return restored; +} + /** * Validate JSON and re-emit source tokens while only changing insignificant * whitespace. Unlike {@link safeJsonFormat}, this keeps duplicate object diff --git a/crates/dbx-core/src/db/elasticsearch_driver.rs b/crates/dbx-core/src/db/elasticsearch_driver.rs index 0baad89472..b0f75894bf 100644 --- a/crates/dbx-core/src/db/elasticsearch_driver.rs +++ b/crates/dbx-core/src/db/elasticsearch_driver.rs @@ -408,6 +408,10 @@ pub async fn find_documents( let result: SearchResponse = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?; + search_response_to_document_result(result) +} + +fn search_response_to_document_result(result: SearchResponse) -> Result { let documents: Vec = result .hits .hits @@ -425,7 +429,16 @@ pub async fn find_documents( }) .collect(); - Ok(MongoDocumentResult { documents, total: result.hits.total.value() }) + // Keep the exact JSON numeric tokens alongside the compatibility value tree. + // Desktop IPC and browser JSON parsing otherwise coerce ES long values through + // JavaScript Number before the document editor can serialize them again. + let raw_documents = documents + .iter() + .map(serde_json::to_string) + .collect::, _>>() + .map_err(|e| format!("Elasticsearch document serialization failed: {e}"))?; + + Ok(MongoDocumentResult { documents, raw_documents: Some(raw_documents), total: result.hits.total.value() }) } fn build_find_documents_body( @@ -1743,6 +1756,21 @@ mod tests { assert_eq!(response.hits.hits[0].routing.as_deref(), Some("tenant-1")); } + #[test] + fn preserves_long_literals_in_document_transport_json() { + let response: SearchResponse = serde_json::from_str( + r#"{"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"doc-1","_source":{"id":2018551659033767937,"string_id":"2018551659033767937","legacy":{"$numberLong":"2018551659033767937"}}}]}}"#, + ) + .unwrap(); + + let result = super::search_response_to_document_result(response).unwrap(); + + assert_eq!( + result.raw_documents.as_deref(), + Some(&[r#"{"id":2018551659033767937,"string_id":"2018551659033767937","legacy":{"$numberLong":"2018551659033767937"},"_id":"doc-1"}"#.to_string()][..]) + ); + } + #[test] fn parses_search_response_rows_with_routing_metadata() { let result = super::parse_elasticsearch_response( diff --git a/crates/dbx-core/src/db/mongo_driver.rs b/crates/dbx-core/src/db/mongo_driver.rs index 4a4bb0e2d7..09e487d335 100644 --- a/crates/dbx-core/src/db/mongo_driver.rs +++ b/crates/dbx-core/src/db/mongo_driver.rs @@ -15,6 +15,8 @@ use std::{collections::HashSet, time::Duration}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MongoDocumentResult { pub documents: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw_documents: Option>, pub total: u64, } @@ -589,7 +591,7 @@ pub async fn find_documents( documents.push(bson_to_json(&Bson::Document(doc))); } - Ok(MongoDocumentResult { documents, total }) + Ok(MongoDocumentResult { documents, raw_documents: None, total }) } #[derive(Debug, Default, Deserialize)] @@ -725,7 +727,7 @@ pub async fn find_documents_extended_json( documents.push(Bson::Document(doc).into_relaxed_extjson()); } - Ok(MongoDocumentResult { documents, total }) + Ok(MongoDocumentResult { documents, raw_documents: None, total }) } pub async fn aggregate_documents( @@ -755,7 +757,7 @@ pub async fn aggregate_documents( if documents.len() > max_rows { documents.truncate(max_rows); } - Ok(MongoDocumentResult { documents, total }) + Ok(MongoDocumentResult { documents, raw_documents: None, total }) } pub async fn create_index( @@ -1111,8 +1113,12 @@ fn find_and_modify_array_filters( fn single_document_result(document: Option) -> MongoDocumentResult { match document { - Some(document) => MongoDocumentResult { documents: vec![bson_to_json(&Bson::Document(document))], total: 1 }, - None => MongoDocumentResult { documents: Vec::new(), total: 0 }, + Some(document) => MongoDocumentResult { + documents: vec![bson_to_json(&Bson::Document(document))], + raw_documents: None, + total: 1, + }, + None => MongoDocumentResult { documents: Vec::new(), raw_documents: None, total: 0 }, } } diff --git a/crates/dbx-core/src/db/vector_driver.rs b/crates/dbx-core/src/db/vector_driver.rs index 59aad278b7..61b9d9416f 100644 --- a/crates/dbx-core/src/db/vector_driver.rs +++ b/crates/dbx-core/src/db/vector_driver.rs @@ -520,7 +520,11 @@ pub async fn find_documents( Value::Object(map) }) .collect(); - return Ok(crate::db::mongo_driver::MongoDocumentResult { documents, total: result.affected_rows }); + return Ok(crate::db::mongo_driver::MongoDocumentResult { + documents, + raw_documents: None, + total: result.affected_rows, + }); } let query = match client.kind { @@ -562,7 +566,7 @@ pub async fn find_documents( Value::Object(map) }) .collect(); - Ok(crate::db::mongo_driver::MongoDocumentResult { documents, total: result.affected_rows }) + Ok(crate::db::mongo_driver::MongoDocumentResult { documents, raw_documents: None, total: result.affected_rows }) } pub async fn execute_rest_query(client: &VectorClient, input: &str) -> Result { diff --git a/packages/app-tests/documentJsonValues.test.ts b/packages/app-tests/documentJsonValues.test.ts new file mode 100644 index 0000000000..4b50575b0e --- /dev/null +++ b/packages/app-tests/documentJsonValues.test.ts @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; +import { documentStoreValueForGrid, parseDocumentStoreInputValue, serializeDocumentStoreId, stringifyDocumentStoreValue } from "../../apps/desktop/src/lib/app/documentJsonValues.ts"; + +test("keeps Elasticsearch long values as native numeric JSON without rounding", () => { + const value = parseDocumentStoreInputValue("2018551659033767937", "elasticsearch"); + + assert.equal(documentStoreValueForGrid(value, "elasticsearch"), "2018551659033767937"); + assert.equal(stringifyDocumentStoreValue({ id: value }, "elasticsearch"), '{"id":2018551659033767937}'); +}); + +test("keeps Elasticsearch string ids distinct from numeric long values", () => { + const value = parseDocumentStoreInputValue('"2018551659033767937"', "elasticsearch"); + + assert.equal(value, "2018551659033767937"); + assert.equal(serializeDocumentStoreId("orders/2018551659033767937", "elasticsearch"), "orders/2018551659033767937"); + assert.equal(stringifyDocumentStoreValue({ id: value }, "elasticsearch"), '{"id":"2018551659033767937"}'); +}); + +test("retains MongoDB Extended JSON parsing for int64 fields", () => { + const value = parseDocumentStoreInputValue("2018551659033767937", "mongodb"); + + assert.deepEqual(value, { $numberLong: "2018551659033767937" }); + assert.equal(serializeDocumentStoreId("2018551659033767937", "mongodb"), '__dbx_mongo_string_id__"2018551659033767937"'); + assert.equal(stringifyDocumentStoreValue({ id: value }, "mongodb"), '{"id":{"$numberLong":"2018551659033767937"}}'); +}); + +test("does not reinterpret Mongo-compatible objects stored in Elasticsearch", () => { + const value = parseDocumentStoreInputValue('{"$numberLong":"2018551659033767937"}', "elasticsearch"); + + assert.deepEqual(value, { $numberLong: "2018551659033767937" }); + assert.equal(stringifyDocumentStoreValue({ legacy: value }, "elasticsearch"), '{"legacy":{"$numberLong":"2018551659033767937"}}'); +});