fix(elasticsearch): preserve document numeric precision
This commit is contained in:
1 parent
977f47a65f
commit
7a434b0674
9 files changed
+203
-34
No files matched your search
@@ -28,6 +28,8 @@ import {
|
|||||||
type DocumentFilterMode,
|
type DocumentFilterMode,
|
||||||
type DocumentFilterRule,
|
type DocumentFilterRule,
|
||||||
} from "@/lib/app/documentStoreProvider";
|
} from "@/lib/app/documentStoreProvider";
|
||||||
|
import { documentStoreValueForGrid, parseDocumentStoreInputValue, serializeDocumentStoreId, stringifyDocumentStoreValue } from "@/lib/app/documentJsonValues";
|
||||||
|
import { isLosslessJsonNumber, parseJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat";
|
||||||
import { buildMongoInsertDocument, buildMongoUpdateDocument, formatMongoShellLiteral, mongoDocumentIdForGrid, parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
|
import { buildMongoInsertDocument, buildMongoUpdateDocument, formatMongoShellLiteral, mongoDocumentIdForGrid, parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
|
||||||
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
|
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
|
||||||
import { useSettingsStore } from "@/stores/settingsStore";
|
import { useSettingsStore } from "@/stores/settingsStore";
|
||||||
@@ -163,8 +165,8 @@ const gridResult = computed<QueryResult>(() => {
|
|||||||
columns.map((col) => {
|
columns.map((col) => {
|
||||||
const val = doc[col];
|
const val = doc[col];
|
||||||
if (val === undefined || val === null) return null;
|
if (val === undefined || val === null) return null;
|
||||||
if (col === "_id") return mongoDocumentIdForGrid(val);
|
if (col === "_id") return documentStoreProvider.value.kind === "elasticsearch" ? documentStoreValueForGrid(val, "elasticsearch") : mongoDocumentIdForGrid(val);
|
||||||
if (typeof val === "object") return JSON.stringify(val);
|
if (typeof val === "object") return documentStoreValueForGrid(val, documentStoreProvider.value.kind);
|
||||||
if (typeof val === "string" || typeof val === "number" || typeof val === "boolean") return val;
|
if (typeof val === "string" || typeof val === "number" || typeof val === "boolean") return val;
|
||||||
return String(val);
|
return String(val);
|
||||||
}),
|
}),
|
||||||
@@ -313,10 +315,10 @@ async function gridSave(changes: DocumentGridChanges) {
|
|||||||
if (newVal === null) {
|
if (newVal === null) {
|
||||||
delete updated[col];
|
delete updated[col];
|
||||||
} else {
|
} else {
|
||||||
updated[col] = parseMongoDocumentInputValue(newVal);
|
updated[col] = parseDocumentStoreInputValue(newVal, "elasticsearch");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, String(id), JSON.stringify(updated), routing);
|
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, String(id), stringifyDocumentStoreValue(updated, "elasticsearch"), routing);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,13 +339,13 @@ async function gridSave(changes: DocumentGridChanges) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (const newRow of changes.newRows) {
|
for (const newRow of changes.newRows) {
|
||||||
const doc = buildMongoInsertDocument(newRow, cols);
|
const doc = isEs ? buildElasticsearchInsertDocument(newRow, cols) : buildMongoInsertDocument(newRow, cols);
|
||||||
if (isEs) {
|
if (isEs) {
|
||||||
const id = documentIdFromGridValue(newRow[idColIdx]);
|
const id = documentIdFromGridValue(newRow[idColIdx]);
|
||||||
if (id) {
|
if (id) {
|
||||||
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, id, JSON.stringify(doc), documentRoutingFromGridRow(newRow, cols));
|
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, id, stringifyDocumentStoreValue(doc, "elasticsearch"), documentRoutingFromGridRow(newRow, cols));
|
||||||
} else {
|
} else {
|
||||||
await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc));
|
await api.documentInsertDocument(props.connectionId, props.database, props.collection, stringifyDocumentStoreValue(doc, "elasticsearch"));
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -353,6 +355,17 @@ async function gridSave(changes: DocumentGridChanges) {
|
|||||||
await load();
|
await load();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildElasticsearchInsertDocument(row: MongoInputValue[], columns: string[]): JsonRecord {
|
||||||
|
const doc: JsonRecord = {};
|
||||||
|
for (let columnIndex = 0; columnIndex < columns.length; columnIndex += 1) {
|
||||||
|
const column = columns[columnIndex];
|
||||||
|
if (!column || column === "_id" || column === "_routing") continue;
|
||||||
|
const value = row[columnIndex];
|
||||||
|
if (value !== null) doc[column] = parseDocumentStoreInputValue(value, "elasticsearch");
|
||||||
|
}
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
|
||||||
function mongoIdPreview(val: unknown): string {
|
function mongoIdPreview(val: unknown): string {
|
||||||
if (val === null || val === undefined) return "null";
|
if (val === null || val === undefined) return "null";
|
||||||
if (typeof val === "string" && /^[a-fA-F0-9]{24}$/.test(val)) return `ObjectId("${val}")`;
|
if (typeof val === "string" && /^[a-fA-F0-9]{24}$/.test(val)) return `ObjectId("${val}")`;
|
||||||
@@ -368,13 +381,13 @@ function elasticsearchRoutingPreview(routing: string | undefined): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildElasticsearchPartialUpdateDocument(changes: Map<number, MongoInputValue>, columns: string[]): Record<string, unknown> {
|
function buildElasticsearchPartialUpdateDocument(changes: Map<number, MongoInputValue>, columns: string[]): Record<string, unknown> {
|
||||||
const filtered = new Map<number, MongoInputValue>();
|
const document: Record<string, unknown> = {};
|
||||||
for (const [colIdx, newVal] of changes) {
|
for (const [colIdx, newVal] of changes) {
|
||||||
const col = columns[colIdx];
|
const col = columns[colIdx];
|
||||||
if (col === "_id" || col === "_routing") continue;
|
if (col === "_id" || col === "_routing") continue;
|
||||||
filtered.set(colIdx, newVal);
|
if (col && newVal !== null) document[col] = parseDocumentStoreInputValue(newVal, "elasticsearch");
|
||||||
}
|
}
|
||||||
return buildMongoUpdateDocument(filtered, columns);
|
return document;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function previewDocumentChanges(changes: DocumentGridChanges): Promise<string[]> {
|
async function previewDocumentChanges(changes: DocumentGridChanges): Promise<string[]> {
|
||||||
@@ -391,7 +404,7 @@ async function previewDocumentChanges(changes: DocumentGridChanges): Promise<str
|
|||||||
if (isEs) {
|
if (isEs) {
|
||||||
const updateDoc = buildElasticsearchPartialUpdateDocument(dirtyCols, columns);
|
const updateDoc = buildElasticsearchPartialUpdateDocument(dirtyCols, columns);
|
||||||
const routing = documentRoutingFromGridRow(row, columns);
|
const routing = documentRoutingFromGridRow(row, columns);
|
||||||
stmts.push(`POST /${coll}/_update/${elasticsearchPathIdPreview(String(id))}${elasticsearchRoutingPreview(routing)}\n${JSON.stringify({ doc: updateDoc.$set ?? updateDoc }, null, 2)}`);
|
stmts.push(`POST /${coll}/_update/${elasticsearchPathIdPreview(String(id))}${elasticsearchRoutingPreview(routing)}\n${stringifyDocumentStoreValue({ doc: updateDoc.$set ?? updateDoc }, "elasticsearch", 2)}`);
|
||||||
} else {
|
} else {
|
||||||
const updateDoc = buildMongoUpdateDocument(dirtyCols, columns, documents.value[rowIdx]);
|
const updateDoc = buildMongoUpdateDocument(dirtyCols, columns, documents.value[rowIdx]);
|
||||||
stmts.push(`db.${coll}.updateOne({_id: ${mongoIdPreview(documents.value[rowIdx]?._id ?? id)}}, ${formatMongoShellLiteral(updateDoc)})`);
|
stmts.push(`db.${coll}.updateOne({_id: ${mongoIdPreview(documents.value[rowIdx]?._id ?? id)}}, ${formatMongoShellLiteral(updateDoc)})`);
|
||||||
@@ -411,13 +424,13 @@ async function previewDocumentChanges(changes: DocumentGridChanges): Promise<str
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (const newRow of newRows) {
|
for (const newRow of newRows) {
|
||||||
const doc = buildMongoInsertDocument(newRow, columns);
|
const doc = isEs ? buildElasticsearchInsertDocument(newRow, columns) : buildMongoInsertDocument(newRow, columns);
|
||||||
if (isEs) {
|
if (isEs) {
|
||||||
const id = idColIdx >= 0 ? documentIdFromGridValue(newRow[idColIdx]) : null;
|
const id = idColIdx >= 0 ? documentIdFromGridValue(newRow[idColIdx]) : null;
|
||||||
if (id) {
|
if (id) {
|
||||||
stmts.push(`PUT /${coll}/_doc/${elasticsearchPathIdPreview(id)}\n${JSON.stringify(doc, null, 2)}`);
|
stmts.push(`PUT /${coll}/_doc/${elasticsearchPathIdPreview(id)}\n${stringifyDocumentStoreValue(doc, "elasticsearch", 2)}`);
|
||||||
} else {
|
} else {
|
||||||
stmts.push(`POST /${coll}/_doc\n${JSON.stringify(doc, null, 2)}`);
|
stmts.push(`POST /${coll}/_doc\n${stringifyDocumentStoreValue(doc, "elasticsearch", 2)}`);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
stmts.push(`db.${coll}.insertOne(${formatMongoShellLiteral(doc)})`);
|
stmts.push(`db.${coll}.insertOne(${formatMongoShellLiteral(doc)})`);
|
||||||
@@ -464,7 +477,16 @@ async function load() {
|
|||||||
const sort = currentDocumentSortJson(sortInput.value);
|
const sort = currentDocumentSortJson(sortInput.value);
|
||||||
const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, page.value * pageSize.value, pageSize.value, filter, undefined, sort, executionId);
|
const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, page.value * pageSize.value, pageSize.value, filter, undefined, sort, executionId);
|
||||||
if (documentLoadExecutionId.value !== executionId) return;
|
if (documentLoadExecutionId.value !== executionId) return;
|
||||||
const nextDocuments = result.documents.map(asRecord);
|
const nextDocuments =
|
||||||
|
documentStoreProvider.value.kind === "elasticsearch" && result.raw_documents?.length === result.documents.length
|
||||||
|
? result.raw_documents.map((raw, index) => {
|
||||||
|
try {
|
||||||
|
return asRecord(parseJsonPreservingLargeNumbers(raw));
|
||||||
|
} catch {
|
||||||
|
return asRecord(result.documents[index]);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
: result.documents.map(asRecord);
|
||||||
documents.value = nextDocuments;
|
documents.value = nextDocuments;
|
||||||
if (nextDocuments.length > 0) {
|
if (nextDocuments.length > 0) {
|
||||||
const keySet = new Set<string>();
|
const keySet = new Set<string>();
|
||||||
@@ -554,13 +576,13 @@ function syncSelectedDocumentAfterLoad(previousSelectedIdx: number | null, previ
|
|||||||
|
|
||||||
selectedIdx.value = nextIdx;
|
selectedIdx.value = nextIdx;
|
||||||
if (!isEditing.value) {
|
if (!isEditing.value) {
|
||||||
editJson.value = JSON.stringify(documents.value[nextIdx], null, 2);
|
editJson.value = stringifyDocumentStoreValue(documents.value[nextIdx], documentStoreProvider.value.kind, 2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function selectDoc(idx: number) {
|
function selectDoc(idx: number) {
|
||||||
selectedIdx.value = idx;
|
selectedIdx.value = idx;
|
||||||
editJson.value = JSON.stringify(documents.value[idx], null, 2);
|
editJson.value = stringifyDocumentStoreValue(documents.value[idx], documentStoreProvider.value.kind, 2);
|
||||||
isEditing.value = false;
|
isEditing.value = false;
|
||||||
isNew.value = false;
|
isNew.value = false;
|
||||||
editFields.value = [];
|
editFields.value = [];
|
||||||
@@ -593,13 +615,24 @@ function cancelEdit() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (selectedDoc.value) {
|
if (selectedDoc.value) {
|
||||||
editJson.value = JSON.stringify(selectedDoc.value, null, 2);
|
editJson.value = stringifyDocumentStoreValue(selectedDoc.value, documentStoreProvider.value.kind, 2);
|
||||||
}
|
}
|
||||||
editFields.value = [];
|
editFields.value = [];
|
||||||
error.value = "";
|
error.value = "";
|
||||||
}
|
}
|
||||||
|
|
||||||
function createEditNode(keyName: string, value: unknown, readonlyKey: boolean, readonlyValue: boolean): EditNode {
|
function createEditNode(keyName: string, value: unknown, readonlyKey: boolean, readonlyValue: boolean): EditNode {
|
||||||
|
if (isLosslessJsonNumber(value)) {
|
||||||
|
return {
|
||||||
|
key: uuid(),
|
||||||
|
keyName,
|
||||||
|
kind: "value",
|
||||||
|
valueText: value.raw,
|
||||||
|
readonlyKey,
|
||||||
|
readonlyValue,
|
||||||
|
children: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
if (Array.isArray(value)) {
|
if (Array.isArray(value)) {
|
||||||
return {
|
return {
|
||||||
key: uuid(),
|
key: uuid(),
|
||||||
@@ -655,12 +688,12 @@ function formatForEdit(value: unknown): string {
|
|||||||
if (value === undefined) return "";
|
if (value === undefined) return "";
|
||||||
if (value === null) return "null";
|
if (value === null) return "null";
|
||||||
if (typeof value === "string") return JSON.stringify(value);
|
if (typeof value === "string") return JSON.stringify(value);
|
||||||
if (typeof value === "object") return JSON.stringify(value, null, 2);
|
if (typeof value === "object") return stringifyDocumentStoreValue(value, documentStoreProvider.value.kind, 2);
|
||||||
return String(value);
|
return String(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseFieldValue(raw: string): unknown {
|
function parseFieldValue(raw: string): unknown {
|
||||||
return parseMongoDocumentInputValue(raw);
|
return parseDocumentStoreInputValue(raw, documentStoreProvider.value.kind);
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildObjectFromNodes(nodes: EditNode[], path: string): JsonRecord {
|
function buildObjectFromNodes(nodes: EditNode[], path: string): JsonRecord {
|
||||||
@@ -695,7 +728,7 @@ async function saveDoc() {
|
|||||||
try {
|
try {
|
||||||
const doc = buildDocumentFromFields();
|
const doc = buildDocumentFromFields();
|
||||||
if (isNew.value) {
|
if (isNew.value) {
|
||||||
await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc));
|
await api.documentInsertDocument(props.connectionId, props.database, props.collection, stringifyDocumentStoreValue(doc, documentStoreProvider.value.kind));
|
||||||
} else if (selectedIdx.value !== null) {
|
} else if (selectedIdx.value !== null) {
|
||||||
const current = documents.value[selectedIdx.value];
|
const current = documents.value[selectedIdx.value];
|
||||||
const id = current?._id;
|
const id = current?._id;
|
||||||
@@ -703,14 +736,14 @@ async function saveDoc() {
|
|||||||
error.value = "No _id field";
|
error.value = "No _id field";
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, serializeMongoDocumentId(id), JSON.stringify(doc), documentRoutingFromDocument(current));
|
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, serializeDocumentStoreId(id, documentStoreProvider.value.kind), stringifyDocumentStoreValue(doc, documentStoreProvider.value.kind), documentRoutingFromDocument(current));
|
||||||
}
|
}
|
||||||
isEditing.value = false;
|
isEditing.value = false;
|
||||||
isNew.value = false;
|
isNew.value = false;
|
||||||
editFields.value = [];
|
editFields.value = [];
|
||||||
await load();
|
await load();
|
||||||
if (selectedIdx.value !== null && documents.value[selectedIdx.value]) {
|
if (selectedIdx.value !== null && documents.value[selectedIdx.value]) {
|
||||||
editJson.value = JSON.stringify(documents.value[selectedIdx.value], null, 2);
|
editJson.value = stringifyDocumentStoreValue(documents.value[selectedIdx.value], documentStoreProvider.value.kind, 2);
|
||||||
}
|
}
|
||||||
} catch (e: unknown) {
|
} catch (e: unknown) {
|
||||||
error.value = e instanceof Error ? e.message : String(e);
|
error.value = e instanceof Error ? e.message : String(e);
|
||||||
@@ -723,7 +756,7 @@ async function applyDeleteDoc(idx: number) {
|
|||||||
if (!id) return;
|
if (!id) return;
|
||||||
error.value = "";
|
error.value = "";
|
||||||
try {
|
try {
|
||||||
await api.documentDeleteDocument(props.connectionId, props.database, props.collection, serializeMongoDocumentId(id), documentRoutingFromDocument(doc));
|
await api.documentDeleteDocument(props.connectionId, props.database, props.collection, serializeDocumentStoreId(id, documentStoreProvider.value.kind), documentRoutingFromDocument(doc));
|
||||||
if (selectedIdx.value === idx) {
|
if (selectedIdx.value === idx) {
|
||||||
selectedIdx.value = null;
|
selectedIdx.value = null;
|
||||||
editJson.value = "";
|
editJson.value = "";
|
||||||
@@ -767,7 +800,7 @@ function docPreview(doc: JsonRecord): string {
|
|||||||
const keys = Object.keys(doc)
|
const keys = Object.keys(doc)
|
||||||
.filter((k) => k !== "_id")
|
.filter((k) => k !== "_id")
|
||||||
.slice(0, 3);
|
.slice(0, 3);
|
||||||
const preview = keys.map((k) => `${k}: ${JSON.stringify(doc[k]).substring(0, 30)}`).join(", ");
|
const preview = keys.map((k) => `${k}: ${stringifyDocumentStoreValue(doc[k], documentStoreProvider.value.kind).substring(0, 30)}`).join(", ");
|
||||||
return `${id} - ${preview}`;
|
return `${id} - ${preview}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import { isLosslessJsonNumber, parseJsonPreservingLargeNumbers, stringifyJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat";
|
||||||
|
import { parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
|
||||||
|
import type { DocumentStoreKind } from "@/lib/app/documentStoreProvider";
|
||||||
|
|
||||||
|
export function parseDocumentStoreInputValue(raw: MongoInputValue, kind: DocumentStoreKind): unknown {
|
||||||
|
if (kind === "mongodb") return parseMongoDocumentInputValue(raw);
|
||||||
|
if (raw === null || typeof raw === "number" || typeof raw === "boolean") return raw;
|
||||||
|
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
if (!trimmed) return "";
|
||||||
|
try {
|
||||||
|
return parseJsonPreservingLargeNumbers(trimmed);
|
||||||
|
} catch {
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stringifyDocumentStoreValue(value: unknown, kind: DocumentStoreKind, indent?: number): string {
|
||||||
|
return kind === "elasticsearch" ? stringifyJsonPreservingLargeNumbers(value, indent) : JSON.stringify(value, null, indent ?? undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function documentStoreValueForGrid(value: unknown, kind: DocumentStoreKind): MongoInputValue {
|
||||||
|
if (kind === "elasticsearch" && isLosslessJsonNumber(value)) return value.raw;
|
||||||
|
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
||||||
|
return stringifyDocumentStoreValue(value, kind);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function serializeDocumentStoreId(value: unknown, kind: DocumentStoreKind): string {
|
||||||
|
return kind === "elasticsearch" ? String(value) : serializeMongoDocumentId(value);
|
||||||
|
}
|
||||||
@@ -1729,6 +1729,7 @@ export async function zookeeperDelete(connectionId: string, key: string): Promis
|
|||||||
// --- MongoDB ---
|
// --- MongoDB ---
|
||||||
export interface MongoDocumentResult {
|
export interface MongoDocumentResult {
|
||||||
documents: any[];
|
documents: any[];
|
||||||
|
raw_documents?: string[];
|
||||||
total: number;
|
total: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, it, expect } from "vitest";
|
||||||
import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat } from "../safeJsonFormat";
|
import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat, stringifyJsonPreservingLargeNumbers } from "../safeJsonFormat";
|
||||||
|
|
||||||
describe("safeJsonFormat", () => {
|
describe("safeJsonFormat", () => {
|
||||||
it("preserves large integers exceeding MAX_SAFE_INTEGER", () => {
|
it("preserves large integers exceeding MAX_SAFE_INTEGER", () => {
|
||||||
@@ -85,6 +85,11 @@ describe("safeJsonFormat", () => {
|
|||||||
expect(safeJsonFormat(input, 2)).toContain("1.234567890123456789e20");
|
expect(safeJsonFormat(input, 2)).toContain("1.234567890123456789e20");
|
||||||
expect(safeJsonFormat(input, 2)).toContain("1e999");
|
expect(safeJsonFormat(input, 2)).toContain("1e999");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("stringifies parsed lossless numbers back to numeric JSON tokens", () => {
|
||||||
|
const parsed = parseJsonPreservingLargeNumbers('{"id":2018551659033767937,"stringId":"2018551659033767937"}');
|
||||||
|
expect(stringifyJsonPreservingLargeNumbers(parsed)).toBe('{"id":2018551659033767937,"stringId":"2018551659033767937"}');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("formatJsonSource", () => {
|
describe("formatJsonSource", () => {
|
||||||
|
|||||||
@@ -46,6 +46,35 @@ export function safeJsonFormat(text: string, indent?: number): string {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stringifies values produced by {@link parseJsonPreservingLargeNumbers}
|
||||||
|
* without routing their numeric literals through JavaScript Number.
|
||||||
|
*/
|
||||||
|
export function stringifyJsonPreservingLargeNumbers(value: unknown, indent?: number): string {
|
||||||
|
let placeholderPrefix = "__DBX_LOSSLESS_NUMBER_";
|
||||||
|
const ordinaryJson = JSON.stringify(value);
|
||||||
|
while (ordinaryJson?.includes(placeholderPrefix)) placeholderPrefix += "_";
|
||||||
|
|
||||||
|
const numbers = new Map<string, string>();
|
||||||
|
const result = JSON.stringify(
|
||||||
|
value,
|
||||||
|
(_key, item) => {
|
||||||
|
if (!isLosslessJsonNumber(item)) return item;
|
||||||
|
const placeholder = `${placeholderPrefix}${numbers.size}__`;
|
||||||
|
numbers.set(placeholder, item.raw);
|
||||||
|
return placeholder;
|
||||||
|
},
|
||||||
|
indent ?? undefined,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result === undefined) throw new TypeError("Value is not JSON serializable");
|
||||||
|
let restored = result;
|
||||||
|
for (const [placeholder, raw] of numbers) {
|
||||||
|
restored = restored.replaceAll(JSON.stringify(placeholder), raw);
|
||||||
|
}
|
||||||
|
return restored;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validate JSON and re-emit source tokens while only changing insignificant
|
* Validate JSON and re-emit source tokens while only changing insignificant
|
||||||
* whitespace. Unlike {@link safeJsonFormat}, this keeps duplicate object
|
* whitespace. Unlike {@link safeJsonFormat}, this keeps duplicate object
|
||||||
|
|||||||
@@ -408,6 +408,10 @@ pub async fn find_documents(
|
|||||||
|
|
||||||
let result: SearchResponse = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
let result: SearchResponse = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
||||||
|
|
||||||
|
search_response_to_document_result(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn search_response_to_document_result(result: SearchResponse) -> Result<MongoDocumentResult, String> {
|
||||||
let documents: Vec<serde_json::Value> = result
|
let documents: Vec<serde_json::Value> = result
|
||||||
.hits
|
.hits
|
||||||
.hits
|
.hits
|
||||||
@@ -425,7 +429,16 @@ pub async fn find_documents(
|
|||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
Ok(MongoDocumentResult { documents, total: result.hits.total.value() })
|
// Keep the exact JSON numeric tokens alongside the compatibility value tree.
|
||||||
|
// Desktop IPC and browser JSON parsing otherwise coerce ES long values through
|
||||||
|
// JavaScript Number before the document editor can serialize them again.
|
||||||
|
let raw_documents = documents
|
||||||
|
.iter()
|
||||||
|
.map(serde_json::to_string)
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map_err(|e| format!("Elasticsearch document serialization failed: {e}"))?;
|
||||||
|
|
||||||
|
Ok(MongoDocumentResult { documents, raw_documents: Some(raw_documents), total: result.hits.total.value() })
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_find_documents_body(
|
fn build_find_documents_body(
|
||||||
@@ -1743,6 +1756,21 @@ mod tests {
|
|||||||
assert_eq!(response.hits.hits[0].routing.as_deref(), Some("tenant-1"));
|
assert_eq!(response.hits.hits[0].routing.as_deref(), Some("tenant-1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn preserves_long_literals_in_document_transport_json() {
|
||||||
|
let response: SearchResponse = serde_json::from_str(
|
||||||
|
r#"{"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"doc-1","_source":{"id":2018551659033767937,"string_id":"2018551659033767937","legacy":{"$numberLong":"2018551659033767937"}}}]}}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let result = super::search_response_to_document_result(response).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
result.raw_documents.as_deref(),
|
||||||
|
Some(&[r#"{"id":2018551659033767937,"string_id":"2018551659033767937","legacy":{"$numberLong":"2018551659033767937"},"_id":"doc-1"}"#.to_string()][..])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parses_search_response_rows_with_routing_metadata() {
|
fn parses_search_response_rows_with_routing_metadata() {
|
||||||
let result = super::parse_elasticsearch_response(
|
let result = super::parse_elasticsearch_response(
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ use std::{collections::HashSet, time::Duration};
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct MongoDocumentResult {
|
pub struct MongoDocumentResult {
|
||||||
pub documents: Vec<serde_json::Value>,
|
pub documents: Vec<serde_json::Value>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub raw_documents: Option<Vec<String>>,
|
||||||
pub total: u64,
|
pub total: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -589,7 +591,7 @@ pub async fn find_documents(
|
|||||||
documents.push(bson_to_json(&Bson::Document(doc)));
|
documents.push(bson_to_json(&Bson::Document(doc)));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(MongoDocumentResult { documents, total })
|
Ok(MongoDocumentResult { documents, raw_documents: None, total })
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Deserialize)]
|
#[derive(Debug, Default, Deserialize)]
|
||||||
@@ -725,7 +727,7 @@ pub async fn find_documents_extended_json(
|
|||||||
documents.push(Bson::Document(doc).into_relaxed_extjson());
|
documents.push(Bson::Document(doc).into_relaxed_extjson());
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(MongoDocumentResult { documents, total })
|
Ok(MongoDocumentResult { documents, raw_documents: None, total })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn aggregate_documents(
|
pub async fn aggregate_documents(
|
||||||
@@ -755,7 +757,7 @@ pub async fn aggregate_documents(
|
|||||||
if documents.len() > max_rows {
|
if documents.len() > max_rows {
|
||||||
documents.truncate(max_rows);
|
documents.truncate(max_rows);
|
||||||
}
|
}
|
||||||
Ok(MongoDocumentResult { documents, total })
|
Ok(MongoDocumentResult { documents, raw_documents: None, total })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn create_index(
|
pub async fn create_index(
|
||||||
@@ -1111,8 +1113,12 @@ fn find_and_modify_array_filters(
|
|||||||
|
|
||||||
fn single_document_result(document: Option<Document>) -> MongoDocumentResult {
|
fn single_document_result(document: Option<Document>) -> MongoDocumentResult {
|
||||||
match document {
|
match document {
|
||||||
Some(document) => MongoDocumentResult { documents: vec![bson_to_json(&Bson::Document(document))], total: 1 },
|
Some(document) => MongoDocumentResult {
|
||||||
None => MongoDocumentResult { documents: Vec::new(), total: 0 },
|
documents: vec![bson_to_json(&Bson::Document(document))],
|
||||||
|
raw_documents: None,
|
||||||
|
total: 1,
|
||||||
|
},
|
||||||
|
None => MongoDocumentResult { documents: Vec::new(), raw_documents: None, total: 0 },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -520,7 +520,11 @@ pub async fn find_documents(
|
|||||||
Value::Object(map)
|
Value::Object(map)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
return Ok(crate::db::mongo_driver::MongoDocumentResult { documents, total: result.affected_rows });
|
return Ok(crate::db::mongo_driver::MongoDocumentResult {
|
||||||
|
documents,
|
||||||
|
raw_documents: None,
|
||||||
|
total: result.affected_rows,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let query = match client.kind {
|
let query = match client.kind {
|
||||||
@@ -562,7 +566,7 @@ pub async fn find_documents(
|
|||||||
Value::Object(map)
|
Value::Object(map)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
Ok(crate::db::mongo_driver::MongoDocumentResult { documents, total: result.affected_rows })
|
Ok(crate::db::mongo_driver::MongoDocumentResult { documents, raw_documents: None, total: result.affected_rows })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn execute_rest_query(client: &VectorClient, input: &str) -> Result<QueryResult, String> {
|
pub async fn execute_rest_query(client: &VectorClient, input: &str) -> Result<QueryResult, String> {
|
||||||
|
|||||||
+33
@@ -0,0 +1,33 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { test } from "vitest";
|
||||||
|
import { documentStoreValueForGrid, parseDocumentStoreInputValue, serializeDocumentStoreId, stringifyDocumentStoreValue } from "../../apps/desktop/src/lib/app/documentJsonValues.ts";
|
||||||
|
|
||||||
|
test("keeps Elasticsearch long values as native numeric JSON without rounding", () => {
|
||||||
|
const value = parseDocumentStoreInputValue("2018551659033767937", "elasticsearch");
|
||||||
|
|
||||||
|
assert.equal(documentStoreValueForGrid(value, "elasticsearch"), "2018551659033767937");
|
||||||
|
assert.equal(stringifyDocumentStoreValue({ id: value }, "elasticsearch"), '{"id":2018551659033767937}');
|
||||||
|
});
|
||||||
|
|
||||||
|
test("keeps Elasticsearch string ids distinct from numeric long values", () => {
|
||||||
|
const value = parseDocumentStoreInputValue('"2018551659033767937"', "elasticsearch");
|
||||||
|
|
||||||
|
assert.equal(value, "2018551659033767937");
|
||||||
|
assert.equal(serializeDocumentStoreId("orders/2018551659033767937", "elasticsearch"), "orders/2018551659033767937");
|
||||||
|
assert.equal(stringifyDocumentStoreValue({ id: value }, "elasticsearch"), '{"id":"2018551659033767937"}');
|
||||||
|
});
|
||||||
|
|
||||||
|
test("retains MongoDB Extended JSON parsing for int64 fields", () => {
|
||||||
|
const value = parseDocumentStoreInputValue("2018551659033767937", "mongodb");
|
||||||
|
|
||||||
|
assert.deepEqual(value, { $numberLong: "2018551659033767937" });
|
||||||
|
assert.equal(serializeDocumentStoreId("2018551659033767937", "mongodb"), '__dbx_mongo_string_id__"2018551659033767937"');
|
||||||
|
assert.equal(stringifyDocumentStoreValue({ id: value }, "mongodb"), '{"id":{"$numberLong":"2018551659033767937"}}');
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not reinterpret Mongo-compatible objects stored in Elasticsearch", () => {
|
||||||
|
const value = parseDocumentStoreInputValue('{"$numberLong":"2018551659033767937"}', "elasticsearch");
|
||||||
|
|
||||||
|
assert.deepEqual(value, { $numberLong: "2018551659033767937" });
|
||||||
|
assert.equal(stringifyDocumentStoreValue({ legacy: value }, "elasticsearch"), '{"legacy":{"$numberLong":"2018551659033767937"}}');
|
||||||
|
});
|
||||||
Reference in new issue
Block a user