fix(elasticsearch): preserve document numeric precision

This commit is contained in:
t8y2 committed 2026-07-15 16:07:05 +08:00
1 parent 977f47a65f
commit 7a434b0674
9 files changed
+203 -34

No files matched your search

@@ -28,6 +28,8 @@ import {
type DocumentFilterMode,
type DocumentFilterRule,
} from "@/lib/app/documentStoreProvider";
import { documentStoreValueForGrid, parseDocumentStoreInputValue, serializeDocumentStoreId, stringifyDocumentStoreValue } from "@/lib/app/documentJsonValues";
import { isLosslessJsonNumber, parseJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat";
import { buildMongoInsertDocument, buildMongoUpdateDocument, formatMongoShellLiteral, mongoDocumentIdForGrid, parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
import { useSettingsStore } from "@/stores/settingsStore";
@@ -163,8 +165,8 @@ const gridResult = computed<QueryResult>(() => {
columns.map((col) => {
const val = doc[col];
if (val === undefined || val === null) return null;
if (col === "_id") return mongoDocumentIdForGrid(val);
if (typeof val === "object") return JSON.stringify(val);
if (col === "_id") return documentStoreProvider.value.kind === "elasticsearch" ? documentStoreValueForGrid(val, "elasticsearch") : mongoDocumentIdForGrid(val);
if (typeof val === "object") return documentStoreValueForGrid(val, documentStoreProvider.value.kind);
if (typeof val === "string" || typeof val === "number" || typeof val === "boolean") return val;
return String(val);
}),
@@ -313,10 +315,10 @@ async function gridSave(changes: DocumentGridChanges) {
if (newVal === null) {
delete updated[col];
} else {
updated[col] = parseMongoDocumentInputValue(newVal);
updated[col] = parseDocumentStoreInputValue(newVal, "elasticsearch");
}
}
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, String(id), JSON.stringify(updated), routing);
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, String(id), stringifyDocumentStoreValue(updated, "elasticsearch"), routing);
continue;
}
@@ -337,13 +339,13 @@ async function gridSave(changes: DocumentGridChanges) {
}
for (const newRow of changes.newRows) {
const doc = buildMongoInsertDocument(newRow, cols);
const doc = isEs ? buildElasticsearchInsertDocument(newRow, cols) : buildMongoInsertDocument(newRow, cols);
if (isEs) {
const id = documentIdFromGridValue(newRow[idColIdx]);
if (id) {
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, id, JSON.stringify(doc), documentRoutingFromGridRow(newRow, cols));
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, id, stringifyDocumentStoreValue(doc, "elasticsearch"), documentRoutingFromGridRow(newRow, cols));
} else {
await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc));
await api.documentInsertDocument(props.connectionId, props.database, props.collection, stringifyDocumentStoreValue(doc, "elasticsearch"));
}
continue;
}
@@ -353,6 +355,17 @@ async function gridSave(changes: DocumentGridChanges) {
await load();
}
function buildElasticsearchInsertDocument(row: MongoInputValue[], columns: string[]): JsonRecord {
const doc: JsonRecord = {};
for (let columnIndex = 0; columnIndex < columns.length; columnIndex += 1) {
const column = columns[columnIndex];
if (!column || column === "_id" || column === "_routing") continue;
const value = row[columnIndex];
if (value !== null) doc[column] = parseDocumentStoreInputValue(value, "elasticsearch");
}
return doc;
}
function mongoIdPreview(val: unknown): string {
if (val === null || val === undefined) return "null";
if (typeof val === "string" && /^[a-fA-F0-9]{24}$/.test(val)) return `ObjectId("${val}")`;
@@ -368,13 +381,13 @@ function elasticsearchRoutingPreview(routing: string | undefined): string {
}
function buildElasticsearchPartialUpdateDocument(changes: Map<number, MongoInputValue>, columns: string[]): Record<string, unknown> {
const filtered = new Map<number, MongoInputValue>();
const document: Record<string, unknown> = {};
for (const [colIdx, newVal] of changes) {
const col = columns[colIdx];
if (col === "_id" || col === "_routing") continue;
filtered.set(colIdx, newVal);
if (col && newVal !== null) document[col] = parseDocumentStoreInputValue(newVal, "elasticsearch");
}
return buildMongoUpdateDocument(filtered, columns);
return document;
}
async function previewDocumentChanges(changes: DocumentGridChanges): Promise<string[]> {
@@ -391,7 +404,7 @@ async function previewDocumentChanges(changes: DocumentGridChanges): Promise<str
if (isEs) {
const updateDoc = buildElasticsearchPartialUpdateDocument(dirtyCols, columns);
const routing = documentRoutingFromGridRow(row, columns);
stmts.push(`POST /${coll}/_update/${elasticsearchPathIdPreview(String(id))}${elasticsearchRoutingPreview(routing)}\n${JSON.stringify({ doc: updateDoc.$set ?? updateDoc }, null, 2)}`);
stmts.push(`POST /${coll}/_update/${elasticsearchPathIdPreview(String(id))}${elasticsearchRoutingPreview(routing)}\n${stringifyDocumentStoreValue({ doc: updateDoc.$set ?? updateDoc }, "elasticsearch", 2)}`);
} else {
const updateDoc = buildMongoUpdateDocument(dirtyCols, columns, documents.value[rowIdx]);
stmts.push(`db.${coll}.updateOne({_id: ${mongoIdPreview(documents.value[rowIdx]?._id ?? id)}}, ${formatMongoShellLiteral(updateDoc)})`);
@@ -411,13 +424,13 @@ async function previewDocumentChanges(changes: DocumentGridChanges): Promise<str
}
for (const newRow of newRows) {
const doc = buildMongoInsertDocument(newRow, columns);
const doc = isEs ? buildElasticsearchInsertDocument(newRow, columns) : buildMongoInsertDocument(newRow, columns);
if (isEs) {
const id = idColIdx >= 0 ? documentIdFromGridValue(newRow[idColIdx]) : null;
if (id) {
stmts.push(`PUT /${coll}/_doc/${elasticsearchPathIdPreview(id)}\n${JSON.stringify(doc, null, 2)}`);
stmts.push(`PUT /${coll}/_doc/${elasticsearchPathIdPreview(id)}\n${stringifyDocumentStoreValue(doc, "elasticsearch", 2)}`);
} else {
stmts.push(`POST /${coll}/_doc\n${JSON.stringify(doc, null, 2)}`);
stmts.push(`POST /${coll}/_doc\n${stringifyDocumentStoreValue(doc, "elasticsearch", 2)}`);
}
} else {
stmts.push(`db.${coll}.insertOne(${formatMongoShellLiteral(doc)})`);
@@ -464,7 +477,16 @@ async function load() {
const sort = currentDocumentSortJson(sortInput.value);
const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, page.value * pageSize.value, pageSize.value, filter, undefined, sort, executionId);
if (documentLoadExecutionId.value !== executionId) return;
const nextDocuments = result.documents.map(asRecord);
const nextDocuments =
documentStoreProvider.value.kind === "elasticsearch" && result.raw_documents?.length === result.documents.length
? result.raw_documents.map((raw, index) => {
try {
return asRecord(parseJsonPreservingLargeNumbers(raw));
} catch {
return asRecord(result.documents[index]);
}
})
: result.documents.map(asRecord);
documents.value = nextDocuments;
if (nextDocuments.length > 0) {
const keySet = new Set<string>();
@@ -554,13 +576,13 @@ function syncSelectedDocumentAfterLoad(previousSelectedIdx: number | null, previ
selectedIdx.value = nextIdx;
if (!isEditing.value) {
editJson.value = JSON.stringify(documents.value[nextIdx], null, 2);
editJson.value = stringifyDocumentStoreValue(documents.value[nextIdx], documentStoreProvider.value.kind, 2);
}
}
function selectDoc(idx: number) {
selectedIdx.value = idx;
editJson.value = JSON.stringify(documents.value[idx], null, 2);
editJson.value = stringifyDocumentStoreValue(documents.value[idx], documentStoreProvider.value.kind, 2);
isEditing.value = false;
isNew.value = false;
editFields.value = [];
@@ -593,13 +615,24 @@ function cancelEdit() {
return;
}
if (selectedDoc.value) {
editJson.value = JSON.stringify(selectedDoc.value, null, 2);
editJson.value = stringifyDocumentStoreValue(selectedDoc.value, documentStoreProvider.value.kind, 2);
}
editFields.value = [];
error.value = "";
}
function createEditNode(keyName: string, value: unknown, readonlyKey: boolean, readonlyValue: boolean): EditNode {
if (isLosslessJsonNumber(value)) {
return {
key: uuid(),
keyName,
kind: "value",
valueText: value.raw,
readonlyKey,
readonlyValue,
children: [],
};
}
if (Array.isArray(value)) {
return {
key: uuid(),
@@ -655,12 +688,12 @@ function formatForEdit(value: unknown): string {
if (value === undefined) return "";
if (value === null) return "null";
if (typeof value === "string") return JSON.stringify(value);
if (typeof value === "object") return JSON.stringify(value, null, 2);
if (typeof value === "object") return stringifyDocumentStoreValue(value, documentStoreProvider.value.kind, 2);
return String(value);
}
function parseFieldValue(raw: string): unknown {
return parseMongoDocumentInputValue(raw);
return parseDocumentStoreInputValue(raw, documentStoreProvider.value.kind);
}
function buildObjectFromNodes(nodes: EditNode[], path: string): JsonRecord {
@@ -695,7 +728,7 @@ async function saveDoc() {
try {
const doc = buildDocumentFromFields();
if (isNew.value) {
await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc));
await api.documentInsertDocument(props.connectionId, props.database, props.collection, stringifyDocumentStoreValue(doc, documentStoreProvider.value.kind));
} else if (selectedIdx.value !== null) {
const current = documents.value[selectedIdx.value];
const id = current?._id;
@@ -703,14 +736,14 @@ async function saveDoc() {
error.value = "No _id field";
return;
}
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, serializeMongoDocumentId(id), JSON.stringify(doc), documentRoutingFromDocument(current));
await api.documentUpdateDocument(props.connectionId, props.database, props.collection, serializeDocumentStoreId(id, documentStoreProvider.value.kind), stringifyDocumentStoreValue(doc, documentStoreProvider.value.kind), documentRoutingFromDocument(current));
}
isEditing.value = false;
isNew.value = false;
editFields.value = [];
await load();
if (selectedIdx.value !== null && documents.value[selectedIdx.value]) {
editJson.value = JSON.stringify(documents.value[selectedIdx.value], null, 2);
editJson.value = stringifyDocumentStoreValue(documents.value[selectedIdx.value], documentStoreProvider.value.kind, 2);
}
} catch (e: unknown) {
error.value = e instanceof Error ? e.message : String(e);
@@ -723,7 +756,7 @@ async function applyDeleteDoc(idx: number) {
if (!id) return;
error.value = "";
try {
await api.documentDeleteDocument(props.connectionId, props.database, props.collection, serializeMongoDocumentId(id), documentRoutingFromDocument(doc));
await api.documentDeleteDocument(props.connectionId, props.database, props.collection, serializeDocumentStoreId(id, documentStoreProvider.value.kind), documentRoutingFromDocument(doc));
if (selectedIdx.value === idx) {
selectedIdx.value = null;
editJson.value = "";
@@ -767,7 +800,7 @@ function docPreview(doc: JsonRecord): string {
const keys = Object.keys(doc)
.filter((k) => k !== "_id")
.slice(0, 3);
const preview = keys.map((k) => `${k}: ${JSON.stringify(doc[k]).substring(0, 30)}`).join(", ");
const preview = keys.map((k) => `${k}: ${stringifyDocumentStoreValue(doc[k], documentStoreProvider.value.kind).substring(0, 30)}`).join(", ");
return `${id} - ${preview}`;
}
@@ -0,0 +1,30 @@
import { isLosslessJsonNumber, parseJsonPreservingLargeNumbers, stringifyJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat";
import { parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
import type { DocumentStoreKind } from "@/lib/app/documentStoreProvider";
export function parseDocumentStoreInputValue(raw: MongoInputValue, kind: DocumentStoreKind): unknown {
if (kind === "mongodb") return parseMongoDocumentInputValue(raw);
if (raw === null || typeof raw === "number" || typeof raw === "boolean") return raw;
const trimmed = raw.trim();
if (!trimmed) return "";
try {
return parseJsonPreservingLargeNumbers(trimmed);
} catch {
return trimmed;
}
}
export function stringifyDocumentStoreValue(value: unknown, kind: DocumentStoreKind, indent?: number): string {
return kind === "elasticsearch" ? stringifyJsonPreservingLargeNumbers(value, indent) : JSON.stringify(value, null, indent ?? undefined);
}
export function documentStoreValueForGrid(value: unknown, kind: DocumentStoreKind): MongoInputValue {
if (kind === "elasticsearch" && isLosslessJsonNumber(value)) return value.raw;
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
return stringifyDocumentStoreValue(value, kind);
}
export function serializeDocumentStoreId(value: unknown, kind: DocumentStoreKind): string {
return kind === "elasticsearch" ? String(value) : serializeMongoDocumentId(value);
}
+1
View File
@@ -1729,6 +1729,7 @@ export async function zookeeperDelete(connectionId: string, key: string): Promis
// --- MongoDB ---
export interface MongoDocumentResult {
documents: any[];
raw_documents?: string[];
total: number;
}
@@ -1,5 +1,5 @@
import { describe, it, expect } from "vitest";
import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat } from "../safeJsonFormat";
import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat, stringifyJsonPreservingLargeNumbers } from "../safeJsonFormat";
describe("safeJsonFormat", () => {
it("preserves large integers exceeding MAX_SAFE_INTEGER", () => {
@@ -85,6 +85,11 @@ describe("safeJsonFormat", () => {
expect(safeJsonFormat(input, 2)).toContain("1.234567890123456789e20");
expect(safeJsonFormat(input, 2)).toContain("1e999");
});
it("stringifies parsed lossless numbers back to numeric JSON tokens", () => {
const parsed = parseJsonPreservingLargeNumbers('{"id":2018551659033767937,"stringId":"2018551659033767937"}');
expect(stringifyJsonPreservingLargeNumbers(parsed)).toBe('{"id":2018551659033767937,"stringId":"2018551659033767937"}');
});
});
describe("formatJsonSource", () => {
@@ -46,6 +46,35 @@ export function safeJsonFormat(text: string, indent?: number): string {
return result;
}
/**
* Stringifies values produced by {@link parseJsonPreservingLargeNumbers}
* without routing their numeric literals through JavaScript Number.
*/
export function stringifyJsonPreservingLargeNumbers(value: unknown, indent?: number): string {
let placeholderPrefix = "__DBX_LOSSLESS_NUMBER_";
const ordinaryJson = JSON.stringify(value);
while (ordinaryJson?.includes(placeholderPrefix)) placeholderPrefix += "_";
const numbers = new Map<string, string>();
const result = JSON.stringify(
value,
(_key, item) => {
if (!isLosslessJsonNumber(item)) return item;
const placeholder = `${placeholderPrefix}${numbers.size}__`;
numbers.set(placeholder, item.raw);
return placeholder;
},
indent ?? undefined,
);
if (result === undefined) throw new TypeError("Value is not JSON serializable");
let restored = result;
for (const [placeholder, raw] of numbers) {
restored = restored.replaceAll(JSON.stringify(placeholder), raw);
}
return restored;
}
/**
* Validate JSON and re-emit source tokens while only changing insignificant
* whitespace. Unlike {@link safeJsonFormat}, this keeps duplicate object