fix(elasticsearch): preserve document numeric precision
This commit is contained in:
1 parent
977f47a65f
commit
7a434b0674
9 files changed
+203
-34
No files matched your search
@@ -0,0 +1,30 @@
|
||||
import { isLosslessJsonNumber, parseJsonPreservingLargeNumbers, stringifyJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat";
|
||||
import { parseMongoDocumentInputValue, serializeMongoDocumentId, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
|
||||
import type { DocumentStoreKind } from "@/lib/app/documentStoreProvider";
|
||||
|
||||
export function parseDocumentStoreInputValue(raw: MongoInputValue, kind: DocumentStoreKind): unknown {
|
||||
if (kind === "mongodb") return parseMongoDocumentInputValue(raw);
|
||||
if (raw === null || typeof raw === "number" || typeof raw === "boolean") return raw;
|
||||
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return "";
|
||||
try {
|
||||
return parseJsonPreservingLargeNumbers(trimmed);
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
export function stringifyDocumentStoreValue(value: unknown, kind: DocumentStoreKind, indent?: number): string {
|
||||
return kind === "elasticsearch" ? stringifyJsonPreservingLargeNumbers(value, indent) : JSON.stringify(value, null, indent ?? undefined);
|
||||
}
|
||||
|
||||
export function documentStoreValueForGrid(value: unknown, kind: DocumentStoreKind): MongoInputValue {
|
||||
if (kind === "elasticsearch" && isLosslessJsonNumber(value)) return value.raw;
|
||||
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
||||
return stringifyDocumentStoreValue(value, kind);
|
||||
}
|
||||
|
||||
export function serializeDocumentStoreId(value: unknown, kind: DocumentStoreKind): string {
|
||||
return kind === "elasticsearch" ? String(value) : serializeMongoDocumentId(value);
|
||||
}
|
||||
@@ -1729,6 +1729,7 @@ export async function zookeeperDelete(connectionId: string, key: string): Promis
|
||||
// --- MongoDB ---
|
||||
export interface MongoDocumentResult {
|
||||
documents: any[];
|
||||
raw_documents?: string[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat } from "../safeJsonFormat";
|
||||
import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat, stringifyJsonPreservingLargeNumbers } from "../safeJsonFormat";
|
||||
|
||||
describe("safeJsonFormat", () => {
|
||||
it("preserves large integers exceeding MAX_SAFE_INTEGER", () => {
|
||||
@@ -85,6 +85,11 @@ describe("safeJsonFormat", () => {
|
||||
expect(safeJsonFormat(input, 2)).toContain("1.234567890123456789e20");
|
||||
expect(safeJsonFormat(input, 2)).toContain("1e999");
|
||||
});
|
||||
|
||||
it("stringifies parsed lossless numbers back to numeric JSON tokens", () => {
|
||||
const parsed = parseJsonPreservingLargeNumbers('{"id":2018551659033767937,"stringId":"2018551659033767937"}');
|
||||
expect(stringifyJsonPreservingLargeNumbers(parsed)).toBe('{"id":2018551659033767937,"stringId":"2018551659033767937"}');
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatJsonSource", () => {
|
||||
|
||||
@@ -46,6 +46,35 @@ export function safeJsonFormat(text: string, indent?: number): string {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stringifies values produced by {@link parseJsonPreservingLargeNumbers}
|
||||
* without routing their numeric literals through JavaScript Number.
|
||||
*/
|
||||
export function stringifyJsonPreservingLargeNumbers(value: unknown, indent?: number): string {
|
||||
let placeholderPrefix = "__DBX_LOSSLESS_NUMBER_";
|
||||
const ordinaryJson = JSON.stringify(value);
|
||||
while (ordinaryJson?.includes(placeholderPrefix)) placeholderPrefix += "_";
|
||||
|
||||
const numbers = new Map<string, string>();
|
||||
const result = JSON.stringify(
|
||||
value,
|
||||
(_key, item) => {
|
||||
if (!isLosslessJsonNumber(item)) return item;
|
||||
const placeholder = `${placeholderPrefix}${numbers.size}__`;
|
||||
numbers.set(placeholder, item.raw);
|
||||
return placeholder;
|
||||
},
|
||||
indent ?? undefined,
|
||||
);
|
||||
|
||||
if (result === undefined) throw new TypeError("Value is not JSON serializable");
|
||||
let restored = result;
|
||||
for (const [placeholder, raw] of numbers) {
|
||||
restored = restored.replaceAll(JSON.stringify(placeholder), raw);
|
||||
}
|
||||
return restored;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate JSON and re-emit source tokens while only changing insignificant
|
||||
* whitespace. Unlike {@link safeJsonFormat}, this keeps duplicate object
|
||||
|
||||
Reference in new issue
Block a user