feat(redis): add decoded Unicode JSON view
This commit is contained in:
1 parent
8b02ba003b
commit
a3eaac3413
12 files changed
+648
-17
No files matched your search
@@ -89,6 +89,289 @@ export function formatJsonSource(text: string, indent?: number): string {
|
||||
return writer.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode non-ASCII unicode escapes (`\uXXXX`) inside JSON string tokens (keys
|
||||
* and values) so text like `{"name":"张三"}` renders as `{"name":"张三"}`.
|
||||
*
|
||||
* - Code points >= 0x80 are decoded to their characters; ASCII control escapes
|
||||
* (U+0000, U+000A, ...) are kept verbatim so the output stays valid JSON.
|
||||
* - Surrogate pairs combine into a single code point; unpaired surrogates stay
|
||||
* escaped rather than emitting invalid lone surrogate characters.
|
||||
* - Escaped backslashes are respected, so a literal `"\\u5f20"` (whose content
|
||||
* is the text `张`) is never mis-decoded, and an escaped quote `\"` is
|
||||
* consumed inside a string token so it never terminates the token.
|
||||
*
|
||||
* This is a view-layer transform only and must never be written back to a
|
||||
* database on its own.
|
||||
*/
|
||||
export function decodeJsonUnicodeEscapes(text: string): string {
|
||||
return decodeJsonSource(text).display;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of walking a JSON source with {@link decodeJsonSource}: the decoded
|
||||
* display text plus the raw-source offset of every display code unit, which lets
|
||||
* callers map display edits back onto the source without touching escapes.
|
||||
*/
|
||||
export interface JsonUnicodeDecodedSource {
|
||||
/** Decoded display text (what the JSON editor shows in decoded mode). */
|
||||
display: string;
|
||||
/**
|
||||
* displayToRaw[i] is the raw-source offset where display code unit i
|
||||
* originates, for i in [0, display.length]; displayToRaw[display.length] is
|
||||
* raw.length. Surrogate-pair display units both point at the pair's start.
|
||||
*/
|
||||
displayToRaw: number[];
|
||||
}
|
||||
|
||||
function decodeJsonSource(text: string): JsonUnicodeDecodedSource {
|
||||
let display = "";
|
||||
const displayToRaw: number[] = [];
|
||||
let index = 0;
|
||||
const push = (ch: string, rawStart: number) => {
|
||||
// Iterate code units, not code points, so surrogate-pair characters record
|
||||
// an offset for each of their two units and stay aligned with display.length.
|
||||
for (let unitIndex = 0; unitIndex < ch.length; unitIndex += 1) {
|
||||
display += ch[unitIndex];
|
||||
displayToRaw.push(rawStart);
|
||||
}
|
||||
};
|
||||
|
||||
while (index < text.length) {
|
||||
if (text[index] !== '"') {
|
||||
push(text[index], index);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Rebuild one JSON string token, decoding escapes inside it.
|
||||
const tokenStart = index;
|
||||
index += 1;
|
||||
push('"', tokenStart);
|
||||
while (index < text.length) {
|
||||
const character = text[index];
|
||||
if (character === '"') {
|
||||
push('"', index);
|
||||
index += 1;
|
||||
break;
|
||||
}
|
||||
if (character !== "\\") {
|
||||
push(character, index);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const next = text[index + 1];
|
||||
if (next === undefined) {
|
||||
push("\\", index);
|
||||
index += 1;
|
||||
break;
|
||||
}
|
||||
const hex = text.slice(index + 2, index + 6);
|
||||
if (next !== "u" || !/^[0-9a-fA-F]{4}$/.test(hex)) {
|
||||
// Not a unicode escape (escaped backslash, escaped quote, ...); copy verbatim.
|
||||
push(`\\${next}`, index);
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
const code = parseInt(hex, 16);
|
||||
const lowEscape = text.slice(index + 6, index + 12);
|
||||
const lowMatch = /^\\u([0-9a-fA-F]{4})$/.exec(lowEscape);
|
||||
if (code >= 0xd800 && code <= 0xdbff && lowMatch) {
|
||||
const low = parseInt(lowMatch[1], 16);
|
||||
if (low >= 0xdc00 && low <= 0xdfff) {
|
||||
push(String.fromCodePoint(0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00)), index);
|
||||
index += 12;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (code >= 0xd800 && code <= 0xdfff) {
|
||||
// Unpaired surrogate: keep the escape rather than emit a lone surrogate char.
|
||||
push(`\\u${hex}`, index);
|
||||
} else if (code >= 0x80) {
|
||||
push(String.fromCharCode(code), index);
|
||||
} else {
|
||||
// ASCII control escapes stay escaped for valid JSON.
|
||||
push(`\\u${hex}`, index);
|
||||
}
|
||||
index += 6;
|
||||
}
|
||||
}
|
||||
displayToRaw.push(text.length);
|
||||
return { display, displayToRaw };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstruct the raw JSON source for a decoded-mode editor draft.
|
||||
*
|
||||
* The editor shows {@link decodeJsonUnicodeEscapes decoded} text; this maps the
|
||||
* user's current draft back onto the canonical raw source so untouched escapes
|
||||
* and the original whitespace/formatting are preserved. Decoded mode only inlines
|
||||
* non-ASCII `\uXXXX` escapes, so every other span of the draft is already JSON
|
||||
* source and is written back verbatim — inserted `\\`, `\"`, and control escapes
|
||||
* keep their meaning instead of being re-escaped. When the draft equals the
|
||||
* decoded baseline the raw source is returned unchanged.
|
||||
*
|
||||
* The result is only valid JSON when the draft is; callers must still validate
|
||||
* with {@link formatJsonSource} before saving.
|
||||
*/
|
||||
export function mapDisplayToRaw(raw: string, displayCurrent: string): string {
|
||||
const { display: baseline, displayToRaw } = decodeJsonSource(raw);
|
||||
if (displayCurrent === baseline) return raw;
|
||||
let out = "";
|
||||
for (const op of diffText(baseline, displayCurrent)) {
|
||||
if (op.kind === "equal") {
|
||||
out += raw.slice(displayToRaw[op.aFrom], displayToRaw[op.aTo]);
|
||||
} else if (op.kind === "delete") {
|
||||
// The deleted display span's raw source is dropped with it.
|
||||
} else {
|
||||
// Inserted display text is JSON source in the decoded draft; write it back
|
||||
// verbatim so its value semantics are preserved.
|
||||
out += displayCurrent.slice(op.bFrom, op.bTo);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
type DisplayDiffOp = { kind: "equal"; aFrom: number; aTo: number; bFrom: number; bTo: number } | { kind: "delete"; aFrom: number; aTo: number; bFrom: number; bTo: number } | { kind: "insert"; aFrom: number; aTo: number; bFrom: number; bTo: number };
|
||||
|
||||
// Beyond this combined size, fall back to a single replace; whole-document
|
||||
// rewrites gain nothing from escape-level preservation.
|
||||
const MAX_DIFF_UNITS = 200_000;
|
||||
// Cap the Myers edit distance so adversarial edits stay bounded in time and trace.
|
||||
const MAX_DIFF_DISTANCE = 2000;
|
||||
|
||||
function diffText(a: string, b: string): DisplayDiffOp[] {
|
||||
const source = splitCodePoints(a);
|
||||
const target = splitCodePoints(b);
|
||||
let prefix = 0;
|
||||
while (prefix < source.units.length && prefix < target.units.length && source.units[prefix] === target.units[prefix]) prefix += 1;
|
||||
let suffix = 0;
|
||||
while (suffix < source.units.length - prefix && suffix < target.units.length - prefix && source.units[source.units.length - 1 - suffix] === target.units[target.units.length - 1 - suffix]) {
|
||||
suffix += 1;
|
||||
}
|
||||
const sourceMiddleEnd = source.units.length - suffix;
|
||||
const targetMiddleEnd = target.units.length - suffix;
|
||||
const midA = source.units.slice(prefix, sourceMiddleEnd);
|
||||
const midB = target.units.slice(prefix, targetMiddleEnd);
|
||||
|
||||
const ops: DisplayDiffOp[] = [];
|
||||
if (prefix > 0) ops.push({ kind: "equal", aFrom: 0, aTo: source.offsets[prefix], bFrom: 0, bTo: target.offsets[prefix] });
|
||||
if (midA.length > 0 || midB.length > 0) {
|
||||
const replace = (): DisplayDiffOp[] => [
|
||||
{ kind: "delete", aFrom: source.offsets[prefix], aTo: source.offsets[sourceMiddleEnd], bFrom: target.offsets[prefix], bTo: target.offsets[prefix] },
|
||||
{ kind: "insert", aFrom: source.offsets[prefix], aTo: source.offsets[prefix], bFrom: target.offsets[prefix], bTo: target.offsets[targetMiddleEnd] },
|
||||
];
|
||||
const middle = midA.length + midB.length <= MAX_DIFF_UNITS ? myersDiff(midA, midB) : null;
|
||||
if (!middle) {
|
||||
ops.push(...replace());
|
||||
} else {
|
||||
for (const op of middle) {
|
||||
ops.push({
|
||||
kind: op.kind,
|
||||
aFrom: source.offsets[prefix + op.aFrom],
|
||||
aTo: source.offsets[prefix + op.aTo],
|
||||
bFrom: target.offsets[prefix + op.bFrom],
|
||||
bTo: target.offsets[prefix + op.bTo],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (suffix > 0) {
|
||||
ops.push({ kind: "equal", aFrom: source.offsets[sourceMiddleEnd], aTo: a.length, bFrom: target.offsets[targetMiddleEnd], bTo: b.length });
|
||||
}
|
||||
return ops;
|
||||
}
|
||||
|
||||
function splitCodePoints(text: string): { units: string[]; offsets: number[] } {
|
||||
const units: string[] = [];
|
||||
const offsets = [0];
|
||||
let offset = 0;
|
||||
for (const unit of text) {
|
||||
units.push(unit);
|
||||
offset += unit.length;
|
||||
offsets.push(offset);
|
||||
}
|
||||
return { units, offsets };
|
||||
}
|
||||
|
||||
type MidDiffOp = { kind: "equal" | "delete" | "insert"; aFrom: number; aTo: number; bFrom: number; bTo: number };
|
||||
|
||||
function myersDiff(a: readonly string[], b: readonly string[]): MidDiffOp[] | null {
|
||||
const n = a.length;
|
||||
const m = b.length;
|
||||
if (n === 0 && m === 0) return [];
|
||||
const max = n + m;
|
||||
const offset = max + 1;
|
||||
const v = new Int32Array(2 * max + 3);
|
||||
const trace: Int32Array[] = [];
|
||||
let foundD = -1;
|
||||
for (let d = 0; d <= max && d <= MAX_DIFF_DISTANCE; d += 1) {
|
||||
// Snapshot V[d-1] for backtracking: reachable k at depth d have parity d,
|
||||
// so the previous frontier lives on k with parity d-1.
|
||||
const level = new Int32Array(2 * d + 1);
|
||||
for (let k = -(d - 1); k <= d - 1; k += 2) level[k + d] = v[k + offset];
|
||||
trace.push(level);
|
||||
for (let k = -d; k <= d; k += 2) {
|
||||
const fromDown = k === -d || (k !== d && v[k - 1 + offset] < v[k + 1 + offset]);
|
||||
let x = fromDown ? v[k + 1 + offset] : v[k - 1 + offset] + 1;
|
||||
let y = x - k;
|
||||
while (x < n && y < m && a[x] === b[y]) {
|
||||
x += 1;
|
||||
y += 1;
|
||||
}
|
||||
v[k + offset] = x;
|
||||
if (x >= n && y >= m) {
|
||||
foundD = d;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (foundD >= 0) break;
|
||||
}
|
||||
if (foundD < 0) return null;
|
||||
|
||||
const ops: MidDiffOp[] = [];
|
||||
let x = n;
|
||||
let y = m;
|
||||
for (let d = foundD; d > 0; d -= 1) {
|
||||
const level = trace[d];
|
||||
const k = x - y;
|
||||
const fromDown = k === -d || (k !== d && level[k - 1 + d] < level[k + 1 + d]);
|
||||
const prevK = fromDown ? k + 1 : k - 1;
|
||||
const prevX = level[prevK + d];
|
||||
const prevY = prevX - prevK;
|
||||
while (x > prevX && y > prevY) {
|
||||
ops.push({ kind: "equal", aFrom: x - 1, aTo: x, bFrom: y - 1, bTo: y });
|
||||
x -= 1;
|
||||
y -= 1;
|
||||
}
|
||||
if (x === prevX) {
|
||||
ops.push({ kind: "insert", aFrom: x, aTo: x, bFrom: prevY, bTo: y });
|
||||
} else {
|
||||
ops.push({ kind: "delete", aFrom: prevX, aTo: x, bFrom: y, bTo: y });
|
||||
}
|
||||
x = prevX;
|
||||
y = prevY;
|
||||
}
|
||||
ops.reverse();
|
||||
return coalesceMidOps(ops);
|
||||
}
|
||||
|
||||
function coalesceMidOps(ops: MidDiffOp[]): MidDiffOp[] {
|
||||
const out: MidDiffOp[] = [];
|
||||
for (const op of ops) {
|
||||
const last = out[out.length - 1];
|
||||
if (last && last.kind === op.kind && last.aTo === op.aFrom && last.bTo === op.bFrom) {
|
||||
last.aTo = op.aTo;
|
||||
last.bTo = op.bTo;
|
||||
} else {
|
||||
out.push({ ...op });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function protectLargeJsonNumbers(text: string): ProtectedJsonNumbers {
|
||||
let placeholderPrefix = "__DBX_LOSSLESS_NUMBER_";
|
||||
while (text.includes(placeholderPrefix)) placeholderPrefix += "_";
|
||||
|
||||
Reference in new issue
Block a user