feat(redis): add decoded Unicode JSON view

apps/desktop/src/lib/common/__tests__/safeJsonFormat.spec.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { formatJsonSource, isLosslessJsonNumber, parseJsonPreservingLargeNumbers, safeJsonFormat, stringifyJsonPreservingLargeNumbers } from "../safeJsonFormat";
+import { decodeJsonUnicodeEscapes, formatJsonSource, isLosslessJsonNumber, mapDisplayToRaw, parseJsonPreservingLargeNumbers, safeJsonFormat, stringifyJsonPreservingLargeNumbers } from "../safeJsonFormat";
 
 describe("safeJsonFormat", () => {
   it("preserves large integers exceeding MAX_SAFE_INTEGER", () => {
@@ -151,3 +151,174 @@ describe("formatJsonSource", () => {
     expect(() => formatJsonSource('{"a":1} trailing')).toThrow(SyntaxError);
   });
 });
+
+describe("decodeJsonUnicodeEscapes", () => {
+  it("decodes non-ASCII escapes in both keys and values", () => {
+    const input = '{"\\u59d3\\u540d":"\\u5f20\\u4e09"}';
+    expect(decodeJsonUnicodeEscapes(input)).toBe('{"姓名":"张三"}');
+  });
+
+  it("combines surrogate pairs into a single code point", () => {
+    expect(decodeJsonUnicodeEscapes('{"e":"\\ud83d\\ude00"}')).toBe('{"e":"😀"}');
+  });
+
+  it("keeps ASCII control escapes verbatim so the output stays valid JSON", () => {
+    const input = '{"a":"\\u0000\\u000a\\u005f"}';
+    expect(decodeJsonUnicodeEscapes(input)).toBe(input);
+  });
+
+  it("keeps an escaped-backslash literal \\uXXXX un-decoded", () => {
+    // JSON source `"\\u5f20"` has content `张` (backslash + "u5f20"), not 张.
+    const input = '{"text":"\\\\u5f20"}';
+    expect(decodeJsonUnicodeEscapes(input)).toBe(input);
+  });
+
+  it("decodes a real escape that follows an escaped backslash", () => {
+    // JSON source `"\\张"` has content `\` + 张; the backslash stays escaped.
+    const input = '{"a":"\\\\\\u5f20"}';
+    expect(decodeJsonUnicodeEscapes(input)).toBe('{"a":"\\\\张"}');
+  });
+
+  it("keeps unpaired surrogate escapes verbatim", () => {
+    expect(decodeJsonUnicodeEscapes('{"a":"\\ud83d"}')).toBe('{"a":"\\ud83d"}');
+    expect(decodeJsonUnicodeEscapes('{"a":"\\ude00"}')).toBe('{"a":"\\ude00"}');
+  });
+
+  it("is a no-op on literal UTF-8 and preserves structure byte-for-byte", () => {
+    const input = '{\n  "name": "张三",\n  "n": 9007199254740992,\n  "b": [true, null]\n}';
+    expect(decodeJsonUnicodeEscapes(input)).toBe(input);
+  });
+
+  it("leaves the source-preserving formatter contract untouched", () => {
+    // formatJsonSource still preserves escapes by default; decode is opt-in.
+    const escaped = '{"name":"\\u5f20\\u4e09"}';
+    expect(formatJsonSource(escaped)).toBe(escaped);
+    expect(decodeJsonUnicodeEscapes(escaped)).toBe('{"name":"张三"}');
+  });
+
+  it("escaped quote then unicode escape inside a string", () => {
+    const raw = '{"v":"say \\"\\u5f20"}';
+    expect(decodeJsonUnicodeEscapes(raw)).toBe('{"v":"say \\"张"}');
+  });
+
+  it("escaped backslash then unicode escape inside a string", () => {
+    const raw = '{"v":"say \\\\\\u5f20"}';
+    expect(decodeJsonUnicodeEscapes(raw)).toBe('{"v":"say \\\\张"}');
+  });
+
+  it("escaped quote inside a string token does not terminate the token", () => {
+    const raw = '{"a":"x\\"y\\u4e2d","b":1}';
+    expect(decodeJsonUnicodeEscapes(raw)).toBe('{"a":"x\\"y中","b":1}');
+  });
+
+  it("reviewer scenario: escaped backslash + escaped quote before a unicode escape", () => {
+    const raw = '{"v":"say \\\\\\"\\u5f20"}';
+    expect(decodeJsonUnicodeEscapes(raw)).toBe('{"v":"say \\\\\\"张"}');
+  });
+});
+
+describe("mapDisplayToRaw", () => {
+  it("returns the raw source unchanged when the draft equals the decoded baseline", () => {
+    const raw = '{"name":"\\u5f20\\u4e09"}';
+    expect(mapDisplayToRaw(raw, decodeJsonUnicodeEscapes(raw))).toBe(raw);
+  });
+
+  it("preserves untouched escapes when editing a sibling field (save-path core case)", () => {
+    const raw = '{"name":"\\u5f20","age":30}';
+    // decoded baseline: {"name":"张","age":30}; user edits age 30 -> 31
+    const current = '{"name":"张","age":31}';
+    const mapped = mapDisplayToRaw(raw, current);
+    expect(mapped).toBe('{"name":"\\u5f20","age":31}');
+    expect(decodeJsonUnicodeEscapes(mapped)).toBe(current);
+  });
+
+  it("preserves untouched escapes in a multi-region edit (change only b)", () => {
+    const raw = '{"a":"\\u5f20","b":"\\u4e2d"}';
+    const edited = '{"a":"张","b":"文"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe('{"a":"\\u5f20","b":"文"}');
+    expect(decodeJsonUnicodeEscapes(mapped)).toBe(edited);
+  });
+
+  it("keeps a structural insert (new key) verbatim", () => {
+    const raw = '{"a":"\\u5f20"}';
+    const edited = '{"a":"张","b":1}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe('{"a":"\\u5f20","b":1}');
+    expect(decodeJsonUnicodeEscapes(mapped)).toBe(edited);
+  });
+
+  it("keeps JSON escape sequences typed in decoded mode verbatim so value semantics survive", () => {
+    // Decoded mode only inlines unicode escapes; an inserted `\\` is already the
+    // JSON source the user wants (value h\i, one literal backslash) and must not
+    // be re-escaped into h\\i (two literal backslashes) on the way to raw.
+    const raw = '{"v":"hi"}';
+    const edited = '{"v":"h\\\\i"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe('{"v":"h\\\\i"}');
+    expect(JSON.parse(mapped)).toEqual({ v: "h\\i" });
+    expect(formatJsonSource(mapped)).toBe(mapped);
+  });
+
+  it("keeps an escaped quote typed in decoded mode verbatim", () => {
+    const raw = '{"v":"a"}';
+    const edited = '{"v":"a \\"b"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe('{"v":"a \\"b"}');
+    expect(JSON.parse(mapped)).toEqual({ v: 'a "b' });
+  });
+
+  it("keeps a control escape typed in decoded mode verbatim", () => {
+    const raw = '{"v":"a"}';
+    const edited = '{"v":"a\\nb"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe('{"v":"a\\nb"}');
+    expect(JSON.parse(mapped)).toEqual({ v: "a\nb" });
+  });
+
+  it('save-path regression: decoded edit with \\\\, \\" and \\n keeps identical JSON semantics', () => {
+    // Open {"name":"张","v":"a"} in decoded mode, then edit v to a value that
+    // contains an escaped backslash, an escaped quote, and an escaped newline.
+    const raw = '{"name":"\\u5f20","v":"a"}';
+    const edited = '{"name":"张","v":"a\\\\b \\"c\\nd"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    // The untouched name escape survives; the v value keeps the user's exact JSON source.
+    expect(mapped).toBe('{"name":"\\u5f20","v":"a\\\\b \\"c\\nd"}');
+    // Identical value semantics between the decoded draft and the bytes that save.
+    expect(JSON.parse(mapped)).toEqual(JSON.parse(edited));
+    // And decoding the stored bytes reproduces the decoded draft exactly.
+    expect(decodeJsonUnicodeEscapes(mapped)).toBe(edited);
+  });
+
+  it("round-trips surrogate pairs and preserves their escapes", () => {
+    const raw = '{"e":"\\ud83d\\ude00"}';
+    const baseline = decodeJsonUnicodeEscapes(raw); // {"e":"😀"}
+    expect(mapDisplayToRaw(raw, baseline)).toBe(raw);
+    const edited = '{"e":"x😀y"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe('{"e":"x\\ud83d\\ude00y"}');
+    expect(decodeJsonUnicodeEscapes(mapped)).toBe(edited);
+  });
+
+  it("keeps surrogate pairs intact when replacing an escaped emoji", () => {
+    const raw = '{"e":"\\ud83d\\ude00"}';
+    const edited = '{"e":"😁"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe(edited);
+    expect(JSON.parse(mapped)).toEqual({ e: "😁" });
+    expect(decodeJsonUnicodeEscapes(mapped)).toBe(edited);
+  });
+
+  it("open+save without edits keeps the exact stored text (bytes preserved)", () => {
+    const raw = '{"name":"\\u5f20","nested":{"list":[1,2],"note":"\\u4e2d"}}';
+    expect(formatJsonSource(mapDisplayToRaw(raw, decodeJsonUnicodeEscapes(raw)))).toBe(formatJsonSource(raw));
+  });
+
+  it("handles a full value replacement in decoded mode", () => {
+    const raw = '{"name":"\\u5f20"}';
+    const edited = '{"name":"李四"}';
+    const mapped = mapDisplayToRaw(raw, edited);
+    expect(mapped).toBe('{"name":"李四"}');
+    expect(decodeJsonUnicodeEscapes(mapped)).toBe(edited);
+  });
+});

apps/desktop/src/lib/common/safeJsonFormat.ts

@@ -89,6 +89,289 @@ export function formatJsonSource(text: string, indent?: number): string {
   return writer.toString();
 }
 
+/**
+ * Decode non-ASCII unicode escapes (`\uXXXX`) inside JSON string tokens (keys
+ * and values) so text like `{"name":"张三"}` renders as `{"name":"张三"}`.
+ *
+ * - Code points >= 0x80 are decoded to their characters; ASCII control escapes
+ *   (U+0000, U+000A, ...) are kept verbatim so the output stays valid JSON.
+ * - Surrogate pairs combine into a single code point; unpaired surrogates stay
+ *   escaped rather than emitting invalid lone surrogate characters.
+ * - Escaped backslashes are respected, so a literal `"\\u5f20"` (whose content
+ *   is the text `张`) is never mis-decoded, and an escaped quote `\"` is
+ *   consumed inside a string token so it never terminates the token.
+ *
+ * This is a view-layer transform only and must never be written back to a
+ * database on its own.
+ */
+export function decodeJsonUnicodeEscapes(text: string): string {
+  return decodeJsonSource(text).display;
+}
+
+/**
+ * Result of walking a JSON source with {@link decodeJsonSource}: the decoded
+ * display text plus the raw-source offset of every display code unit, which lets
+ * callers map display edits back onto the source without touching escapes.
+ */
+export interface JsonUnicodeDecodedSource {
+  /** Decoded display text (what the JSON editor shows in decoded mode). */
+  display: string;
+  /**
+   * displayToRaw[i] is the raw-source offset where display code unit i
+   * originates, for i in [0, display.length]; displayToRaw[display.length] is
+   * raw.length. Surrogate-pair display units both point at the pair's start.
+   */
+  displayToRaw: number[];
+}
+
+function decodeJsonSource(text: string): JsonUnicodeDecodedSource {
+  let display = "";
+  const displayToRaw: number[] = [];
+  let index = 0;
+  const push = (ch: string, rawStart: number) => {
+    // Iterate code units, not code points, so surrogate-pair characters record
+    // an offset for each of their two units and stay aligned with display.length.
+    for (let unitIndex = 0; unitIndex < ch.length; unitIndex += 1) {
+      display += ch[unitIndex];
+      displayToRaw.push(rawStart);
+    }
+  };
+
+  while (index < text.length) {
+    if (text[index] !== '"') {
+      push(text[index], index);
+      index += 1;
+      continue;
+    }
+
+    // Rebuild one JSON string token, decoding escapes inside it.
+    const tokenStart = index;
+    index += 1;
+    push('"', tokenStart);
+    while (index < text.length) {
+      const character = text[index];
+      if (character === '"') {
+        push('"', index);
+        index += 1;
+        break;
+      }
+      if (character !== "\\") {
+        push(character, index);
+        index += 1;
+        continue;
+      }
+
+      const next = text[index + 1];
+      if (next === undefined) {
+        push("\\", index);
+        index += 1;
+        break;
+      }
+      const hex = text.slice(index + 2, index + 6);
+      if (next !== "u" || !/^[0-9a-fA-F]{4}$/.test(hex)) {
+        // Not a unicode escape (escaped backslash, escaped quote, ...); copy verbatim.
+        push(`\\${next}`, index);
+        index += 2;
+        continue;
+      }
+
+      const code = parseInt(hex, 16);
+      const lowEscape = text.slice(index + 6, index + 12);
+      const lowMatch = /^\\u([0-9a-fA-F]{4})$/.exec(lowEscape);
+      if (code >= 0xd800 && code <= 0xdbff && lowMatch) {
+        const low = parseInt(lowMatch[1], 16);
+        if (low >= 0xdc00 && low <= 0xdfff) {
+          push(String.fromCodePoint(0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00)), index);
+          index += 12;
+          continue;
+        }
+      }
+      if (code >= 0xd800 && code <= 0xdfff) {
+        // Unpaired surrogate: keep the escape rather than emit a lone surrogate char.
+        push(`\\u${hex}`, index);
+      } else if (code >= 0x80) {
+        push(String.fromCharCode(code), index);
+      } else {
+        // ASCII control escapes stay escaped for valid JSON.
+        push(`\\u${hex}`, index);
+      }
+      index += 6;
+    }
+  }
+  displayToRaw.push(text.length);
+  return { display, displayToRaw };
+}
+
+/**
+ * Reconstruct the raw JSON source for a decoded-mode editor draft.
+ *
+ * The editor shows {@link decodeJsonUnicodeEscapes decoded} text; this maps the
+ * user's current draft back onto the canonical raw source so untouched escapes
+ * and the original whitespace/formatting are preserved. Decoded mode only inlines
+ * non-ASCII `\uXXXX` escapes, so every other span of the draft is already JSON
+ * source and is written back verbatim — inserted `\\`, `\"`, and control escapes
+ * keep their meaning instead of being re-escaped. When the draft equals the
+ * decoded baseline the raw source is returned unchanged.
+ *
+ * The result is only valid JSON when the draft is; callers must still validate
+ * with {@link formatJsonSource} before saving.
+ */
+export function mapDisplayToRaw(raw: string, displayCurrent: string): string {
+  const { display: baseline, displayToRaw } = decodeJsonSource(raw);
+  if (displayCurrent === baseline) return raw;
+  let out = "";
+  for (const op of diffText(baseline, displayCurrent)) {
+    if (op.kind === "equal") {
+      out += raw.slice(displayToRaw[op.aFrom], displayToRaw[op.aTo]);
+    } else if (op.kind === "delete") {
+      // The deleted display span's raw source is dropped with it.
+    } else {
+      // Inserted display text is JSON source in the decoded draft; write it back
+      // verbatim so its value semantics are preserved.
+      out += displayCurrent.slice(op.bFrom, op.bTo);
+    }
+  }
+  return out;
+}
+
+type DisplayDiffOp = { kind: "equal"; aFrom: number; aTo: number; bFrom: number; bTo: number } | { kind: "delete"; aFrom: number; aTo: number; bFrom: number; bTo: number } | { kind: "insert"; aFrom: number; aTo: number; bFrom: number; bTo: number };
+
+// Beyond this combined size, fall back to a single replace; whole-document
+// rewrites gain nothing from escape-level preservation.
+const MAX_DIFF_UNITS = 200_000;
+// Cap the Myers edit distance so adversarial edits stay bounded in time and trace.
+const MAX_DIFF_DISTANCE = 2000;
+
+function diffText(a: string, b: string): DisplayDiffOp[] {
+  const source = splitCodePoints(a);
+  const target = splitCodePoints(b);
+  let prefix = 0;
+  while (prefix < source.units.length && prefix < target.units.length && source.units[prefix] === target.units[prefix]) prefix += 1;
+  let suffix = 0;
+  while (suffix < source.units.length - prefix && suffix < target.units.length - prefix && source.units[source.units.length - 1 - suffix] === target.units[target.units.length - 1 - suffix]) {
+    suffix += 1;
+  }
+  const sourceMiddleEnd = source.units.length - suffix;
+  const targetMiddleEnd = target.units.length - suffix;
+  const midA = source.units.slice(prefix, sourceMiddleEnd);
+  const midB = target.units.slice(prefix, targetMiddleEnd);
+
+  const ops: DisplayDiffOp[] = [];
+  if (prefix > 0) ops.push({ kind: "equal", aFrom: 0, aTo: source.offsets[prefix], bFrom: 0, bTo: target.offsets[prefix] });
+  if (midA.length > 0 || midB.length > 0) {
+    const replace = (): DisplayDiffOp[] => [
+      { kind: "delete", aFrom: source.offsets[prefix], aTo: source.offsets[sourceMiddleEnd], bFrom: target.offsets[prefix], bTo: target.offsets[prefix] },
+      { kind: "insert", aFrom: source.offsets[prefix], aTo: source.offsets[prefix], bFrom: target.offsets[prefix], bTo: target.offsets[targetMiddleEnd] },
+    ];
+    const middle = midA.length + midB.length <= MAX_DIFF_UNITS ? myersDiff(midA, midB) : null;
+    if (!middle) {
+      ops.push(...replace());
+    } else {
+      for (const op of middle) {
+        ops.push({
+          kind: op.kind,
+          aFrom: source.offsets[prefix + op.aFrom],
+          aTo: source.offsets[prefix + op.aTo],
+          bFrom: target.offsets[prefix + op.bFrom],
+          bTo: target.offsets[prefix + op.bTo],
+        });
+      }
+    }
+  }
+  if (suffix > 0) {
+    ops.push({ kind: "equal", aFrom: source.offsets[sourceMiddleEnd], aTo: a.length, bFrom: target.offsets[targetMiddleEnd], bTo: b.length });
+  }
+  return ops;
+}
+
+function splitCodePoints(text: string): { units: string[]; offsets: number[] } {
+  const units: string[] = [];
+  const offsets = [0];
+  let offset = 0;
+  for (const unit of text) {
+    units.push(unit);
+    offset += unit.length;
+    offsets.push(offset);
+  }
+  return { units, offsets };
+}
+
+type MidDiffOp = { kind: "equal" | "delete" | "insert"; aFrom: number; aTo: number; bFrom: number; bTo: number };
+
+function myersDiff(a: readonly string[], b: readonly string[]): MidDiffOp[] | null {
+  const n = a.length;
+  const m = b.length;
+  if (n === 0 && m === 0) return [];
+  const max = n + m;
+  const offset = max + 1;
+  const v = new Int32Array(2 * max + 3);
+  const trace: Int32Array[] = [];
+  let foundD = -1;
+  for (let d = 0; d <= max && d <= MAX_DIFF_DISTANCE; d += 1) {
+    // Snapshot V[d-1] for backtracking: reachable k at depth d have parity d,
+    // so the previous frontier lives on k with parity d-1.
+    const level = new Int32Array(2 * d + 1);
+    for (let k = -(d - 1); k <= d - 1; k += 2) level[k + d] = v[k + offset];
+    trace.push(level);
+    for (let k = -d; k <= d; k += 2) {
+      const fromDown = k === -d || (k !== d && v[k - 1 + offset] < v[k + 1 + offset]);
+      let x = fromDown ? v[k + 1 + offset] : v[k - 1 + offset] + 1;
+      let y = x - k;
+      while (x < n && y < m && a[x] === b[y]) {
+        x += 1;
+        y += 1;
+      }
+      v[k + offset] = x;
+      if (x >= n && y >= m) {
+        foundD = d;
+        break;
+      }
+    }
+    if (foundD >= 0) break;
+  }
+  if (foundD < 0) return null;
+
+  const ops: MidDiffOp[] = [];
+  let x = n;
+  let y = m;
+  for (let d = foundD; d > 0; d -= 1) {
+    const level = trace[d];
+    const k = x - y;
+    const fromDown = k === -d || (k !== d && level[k - 1 + d] < level[k + 1 + d]);
+    const prevK = fromDown ? k + 1 : k - 1;
+    const prevX = level[prevK + d];
+    const prevY = prevX - prevK;
+    while (x > prevX && y > prevY) {
+      ops.push({ kind: "equal", aFrom: x - 1, aTo: x, bFrom: y - 1, bTo: y });
+      x -= 1;
+      y -= 1;
+    }
+    if (x === prevX) {
+      ops.push({ kind: "insert", aFrom: x, aTo: x, bFrom: prevY, bTo: y });
+    } else {
+      ops.push({ kind: "delete", aFrom: prevX, aTo: x, bFrom: y, bTo: y });
+    }
+    x = prevX;
+    y = prevY;
+  }
+  ops.reverse();
+  return coalesceMidOps(ops);
+}
+
+function coalesceMidOps(ops: MidDiffOp[]): MidDiffOp[] {
+  const out: MidDiffOp[] = [];
+  for (const op of ops) {
+    const last = out[out.length - 1];
+    if (last && last.kind === op.kind && last.aTo === op.aFrom && last.bTo === op.bFrom) {
+      last.aTo = op.aTo;
+      last.bTo = op.bTo;
+    } else {
+      out.push({ ...op });
+    }
+  }
+  return out;
+}
+
 function protectLargeJsonNumbers(text: string): ProtectedJsonNumbers {
   let placeholderPrefix = "__DBX_LOSSLESS_NUMBER_";
   while (text.includes(placeholderPrefix)) placeholderPrefix += "_";