feat(redis): add decompressed view for compressed values

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-08-03 20:59:39 +08:00
1 parent 06838c422b
commit ec44e09ec8
15 files changed
+551 -8

No files matched your search

@@ -0,0 +1,163 @@
import { describe, expect, it } from "vitest";
import { deflateRawSync, deflateSync, gzipSync } from "zlib";
import { decompressRedisValue, isGzipMagic, REDIS_DECOMPRESS_MAX_OUTPUT_BYTES } from "../redisCompression";
// Decompression runs against the real pako implementation (pure JS), so these
// tests exercise the actual production path in Node the same way it runs in the
// Tauri WebView2 / WKWebView renderer — no DecompressionStream mock, no
// environment-specific behavior to paper over. node:zlib is used only to build
// fixtures (it is byte-compatible with pako for gzip/zlib/raw DEFLATE).
function gzipOf(text: string): Uint8Array {
return new Uint8Array(gzipSync(text));
}
function zlibOf(text: string): Uint8Array {
return new Uint8Array(deflateSync(text));
}
function deflateRawOf(data: Uint8Array | string): Uint8Array {
return new Uint8Array(deflateRawSync(data));
}
describe("isGzipMagic", () => {
it("detects the gzip header", () => {
expect(isGzipMagic(gzipOf("hello"))).toBe(true);
});
it("rejects non-gzip payloads", () => {
expect(isGzipMagic(new Uint8Array([0x78, 0x9c, 0x01, 0x00]))).toBe(false);
expect(isGzipMagic(new Uint8Array(0))).toBe(false);
expect(isGzipMagic(new Uint8Array([0x1f]))).toBe(false);
});
});
describe("decompressRedisValue", () => {
it("decompresses gzip via magic detection", async () => {
const result = await decompressRedisValue(gzipOf('{"a":1}'));
expect(result).toEqual({ ok: true, text: '{"a":1}', algorithm: "gzip" });
});
it("decompresses zlib-wrapped deflate", async () => {
const result = await decompressRedisValue(zlibOf("hello world"));
expect(result).toEqual({ ok: true, text: "hello world", algorithm: "zlib" });
});
it("does not auto-detect raw deflate from a valid raw-deflate stream", async () => {
// A valid raw DEFLATE stream has no framing and no checksum, so arbitrary
// binary can look like it. Auto-detection must not accept it — regression
// for the old behavior that fell back to raw deflate after zlib failed and
// displayed the "decompressed" garbage as real content.
const rawDeflate = deflateRawOf("raw deflate payload");
const result = await decompressRedisValue(rawDeflate);
expect(result).toEqual({ ok: false, reason: "corrupt" });
});
it("rejects arbitrary binary that happens to be a valid raw-deflate stream", async () => {
// This is the exact false-positive the reviewer flagged: a value that was
// never meant to be compressed data (a binary payload — a PNG header plus
// non-UTF-8 bytes) gets raw-deflated, producing a structurally valid
// bitstream. Under the old code it was accepted and shown as decompressed
// garbage; auto-detection must now reject it.
const binaryPayload = new Uint8Array([
0x89,
0x50,
0x4e,
0x47,
0x0d,
0x0a,
0x1a,
0x0a, // PNG signature
0x00,
0x00,
0x00,
0x0d,
0x49,
0x48,
0x44,
0x52,
0xff,
0x00,
0xff,
0x00,
0x01,
0x02,
0x03,
0x7f,
0x80,
0xc0,
0xe0,
0xfe,
]);
const rawDeflate = deflateRawOf(binaryPayload);
const result = await decompressRedisValue(rawDeflate);
expect(result.ok).toBe(false);
});
it("decompresses raw deflate only when the algorithm is explicitly requested", async () => {
const rawDeflate = deflateRawOf("explicit raw deflate payload");
expect(await decompressRedisValue(rawDeflate)).toEqual({ ok: false, reason: "corrupt" });
const result = await decompressRedisValue(rawDeflate, { algorithm: "deflate" });
expect(result).toEqual({ ok: true, text: "explicit raw deflate payload", algorithm: "deflate" });
});
it("forces a single algorithm when requested, bypassing magic detection", async () => {
const gz = gzipOf("gzip data");
const zl = zlibOf("zlib data");
// A gzip value forced as zlib must fail, and vice versa — no cross-format
// guessing when the caller commits to an algorithm.
expect(await decompressRedisValue(gz, { algorithm: "zlib" })).toEqual({ ok: false, reason: "corrupt" });
expect(await decompressRedisValue(zl, { algorithm: "gzip" })).toEqual({ ok: false, reason: "corrupt" });
expect(await decompressRedisValue(gz, { algorithm: "gzip" })).toEqual({ ok: true, text: "gzip data", algorithm: "gzip" });
expect(await decompressRedisValue(zl, { algorithm: "zlib" })).toEqual({ ok: true, text: "zlib data", algorithm: "zlib" });
});
it("reports corrupt data without throwing", async () => {
const result = await decompressRedisValue(new Uint8Array([0x1f, 0x8b, 0x00, 0x00, 0x00]));
expect(result).toEqual({ ok: false, reason: "corrupt" });
});
it("reports corrupt for plain text that is not compressed", async () => {
const result = await decompressRedisValue(new TextEncoder().encode("just plain text"));
expect(result.ok).toBe(false);
});
it("reports corrupt for empty input", async () => {
const result = await decompressRedisValue(new Uint8Array(0));
expect(result).toEqual({ ok: false, reason: "corrupt" });
});
it("enforces the output cap during decompression for high-ratio input", async () => {
// 64 MiB of zeros compresses to ~64 KiB (~1000:1). The bounded inflate
// streams output in chunks and aborts as soon as cumulative output exceeds
// the cap — the 64 MiB is never materialized. Runs against real pako, so it
// reflects actual platform behavior rather than a mock that buffers the
// whole result first.
const payload = new Uint8Array(64 * 1024 * 1024);
const cases: Array<[Uint8Array, Parameters<typeof decompressRedisValue>[1]]> = [
[new Uint8Array(gzipSync(payload)), undefined],
[new Uint8Array(deflateSync(payload)), undefined],
[deflateRawOf(payload), { algorithm: "deflate" }],
];
for (const [compressed, options] of cases) {
const result = await decompressRedisValue(compressed, { ...options, maxOutputBytes: 1024 });
expect(result).toEqual({ ok: false, reason: "limit" });
}
});
it("allows output up to exactly the cap", async () => {
const payload = new Uint8Array(1024);
const result = await decompressRedisValue(new Uint8Array(gzipSync(payload)), { maxOutputBytes: 1024 });
expect(result.ok).toBe(true);
if (result.ok) expect(result.algorithm).toBe("gzip");
});
it("respects the shared default cap constant", () => {
expect(REDIS_DECOMPRESS_MAX_OUTPUT_BYTES).toBe(50 * 1024 * 1024);
});
it("honors a custom cap below the default", async () => {
const result = await decompressRedisValue(gzipOf("small payload"), { maxOutputBytes: 8 });
expect(result).toEqual({ ok: false, reason: "limit" });
});
});
@@ -0,0 +1,134 @@
import { Inflate as PakoInflate, Z_OK } from "pako";
export const REDIS_DECOMPRESS_MAX_OUTPUT_BYTES = 50 * 1024 * 1024;
export type RedisDecompressAlgorithm = "gzip" | "zlib" | "deflate";
/** Reliable gzip header — the only compression signature we trust for auto-detection. */
export function isGzipMagic(bytes: Uint8Array): boolean {
return bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b;
}
export type RedisDecompressResult =
| {
ok: true;
text: string;
algorithm: RedisDecompressAlgorithm;
}
| {
ok: false;
reason: "corrupt" | "limit";
};
export type RedisDecompressOptions = {
maxOutputBytes?: number;
/**
* Force a specific compression format instead of auto-detecting.
* Auto-detection recognizes only gzip (magic bytes) and zlib (RFC 1950 header
* + ADLER32 checksum). Raw deflate (`"deflate"`, RFC 1951) has no framing or
* checksum, so arbitrary binary data can be misread as valid output — pass
* this only when the value is known to be raw DEFLATE.
*/
algorithm?: RedisDecompressAlgorithm;
};
class DecompressionLimitError extends Error {
readonly limitBytes: number;
constructor(limitBytes: number) {
super(`Decompressed output exceeds ${limitBytes} bytes`);
this.name = "DecompressionLimitError";
this.limitBytes = limitBytes;
}
}
// pako windowBits per format: gzip = 31 (RFC 1952), zlib = 15 (RFC 1950),
// raw deflate = -15 (RFC 1951).
const WINDOW_BITS: Record<RedisDecompressAlgorithm, number> = {
gzip: 31,
zlib: 15,
deflate: -15,
};
/**
* Bounded inflate via pako's streaming `Inflate`. Output arrives chunk by chunk
* through `onData`; we count it and throw the moment cumulative output exceeds
* `maxOutputBytes`. Because pako emits chunks incrementally (its zlib strm uses
* a fixed-size output buffer), a zip bomb aborts after the first chunks past
* the cap — the full output is never materialized, so peak memory stays
* ~cap + chunk + input instead of unbounded. This is an allocation-time limit,
* unlike counting bytes only after a decompressor has already buffered the
* whole result internally.
*/
function inflateBounded(bytes: Uint8Array, algorithm: RedisDecompressAlgorithm, maxOutputBytes: number): Uint8Array {
const inflater = new PakoInflate({ windowBits: WINDOW_BITS[algorithm] });
const chunks: Uint8Array[] = [];
let total = 0;
inflater.onData = (chunk) => {
total += chunk.byteLength;
if (total > maxOutputBytes) throw new DecompressionLimitError(maxOutputBytes);
chunks.push(chunk);
};
inflater.push(bytes, true);
if (inflater.err !== Z_OK) {
throw new Error(inflater.msg || `decompression failed (status ${inflater.err})`);
}
return concatBytes(chunks);
}
function concatBytes(chunks: Uint8Array[]): Uint8Array {
let total = 0;
for (const chunk of chunks) total += chunk.byteLength;
const merged = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.byteLength;
}
return merged;
}
function decodeText(bytes: Uint8Array): string {
return new TextDecoder("utf-8").decode(bytes);
}
/**
* Decompress Redis value bytes with the standard web formats.
*
* Detection order (when no `algorithm` is forced):
* 1. gzip — reliable magic (`1f 8b`); a failure here means corrupt data.
* 2. zlib — RFC 1950 header + ADLER32 trailer validate on success, so a
* successful zlib decode is checksum-verified.
* Raw deflate (RFC 1951) is NEVER auto-detected: it has no framing or checksum,
* so arbitrary binary can be accepted and shown, searched, or copied as
* "decompressed" content. Callers that know a value is raw DEFLATE must pass
* `{ algorithm: "deflate" }` explicitly.
*
* The `ok` result carries the algorithm that actually succeeded so the UI can
* label it (e.g. "Decompressed (zlib)").
*/
export async function decompressRedisValue(bytes: Uint8Array, options: RedisDecompressOptions = {}): Promise<RedisDecompressResult> {
const maxOutputBytes = options.maxOutputBytes ?? REDIS_DECOMPRESS_MAX_OUTPUT_BYTES;
if (bytes.length === 0) return { ok: false, reason: "corrupt" };
if (options.algorithm) {
return decompressOne(bytes, options.algorithm, maxOutputBytes);
}
if (isGzipMagic(bytes)) {
return decompressOne(bytes, "gzip", maxOutputBytes);
}
return decompressOne(bytes, "zlib", maxOutputBytes);
}
async function decompressOne(bytes: Uint8Array, algorithm: RedisDecompressAlgorithm, maxOutputBytes: number): Promise<RedisDecompressResult> {
// Yield so the UI can paint the loading state before the synchronous inflate.
await new Promise((resolve) => setTimeout(resolve, 0));
try {
const output = inflateBounded(bytes, algorithm, maxOutputBytes);
return { ok: true, text: decodeText(output), algorithm };
} catch (error) {
if (error instanceof DecompressionLimitError) return { ok: false, reason: "limit" };
return { ok: false, reason: "corrupt" };
}
}
@@ -4,10 +4,10 @@ import { formatJsonSource, parseJsonPreservingLargeNumbers } from "@/lib/common/
import type { RedisBlob, RedisCollectionPage, RedisHashItem, RedisListItem, RedisSetItem, RedisValue, RedisZsetItem } from "@/lib/backend/api";
import { parseJavaSerializedDetail, type RedisJavaSerializedDetail } from "@/lib/redis/javaSerialized";
export type RedisValueFormat = "utf8" | "ascii" | "binary" | "json" | "javaserialize" | "hex" | "base64";
export type RedisValueFormat = "utf8" | "ascii" | "binary" | "json" | "javaserialize" | "hex" | "base64" | "decompressed";
export type RedisMemberDetailFormat = "json" | "text";
export const REDIS_VALUE_FORMAT_DISPLAY_ORDER: RedisValueFormat[] = ["utf8", "ascii", "binary", "json", "javaserialize", "hex", "base64"];
export const REDIS_VALUE_FORMAT_DISPLAY_ORDER: RedisValueFormat[] = ["utf8", "ascii", "binary", "json", "javaserialize", "hex", "base64", "decompressed"];
export interface RedisMemberDetail {
text: string;