feat(redis): add decompressed view for compressed values
This commit is contained in:
1 parent
06838c422b
commit
ec44e09ec8
15 files changed
+551
-8
No files matched your search
@@ -0,0 +1,163 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { deflateRawSync, deflateSync, gzipSync } from "zlib";
|
||||
import { decompressRedisValue, isGzipMagic, REDIS_DECOMPRESS_MAX_OUTPUT_BYTES } from "../redisCompression";
|
||||
|
||||
// Decompression runs against the real pako implementation (pure JS), so these
|
||||
// tests exercise the actual production path in Node the same way it runs in the
|
||||
// Tauri WebView2 / WKWebView renderer — no DecompressionStream mock, no
|
||||
// environment-specific behavior to paper over. node:zlib is used only to build
|
||||
// fixtures (it is byte-compatible with pako for gzip/zlib/raw DEFLATE).
|
||||
|
||||
function gzipOf(text: string): Uint8Array {
|
||||
return new Uint8Array(gzipSync(text));
|
||||
}
|
||||
|
||||
function zlibOf(text: string): Uint8Array {
|
||||
return new Uint8Array(deflateSync(text));
|
||||
}
|
||||
|
||||
function deflateRawOf(data: Uint8Array | string): Uint8Array {
|
||||
return new Uint8Array(deflateRawSync(data));
|
||||
}
|
||||
|
||||
describe("isGzipMagic", () => {
|
||||
it("detects the gzip header", () => {
|
||||
expect(isGzipMagic(gzipOf("hello"))).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects non-gzip payloads", () => {
|
||||
expect(isGzipMagic(new Uint8Array([0x78, 0x9c, 0x01, 0x00]))).toBe(false);
|
||||
expect(isGzipMagic(new Uint8Array(0))).toBe(false);
|
||||
expect(isGzipMagic(new Uint8Array([0x1f]))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decompressRedisValue", () => {
|
||||
it("decompresses gzip via magic detection", async () => {
|
||||
const result = await decompressRedisValue(gzipOf('{"a":1}'));
|
||||
expect(result).toEqual({ ok: true, text: '{"a":1}', algorithm: "gzip" });
|
||||
});
|
||||
|
||||
it("decompresses zlib-wrapped deflate", async () => {
|
||||
const result = await decompressRedisValue(zlibOf("hello world"));
|
||||
expect(result).toEqual({ ok: true, text: "hello world", algorithm: "zlib" });
|
||||
});
|
||||
|
||||
it("does not auto-detect raw deflate from a valid raw-deflate stream", async () => {
|
||||
// A valid raw DEFLATE stream has no framing and no checksum, so arbitrary
|
||||
// binary can look like it. Auto-detection must not accept it — regression
|
||||
// for the old behavior that fell back to raw deflate after zlib failed and
|
||||
// displayed the "decompressed" garbage as real content.
|
||||
const rawDeflate = deflateRawOf("raw deflate payload");
|
||||
const result = await decompressRedisValue(rawDeflate);
|
||||
expect(result).toEqual({ ok: false, reason: "corrupt" });
|
||||
});
|
||||
|
||||
it("rejects arbitrary binary that happens to be a valid raw-deflate stream", async () => {
|
||||
// This is the exact false-positive the reviewer flagged: a value that was
|
||||
// never meant to be compressed data (a binary payload — a PNG header plus
|
||||
// non-UTF-8 bytes) gets raw-deflated, producing a structurally valid
|
||||
// bitstream. Under the old code it was accepted and shown as decompressed
|
||||
// garbage; auto-detection must now reject it.
|
||||
const binaryPayload = new Uint8Array([
|
||||
0x89,
|
||||
0x50,
|
||||
0x4e,
|
||||
0x47,
|
||||
0x0d,
|
||||
0x0a,
|
||||
0x1a,
|
||||
0x0a, // PNG signature
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x0d,
|
||||
0x49,
|
||||
0x48,
|
||||
0x44,
|
||||
0x52,
|
||||
0xff,
|
||||
0x00,
|
||||
0xff,
|
||||
0x00,
|
||||
0x01,
|
||||
0x02,
|
||||
0x03,
|
||||
0x7f,
|
||||
0x80,
|
||||
0xc0,
|
||||
0xe0,
|
||||
0xfe,
|
||||
]);
|
||||
const rawDeflate = deflateRawOf(binaryPayload);
|
||||
const result = await decompressRedisValue(rawDeflate);
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("decompresses raw deflate only when the algorithm is explicitly requested", async () => {
|
||||
const rawDeflate = deflateRawOf("explicit raw deflate payload");
|
||||
expect(await decompressRedisValue(rawDeflate)).toEqual({ ok: false, reason: "corrupt" });
|
||||
const result = await decompressRedisValue(rawDeflate, { algorithm: "deflate" });
|
||||
expect(result).toEqual({ ok: true, text: "explicit raw deflate payload", algorithm: "deflate" });
|
||||
});
|
||||
|
||||
it("forces a single algorithm when requested, bypassing magic detection", async () => {
|
||||
const gz = gzipOf("gzip data");
|
||||
const zl = zlibOf("zlib data");
|
||||
// A gzip value forced as zlib must fail, and vice versa — no cross-format
|
||||
// guessing when the caller commits to an algorithm.
|
||||
expect(await decompressRedisValue(gz, { algorithm: "zlib" })).toEqual({ ok: false, reason: "corrupt" });
|
||||
expect(await decompressRedisValue(zl, { algorithm: "gzip" })).toEqual({ ok: false, reason: "corrupt" });
|
||||
expect(await decompressRedisValue(gz, { algorithm: "gzip" })).toEqual({ ok: true, text: "gzip data", algorithm: "gzip" });
|
||||
expect(await decompressRedisValue(zl, { algorithm: "zlib" })).toEqual({ ok: true, text: "zlib data", algorithm: "zlib" });
|
||||
});
|
||||
|
||||
it("reports corrupt data without throwing", async () => {
|
||||
const result = await decompressRedisValue(new Uint8Array([0x1f, 0x8b, 0x00, 0x00, 0x00]));
|
||||
expect(result).toEqual({ ok: false, reason: "corrupt" });
|
||||
});
|
||||
|
||||
it("reports corrupt for plain text that is not compressed", async () => {
|
||||
const result = await decompressRedisValue(new TextEncoder().encode("just plain text"));
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("reports corrupt for empty input", async () => {
|
||||
const result = await decompressRedisValue(new Uint8Array(0));
|
||||
expect(result).toEqual({ ok: false, reason: "corrupt" });
|
||||
});
|
||||
|
||||
it("enforces the output cap during decompression for high-ratio input", async () => {
|
||||
// 64 MiB of zeros compresses to ~64 KiB (~1000:1). The bounded inflate
|
||||
// streams output in chunks and aborts as soon as cumulative output exceeds
|
||||
// the cap — the 64 MiB is never materialized. Runs against real pako, so it
|
||||
// reflects actual platform behavior rather than a mock that buffers the
|
||||
// whole result first.
|
||||
const payload = new Uint8Array(64 * 1024 * 1024);
|
||||
const cases: Array<[Uint8Array, Parameters<typeof decompressRedisValue>[1]]> = [
|
||||
[new Uint8Array(gzipSync(payload)), undefined],
|
||||
[new Uint8Array(deflateSync(payload)), undefined],
|
||||
[deflateRawOf(payload), { algorithm: "deflate" }],
|
||||
];
|
||||
for (const [compressed, options] of cases) {
|
||||
const result = await decompressRedisValue(compressed, { ...options, maxOutputBytes: 1024 });
|
||||
expect(result).toEqual({ ok: false, reason: "limit" });
|
||||
}
|
||||
});
|
||||
|
||||
it("allows output up to exactly the cap", async () => {
|
||||
const payload = new Uint8Array(1024);
|
||||
const result = await decompressRedisValue(new Uint8Array(gzipSync(payload)), { maxOutputBytes: 1024 });
|
||||
expect(result.ok).toBe(true);
|
||||
if (result.ok) expect(result.algorithm).toBe("gzip");
|
||||
});
|
||||
|
||||
it("respects the shared default cap constant", () => {
|
||||
expect(REDIS_DECOMPRESS_MAX_OUTPUT_BYTES).toBe(50 * 1024 * 1024);
|
||||
});
|
||||
|
||||
it("honors a custom cap below the default", async () => {
|
||||
const result = await decompressRedisValue(gzipOf("small payload"), { maxOutputBytes: 8 });
|
||||
expect(result).toEqual({ ok: false, reason: "limit" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Inflate as PakoInflate, Z_OK } from "pako";
|
||||
|
||||
export const REDIS_DECOMPRESS_MAX_OUTPUT_BYTES = 50 * 1024 * 1024;
|
||||
|
||||
export type RedisDecompressAlgorithm = "gzip" | "zlib" | "deflate";
|
||||
|
||||
/** Reliable gzip header — the only compression signature we trust for auto-detection. */
|
||||
export function isGzipMagic(bytes: Uint8Array): boolean {
|
||||
return bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b;
|
||||
}
|
||||
|
||||
export type RedisDecompressResult =
|
||||
| {
|
||||
ok: true;
|
||||
text: string;
|
||||
algorithm: RedisDecompressAlgorithm;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
reason: "corrupt" | "limit";
|
||||
};
|
||||
|
||||
export type RedisDecompressOptions = {
|
||||
maxOutputBytes?: number;
|
||||
/**
|
||||
* Force a specific compression format instead of auto-detecting.
|
||||
* Auto-detection recognizes only gzip (magic bytes) and zlib (RFC 1950 header
|
||||
* + ADLER32 checksum). Raw deflate (`"deflate"`, RFC 1951) has no framing or
|
||||
* checksum, so arbitrary binary data can be misread as valid output — pass
|
||||
* this only when the value is known to be raw DEFLATE.
|
||||
*/
|
||||
algorithm?: RedisDecompressAlgorithm;
|
||||
};
|
||||
|
||||
class DecompressionLimitError extends Error {
|
||||
readonly limitBytes: number;
|
||||
constructor(limitBytes: number) {
|
||||
super(`Decompressed output exceeds ${limitBytes} bytes`);
|
||||
this.name = "DecompressionLimitError";
|
||||
this.limitBytes = limitBytes;
|
||||
}
|
||||
}
|
||||
|
||||
// pako windowBits per format: gzip = 31 (RFC 1952), zlib = 15 (RFC 1950),
|
||||
// raw deflate = -15 (RFC 1951).
|
||||
const WINDOW_BITS: Record<RedisDecompressAlgorithm, number> = {
|
||||
gzip: 31,
|
||||
zlib: 15,
|
||||
deflate: -15,
|
||||
};
|
||||
|
||||
/**
|
||||
* Bounded inflate via pako's streaming `Inflate`. Output arrives chunk by chunk
|
||||
* through `onData`; we count it and throw the moment cumulative output exceeds
|
||||
* `maxOutputBytes`. Because pako emits chunks incrementally (its zlib strm uses
|
||||
* a fixed-size output buffer), a zip bomb aborts after the first chunks past
|
||||
* the cap — the full output is never materialized, so peak memory stays
|
||||
* ~cap + chunk + input instead of unbounded. This is an allocation-time limit,
|
||||
* unlike counting bytes only after a decompressor has already buffered the
|
||||
* whole result internally.
|
||||
*/
|
||||
function inflateBounded(bytes: Uint8Array, algorithm: RedisDecompressAlgorithm, maxOutputBytes: number): Uint8Array {
|
||||
const inflater = new PakoInflate({ windowBits: WINDOW_BITS[algorithm] });
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
inflater.onData = (chunk) => {
|
||||
total += chunk.byteLength;
|
||||
if (total > maxOutputBytes) throw new DecompressionLimitError(maxOutputBytes);
|
||||
chunks.push(chunk);
|
||||
};
|
||||
inflater.push(bytes, true);
|
||||
if (inflater.err !== Z_OK) {
|
||||
throw new Error(inflater.msg || `decompression failed (status ${inflater.err})`);
|
||||
}
|
||||
return concatBytes(chunks);
|
||||
}
|
||||
|
||||
function concatBytes(chunks: Uint8Array[]): Uint8Array {
|
||||
let total = 0;
|
||||
for (const chunk of chunks) total += chunk.byteLength;
|
||||
const merged = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
merged.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
function decodeText(bytes: Uint8Array): string {
|
||||
return new TextDecoder("utf-8").decode(bytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress Redis value bytes with the standard web formats.
|
||||
*
|
||||
* Detection order (when no `algorithm` is forced):
|
||||
* 1. gzip — reliable magic (`1f 8b`); a failure here means corrupt data.
|
||||
* 2. zlib — RFC 1950 header + ADLER32 trailer validate on success, so a
|
||||
* successful zlib decode is checksum-verified.
|
||||
* Raw deflate (RFC 1951) is NEVER auto-detected: it has no framing or checksum,
|
||||
* so arbitrary binary can be accepted and shown, searched, or copied as
|
||||
* "decompressed" content. Callers that know a value is raw DEFLATE must pass
|
||||
* `{ algorithm: "deflate" }` explicitly.
|
||||
*
|
||||
* The `ok` result carries the algorithm that actually succeeded so the UI can
|
||||
* label it (e.g. "Decompressed (zlib)").
|
||||
*/
|
||||
export async function decompressRedisValue(bytes: Uint8Array, options: RedisDecompressOptions = {}): Promise<RedisDecompressResult> {
|
||||
const maxOutputBytes = options.maxOutputBytes ?? REDIS_DECOMPRESS_MAX_OUTPUT_BYTES;
|
||||
if (bytes.length === 0) return { ok: false, reason: "corrupt" };
|
||||
|
||||
if (options.algorithm) {
|
||||
return decompressOne(bytes, options.algorithm, maxOutputBytes);
|
||||
}
|
||||
|
||||
if (isGzipMagic(bytes)) {
|
||||
return decompressOne(bytes, "gzip", maxOutputBytes);
|
||||
}
|
||||
|
||||
return decompressOne(bytes, "zlib", maxOutputBytes);
|
||||
}
|
||||
|
||||
async function decompressOne(bytes: Uint8Array, algorithm: RedisDecompressAlgorithm, maxOutputBytes: number): Promise<RedisDecompressResult> {
|
||||
// Yield so the UI can paint the loading state before the synchronous inflate.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
try {
|
||||
const output = inflateBounded(bytes, algorithm, maxOutputBytes);
|
||||
return { ok: true, text: decodeText(output), algorithm };
|
||||
} catch (error) {
|
||||
if (error instanceof DecompressionLimitError) return { ok: false, reason: "limit" };
|
||||
return { ok: false, reason: "corrupt" };
|
||||
}
|
||||
}
|
||||
@@ -4,10 +4,10 @@ import { formatJsonSource, parseJsonPreservingLargeNumbers } from "@/lib/common/
|
||||
import type { RedisBlob, RedisCollectionPage, RedisHashItem, RedisListItem, RedisSetItem, RedisValue, RedisZsetItem } from "@/lib/backend/api";
|
||||
import { parseJavaSerializedDetail, type RedisJavaSerializedDetail } from "@/lib/redis/javaSerialized";
|
||||
|
||||
export type RedisValueFormat = "utf8" | "ascii" | "binary" | "json" | "javaserialize" | "hex" | "base64";
|
||||
export type RedisValueFormat = "utf8" | "ascii" | "binary" | "json" | "javaserialize" | "hex" | "base64" | "decompressed";
|
||||
export type RedisMemberDetailFormat = "json" | "text";
|
||||
|
||||
export const REDIS_VALUE_FORMAT_DISPLAY_ORDER: RedisValueFormat[] = ["utf8", "ascii", "binary", "json", "javaserialize", "hex", "base64"];
|
||||
export const REDIS_VALUE_FORMAT_DISPLAY_ORDER: RedisValueFormat[] = ["utf8", "ascii", "binary", "json", "javaserialize", "hex", "base64", "decompressed"];
|
||||
|
||||
export interface RedisMemberDetail {
|
||||
text: string;
|
||||
|
||||
Reference in new issue
Block a user