export interface RedisPickleDetail { formattedText: string; value: unknown; } const PROTO = 0x80; const NEWOBJ = 0x81; const TUPLE1 = 0x85; const TUPLE2 = 0x86; const TUPLE3 = 0x87; const NEWTRUE = 0x88; const NEWFALSE = 0x89; const LONG1 = 0x8a; const LONG4 = 0x8b; const SHORT_BINUNICODE = 0x8c; const BINUNICODE8 = 0x8d; const BINBYTES8 = 0x8e; const EMPTY_SET = 0x8f; const ADDITEMS = 0x90; const FROZENSET = 0x91; const NEWOBJ_EX = 0x92; const STACK_GLOBAL = 0x93; const MEMOIZE = 0x94; const FRAME = 0x95; const BYTEARRAY8 = 0x96; const SHORT_BINBYTES = 0x43; const BINBYTES = 0x42; const BININT = 0x4a; const BININT1 = 0x4b; const BININT2 = 0x4d; const BINFLOAT = 0x47; const BINUNICODE = 0x58; const BINSTRING = 0x54; const SHORT_BINSTRING = 0x55; const BINPUT = 0x71; const LONG_BINPUT = 0x72; const BINGET = 0x68; const LONG_BINGET = 0x6a; const MARK = 0x28; const STOP = 0x2e; const POP = 0x30; const POP_MARK = 0x31; const DUP = 0x32; const EMPTY_TUPLE = 0x29; const EMPTY_LIST = 0x5d; const EMPTY_DICT = 0x7d; const APPEND = 0x61; const BUILD = 0x62; const GLOBAL = 0x63; const DICT = 0x64; const APPENDS = 0x65; const GET = 0x67; const INST = 0x69; const LIST = 0x6c; const PUT = 0x70; const SETITEM = 0x73; const TUPLE = 0x74; const SETITEMS = 0x75; const NONE = 0x4e; const REDUCE = 0x52; const FLOAT = 0x46; const INT = 0x49; const LONG = 0x4c; const STRING = 0x53; const UNICODE = 0x56; const OBJ = 0x6f; const MAX_OPS = 1_000_000; class PickleGlobal { constructor( readonly module: string, readonly name: string, ) {} get id(): string { return `${this.module}.${this.name}`; } } class PickleInstance { state: unknown = null; constructor( readonly className: string, readonly args: unknown, ) {} } class PickleUnpickler { private readonly view: DataView; private pos = 0; private readonly stack: unknown[] = []; private readonly metastack: unknown[][] = []; private readonly memo = new Map(); private nextMemoId = 0; private ops = 0; constructor(private readonly bytes: Uint8Array) { this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); } load(): unknown { while (this.pos < this.bytes.length) { if (++this.ops > MAX_OPS) throw new Error("pickle too complex"); const opcode = this.readByte(); if (opcode === STOP) break; this.dispatch(opcode); } if (this.stack.length === 0) throw new Error("empty pickle"); return this.stack[this.stack.length - 1]; } private dispatch(opcode: number): void { switch (opcode) { case PROTO: if (this.readByte() > 5) throw new Error("unsupported pickle protocol"); return; case FRAME: this.read(8); return; case NONE: this.stack.push(null); return; case NEWTRUE: this.stack.push(true); return; case NEWFALSE: this.stack.push(false); return; case INT: this.stack.push(parsePickleInt(this.readLine())); return; case BININT: this.stack.push(this.view.getInt32(this.advance(4), true)); return; case BININT1: this.stack.push(this.readByte()); return; case BININT2: this.stack.push(this.view.getUint16(this.advance(2), true)); return; case LONG: this.stack.push(parsePickleLong(this.readLine())); return; case LONG1: this.stack.push(this.readLong(this.readByte())); return; case LONG4: this.stack.push(this.readLong(this.view.getInt32(this.advance(4), true))); return; case FLOAT: this.stack.push(Number(this.readLine())); return; case BINFLOAT: this.stack.push(this.view.getFloat64(this.advance(8), false)); return; case BINUNICODE: this.stack.push(this.readUtf8(this.view.getUint32(this.advance(4), true))); return; case SHORT_BINUNICODE: this.stack.push(this.readUtf8(this.readByte())); return; case BINUNICODE8: this.stack.push(this.readUtf8(this.readUint64())); return; case UNICODE: this.stack.push(decodeUnicodeEscape(this.readLine())); return; case STRING: this.stack.push(parsePickleString(this.readLine())); return; case BINSTRING: this.stack.push(latin1FromBytes(this.read(this.view.getUint32(this.advance(4), true)))); return; case SHORT_BINSTRING: this.stack.push(latin1FromBytes(this.read(this.readByte()))); return; case BINBYTES: this.stack.push(this.read(this.view.getUint32(this.advance(4), true))); return; case SHORT_BINBYTES: this.stack.push(this.read(this.readByte())); return; case BINBYTES8: case BYTEARRAY8: this.stack.push(this.read(this.readUint64())); return; case EMPTY_LIST: this.stack.push([]); return; case EMPTY_DICT: this.stack.push(Object.create(null)); return; case EMPTY_TUPLE: this.stack.push([]); return; case EMPTY_SET: this.stack.push([]); return; case MARK: this.metastack.push(this.stack.splice(0, this.stack.length)); return; case POP: this.pop(); return; case POP_MARK: this.popMark(); return; case DUP: this.stack.push(this.stack[this.stack.length - 1]); return; case APPEND: this.appendOne(); return; case APPENDS: this.appends(); return; case SETITEM: this.setItem(); return; case SETITEMS: this.setItems(); return; case ADDITEMS: this.addItems(); return; case LIST: case TUPLE: case FROZENSET: this.stack.push(this.popMark()); return; case DICT: this.stack.push(pairsToObject(this.popMark())); return; case TUPLE1: this.stack.push([this.pop()]); return; case TUPLE2: { const b = this.pop(); const a = this.pop(); this.stack.push([a, b]); return; } case TUPLE3: { const c = this.pop(); const b = this.pop(); const a = this.pop(); this.stack.push([a, b, c]); return; } case GLOBAL: this.stack.push(new PickleGlobal(this.readLine(), this.readLine())); return; case STACK_GLOBAL: { const name = this.pop(); const module = this.pop(); this.stack.push(new PickleGlobal(String(module), String(name))); return; } case REDUCE: this.reduce(); return; case NEWOBJ: this.newObj(false); return; case NEWOBJ_EX: this.newObj(true); return; case BUILD: this.build(); return; case INST: this.stack.push(new PickleInstance(`${this.readLine()}.${this.readLine()}`, this.popMark())); return; case OBJ: { const items = this.popMark(); const cls = items.shift(); this.stack.push(new PickleInstance(classNameOf(cls), items)); return; } case MEMOIZE: this.memo.set(this.nextMemoId++, this.peek()); return; case BINPUT: this.memo.set(this.readByte(), this.peek()); return; case LONG_BINPUT: this.memo.set(this.view.getUint32(this.advance(4), true), this.peek()); return; case BINGET: this.stack.push(this.memoGet(this.readByte())); return; case LONG_BINGET: this.stack.push(this.memoGet(this.view.getUint32(this.advance(4), true))); return; case PUT: this.memo.set(Number(this.readLine()), this.peek()); return; case GET: this.stack.push(this.memoGet(Number(this.readLine()))); return; default: throw new Error(`unsupported pickle opcode 0x${opcode.toString(16)}`); } } private reduce(): void { const args = this.pop(); const callable = this.pop(); this.stack.push(applyReduce(callable, Array.isArray(args) ? args : [args])); } private newObj(withKwargs: boolean): void { const kwargs = withKwargs ? this.pop() : undefined; const args = this.pop(); const cls = this.pop(); const instance = new PickleInstance(classNameOf(cls), Array.isArray(args) ? args : [args]); if (kwargs != null) instance.state = kwargs; this.stack.push(instance); } private build(): void { const state = this.pop(); this.stack[this.stack.length - 1] = applyBuild(this.peek(), state); } private appendOne(): void { const value = this.pop(); const list = this.peek(); if (!Array.isArray(list)) throw new Error("APPEND on non-list"); list.push(value); } private appends(): void { const items = this.popMark(); const list = this.peek(); if (!Array.isArray(list)) throw new Error("APPENDS on non-list"); list.push(...items); } private addItems(): void { const items = this.popMark(); const list = this.peek(); if (!Array.isArray(list)) throw new Error("ADDITEMS on non-set"); list.push(...items); } private setItem(): void { const value = this.pop(); const key = this.pop(); writeKey(this.peek(), key, value); } private setItems(): void { const items = this.popMark(); const dict = this.peek(); for (let i = 0; i + 1 < items.length; i += 2) writeKey(dict, items[i], items[i + 1]); } private popMark(): unknown[] { const items = this.stack.splice(0, this.stack.length); const parent = this.metastack.pop(); if (!parent) throw new Error("MARK underflow"); this.stack.push(...parent); return items; } private pop(): unknown { if (this.stack.length === 0) throw new Error("stack underflow"); return this.stack.pop(); } private peek(): unknown { if (this.stack.length === 0) throw new Error("stack underflow"); return this.stack[this.stack.length - 1]; } private memoGet(id: number): unknown { if (!this.memo.has(id)) throw new Error("missing pickle memo"); return this.memo.get(id); } private readByte(): number { if (this.pos >= this.bytes.length) throw new Error("unexpected end of pickle"); return this.bytes[this.pos++]; } private advance(n: number): number { const start = this.pos; if (start + n > this.bytes.length) throw new Error("unexpected end of pickle"); this.pos += n; return start; } private read(n: number): Uint8Array { const start = this.advance(n); return this.bytes.subarray(start, start + n); } private readUint64(): number { const start = this.advance(8); const lo = this.view.getUint32(start, true); const hi = this.view.getUint32(start + 4, true); if (hi > 0x1fffff) throw new Error("pickle length too large"); return lo + hi * 2 ** 32; } private readUtf8(n: number): string { return new TextDecoder("utf-8", { fatal: true }).decode(this.read(n)); } private readLine(): string { const start = this.pos; while (this.pos < this.bytes.length && this.bytes[this.pos] !== 0x0a) this.pos++; if (this.pos >= this.bytes.length) throw new Error("unterminated pickle line"); const line = this.bytes.subarray(start, this.pos); this.pos++; return new TextDecoder("latin1").decode(line); } private readLong(length: number): number | string { if (length < 0) throw new Error("invalid pickle long"); return decodeTwoComplement(this.read(length)); } } function parsePickleInt(line: string): boolean | number { if (line === "01") return true; if (line === "00") return false; const value = Number(line); if (!Number.isFinite(value)) throw new Error("invalid pickle int"); return value; } function parsePickleLong(line: string): number | string { const trimmed = line.endsWith("L") ? line.slice(0, -1) : line; return bigintToJson(BigInt(trimmed || "0")); } function parsePickleString(line: string): string { const trimmed = line.trim(); if ((trimmed.startsWith("'") && trimmed.endsWith("'")) || (trimmed.startsWith('"') && trimmed.endsWith('"'))) { return trimmed.slice(1, -1); } return trimmed; } function decodeUnicodeEscape(line: string): string { return line.replace(/\\u([0-9a-fA-F]{4})/g, (_, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))); } function classNameOf(value: unknown): string { return value instanceof PickleGlobal ? value.id : String(value); } function applyReduce(callable: unknown, args: unknown[]): unknown { const id = callable instanceof PickleGlobal ? callable.id : ""; switch (id) { case "datetime.datetime": return reduceDatetime(args); case "datetime.date": return reduceDate(args); case "datetime.time": return reduceTime(args); case "_codecs.encode": return reduceCodecsEncode(args); case "builtins.bytes": case "__builtin__.bytes": case "builtins.bytearray": return reduceBytes(args); case "uuid.UUID": return new PickleInstance(id, args); default: return { $class: id || String(callable), $args: toJsonable(args) }; } } function applyBuild(obj: unknown, state: unknown): unknown { if (obj instanceof PickleInstance) { obj.state = state; if (obj.className === "uuid.UUID") return reduceUuidState(state); return { $class: obj.className, $args: toJsonable(obj.args), $state: toJsonable(state) }; } if (obj && typeof obj === "object" && !Array.isArray(obj) && state && typeof state === "object" && !Array.isArray(state)) { return { ...(obj as object), ...(state as object) }; } return obj; } function reduceDatetime(args: unknown[]): string { if (args.length === 1 && args[0] instanceof Uint8Array) return datetimeFromPayload(args[0]); if (typeof args[0] === "string") return datetimeFromPayload(latin1ToBytes(args[0])); const [year, month, day, hour = 0, minute = 0, second = 0, micro = 0] = args.map((value) => Number(value)); return formatDatetime(year, month, day, hour, minute, second, micro); } function reduceDate(args: unknown[]): string { const [year, month, day] = args.map((value) => Number(value)); return `${pad(year, 4)}-${pad(month, 2)}-${pad(day, 2)}`; } function reduceTime(args: unknown[]): string { const [hour = 0, minute = 0, second = 0, micro = 0] = args.map((value) => Number(value)); return formatTime(hour, minute, second, micro); } function reduceCodecsEncode(args: unknown[]): unknown { const [data, encoding] = args; if (typeof data === "string" && encoding === "latin1") return latin1ToBytes(data); return { $class: "_codecs.encode", $args: toJsonable(args) }; } function reduceBytes(args: unknown[]): unknown { if (args.length === 1 && args[0] instanceof Uint8Array) return args[0]; if (args.length === 1 && typeof args[0] === "string") return latin1ToBytes(args[0]); return { $class: "bytes", $args: toJsonable(args) }; } function reduceUuidState(state: unknown): unknown { if (!state || typeof state !== "object") return { $class: "uuid.UUID", $state: toJsonable(state) }; const record = state as Record; if (record.int != null) return uuidFromInt(toBigInt(record.int)); if (typeof record.hex === "string") return normalizeUuid(record.hex); return { $class: "uuid.UUID", $state: toJsonable(state) }; } function datetimeFromPayload(bytes: Uint8Array): string { if (bytes.length !== 10) throw new Error("invalid datetime payload"); const year = (bytes[0] << 8) | bytes[1]; const micro = (bytes[7] << 16) | (bytes[8] << 8) | bytes[9]; return formatDatetime(year, bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], micro); } function formatDatetime(year: number, month: number, day: number, hour: number, minute: number, second: number, micro: number): string { return `${pad(year, 4)}-${pad(month, 2)}-${pad(day, 2)}T${formatTime(hour, minute, second, micro)}`; } function formatTime(hour: number, minute: number, second: number, micro: number): string { const base = `${pad(hour, 2)}:${pad(minute, 2)}:${pad(second, 2)}`; return micro ? `${base}.${pad(micro, 6)}` : base; } function pad(value: number, width: number): string { return String(Math.trunc(value)).padStart(width, "0"); } function uuidFromInt(value: bigint): string { return normalizeUuid(value.toString(16).padStart(32, "0")); } function normalizeUuid(hex: string): string { const digits = hex.replace(/-/g, "").padStart(32, "0").slice(-32); return `${digits.slice(0, 8)}-${digits.slice(8, 12)}-${digits.slice(12, 16)}-${digits.slice(16, 20)}-${digits.slice(20)}`; } function writeKey(dict: unknown, key: unknown, value: unknown): void { if (!dict || typeof dict !== "object" || Array.isArray(dict)) throw new Error("SETITEM on non-dict"); (dict as Record)[keyToString(key)] = value; } function pairsToObject(items: unknown[]): Record { const dict = Object.create(null) as Record; for (let i = 0; i + 1 < items.length; i += 2) dict[keyToString(items[i])] = items[i + 1]; return dict; } function keyToString(key: unknown): string { if (typeof key === "string") return key; if (typeof key === "number" || typeof key === "boolean") return String(key); return JSON.stringify(toJsonable(key)) ?? String(key); } function latin1FromBytes(bytes: Uint8Array): string { return new TextDecoder("latin1").decode(bytes); } function latin1ToBytes(value: string): Uint8Array { const bytes = new Uint8Array(value.length); for (let i = 0; i < value.length; i++) bytes[i] = value.charCodeAt(i) & 0xff; return bytes; } function decodeTwoComplement(bytes: Uint8Array): number | string { if (bytes.length === 0) return 0; let value = 0n; for (let i = bytes.length - 1; i >= 0; i--) value = (value << 8n) | BigInt(bytes[i]); const bits = BigInt(bytes.length * 8); if (bytes[bytes.length - 1] & 0x80) value -= 1n << bits; return bigintToJson(value); } function bigintToJson(value: bigint): number | string { if (value <= BigInt(Number.MAX_SAFE_INTEGER) && value >= BigInt(Number.MIN_SAFE_INTEGER)) return Number(value); return value.toString(); } function toBigInt(value: unknown): bigint { if (typeof value === "bigint") return value; if (typeof value === "number") return BigInt(value); if (typeof value === "string") return BigInt(value); throw new Error("invalid uuid int"); } export function toJsonable(value: unknown, seen = new WeakSet()): unknown { if (value instanceof Uint8Array) { try { return new TextDecoder("utf-8", { fatal: true }).decode(value); } catch { return { $bytes: bytesToBase64(value) }; } } if (value instanceof PickleGlobal) return { $class: value.id }; if (value instanceof PickleInstance) { return { $class: value.className, $args: toJsonable(value.args, seen), $state: toJsonable(value.state, seen) }; } if (Array.isArray(value)) return value.map((item) => toJsonable(item, seen)); if (value && typeof value === "object") { if (seen.has(value)) return { $ref: true }; seen.add(value); const output: Record = {}; for (const [key, item] of Object.entries(value)) output[key] = toJsonable(item, seen); return output; } if (typeof value === "bigint") return bigintToJson(value); return value; } function bytesToBase64(bytes: Uint8Array): string { let binary = ""; bytes.forEach((byte) => { binary += String.fromCharCode(byte); }); return btoa(binary); } export function isPickleMagic(bytes: Uint8Array): boolean { return bytes.length >= 3 && bytes[0] === PROTO && bytes[1] <= 5 && bytes[bytes.length - 1] === STOP; } export function decodePickle(bytes: Uint8Array): RedisPickleDetail | null { try { const value = toJsonable(new PickleUnpickler(bytes).load()); return { value, formattedText: typeof value === "string" ? value : JSON.stringify(value, null, 2), }; } catch { return null; } }