fix(redis): paginate stream entries

This commit is contained in:
onenewcode authored and GitHub committed 2026-07-30 15:56:56 +08:00
1 parent 7775bbf8f8
commit 223e9b3a2f
21 files changed
+479 -33

No files matched your search

@@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({ const mocks = vi.hoisted(() => ({
redisGetValue: vi.fn(), redisGetValue: vi.fn(),
redisGetStreamEntries: vi.fn(),
redisGetStreamGroups: vi.fn(), redisGetStreamGroups: vi.fn(),
redisGetStreamConsumers: vi.fn(), redisGetStreamConsumers: vi.fn(),
redisGetStreamPending: vi.fn(), redisGetStreamPending: vi.fn(),
@@ -16,6 +17,7 @@ const mocks = vi.hoisted(() => ({
vi.mock("@/lib/backend/api", () => ({ vi.mock("@/lib/backend/api", () => ({
redisGetValue: mocks.redisGetValue, redisGetValue: mocks.redisGetValue,
redisGetStreamEntries: mocks.redisGetStreamEntries,
redisGetStreamGroups: mocks.redisGetStreamGroups, redisGetStreamGroups: mocks.redisGetStreamGroups,
redisGetStreamConsumers: mocks.redisGetStreamConsumers, redisGetStreamConsumers: mocks.redisGetStreamConsumers,
redisGetStreamPending: mocks.redisGetStreamPending, redisGetStreamPending: mocks.redisGetStreamPending,
@@ -40,11 +42,7 @@ vi.mock("vue-virtual-scroller", async () => {
const DynamicScroller = defineComponent({ const DynamicScroller = defineComponent({
props: { items: { type: Array, default: () => [] } }, props: { items: { type: Array, default: () => [] } },
setup(props, { slots }) { setup(props, { slots }) {
return () => return () => h("div", [...(props.items as unknown[]).map((item, index) => slots.default?.({ item, active: true, index })), ...(slots.after?.() ?? [])]);
h(
"div",
(props.items as unknown[]).map((item, index) => slots.default?.({ item, active: true, index })),
);
}, },
}); });
const DynamicScrollerItem = defineComponent({ const DynamicScrollerItem = defineComponent({
@@ -84,13 +82,17 @@ async function settle() {
await nextTick(); await nextTick();
} }
function streamValue() { function streamEntry(id: string) {
return { id, fields: [{ field: "event", value: "login" }] };
}
function streamValue(entries = [] as ReturnType<typeof streamEntry>[], nextCursor?: string, total = entries.length) {
return { return {
key_display: "orders", key_display: "orders",
key_raw: "b3JkZXJz", key_raw: "b3JkZXJz",
ttl: -1, ttl: -1,
redis_type: "stream", redis_type: "stream",
data: { kind: "stream" as const, entries: [] }, data: { kind: "stream" as const, entries, total, ...(nextCursor ? { next_cursor: nextCursor } : {}) },
}; };
} }
@@ -158,6 +160,21 @@ function openGroups(host: HTMLElement) {
} }
describe("RedisValueViewer stream monitoring", () => { describe("RedisValueViewer stream monitoring", () => {
it("loads more Stream entries from the returned cursor", async () => {
mocks.redisGetValue.mockResolvedValue(streamValue([streamEntry("1714470000000-0")], "1714470000000-0"));
mocks.redisGetStreamEntries.mockResolvedValue({ entries: [streamEntry("1714470000001-0")] });
const host = mountViewer();
await settle();
expect(host.querySelectorAll("[data-redis-stream-entry]")).toHaveLength(1);
host.querySelector<HTMLButtonElement>("[data-redis-stream-entries-more]")!.click();
await settle();
expect(mocks.redisGetStreamEntries).toHaveBeenCalledWith("redis-1", 2, "b3JkZXJz", "1714470000000-0");
expect(host.querySelectorAll("[data-redis-stream-entry]")).toHaveLength(2);
expect(host.querySelector("[data-redis-stream-entries-more]")).toBeNull();
});
it("renders large counter values transported as decimal strings", async () => { it("renders large counter values transported as decimal strings", async () => {
const unsafeMetric = "9007199254740992"; const unsafeMetric = "9007199254740992";
mocks.redisGetValue.mockResolvedValue(streamValue()); mocks.redisGetValue.mockResolvedValue(streamValue());
@@ -82,6 +82,9 @@ const loading = ref(false);
const loadingMore = ref(false); const loadingMore = ref(false);
let loadRequestId = 0; let loadRequestId = 0;
const streamTab = ref<"entries" | "groups">("entries"); const streamTab = ref<"entries" | "groups">("entries");
const streamEntries = ref<RedisStreamEntry[]>([]);
const streamEntriesCursor = ref<string | undefined>();
const streamEntriesLoadingMore = ref(false);
const streamGroups = ref<RedisStreamGroup[]>([]); const streamGroups = ref<RedisStreamGroup[]>([]);
const streamGroupsLoaded = ref(false); const streamGroupsLoaded = ref(false);
const streamGroupsLoading = ref(false); const streamGroupsLoading = ref(false);
@@ -112,6 +115,7 @@ let streamGroupsRequestId = 0;
let streamGroupDetailRequestId = 0; let streamGroupDetailRequestId = 0;
let streamConsumersRequestId = 0; let streamConsumersRequestId = 0;
let streamPendingRequestId = 0; let streamPendingRequestId = 0;
let streamEntriesRequestId = 0;
const editValue = ref(""); const editValue = ref("");
const savingString = ref(false); const savingString = ref(false);
const savingJson = ref(false); const savingJson = ref(false);
@@ -316,8 +320,8 @@ const metadataSizeLabel = computed(() => {
return String(size); return String(size);
}); });
const streamRows = computed<RedisStreamRow[]>(() => { const streamRows = computed<RedisStreamRow[]>(() => {
if (data.value?.data.kind !== "stream") return []; if (redisKind.value !== "stream") return [];
return data.value.data.entries.map((entry, index) => ({ return streamEntries.value.map((entry, index) => ({
id: `${index}:${entry.id}`, id: `${index}:${entry.id}`,
index, index,
entry, entry,
@@ -434,6 +438,25 @@ type RedisStreamRow = {
entry: RedisStreamEntry; entry: RedisStreamEntry;
}; };
function replaceStreamEntries(value: RedisValue) {
streamEntriesRequestId++;
streamEntriesLoadingMore.value = false;
if (value.data.kind === "stream") {
streamEntries.value = [...value.data.entries];
streamEntriesCursor.value = value.data.next_cursor;
return;
}
streamEntries.value = [];
streamEntriesCursor.value = undefined;
}
function resetStreamEntries() {
streamEntriesRequestId++;
streamEntries.value = [];
streamEntriesCursor.value = undefined;
streamEntriesLoadingMore.value = false;
}
function isSelectedStreamGroup(group: RedisStreamGroup, requestId = streamGroupDetailRequestId): boolean { function isSelectedStreamGroup(group: RedisStreamGroup, requestId = streamGroupDetailRequestId): boolean {
return requestId === streamGroupDetailRequestId && selectedStreamGroup.value?.name.raw_base64 === group.name.raw_base64; return requestId === streamGroupDetailRequestId && selectedStreamGroup.value?.name.raw_base64 === group.name.raw_base64;
} }
@@ -464,6 +487,25 @@ function resetStreamMonitoring() {
streamGroupsError.value = ""; streamGroupsError.value = "";
} }
async function loadMoreStreamEntries() {
const cursor = streamEntriesCursor.value;
if (redisKind.value !== "stream" || !cursor || loading.value || streamEntriesLoadingMore.value) return;
const requestId = ++streamEntriesRequestId;
streamEntriesLoadingMore.value = true;
try {
const page = await api.redisGetStreamEntries(props.connectionId, props.db, props.keyRaw, cursor);
if (requestId !== streamEntriesRequestId || redisKind.value !== "stream" || streamEntriesCursor.value !== cursor) return;
streamEntries.value = [...streamEntries.value, ...page.entries];
streamEntriesCursor.value = page.next_cursor;
} catch (error) {
if (requestId === streamEntriesRequestId) toast(errorMessage(error), 3000);
} finally {
if (requestId === streamEntriesRequestId) streamEntriesLoadingMore.value = false;
}
}
async function loadStreamGroups(force = false): Promise<boolean> { async function loadStreamGroups(force = false): Promise<boolean> {
if (redisKind.value !== "stream") return false; if (redisKind.value !== "stream") return false;
if (!force && (streamGroupsLoaded.value || streamGroupsLoading.value)) return streamGroupsLoaded.value; if (!force && (streamGroupsLoaded.value || streamGroupsLoading.value)) return streamGroupsLoaded.value;
@@ -902,6 +944,7 @@ async function load(options: { selectDefaultMember?: boolean; preserveDraft?: bo
data.value = null; data.value = null;
collectionItems.value = []; collectionItems.value = [];
scanCursor.value = undefined; scanCursor.value = undefined;
resetStreamEntries();
resetStreamMonitoring(); resetStreamMonitoring();
stopAutoRefresh(); stopAutoRefresh();
emit("deleted", props.keyRaw); emit("deleted", props.keyRaw);
@@ -929,6 +972,7 @@ async function load(options: { selectDefaultMember?: boolean; preserveDraft?: bo
emit("loaded", loadedValue); emit("loaded", loadedValue);
scanCursor.value = redisValueCollectionScanCursor(loadedValue); scanCursor.value = redisValueCollectionScanCursor(loadedValue);
collectionItems.value = redisValueCollectionItems(loadedValue); collectionItems.value = redisValueCollectionItems(loadedValue);
replaceStreamEntries(loadedValue);
if (loadedValue.data.kind !== "stream") resetStreamMonitoring(); if (loadedValue.data.kind !== "stream") resetStreamMonitoring();
// A foreground load replaces the current value, so it also starts a new // A foreground load replaces the current value, so it also starts a new
@@ -1050,7 +1094,8 @@ function requestDeleteKey() {
async function copyValue() { async function copyValue() {
if (!data.value) return; if (!data.value) return;
const text = redisValueCopyText(data.value, collectionItems.value); const value = data.value.data.kind === "stream" ? { ...data.value, data: { ...data.value.data, entries: streamEntries.value } } : data.value;
const text = redisValueCopyText(value, collectionItems.value);
try { try {
await copyToClipboard(text); await copyToClipboard(text);
toast(t("redis.copied"), 2000); toast(t("redis.copied"), 2000);
@@ -1129,7 +1174,7 @@ function generateInsertStatements(): string | null {
break; break;
} }
case "stream": { case "stream": {
for (const entry of data.value.data.entries) { for (const entry of streamEntries.value) {
const fields = entry.fields.map(({ field, value }) => `${escapeRedisArg(field)} ${escapeRedisArg(value)}`).join(" "); const fields = entry.fields.map(({ field, value }) => `${escapeRedisArg(field)} ${escapeRedisArg(value)}`).join(" ");
commands.push(`XADD ${escapeRedisArg(key)} * ${fields}`); commands.push(`XADD ${escapeRedisArg(key)} * ${fields}`);
} }
@@ -1772,6 +1817,7 @@ watch(
() => { () => {
resetValueSearch(); resetValueSearch();
valueViewerSearchActive.value = false; valueViewerSearchActive.value = false;
resetStreamEntries();
resetStreamMonitoring(); resetStreamMonitoring();
}, },
); );
@@ -2227,9 +2273,6 @@ defineExpose({ focusSearch });
</div> </div>
<TabsContent value="entries" class="m-0 min-h-0 flex-1 flex flex-col"> <TabsContent value="entries" class="m-0 min-h-0 flex-1 flex flex-col">
<div class="px-4 py-1 text-xs text-muted-foreground border-b shrink-0">
{{ t("redis.entries", { count: streamRows.length }) }}
</div>
<DynamicScroller class="flex-1 overflow-y-auto" :items="streamRows" :min-item-size="REDIS_STREAM_MIN_ROW_HEIGHT" :buffer="600" key-field="id"> <DynamicScroller class="flex-1 overflow-y-auto" :items="streamRows" :min-item-size="REDIS_STREAM_MIN_ROW_HEIGHT" :buffer="600" key-field="id">
<template #default="{ item: row, active }"> <template #default="{ item: row, active }">
<DynamicScrollerItem :item="row" :active="active" :size-dependencies="[streamFieldCount(row)]" :data-index="row.index"> <DynamicScrollerItem :item="row" :active="active" :size-dependencies="[streamFieldCount(row)]" :data-index="row.index">
@@ -2259,6 +2302,14 @@ defineExpose({ focusSearch });
</div> </div>
</DynamicScrollerItem> </DynamicScrollerItem>
</template> </template>
<template #after>
<div v-if="streamEntriesCursor" class="border-t p-2">
<Button data-redis-stream-entries-more variant="outline" size="sm" class="h-7 w-full text-xs" :disabled="loading || streamEntriesLoadingMore" @click="loadMoreStreamEntries">
<Loader2 v-if="streamEntriesLoadingMore" class="mr-1.5 h-3 w-3 animate-spin" />
{{ t("redis.loadMoreEntries") }}
</Button>
</div>
</template>
</DynamicScroller> </DynamicScroller>
</TabsContent> </TabsContent>
+1
View File
@@ -3069,6 +3069,7 @@ export default {
noPendingEntries: "No pending entries", noPendingEntries: "No pending entries",
entryId: "Entry ID", entryId: "Entry ID",
deliveries: "Deliveries", deliveries: "Deliveries",
loadMoreEntries: "Load more entries",
loadMorePending: "Load more pending entries", loadMorePending: "Load more pending entries",
noExpiry: "no expiry", noExpiry: "no expiry",
expiry: "Expiration", expiry: "Expiration",
+1
View File
@@ -2930,6 +2930,7 @@ export default withEnglishFallback({
noPendingEntries: "No hay entradas pendientes", noPendingEntries: "No hay entradas pendientes",
entryId: "ID de entrada", entryId: "ID de entrada",
deliveries: "Entregas", deliveries: "Entregas",
loadMoreEntries: "Cargar más entradas",
loadMorePending: "Cargar más entradas pendientes", loadMorePending: "Cargar más entradas pendientes",
noExpiry: "sin expiración", noExpiry: "sin expiración",
expiry: "Expiración", expiry: "Expiración",
+1
View File
@@ -2928,6 +2928,7 @@ export default withEnglishFallback({
noPendingEntries: "Nessuna voce in sospeso", noPendingEntries: "Nessuna voce in sospeso",
entryId: "ID voce", entryId: "ID voce",
deliveries: "Consegne", deliveries: "Consegne",
loadMoreEntries: "Carica altre voci",
loadMorePending: "Carica altre voci in sospeso", loadMorePending: "Carica altre voci in sospeso",
noExpiry: "nessuna scadenza", noExpiry: "nessuna scadenza",
expiry: "Scadenza", expiry: "Scadenza",
+1
View File
@@ -2929,6 +2929,7 @@ export default withEnglishFallback({
noPendingEntries: "保留中のエントリはありません", noPendingEntries: "保留中のエントリはありません",
entryId: "エントリ ID", entryId: "エントリ ID",
deliveries: "配信回数", deliveries: "配信回数",
loadMoreEntries: "さらにエントリを読み込む",
loadMorePending: "保留中のエントリをさらに読み込む", loadMorePending: "保留中のエントリをさらに読み込む",
noExpiry: "期限なし", noExpiry: "期限なし",
expiry: "有効期限", expiry: "有効期限",
+1
View File
@@ -3032,6 +3032,7 @@ export default withEnglishFallback({
noPendingEntries: "대기 항목이 없습니다", noPendingEntries: "대기 항목이 없습니다",
entryId: "항목 ID", entryId: "항목 ID",
deliveries: "전달", deliveries: "전달",
loadMoreEntries: "항목 더 불러오기",
loadMorePending: "대기 항목 더 불러오기", loadMorePending: "대기 항목 더 불러오기",
noExpiry: "만료 없음", noExpiry: "만료 없음",
expiry: "만료", expiry: "만료",
+1
View File
@@ -2930,6 +2930,7 @@ export default withEnglishFallback({
noPendingEntries: "Nenhuma entrada pendente", noPendingEntries: "Nenhuma entrada pendente",
entryId: "ID da entrada", entryId: "ID da entrada",
deliveries: "Entregas", deliveries: "Entregas",
loadMoreEntries: "Carregar mais entradas",
loadMorePending: "Carregar mais entradas pendentes", loadMorePending: "Carregar mais entradas pendentes",
noExpiry: "sem expiração", noExpiry: "sem expiração",
expiry: "Expiração", expiry: "Expiração",
+1
View File
@@ -3069,6 +3069,7 @@ export default withEnglishFallback({
noPendingEntries: "暂无待处理条目", noPendingEntries: "暂无待处理条目",
entryId: "条目 ID", entryId: "条目 ID",
deliveries: "投递次数", deliveries: "投递次数",
loadMoreEntries: "加载更多条目",
loadMorePending: "加载更多待处理条目", loadMorePending: "加载更多待处理条目",
noExpiry: "永不过期", noExpiry: "永不过期",
expiry: "过期时间", expiry: "过期时间",
+1
View File
@@ -2601,6 +2601,7 @@ export default withEnglishFallback({
noPendingEntries: "暫無待處理項目", noPendingEntries: "暫無待處理項目",
entryId: "項目 ID", entryId: "項目 ID",
deliveries: "投遞次數", deliveries: "投遞次數",
loadMoreEntries: "載入更多項目",
loadMorePending: "載入更多待處理項目", loadMorePending: "載入更多待處理項目",
noExpiry: "永不過期", noExpiry: "永不過期",
expiry: "到期時間", expiry: "到期時間",
@@ -146,6 +146,38 @@ describe("redisValuePresentation", () => {
}`); }`);
}); });
it("uses the Stream's Redis length instead of the loaded page size", () => {
const value = {
key_display: "orders",
key_raw: "b3JkZXJz",
ttl: -1,
redis_type: "stream",
data: {
kind: "stream" as const,
entries: [{ id: "1714470000000-0", fields: [{ field: "event", value: "login" }] }],
total: 177,
next_cursor: "1714470000000-0",
},
};
expect(redisValueSize(value)).toBe(177);
});
it("uses loaded Stream entries when the total is unavailable", () => {
const value = {
key_display: "orders",
key_raw: "b3JkZXJz",
ttl: -1,
redis_type: "stream",
data: {
kind: "stream" as const,
entries: [{ id: "1714470000000-0", fields: [{ field: "event", value: "login" }] }],
},
};
expect(redisValueSize(value)).toBe(1);
});
it("labels raw text views by encoding instead of generic raw text", () => { it("labels raw text views by encoding instead of generic raw text", () => {
expect(formatRedisMemberDetail("plain-text").rawLabel).toBe("ASCII"); expect(formatRedisMemberDetail("plain-text").rawLabel).toBe("ASCII");
expect( expect(
+2
View File
@@ -392,6 +392,7 @@ export const redisScanKeys = forward("redisScanKeys");
export const redisScanKeysBatch = forward("redisScanKeysBatch"); export const redisScanKeysBatch = forward("redisScanKeysBatch");
export const redisScanValues = forward("redisScanValues"); export const redisScanValues = forward("redisScanValues");
export const redisGetValue = forward("redisGetValue"); export const redisGetValue = forward("redisGetValue");
export const redisGetStreamEntries = forward("redisGetStreamEntries");
export const redisGetStreamGroups = forward("redisGetStreamGroups"); export const redisGetStreamGroups = forward("redisGetStreamGroups");
export const redisGetStreamConsumers = forward("redisGetStreamConsumers"); export const redisGetStreamConsumers = forward("redisGetStreamConsumers");
export const redisGetStreamPending = forward("redisGetStreamPending"); export const redisGetStreamPending = forward("redisGetStreamPending");
@@ -640,6 +641,7 @@ export type {
RedisStreamField, RedisStreamField,
RedisStreamGroup, RedisStreamGroup,
RedisStreamMetric, RedisStreamMetric,
RedisStreamPage,
RedisStreamPendingEntry, RedisStreamPendingEntry,
RedisStreamPendingPage, RedisStreamPendingPage,
RedisValue, RedisValue,
+5
View File
@@ -70,6 +70,7 @@ import type {
RedisDatabaseInfo, RedisDatabaseInfo,
RedisStreamConsumer, RedisStreamConsumer,
RedisStreamGroup, RedisStreamGroup,
RedisStreamPage,
RedisStreamPendingPage, RedisStreamPendingPage,
RedisValue, RedisValue,
RedisScanResult, RedisScanResult,
@@ -2253,6 +2254,10 @@ export async function redisGetValue(connectionId: string, db: number, keyRaw: st
return post("/api/redis/get-value", { connectionId, db, keyRaw }); return post("/api/redis/get-value", { connectionId, db, keyRaw });
} }
export async function redisGetStreamEntries(connectionId: string, db: number, keyRaw: string, cursor?: string): Promise<RedisStreamPage> {
return post("/api/redis/get-stream-entries", { connectionId, db, keyRaw, cursor });
}
export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> { export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> {
return post("/api/redis/get-stream-groups", { connectionId, db, keyRaw }); return post("/api/redis/get-stream-groups", { connectionId, db, keyRaw });
} }
+10 -1
View File
@@ -1949,6 +1949,11 @@ export interface RedisStreamEntry {
fields: RedisStreamField[]; fields: RedisStreamField[];
} }
export interface RedisStreamPage {
entries: RedisStreamEntry[];
next_cursor?: string;
}
// Redis counters above Number.MAX_SAFE_INTEGER are transported as decimal strings. // Redis counters above Number.MAX_SAFE_INTEGER are transported as decimal strings.
export type RedisStreamMetric = number | string; export type RedisStreamMetric = number | string;
@@ -2002,7 +2007,7 @@ export type RedisValueData =
total: number; total: number;
scan_cursor?: number; scan_cursor?: number;
} }
| { kind: "stream"; entries: RedisStreamEntry[] } | { kind: "stream"; entries: RedisStreamEntry[]; total?: number; next_cursor?: string }
| { kind: "unknown" }; | { kind: "unknown" };
export interface RedisValue { export interface RedisValue {
@@ -2085,6 +2090,10 @@ export async function redisGetValue(connectionId: string, db: number, keyRaw: st
return invoke("redis_get_value", { connectionId, db, keyRaw }); return invoke("redis_get_value", { connectionId, db, keyRaw });
} }
export async function redisGetStreamEntries(connectionId: string, db: number, keyRaw: string, cursor?: string): Promise<RedisStreamPage> {
return invoke("redis_get_stream_entries", { connectionId, db, keyRaw, cursor });
}
export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> { export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> {
return invoke("redis_get_stream_groups", { connectionId, db, keyRaw }); return invoke("redis_get_stream_groups", { connectionId, db, keyRaw });
} }
@@ -327,7 +327,7 @@ export function redisValueSize(value: RedisValue): number {
case "zset": case "zset":
return value.data.total; return value.data.total;
case "stream": case "stream":
return value.data.entries.length; return value.data.total ?? value.data.entries.length;
default: default:
return 0; return 0;
} }
+268 -15
View File
@@ -14,7 +14,7 @@ use tokio::sync::{Mutex, MutexGuard};
use super::json_value_for_js; use super::json_value_for_js;
const STREAM_ENTRY_LIMIT: usize = 100; const STREAM_ENTRY_PAGE_SIZE: usize = 50;
const STREAM_PENDING_PAGE_SIZE: usize = 100; const STREAM_PENDING_PAGE_SIZE: usize = 100;
const COLLECTION_PAGE_SIZE: usize = 200; const COLLECTION_PAGE_SIZE: usize = 200;
const HASH_FILTER_SCAN_MAX_ITERATIONS: usize = 10; const HASH_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
@@ -124,6 +124,13 @@ pub struct RedisStreamEntry {
pub fields: Vec<RedisStreamField>, pub fields: Vec<RedisStreamField>,
} }
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RedisStreamPage {
pub entries: Vec<RedisStreamEntry>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_cursor: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RedisStreamGroup { pub struct RedisStreamGroup {
pub name: RedisBlob, pub name: RedisBlob,
@@ -224,6 +231,14 @@ pub enum RedisValueData {
}, },
Stream { Stream {
entries: Vec<RedisStreamEntry>, entries: Vec<RedisStreamEntry>,
#[serde(
default,
skip_serializing_if = "Option::is_none",
serialize_with = "serialize_optional_redis_u64_for_js"
)]
total: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
next_cursor: Option<String>,
}, },
Unknown, Unknown,
} }
@@ -2281,7 +2296,14 @@ where
{ {
let redis_type: String = redis::cmd("TYPE").arg(key).query_async(con).await.map_err(|e| e.to_string())?; let redis_type: String = redis::cmd("TYPE").arg(key).query_async(con).await.map_err(|e| e.to_string())?;
let ttl: i64 = redis::cmd("TTL").arg(key).query_async(con).await.unwrap_or(-1); // For Streams, fetch metadata and the initial page together after TYPE.
// This keeps the accurate XLEN without adding another Redis round trip.
let stream_initial_page =
if redis_type == "stream" { Some(get_stream_initial_page(con, key).await?) } else { None };
let ttl: i64 = match stream_initial_page.as_ref() {
Some((ttl, _, _)) => *ttl,
None => redis::cmd("TTL").arg(key).query_async(con).await.unwrap_or(-1),
};
let data = match redis_type.as_str() { let data = match redis_type.as_str() {
"string" => { "string" => {
@@ -2315,7 +2337,10 @@ where
let (cursor, items) = hscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE, None).await?; let (cursor, items) = hscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE, None).await?;
RedisValueData::Hash { items, total: len, scan_cursor: (cursor > 0).then_some(cursor) } RedisValueData::Hash { items, total: len, scan_cursor: (cursor > 0).then_some(cursor) }
} }
"stream" => RedisValueData::Stream { entries: get_stream_entries(con, key).await? }, "stream" => {
let (_, total, page) = stream_initial_page.expect("Stream page is loaded with its Stream type");
RedisValueData::Stream { entries: page.entries, total, next_cursor: page.next_cursor }
}
key_type if is_redis_json_type(key_type) => { key_type if is_redis_json_type(key_type) => {
let raw: RedisRawValue = let raw: RedisRawValue =
redis::cmd("JSON.GET").arg(key).query_async(con).await.map_err(|e| e.to_string())?; redis::cmd("JSON.GET").arg(key).query_async(con).await.map_err(|e| e.to_string())?;
@@ -2370,7 +2395,7 @@ fn redis_search_value_text(value: &RedisValueData) -> String {
.flat_map(|item| [item.score.clone(), redis_blob_display_text(&item.member)]) .flat_map(|item| [item.score.clone(), redis_blob_display_text(&item.member)])
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join(" "), .join(" "),
RedisValueData::Stream { entries } => entries RedisValueData::Stream { entries, .. } => entries
.iter() .iter()
.flat_map(|entry| { .flat_map(|entry| {
entry.fields.iter().flat_map(|field| [field.field.clone(), field.value.clone()]).collect::<Vec<_>>() entry.fields.iter().flat_map(|field| [field.field.clone(), field.value.clone()]).collect::<Vec<_>>()
@@ -2403,26 +2428,77 @@ fn redis_search_value_size(value: &RedisValue) -> u64 {
| RedisValueData::Set { total, .. } | RedisValueData::Set { total, .. }
| RedisValueData::Hash { total, .. } | RedisValueData::Hash { total, .. }
| RedisValueData::Zset { total, .. } => *total, | RedisValueData::Zset { total, .. } => *total,
RedisValueData::Stream { entries } => entries.len() as u64, RedisValueData::Stream { entries, total, .. } => total.unwrap_or(entries.len() as u64),
RedisValueData::Unknown => 0, RedisValueData::Unknown => 0,
} }
} }
async fn get_stream_entries<C>(con: &mut C, key: &[u8]) -> Result<Vec<RedisStreamEntry>, String> pub async fn get_stream_entries_page<C>(
con: &mut C,
key: &[u8],
cursor: Option<&str>,
) -> Result<RedisStreamPage, String>
where where
C: ConnectionLike + Send + Sync + Unpin, C: ConnectionLike + Send + Sync + Unpin,
{ {
let cursor = cursor.filter(|value| !value.is_empty());
let start = cursor.unwrap_or("-");
// XRANGE starts inclusively. Fetch a duplicate candidate and a lookahead
// row so this works with Redis 5 and does not skip records after trimming.
let requested_count = stream_entry_page_request_count(cursor);
let raw: RedisRawValue = redis::cmd("XRANGE") let raw: RedisRawValue = redis::cmd("XRANGE")
.arg(key) .arg(key)
.arg("-") .arg(start)
.arg("+") .arg("+")
.arg("COUNT") .arg("COUNT")
.arg(STREAM_ENTRY_LIMIT) .arg(requested_count)
.query_async(con) .query_async(con)
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
Ok(parse_stream_entries(raw)) Ok(stream_entries_page_from_raw(raw, cursor))
}
async fn get_stream_initial_page<C>(con: &mut C, key: &[u8]) -> Result<(i64, Option<u64>, RedisStreamPage), String>
where
C: ConnectionLike + Send + Sync + Unpin,
{
let mut pipe = redis::pipe();
pipe.cmd("TTL").arg(key);
pipe.cmd("XLEN").arg(key);
pipe.cmd("XRANGE").arg(key).arg("-").arg("+").arg("COUNT").arg(stream_entry_page_request_count(None));
match pipe.query_async::<(i64, u64, RedisRawValue)>(con).await {
Ok((ttl, total, raw)) => Ok((ttl, Some(total), stream_entries_page_from_raw(raw, None))),
Err(_) => {
// An ACL may allow XRANGE while rejecting the optional XLEN
// metadata. Keep the Stream readable and omit the unknown size.
let ttl: i64 = redis::cmd("TTL").arg(key).query_async(con).await.unwrap_or(-1);
let total = redis::cmd("XLEN").arg(key).query_async::<u64>(con).await.ok();
let page = get_stream_entries_page(con, key, None).await?;
Ok((ttl, total, page))
}
}
}
fn stream_entry_page_request_count(cursor: Option<&str>) -> usize {
STREAM_ENTRY_PAGE_SIZE + 1 + usize::from(cursor.is_some())
}
fn stream_entries_page_from_raw(raw: RedisRawValue, cursor: Option<&str>) -> RedisStreamPage {
let mut entries = parse_stream_entries(raw);
if let Some(cursor) = cursor {
if entries.first().is_some_and(|entry| entry.id == cursor) {
entries.remove(0);
}
}
let next_cursor = if entries.len() > STREAM_ENTRY_PAGE_SIZE {
entries.truncate(STREAM_ENTRY_PAGE_SIZE);
entries.last().map(|entry| entry.id.clone())
} else {
None
};
RedisStreamPage { entries, next_cursor }
} }
pub async fn get_stream_groups<C>(con: &mut C, key: &[u8]) -> Result<Vec<RedisStreamGroup>, String> pub async fn get_stream_groups<C>(con: &mut C, key: &[u8]) -> Result<Vec<RedisStreamGroup>, String>
@@ -3216,11 +3292,15 @@ mod tests {
fn req_packed_commands<'a>( fn req_packed_commands<'a>(
&'a mut self, &'a mut self,
_cmd: &'a Pipeline, cmd: &'a Pipeline,
_offset: usize, _offset: usize,
_count: usize, count: usize,
) -> RedisFuture<'a, Vec<RedisRawValue>> { ) -> RedisFuture<'a, Vec<RedisRawValue>> {
Box::pin(async move { Ok(Vec::new()) }) self.commands.push(String::from_utf8_lossy(&cmd.get_packed_pipeline()).into_owned());
let responses = (0..count)
.map(|_| self.responses.pop_front().unwrap_or(Ok(RedisRawValue::Nil)))
.collect::<redis::RedisResult<Vec<_>>>();
Box::pin(async move { responses })
} }
fn get_db(&self) -> i64 { fn get_db(&self) -> i64 {
@@ -3248,11 +3328,15 @@ mod tests {
fn req_packed_commands<'a>( fn req_packed_commands<'a>(
&'a mut self, &'a mut self,
_cmd: &'a Pipeline, cmd: &'a Pipeline,
_offset: usize, _offset: usize,
_count: usize, count: usize,
) -> RedisFuture<'a, Vec<RedisRawValue>> { ) -> RedisFuture<'a, Vec<RedisRawValue>> {
Box::pin(async move { Ok(Vec::new()) }) self.commands.lock().unwrap().push(String::from_utf8_lossy(&cmd.get_packed_pipeline()).into_owned());
let responses = (0..count)
.map(|_| self.responses.pop_front().unwrap_or(Ok(RedisRawValue::Nil)))
.collect::<redis::RedisResult<Vec<_>>>();
Box::pin(async move { responses })
} }
fn get_db(&self) -> i64 { fn get_db(&self) -> i64 {
@@ -3514,6 +3598,175 @@ mod tests {
assert_eq!(entry_json["deliveries"], unsafe_value.to_string()); assert_eq!(entry_json["deliveries"], unsafe_value.to_string());
} }
#[tokio::test]
async fn stream_value_uses_xlen_for_its_total_size() {
let entries = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
bulk("1714470000000-0"),
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
])]);
let mut con =
FakeRedisConnection::new(vec![bulk("stream"), RedisRawValue::Int(-1), RedisRawValue::Int(177), entries]);
let value = super::get_value(&mut con, b"orders").await.unwrap();
let RedisValueData::Stream { entries, total, next_cursor } = &value.data else {
panic!("expected a Stream value");
};
assert_eq!(*total, Some(177));
assert_eq!(entries.len(), 1);
assert!(next_cursor.is_none());
assert_eq!(super::redis_search_value_size(&value), 177);
assert_eq!(con.command_count("TYPE"), 1);
assert_eq!(con.command_count("TTL"), 1);
assert_eq!(con.command_count("XLEN"), 1);
assert_eq!(con.command_count("XRANGE"), 1);
assert_eq!(con.commands.len(), 2);
}
#[tokio::test]
async fn stream_value_still_loads_when_xlen_is_not_permitted() {
let entries = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
bulk("1714470000000-0"),
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
])]);
let xlen_denied = || {
redis::make_extension_error(
"NOPERM".to_string(),
Some("this user has no permissions to run the 'xlen' command".to_string()),
)
};
let mut con = FakeRedisConnection::with_results(vec![
Ok(bulk("stream")),
Ok(RedisRawValue::Int(-1)),
Err(xlen_denied()),
Ok(RedisRawValue::Int(-1)),
Err(xlen_denied()),
Ok(entries),
]);
let value = super::get_value(&mut con, b"orders").await.unwrap();
let RedisValueData::Stream { entries, total, .. } = &value.data else {
panic!("expected a Stream value");
};
assert_eq!(entries.len(), 1);
assert_eq!(*total, None);
assert_eq!(super::redis_search_value_size(&value), 1);
assert!(serde_json::to_value(&value).unwrap()["data"].get("total").is_none());
assert_eq!(con.command_count("XLEN"), 2);
assert_eq!(con.command_count("XRANGE"), 2);
}
#[tokio::test]
async fn stream_entry_first_page_exposes_a_cursor_when_more_entries_exist() {
let entries = RedisRawValue::Array(
(0..=50)
.map(|index| {
RedisRawValue::Array(vec![
bulk(&format!("1714470000000-{index}")),
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
])
})
.collect(),
);
let mut con = FakeRedisConnection::new(vec![entries]);
let page = super::get_stream_entries_page(&mut con, b"orders", None).await.unwrap();
assert_eq!(page.entries.len(), 50);
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-0"));
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-49"));
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-49"));
assert_eq!(con.command_count("XRANGE"), 1);
assert!(con.commands[0].contains("\r\n-\r\n"));
assert!(con.commands[0].contains("\r\n51\r\n"));
}
#[tokio::test]
async fn stream_entry_pagination_reads_all_entries_across_sequential_pages() {
let stream_entries = |start: u64, end: u64| {
RedisRawValue::Array(
(start..=end)
.map(|index| {
RedisRawValue::Array(vec![
bulk(&format!("1714470000000-{index}")),
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
])
})
.collect(),
)
};
let mut con =
FakeRedisConnection::new(vec![stream_entries(0, 50), stream_entries(49, 100), stream_entries(99, 120)]);
let first = super::get_stream_entries_page(&mut con, b"orders", None).await.unwrap();
let second = super::get_stream_entries_page(&mut con, b"orders", first.next_cursor.as_deref()).await.unwrap();
let third = super::get_stream_entries_page(&mut con, b"orders", second.next_cursor.as_deref()).await.unwrap();
let ids = first
.entries
.iter()
.chain(&second.entries)
.chain(&third.entries)
.map(|entry| entry.id.clone())
.collect::<Vec<_>>();
let expected = (0..=120).map(|index| format!("1714470000000-{index}")).collect::<Vec<_>>();
assert_eq!(first.next_cursor.as_deref(), Some("1714470000000-49"));
assert_eq!(second.next_cursor.as_deref(), Some("1714470000000-99"));
assert_eq!(third.next_cursor, None);
assert_eq!(ids, expected);
assert_eq!(con.command_count("XRANGE"), 3);
}
#[tokio::test]
async fn stream_entry_pagination_uses_a_lookahead_and_skips_the_duplicate_cursor() {
let entries = RedisRawValue::Array(
(17..=68)
.map(|index| {
RedisRawValue::Array(vec![
bulk(&format!("1714470000000-{index}")),
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
])
})
.collect(),
);
let mut con = FakeRedisConnection::new(vec![entries]);
let page = super::get_stream_entries_page(&mut con, b"orders", Some("1714470000000-17")).await.unwrap();
assert_eq!(page.entries.len(), 50);
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-18"));
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-67"));
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-67"));
assert_eq!(con.command_count("XRANGE"), 1);
assert!(con.commands[0].contains("\r\n1714470000000-17\r\n"));
assert!(!con.commands[0].contains("\r\n(1714470000000-17\r\n"));
assert!(con.commands[0].contains("\r\n52\r\n"));
}
#[tokio::test]
async fn stream_entry_pagination_keeps_the_first_entry_when_the_cursor_was_trimmed() {
let entries = RedisRawValue::Array(
(18..=68)
.map(|index| {
RedisRawValue::Array(vec![
bulk(&format!("1714470000000-{index}")),
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
])
})
.collect(),
);
let mut con = FakeRedisConnection::new(vec![entries]);
let page = super::get_stream_entries_page(&mut con, b"orders", Some("1714470000000-17")).await.unwrap();
assert_eq!(page.entries.len(), 50);
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-18"));
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-67"));
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-67"));
}
#[tokio::test] #[tokio::test]
async fn stream_monitoring_commands_are_read_only_and_pending_pagination_supports_redis_5() { async fn stream_monitoring_commands_are_read_only_and_pending_pagination_supports_redis_5() {
let groups = RedisRawValue::Array(vec![RedisRawValue::Array(vec![ let groups = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
+31 -1
View File
@@ -1,7 +1,7 @@
use crate::connection::{AppState, PoolKind}; use crate::connection::{AppState, PoolKind};
use crate::db::redis_driver::{ use crate::db::redis_driver::{
self, RedisCollectionPage, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult, self, RedisCollectionPage, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult,
RedisStreamConsumer, RedisStreamGroup, RedisStreamPendingPage, RedisValue, RedisStreamConsumer, RedisStreamGroup, RedisStreamPage, RedisStreamPendingPage, RedisValue,
}; };
async fn ensure_redis_pool(state: &AppState, connection_id: &str) -> Result<(), String> { async fn ensure_redis_pool(state: &AppState, connection_id: &str) -> Result<(), String> {
@@ -137,6 +137,36 @@ pub async fn redis_get_value_in_db_core(
} }
} }
pub async fn redis_stream_entries_in_db_core(
state: &AppState,
connection_id: &str,
db: u32,
key_raw: &str,
cursor: Option<&str>,
) -> Result<RedisStreamPage, String> {
ensure_redis_pool(state, connection_id).await?;
let connections = state.connections.read().await;
let pool = connections.get(connection_id).ok_or("Connection not found")?;
match pool {
PoolKind::Redis(redis) => {
let key = redis_driver::redis_key_raw_to_bytes(key_raw)?;
match redis {
RedisConnection::Direct(con) => {
let mut con = con.lock().await;
redis_driver::select_db(&mut *con, db).await?;
redis_driver::get_stream_entries_page(&mut *con, &key, cursor).await
}
RedisConnection::Cluster(cluster) => {
redis_driver::ensure_cluster_db(db)?;
let mut con = redis_driver::cluster_key_connection(cluster, &key).await?;
redis_driver::get_stream_entries_page(&mut con, &key, cursor).await
}
}
}
_ => Err("Not a Redis connection".to_string()),
}
}
pub async fn redis_stream_groups_in_db_core( pub async fn redis_stream_groups_in_db_core(
state: &AppState, state: &AppState,
connection_id: &str, connection_id: &str,
+1
View File
@@ -493,6 +493,7 @@ async fn main() {
.route("/redis/scan-keys-batch", post(routes::redis::scan_keys_batch)) .route("/redis/scan-keys-batch", post(routes::redis::scan_keys_batch))
.route("/redis/scan-values", post(routes::redis::scan_values)) .route("/redis/scan-values", post(routes::redis::scan_values))
.route("/redis/get-value", post(routes::redis::get_value)) .route("/redis/get-value", post(routes::redis::get_value))
.route("/redis/get-stream-entries", post(routes::redis::get_stream_entries))
.route("/redis/get-stream-groups", post(routes::redis::get_stream_groups)) .route("/redis/get-stream-groups", post(routes::redis::get_stream_groups))
.route("/redis/get-stream-consumers", post(routes::redis::get_stream_consumers)) .route("/redis/get-stream-consumers", post(routes::redis::get_stream_consumers))
.route("/redis/get-stream-pending", post(routes::redis::get_stream_pending)) .route("/redis/get-stream-pending", post(routes::redis::get_stream_pending))
+25
View File
@@ -76,6 +76,15 @@ pub struct RedisKeyRequest {
pub key_raw: String, pub key_raw: String,
} }
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RedisStreamEntriesRequest {
pub connection_id: String,
pub db: u32,
pub key_raw: String,
pub cursor: Option<String>,
}
#[derive(Deserialize)] #[derive(Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct RedisStreamGroupRequest { pub struct RedisStreamGroupRequest {
@@ -322,6 +331,22 @@ pub async fn get_value(
Ok(Json(serde_json::to_value(result).map_err(|e| AppError::from(e.to_string()))?)) Ok(Json(serde_json::to_value(result).map_err(|e| AppError::from(e.to_string()))?))
} }
pub async fn get_stream_entries(
State(state): State<Arc<WebState>>,
Json(req): Json<RedisStreamEntriesRequest>,
) -> Result<Json<serde_json::Value>, AppError> {
let result = dbx_core::redis_ops::redis_stream_entries_in_db_core(
&state.app,
&req.connection_id,
req.db,
&req.key_raw,
req.cursor.as_deref(),
)
.await
.map_err(AppError::from)?;
Ok(Json(serde_json::to_value(result).map_err(|e| AppError::from(e.to_string()))?))
}
pub async fn get_stream_groups( pub async fn get_stream_groups(
State(state): State<Arc<WebState>>, State(state): State<Arc<WebState>>,
Json(req): Json<RedisKeyRequest>, Json(req): Json<RedisKeyRequest>,
+13 -1
View File
@@ -4,7 +4,8 @@ use tauri::State;
use crate::commands::connection::{ensure_connection_writable, AppState}; use crate::commands::connection::{ensure_connection_writable, AppState};
use dbx_core::db::redis_driver::{ use dbx_core::db::redis_driver::{
classify_command, parse_command_argv, RedisCollectionPage, RedisCommandResult, RedisCommandSafety, classify_command, parse_command_argv, RedisCollectionPage, RedisCommandResult, RedisCommandSafety,
RedisDatabaseInfo, RedisScanResult, RedisStreamConsumer, RedisStreamGroup, RedisStreamPendingPage, RedisValue, RedisDatabaseInfo, RedisScanResult, RedisStreamConsumer, RedisStreamGroup, RedisStreamPage, RedisStreamPendingPage,
RedisValue,
}; };
#[tauri::command] #[tauri::command]
@@ -86,6 +87,17 @@ pub async fn redis_get_value(
dbx_core::redis_ops::redis_get_value_in_db_core(&state, &connection_id, db, &key_raw).await dbx_core::redis_ops::redis_get_value_in_db_core(&state, &connection_id, db, &key_raw).await
} }
#[tauri::command]
pub async fn redis_get_stream_entries(
state: State<'_, Arc<AppState>>,
connection_id: String,
db: u32,
key_raw: String,
cursor: Option<String>,
) -> Result<RedisStreamPage, String> {
dbx_core::redis_ops::redis_stream_entries_in_db_core(&state, &connection_id, db, &key_raw, cursor.as_deref()).await
}
#[tauri::command] #[tauri::command]
pub async fn redis_get_stream_groups( pub async fn redis_get_stream_groups(
state: State<'_, Arc<AppState>>, state: State<'_, Arc<AppState>>,
+1
View File
@@ -1765,6 +1765,7 @@ pub fn run() {
commands::redis_cmd::redis_scan_keys_batch, commands::redis_cmd::redis_scan_keys_batch,
commands::redis_cmd::redis_scan_values, commands::redis_cmd::redis_scan_values,
commands::redis_cmd::redis_get_value, commands::redis_cmd::redis_get_value,
commands::redis_cmd::redis_get_stream_entries,
commands::redis_cmd::redis_get_stream_groups, commands::redis_cmd::redis_get_stream_groups,
commands::redis_cmd::redis_get_stream_consumers, commands::redis_cmd::redis_get_stream_consumers,
commands::redis_cmd::redis_get_stream_pending, commands::redis_cmd::redis_get_stream_pending,