fix(redis): paginate stream entries
This commit is contained in:
1 parent
7775bbf8f8
commit
223e9b3a2f
21 files changed
+479
-33
No files matched your search
+24
-7
@@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|||||||
|
|
||||||
const mocks = vi.hoisted(() => ({
|
const mocks = vi.hoisted(() => ({
|
||||||
redisGetValue: vi.fn(),
|
redisGetValue: vi.fn(),
|
||||||
|
redisGetStreamEntries: vi.fn(),
|
||||||
redisGetStreamGroups: vi.fn(),
|
redisGetStreamGroups: vi.fn(),
|
||||||
redisGetStreamConsumers: vi.fn(),
|
redisGetStreamConsumers: vi.fn(),
|
||||||
redisGetStreamPending: vi.fn(),
|
redisGetStreamPending: vi.fn(),
|
||||||
@@ -16,6 +17,7 @@ const mocks = vi.hoisted(() => ({
|
|||||||
|
|
||||||
vi.mock("@/lib/backend/api", () => ({
|
vi.mock("@/lib/backend/api", () => ({
|
||||||
redisGetValue: mocks.redisGetValue,
|
redisGetValue: mocks.redisGetValue,
|
||||||
|
redisGetStreamEntries: mocks.redisGetStreamEntries,
|
||||||
redisGetStreamGroups: mocks.redisGetStreamGroups,
|
redisGetStreamGroups: mocks.redisGetStreamGroups,
|
||||||
redisGetStreamConsumers: mocks.redisGetStreamConsumers,
|
redisGetStreamConsumers: mocks.redisGetStreamConsumers,
|
||||||
redisGetStreamPending: mocks.redisGetStreamPending,
|
redisGetStreamPending: mocks.redisGetStreamPending,
|
||||||
@@ -40,11 +42,7 @@ vi.mock("vue-virtual-scroller", async () => {
|
|||||||
const DynamicScroller = defineComponent({
|
const DynamicScroller = defineComponent({
|
||||||
props: { items: { type: Array, default: () => [] } },
|
props: { items: { type: Array, default: () => [] } },
|
||||||
setup(props, { slots }) {
|
setup(props, { slots }) {
|
||||||
return () =>
|
return () => h("div", [...(props.items as unknown[]).map((item, index) => slots.default?.({ item, active: true, index })), ...(slots.after?.() ?? [])]);
|
||||||
h(
|
|
||||||
"div",
|
|
||||||
(props.items as unknown[]).map((item, index) => slots.default?.({ item, active: true, index })),
|
|
||||||
);
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const DynamicScrollerItem = defineComponent({
|
const DynamicScrollerItem = defineComponent({
|
||||||
@@ -84,13 +82,17 @@ async function settle() {
|
|||||||
await nextTick();
|
await nextTick();
|
||||||
}
|
}
|
||||||
|
|
||||||
function streamValue() {
|
function streamEntry(id: string) {
|
||||||
|
return { id, fields: [{ field: "event", value: "login" }] };
|
||||||
|
}
|
||||||
|
|
||||||
|
function streamValue(entries = [] as ReturnType<typeof streamEntry>[], nextCursor?: string, total = entries.length) {
|
||||||
return {
|
return {
|
||||||
key_display: "orders",
|
key_display: "orders",
|
||||||
key_raw: "b3JkZXJz",
|
key_raw: "b3JkZXJz",
|
||||||
ttl: -1,
|
ttl: -1,
|
||||||
redis_type: "stream",
|
redis_type: "stream",
|
||||||
data: { kind: "stream" as const, entries: [] },
|
data: { kind: "stream" as const, entries, total, ...(nextCursor ? { next_cursor: nextCursor } : {}) },
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,6 +160,21 @@ function openGroups(host: HTMLElement) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("RedisValueViewer stream monitoring", () => {
|
describe("RedisValueViewer stream monitoring", () => {
|
||||||
|
it("loads more Stream entries from the returned cursor", async () => {
|
||||||
|
mocks.redisGetValue.mockResolvedValue(streamValue([streamEntry("1714470000000-0")], "1714470000000-0"));
|
||||||
|
mocks.redisGetStreamEntries.mockResolvedValue({ entries: [streamEntry("1714470000001-0")] });
|
||||||
|
const host = mountViewer();
|
||||||
|
await settle();
|
||||||
|
|
||||||
|
expect(host.querySelectorAll("[data-redis-stream-entry]")).toHaveLength(1);
|
||||||
|
host.querySelector<HTMLButtonElement>("[data-redis-stream-entries-more]")!.click();
|
||||||
|
await settle();
|
||||||
|
|
||||||
|
expect(mocks.redisGetStreamEntries).toHaveBeenCalledWith("redis-1", 2, "b3JkZXJz", "1714470000000-0");
|
||||||
|
expect(host.querySelectorAll("[data-redis-stream-entry]")).toHaveLength(2);
|
||||||
|
expect(host.querySelector("[data-redis-stream-entries-more]")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it("renders large counter values transported as decimal strings", async () => {
|
it("renders large counter values transported as decimal strings", async () => {
|
||||||
const unsafeMetric = "9007199254740992";
|
const unsafeMetric = "9007199254740992";
|
||||||
mocks.redisGetValue.mockResolvedValue(streamValue());
|
mocks.redisGetValue.mockResolvedValue(streamValue());
|
||||||
|
|||||||
@@ -82,6 +82,9 @@ const loading = ref(false);
|
|||||||
const loadingMore = ref(false);
|
const loadingMore = ref(false);
|
||||||
let loadRequestId = 0;
|
let loadRequestId = 0;
|
||||||
const streamTab = ref<"entries" | "groups">("entries");
|
const streamTab = ref<"entries" | "groups">("entries");
|
||||||
|
const streamEntries = ref<RedisStreamEntry[]>([]);
|
||||||
|
const streamEntriesCursor = ref<string | undefined>();
|
||||||
|
const streamEntriesLoadingMore = ref(false);
|
||||||
const streamGroups = ref<RedisStreamGroup[]>([]);
|
const streamGroups = ref<RedisStreamGroup[]>([]);
|
||||||
const streamGroupsLoaded = ref(false);
|
const streamGroupsLoaded = ref(false);
|
||||||
const streamGroupsLoading = ref(false);
|
const streamGroupsLoading = ref(false);
|
||||||
@@ -112,6 +115,7 @@ let streamGroupsRequestId = 0;
|
|||||||
let streamGroupDetailRequestId = 0;
|
let streamGroupDetailRequestId = 0;
|
||||||
let streamConsumersRequestId = 0;
|
let streamConsumersRequestId = 0;
|
||||||
let streamPendingRequestId = 0;
|
let streamPendingRequestId = 0;
|
||||||
|
let streamEntriesRequestId = 0;
|
||||||
const editValue = ref("");
|
const editValue = ref("");
|
||||||
const savingString = ref(false);
|
const savingString = ref(false);
|
||||||
const savingJson = ref(false);
|
const savingJson = ref(false);
|
||||||
@@ -316,8 +320,8 @@ const metadataSizeLabel = computed(() => {
|
|||||||
return String(size);
|
return String(size);
|
||||||
});
|
});
|
||||||
const streamRows = computed<RedisStreamRow[]>(() => {
|
const streamRows = computed<RedisStreamRow[]>(() => {
|
||||||
if (data.value?.data.kind !== "stream") return [];
|
if (redisKind.value !== "stream") return [];
|
||||||
return data.value.data.entries.map((entry, index) => ({
|
return streamEntries.value.map((entry, index) => ({
|
||||||
id: `${index}:${entry.id}`,
|
id: `${index}:${entry.id}`,
|
||||||
index,
|
index,
|
||||||
entry,
|
entry,
|
||||||
@@ -434,6 +438,25 @@ type RedisStreamRow = {
|
|||||||
entry: RedisStreamEntry;
|
entry: RedisStreamEntry;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function replaceStreamEntries(value: RedisValue) {
|
||||||
|
streamEntriesRequestId++;
|
||||||
|
streamEntriesLoadingMore.value = false;
|
||||||
|
if (value.data.kind === "stream") {
|
||||||
|
streamEntries.value = [...value.data.entries];
|
||||||
|
streamEntriesCursor.value = value.data.next_cursor;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
streamEntries.value = [];
|
||||||
|
streamEntriesCursor.value = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetStreamEntries() {
|
||||||
|
streamEntriesRequestId++;
|
||||||
|
streamEntries.value = [];
|
||||||
|
streamEntriesCursor.value = undefined;
|
||||||
|
streamEntriesLoadingMore.value = false;
|
||||||
|
}
|
||||||
|
|
||||||
function isSelectedStreamGroup(group: RedisStreamGroup, requestId = streamGroupDetailRequestId): boolean {
|
function isSelectedStreamGroup(group: RedisStreamGroup, requestId = streamGroupDetailRequestId): boolean {
|
||||||
return requestId === streamGroupDetailRequestId && selectedStreamGroup.value?.name.raw_base64 === group.name.raw_base64;
|
return requestId === streamGroupDetailRequestId && selectedStreamGroup.value?.name.raw_base64 === group.name.raw_base64;
|
||||||
}
|
}
|
||||||
@@ -464,6 +487,25 @@ function resetStreamMonitoring() {
|
|||||||
streamGroupsError.value = "";
|
streamGroupsError.value = "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadMoreStreamEntries() {
|
||||||
|
const cursor = streamEntriesCursor.value;
|
||||||
|
if (redisKind.value !== "stream" || !cursor || loading.value || streamEntriesLoadingMore.value) return;
|
||||||
|
|
||||||
|
const requestId = ++streamEntriesRequestId;
|
||||||
|
streamEntriesLoadingMore.value = true;
|
||||||
|
try {
|
||||||
|
const page = await api.redisGetStreamEntries(props.connectionId, props.db, props.keyRaw, cursor);
|
||||||
|
if (requestId !== streamEntriesRequestId || redisKind.value !== "stream" || streamEntriesCursor.value !== cursor) return;
|
||||||
|
|
||||||
|
streamEntries.value = [...streamEntries.value, ...page.entries];
|
||||||
|
streamEntriesCursor.value = page.next_cursor;
|
||||||
|
} catch (error) {
|
||||||
|
if (requestId === streamEntriesRequestId) toast(errorMessage(error), 3000);
|
||||||
|
} finally {
|
||||||
|
if (requestId === streamEntriesRequestId) streamEntriesLoadingMore.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadStreamGroups(force = false): Promise<boolean> {
|
async function loadStreamGroups(force = false): Promise<boolean> {
|
||||||
if (redisKind.value !== "stream") return false;
|
if (redisKind.value !== "stream") return false;
|
||||||
if (!force && (streamGroupsLoaded.value || streamGroupsLoading.value)) return streamGroupsLoaded.value;
|
if (!force && (streamGroupsLoaded.value || streamGroupsLoading.value)) return streamGroupsLoaded.value;
|
||||||
@@ -902,6 +944,7 @@ async function load(options: { selectDefaultMember?: boolean; preserveDraft?: bo
|
|||||||
data.value = null;
|
data.value = null;
|
||||||
collectionItems.value = [];
|
collectionItems.value = [];
|
||||||
scanCursor.value = undefined;
|
scanCursor.value = undefined;
|
||||||
|
resetStreamEntries();
|
||||||
resetStreamMonitoring();
|
resetStreamMonitoring();
|
||||||
stopAutoRefresh();
|
stopAutoRefresh();
|
||||||
emit("deleted", props.keyRaw);
|
emit("deleted", props.keyRaw);
|
||||||
@@ -929,6 +972,7 @@ async function load(options: { selectDefaultMember?: boolean; preserveDraft?: bo
|
|||||||
emit("loaded", loadedValue);
|
emit("loaded", loadedValue);
|
||||||
scanCursor.value = redisValueCollectionScanCursor(loadedValue);
|
scanCursor.value = redisValueCollectionScanCursor(loadedValue);
|
||||||
collectionItems.value = redisValueCollectionItems(loadedValue);
|
collectionItems.value = redisValueCollectionItems(loadedValue);
|
||||||
|
replaceStreamEntries(loadedValue);
|
||||||
if (loadedValue.data.kind !== "stream") resetStreamMonitoring();
|
if (loadedValue.data.kind !== "stream") resetStreamMonitoring();
|
||||||
|
|
||||||
// A foreground load replaces the current value, so it also starts a new
|
// A foreground load replaces the current value, so it also starts a new
|
||||||
@@ -1050,7 +1094,8 @@ function requestDeleteKey() {
|
|||||||
|
|
||||||
async function copyValue() {
|
async function copyValue() {
|
||||||
if (!data.value) return;
|
if (!data.value) return;
|
||||||
const text = redisValueCopyText(data.value, collectionItems.value);
|
const value = data.value.data.kind === "stream" ? { ...data.value, data: { ...data.value.data, entries: streamEntries.value } } : data.value;
|
||||||
|
const text = redisValueCopyText(value, collectionItems.value);
|
||||||
try {
|
try {
|
||||||
await copyToClipboard(text);
|
await copyToClipboard(text);
|
||||||
toast(t("redis.copied"), 2000);
|
toast(t("redis.copied"), 2000);
|
||||||
@@ -1129,7 +1174,7 @@ function generateInsertStatements(): string | null {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "stream": {
|
case "stream": {
|
||||||
for (const entry of data.value.data.entries) {
|
for (const entry of streamEntries.value) {
|
||||||
const fields = entry.fields.map(({ field, value }) => `${escapeRedisArg(field)} ${escapeRedisArg(value)}`).join(" ");
|
const fields = entry.fields.map(({ field, value }) => `${escapeRedisArg(field)} ${escapeRedisArg(value)}`).join(" ");
|
||||||
commands.push(`XADD ${escapeRedisArg(key)} * ${fields}`);
|
commands.push(`XADD ${escapeRedisArg(key)} * ${fields}`);
|
||||||
}
|
}
|
||||||
@@ -1772,6 +1817,7 @@ watch(
|
|||||||
() => {
|
() => {
|
||||||
resetValueSearch();
|
resetValueSearch();
|
||||||
valueViewerSearchActive.value = false;
|
valueViewerSearchActive.value = false;
|
||||||
|
resetStreamEntries();
|
||||||
resetStreamMonitoring();
|
resetStreamMonitoring();
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -2227,9 +2273,6 @@ defineExpose({ focusSearch });
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<TabsContent value="entries" class="m-0 min-h-0 flex-1 flex flex-col">
|
<TabsContent value="entries" class="m-0 min-h-0 flex-1 flex flex-col">
|
||||||
<div class="px-4 py-1 text-xs text-muted-foreground border-b shrink-0">
|
|
||||||
{{ t("redis.entries", { count: streamRows.length }) }}
|
|
||||||
</div>
|
|
||||||
<DynamicScroller class="flex-1 overflow-y-auto" :items="streamRows" :min-item-size="REDIS_STREAM_MIN_ROW_HEIGHT" :buffer="600" key-field="id">
|
<DynamicScroller class="flex-1 overflow-y-auto" :items="streamRows" :min-item-size="REDIS_STREAM_MIN_ROW_HEIGHT" :buffer="600" key-field="id">
|
||||||
<template #default="{ item: row, active }">
|
<template #default="{ item: row, active }">
|
||||||
<DynamicScrollerItem :item="row" :active="active" :size-dependencies="[streamFieldCount(row)]" :data-index="row.index">
|
<DynamicScrollerItem :item="row" :active="active" :size-dependencies="[streamFieldCount(row)]" :data-index="row.index">
|
||||||
@@ -2259,6 +2302,14 @@ defineExpose({ focusSearch });
|
|||||||
</div>
|
</div>
|
||||||
</DynamicScrollerItem>
|
</DynamicScrollerItem>
|
||||||
</template>
|
</template>
|
||||||
|
<template #after>
|
||||||
|
<div v-if="streamEntriesCursor" class="border-t p-2">
|
||||||
|
<Button data-redis-stream-entries-more variant="outline" size="sm" class="h-7 w-full text-xs" :disabled="loading || streamEntriesLoadingMore" @click="loadMoreStreamEntries">
|
||||||
|
<Loader2 v-if="streamEntriesLoadingMore" class="mr-1.5 h-3 w-3 animate-spin" />
|
||||||
|
{{ t("redis.loadMoreEntries") }}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
</DynamicScroller>
|
</DynamicScroller>
|
||||||
</TabsContent>
|
</TabsContent>
|
||||||
|
|
||||||
|
|||||||
@@ -3069,6 +3069,7 @@ export default {
|
|||||||
noPendingEntries: "No pending entries",
|
noPendingEntries: "No pending entries",
|
||||||
entryId: "Entry ID",
|
entryId: "Entry ID",
|
||||||
deliveries: "Deliveries",
|
deliveries: "Deliveries",
|
||||||
|
loadMoreEntries: "Load more entries",
|
||||||
loadMorePending: "Load more pending entries",
|
loadMorePending: "Load more pending entries",
|
||||||
noExpiry: "no expiry",
|
noExpiry: "no expiry",
|
||||||
expiry: "Expiration",
|
expiry: "Expiration",
|
||||||
|
|||||||
@@ -2930,6 +2930,7 @@ export default withEnglishFallback({
|
|||||||
noPendingEntries: "No hay entradas pendientes",
|
noPendingEntries: "No hay entradas pendientes",
|
||||||
entryId: "ID de entrada",
|
entryId: "ID de entrada",
|
||||||
deliveries: "Entregas",
|
deliveries: "Entregas",
|
||||||
|
loadMoreEntries: "Cargar más entradas",
|
||||||
loadMorePending: "Cargar más entradas pendientes",
|
loadMorePending: "Cargar más entradas pendientes",
|
||||||
noExpiry: "sin expiración",
|
noExpiry: "sin expiración",
|
||||||
expiry: "Expiración",
|
expiry: "Expiración",
|
||||||
|
|||||||
@@ -2928,6 +2928,7 @@ export default withEnglishFallback({
|
|||||||
noPendingEntries: "Nessuna voce in sospeso",
|
noPendingEntries: "Nessuna voce in sospeso",
|
||||||
entryId: "ID voce",
|
entryId: "ID voce",
|
||||||
deliveries: "Consegne",
|
deliveries: "Consegne",
|
||||||
|
loadMoreEntries: "Carica altre voci",
|
||||||
loadMorePending: "Carica altre voci in sospeso",
|
loadMorePending: "Carica altre voci in sospeso",
|
||||||
noExpiry: "nessuna scadenza",
|
noExpiry: "nessuna scadenza",
|
||||||
expiry: "Scadenza",
|
expiry: "Scadenza",
|
||||||
|
|||||||
@@ -2929,6 +2929,7 @@ export default withEnglishFallback({
|
|||||||
noPendingEntries: "保留中のエントリはありません",
|
noPendingEntries: "保留中のエントリはありません",
|
||||||
entryId: "エントリ ID",
|
entryId: "エントリ ID",
|
||||||
deliveries: "配信回数",
|
deliveries: "配信回数",
|
||||||
|
loadMoreEntries: "さらにエントリを読み込む",
|
||||||
loadMorePending: "保留中のエントリをさらに読み込む",
|
loadMorePending: "保留中のエントリをさらに読み込む",
|
||||||
noExpiry: "期限なし",
|
noExpiry: "期限なし",
|
||||||
expiry: "有効期限",
|
expiry: "有効期限",
|
||||||
|
|||||||
@@ -3032,6 +3032,7 @@ export default withEnglishFallback({
|
|||||||
noPendingEntries: "대기 항목이 없습니다",
|
noPendingEntries: "대기 항목이 없습니다",
|
||||||
entryId: "항목 ID",
|
entryId: "항목 ID",
|
||||||
deliveries: "전달",
|
deliveries: "전달",
|
||||||
|
loadMoreEntries: "항목 더 불러오기",
|
||||||
loadMorePending: "대기 항목 더 불러오기",
|
loadMorePending: "대기 항목 더 불러오기",
|
||||||
noExpiry: "만료 없음",
|
noExpiry: "만료 없음",
|
||||||
expiry: "만료",
|
expiry: "만료",
|
||||||
|
|||||||
@@ -2930,6 +2930,7 @@ export default withEnglishFallback({
|
|||||||
noPendingEntries: "Nenhuma entrada pendente",
|
noPendingEntries: "Nenhuma entrada pendente",
|
||||||
entryId: "ID da entrada",
|
entryId: "ID da entrada",
|
||||||
deliveries: "Entregas",
|
deliveries: "Entregas",
|
||||||
|
loadMoreEntries: "Carregar mais entradas",
|
||||||
loadMorePending: "Carregar mais entradas pendentes",
|
loadMorePending: "Carregar mais entradas pendentes",
|
||||||
noExpiry: "sem expiração",
|
noExpiry: "sem expiração",
|
||||||
expiry: "Expiração",
|
expiry: "Expiração",
|
||||||
|
|||||||
@@ -3069,6 +3069,7 @@ export default withEnglishFallback({
|
|||||||
noPendingEntries: "暂无待处理条目",
|
noPendingEntries: "暂无待处理条目",
|
||||||
entryId: "条目 ID",
|
entryId: "条目 ID",
|
||||||
deliveries: "投递次数",
|
deliveries: "投递次数",
|
||||||
|
loadMoreEntries: "加载更多条目",
|
||||||
loadMorePending: "加载更多待处理条目",
|
loadMorePending: "加载更多待处理条目",
|
||||||
noExpiry: "永不过期",
|
noExpiry: "永不过期",
|
||||||
expiry: "过期时间",
|
expiry: "过期时间",
|
||||||
|
|||||||
@@ -2601,6 +2601,7 @@ export default withEnglishFallback({
|
|||||||
noPendingEntries: "暫無待處理項目",
|
noPendingEntries: "暫無待處理項目",
|
||||||
entryId: "項目 ID",
|
entryId: "項目 ID",
|
||||||
deliveries: "投遞次數",
|
deliveries: "投遞次數",
|
||||||
|
loadMoreEntries: "載入更多項目",
|
||||||
loadMorePending: "載入更多待處理項目",
|
loadMorePending: "載入更多待處理項目",
|
||||||
noExpiry: "永不過期",
|
noExpiry: "永不過期",
|
||||||
expiry: "到期時間",
|
expiry: "到期時間",
|
||||||
|
|||||||
@@ -146,6 +146,38 @@ describe("redisValuePresentation", () => {
|
|||||||
}`);
|
}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses the Stream's Redis length instead of the loaded page size", () => {
|
||||||
|
const value = {
|
||||||
|
key_display: "orders",
|
||||||
|
key_raw: "b3JkZXJz",
|
||||||
|
ttl: -1,
|
||||||
|
redis_type: "stream",
|
||||||
|
data: {
|
||||||
|
kind: "stream" as const,
|
||||||
|
entries: [{ id: "1714470000000-0", fields: [{ field: "event", value: "login" }] }],
|
||||||
|
total: 177,
|
||||||
|
next_cursor: "1714470000000-0",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(redisValueSize(value)).toBe(177);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses loaded Stream entries when the total is unavailable", () => {
|
||||||
|
const value = {
|
||||||
|
key_display: "orders",
|
||||||
|
key_raw: "b3JkZXJz",
|
||||||
|
ttl: -1,
|
||||||
|
redis_type: "stream",
|
||||||
|
data: {
|
||||||
|
kind: "stream" as const,
|
||||||
|
entries: [{ id: "1714470000000-0", fields: [{ field: "event", value: "login" }] }],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(redisValueSize(value)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("labels raw text views by encoding instead of generic raw text", () => {
|
it("labels raw text views by encoding instead of generic raw text", () => {
|
||||||
expect(formatRedisMemberDetail("plain-text").rawLabel).toBe("ASCII");
|
expect(formatRedisMemberDetail("plain-text").rawLabel).toBe("ASCII");
|
||||||
expect(
|
expect(
|
||||||
|
|||||||
@@ -392,6 +392,7 @@ export const redisScanKeys = forward("redisScanKeys");
|
|||||||
export const redisScanKeysBatch = forward("redisScanKeysBatch");
|
export const redisScanKeysBatch = forward("redisScanKeysBatch");
|
||||||
export const redisScanValues = forward("redisScanValues");
|
export const redisScanValues = forward("redisScanValues");
|
||||||
export const redisGetValue = forward("redisGetValue");
|
export const redisGetValue = forward("redisGetValue");
|
||||||
|
export const redisGetStreamEntries = forward("redisGetStreamEntries");
|
||||||
export const redisGetStreamGroups = forward("redisGetStreamGroups");
|
export const redisGetStreamGroups = forward("redisGetStreamGroups");
|
||||||
export const redisGetStreamConsumers = forward("redisGetStreamConsumers");
|
export const redisGetStreamConsumers = forward("redisGetStreamConsumers");
|
||||||
export const redisGetStreamPending = forward("redisGetStreamPending");
|
export const redisGetStreamPending = forward("redisGetStreamPending");
|
||||||
@@ -640,6 +641,7 @@ export type {
|
|||||||
RedisStreamField,
|
RedisStreamField,
|
||||||
RedisStreamGroup,
|
RedisStreamGroup,
|
||||||
RedisStreamMetric,
|
RedisStreamMetric,
|
||||||
|
RedisStreamPage,
|
||||||
RedisStreamPendingEntry,
|
RedisStreamPendingEntry,
|
||||||
RedisStreamPendingPage,
|
RedisStreamPendingPage,
|
||||||
RedisValue,
|
RedisValue,
|
||||||
|
|||||||
@@ -70,6 +70,7 @@ import type {
|
|||||||
RedisDatabaseInfo,
|
RedisDatabaseInfo,
|
||||||
RedisStreamConsumer,
|
RedisStreamConsumer,
|
||||||
RedisStreamGroup,
|
RedisStreamGroup,
|
||||||
|
RedisStreamPage,
|
||||||
RedisStreamPendingPage,
|
RedisStreamPendingPage,
|
||||||
RedisValue,
|
RedisValue,
|
||||||
RedisScanResult,
|
RedisScanResult,
|
||||||
@@ -2253,6 +2254,10 @@ export async function redisGetValue(connectionId: string, db: number, keyRaw: st
|
|||||||
return post("/api/redis/get-value", { connectionId, db, keyRaw });
|
return post("/api/redis/get-value", { connectionId, db, keyRaw });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function redisGetStreamEntries(connectionId: string, db: number, keyRaw: string, cursor?: string): Promise<RedisStreamPage> {
|
||||||
|
return post("/api/redis/get-stream-entries", { connectionId, db, keyRaw, cursor });
|
||||||
|
}
|
||||||
|
|
||||||
export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> {
|
export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> {
|
||||||
return post("/api/redis/get-stream-groups", { connectionId, db, keyRaw });
|
return post("/api/redis/get-stream-groups", { connectionId, db, keyRaw });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1949,6 +1949,11 @@ export interface RedisStreamEntry {
|
|||||||
fields: RedisStreamField[];
|
fields: RedisStreamField[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RedisStreamPage {
|
||||||
|
entries: RedisStreamEntry[];
|
||||||
|
next_cursor?: string;
|
||||||
|
}
|
||||||
|
|
||||||
// Redis counters above Number.MAX_SAFE_INTEGER are transported as decimal strings.
|
// Redis counters above Number.MAX_SAFE_INTEGER are transported as decimal strings.
|
||||||
export type RedisStreamMetric = number | string;
|
export type RedisStreamMetric = number | string;
|
||||||
|
|
||||||
@@ -2002,7 +2007,7 @@ export type RedisValueData =
|
|||||||
total: number;
|
total: number;
|
||||||
scan_cursor?: number;
|
scan_cursor?: number;
|
||||||
}
|
}
|
||||||
| { kind: "stream"; entries: RedisStreamEntry[] }
|
| { kind: "stream"; entries: RedisStreamEntry[]; total?: number; next_cursor?: string }
|
||||||
| { kind: "unknown" };
|
| { kind: "unknown" };
|
||||||
|
|
||||||
export interface RedisValue {
|
export interface RedisValue {
|
||||||
@@ -2085,6 +2090,10 @@ export async function redisGetValue(connectionId: string, db: number, keyRaw: st
|
|||||||
return invoke("redis_get_value", { connectionId, db, keyRaw });
|
return invoke("redis_get_value", { connectionId, db, keyRaw });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function redisGetStreamEntries(connectionId: string, db: number, keyRaw: string, cursor?: string): Promise<RedisStreamPage> {
|
||||||
|
return invoke("redis_get_stream_entries", { connectionId, db, keyRaw, cursor });
|
||||||
|
}
|
||||||
|
|
||||||
export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> {
|
export async function redisGetStreamGroups(connectionId: string, db: number, keyRaw: string): Promise<RedisStreamGroup[]> {
|
||||||
return invoke("redis_get_stream_groups", { connectionId, db, keyRaw });
|
return invoke("redis_get_stream_groups", { connectionId, db, keyRaw });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -327,7 +327,7 @@ export function redisValueSize(value: RedisValue): number {
|
|||||||
case "zset":
|
case "zset":
|
||||||
return value.data.total;
|
return value.data.total;
|
||||||
case "stream":
|
case "stream":
|
||||||
return value.data.entries.length;
|
return value.data.total ?? value.data.entries.length;
|
||||||
default:
|
default:
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use tokio::sync::{Mutex, MutexGuard};
|
|||||||
|
|
||||||
use super::json_value_for_js;
|
use super::json_value_for_js;
|
||||||
|
|
||||||
const STREAM_ENTRY_LIMIT: usize = 100;
|
const STREAM_ENTRY_PAGE_SIZE: usize = 50;
|
||||||
const STREAM_PENDING_PAGE_SIZE: usize = 100;
|
const STREAM_PENDING_PAGE_SIZE: usize = 100;
|
||||||
const COLLECTION_PAGE_SIZE: usize = 200;
|
const COLLECTION_PAGE_SIZE: usize = 200;
|
||||||
const HASH_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
|
const HASH_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
|
||||||
@@ -124,6 +124,13 @@ pub struct RedisStreamEntry {
|
|||||||
pub fields: Vec<RedisStreamField>,
|
pub fields: Vec<RedisStreamField>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct RedisStreamPage {
|
||||||
|
pub entries: Vec<RedisStreamEntry>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub next_cursor: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct RedisStreamGroup {
|
pub struct RedisStreamGroup {
|
||||||
pub name: RedisBlob,
|
pub name: RedisBlob,
|
||||||
@@ -224,6 +231,14 @@ pub enum RedisValueData {
|
|||||||
},
|
},
|
||||||
Stream {
|
Stream {
|
||||||
entries: Vec<RedisStreamEntry>,
|
entries: Vec<RedisStreamEntry>,
|
||||||
|
#[serde(
|
||||||
|
default,
|
||||||
|
skip_serializing_if = "Option::is_none",
|
||||||
|
serialize_with = "serialize_optional_redis_u64_for_js"
|
||||||
|
)]
|
||||||
|
total: Option<u64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
next_cursor: Option<String>,
|
||||||
},
|
},
|
||||||
Unknown,
|
Unknown,
|
||||||
}
|
}
|
||||||
@@ -2281,7 +2296,14 @@ where
|
|||||||
{
|
{
|
||||||
let redis_type: String = redis::cmd("TYPE").arg(key).query_async(con).await.map_err(|e| e.to_string())?;
|
let redis_type: String = redis::cmd("TYPE").arg(key).query_async(con).await.map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
let ttl: i64 = redis::cmd("TTL").arg(key).query_async(con).await.unwrap_or(-1);
|
// For Streams, fetch metadata and the initial page together after TYPE.
|
||||||
|
// This keeps the accurate XLEN without adding another Redis round trip.
|
||||||
|
let stream_initial_page =
|
||||||
|
if redis_type == "stream" { Some(get_stream_initial_page(con, key).await?) } else { None };
|
||||||
|
let ttl: i64 = match stream_initial_page.as_ref() {
|
||||||
|
Some((ttl, _, _)) => *ttl,
|
||||||
|
None => redis::cmd("TTL").arg(key).query_async(con).await.unwrap_or(-1),
|
||||||
|
};
|
||||||
|
|
||||||
let data = match redis_type.as_str() {
|
let data = match redis_type.as_str() {
|
||||||
"string" => {
|
"string" => {
|
||||||
@@ -2315,7 +2337,10 @@ where
|
|||||||
let (cursor, items) = hscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE, None).await?;
|
let (cursor, items) = hscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE, None).await?;
|
||||||
RedisValueData::Hash { items, total: len, scan_cursor: (cursor > 0).then_some(cursor) }
|
RedisValueData::Hash { items, total: len, scan_cursor: (cursor > 0).then_some(cursor) }
|
||||||
}
|
}
|
||||||
"stream" => RedisValueData::Stream { entries: get_stream_entries(con, key).await? },
|
"stream" => {
|
||||||
|
let (_, total, page) = stream_initial_page.expect("Stream page is loaded with its Stream type");
|
||||||
|
RedisValueData::Stream { entries: page.entries, total, next_cursor: page.next_cursor }
|
||||||
|
}
|
||||||
key_type if is_redis_json_type(key_type) => {
|
key_type if is_redis_json_type(key_type) => {
|
||||||
let raw: RedisRawValue =
|
let raw: RedisRawValue =
|
||||||
redis::cmd("JSON.GET").arg(key).query_async(con).await.map_err(|e| e.to_string())?;
|
redis::cmd("JSON.GET").arg(key).query_async(con).await.map_err(|e| e.to_string())?;
|
||||||
@@ -2370,7 +2395,7 @@ fn redis_search_value_text(value: &RedisValueData) -> String {
|
|||||||
.flat_map(|item| [item.score.clone(), redis_blob_display_text(&item.member)])
|
.flat_map(|item| [item.score.clone(), redis_blob_display_text(&item.member)])
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(" "),
|
.join(" "),
|
||||||
RedisValueData::Stream { entries } => entries
|
RedisValueData::Stream { entries, .. } => entries
|
||||||
.iter()
|
.iter()
|
||||||
.flat_map(|entry| {
|
.flat_map(|entry| {
|
||||||
entry.fields.iter().flat_map(|field| [field.field.clone(), field.value.clone()]).collect::<Vec<_>>()
|
entry.fields.iter().flat_map(|field| [field.field.clone(), field.value.clone()]).collect::<Vec<_>>()
|
||||||
@@ -2403,26 +2428,77 @@ fn redis_search_value_size(value: &RedisValue) -> u64 {
|
|||||||
| RedisValueData::Set { total, .. }
|
| RedisValueData::Set { total, .. }
|
||||||
| RedisValueData::Hash { total, .. }
|
| RedisValueData::Hash { total, .. }
|
||||||
| RedisValueData::Zset { total, .. } => *total,
|
| RedisValueData::Zset { total, .. } => *total,
|
||||||
RedisValueData::Stream { entries } => entries.len() as u64,
|
RedisValueData::Stream { entries, total, .. } => total.unwrap_or(entries.len() as u64),
|
||||||
RedisValueData::Unknown => 0,
|
RedisValueData::Unknown => 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_stream_entries<C>(con: &mut C, key: &[u8]) -> Result<Vec<RedisStreamEntry>, String>
|
pub async fn get_stream_entries_page<C>(
|
||||||
|
con: &mut C,
|
||||||
|
key: &[u8],
|
||||||
|
cursor: Option<&str>,
|
||||||
|
) -> Result<RedisStreamPage, String>
|
||||||
where
|
where
|
||||||
C: ConnectionLike + Send + Sync + Unpin,
|
C: ConnectionLike + Send + Sync + Unpin,
|
||||||
{
|
{
|
||||||
|
let cursor = cursor.filter(|value| !value.is_empty());
|
||||||
|
let start = cursor.unwrap_or("-");
|
||||||
|
// XRANGE starts inclusively. Fetch a duplicate candidate and a lookahead
|
||||||
|
// row so this works with Redis 5 and does not skip records after trimming.
|
||||||
|
let requested_count = stream_entry_page_request_count(cursor);
|
||||||
let raw: RedisRawValue = redis::cmd("XRANGE")
|
let raw: RedisRawValue = redis::cmd("XRANGE")
|
||||||
.arg(key)
|
.arg(key)
|
||||||
.arg("-")
|
.arg(start)
|
||||||
.arg("+")
|
.arg("+")
|
||||||
.arg("COUNT")
|
.arg("COUNT")
|
||||||
.arg(STREAM_ENTRY_LIMIT)
|
.arg(requested_count)
|
||||||
.query_async(con)
|
.query_async(con)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| e.to_string())?;
|
.map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
Ok(parse_stream_entries(raw))
|
Ok(stream_entries_page_from_raw(raw, cursor))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_stream_initial_page<C>(con: &mut C, key: &[u8]) -> Result<(i64, Option<u64>, RedisStreamPage), String>
|
||||||
|
where
|
||||||
|
C: ConnectionLike + Send + Sync + Unpin,
|
||||||
|
{
|
||||||
|
let mut pipe = redis::pipe();
|
||||||
|
pipe.cmd("TTL").arg(key);
|
||||||
|
pipe.cmd("XLEN").arg(key);
|
||||||
|
pipe.cmd("XRANGE").arg(key).arg("-").arg("+").arg("COUNT").arg(stream_entry_page_request_count(None));
|
||||||
|
match pipe.query_async::<(i64, u64, RedisRawValue)>(con).await {
|
||||||
|
Ok((ttl, total, raw)) => Ok((ttl, Some(total), stream_entries_page_from_raw(raw, None))),
|
||||||
|
Err(_) => {
|
||||||
|
// An ACL may allow XRANGE while rejecting the optional XLEN
|
||||||
|
// metadata. Keep the Stream readable and omit the unknown size.
|
||||||
|
let ttl: i64 = redis::cmd("TTL").arg(key).query_async(con).await.unwrap_or(-1);
|
||||||
|
let total = redis::cmd("XLEN").arg(key).query_async::<u64>(con).await.ok();
|
||||||
|
let page = get_stream_entries_page(con, key, None).await?;
|
||||||
|
Ok((ttl, total, page))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stream_entry_page_request_count(cursor: Option<&str>) -> usize {
|
||||||
|
STREAM_ENTRY_PAGE_SIZE + 1 + usize::from(cursor.is_some())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stream_entries_page_from_raw(raw: RedisRawValue, cursor: Option<&str>) -> RedisStreamPage {
|
||||||
|
let mut entries = parse_stream_entries(raw);
|
||||||
|
if let Some(cursor) = cursor {
|
||||||
|
if entries.first().is_some_and(|entry| entry.id == cursor) {
|
||||||
|
entries.remove(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let next_cursor = if entries.len() > STREAM_ENTRY_PAGE_SIZE {
|
||||||
|
entries.truncate(STREAM_ENTRY_PAGE_SIZE);
|
||||||
|
entries.last().map(|entry| entry.id.clone())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
RedisStreamPage { entries, next_cursor }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_stream_groups<C>(con: &mut C, key: &[u8]) -> Result<Vec<RedisStreamGroup>, String>
|
pub async fn get_stream_groups<C>(con: &mut C, key: &[u8]) -> Result<Vec<RedisStreamGroup>, String>
|
||||||
@@ -3216,11 +3292,15 @@ mod tests {
|
|||||||
|
|
||||||
fn req_packed_commands<'a>(
|
fn req_packed_commands<'a>(
|
||||||
&'a mut self,
|
&'a mut self,
|
||||||
_cmd: &'a Pipeline,
|
cmd: &'a Pipeline,
|
||||||
_offset: usize,
|
_offset: usize,
|
||||||
_count: usize,
|
count: usize,
|
||||||
) -> RedisFuture<'a, Vec<RedisRawValue>> {
|
) -> RedisFuture<'a, Vec<RedisRawValue>> {
|
||||||
Box::pin(async move { Ok(Vec::new()) })
|
self.commands.push(String::from_utf8_lossy(&cmd.get_packed_pipeline()).into_owned());
|
||||||
|
let responses = (0..count)
|
||||||
|
.map(|_| self.responses.pop_front().unwrap_or(Ok(RedisRawValue::Nil)))
|
||||||
|
.collect::<redis::RedisResult<Vec<_>>>();
|
||||||
|
Box::pin(async move { responses })
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_db(&self) -> i64 {
|
fn get_db(&self) -> i64 {
|
||||||
@@ -3248,11 +3328,15 @@ mod tests {
|
|||||||
|
|
||||||
fn req_packed_commands<'a>(
|
fn req_packed_commands<'a>(
|
||||||
&'a mut self,
|
&'a mut self,
|
||||||
_cmd: &'a Pipeline,
|
cmd: &'a Pipeline,
|
||||||
_offset: usize,
|
_offset: usize,
|
||||||
_count: usize,
|
count: usize,
|
||||||
) -> RedisFuture<'a, Vec<RedisRawValue>> {
|
) -> RedisFuture<'a, Vec<RedisRawValue>> {
|
||||||
Box::pin(async move { Ok(Vec::new()) })
|
self.commands.lock().unwrap().push(String::from_utf8_lossy(&cmd.get_packed_pipeline()).into_owned());
|
||||||
|
let responses = (0..count)
|
||||||
|
.map(|_| self.responses.pop_front().unwrap_or(Ok(RedisRawValue::Nil)))
|
||||||
|
.collect::<redis::RedisResult<Vec<_>>>();
|
||||||
|
Box::pin(async move { responses })
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_db(&self) -> i64 {
|
fn get_db(&self) -> i64 {
|
||||||
@@ -3514,6 +3598,175 @@ mod tests {
|
|||||||
assert_eq!(entry_json["deliveries"], unsafe_value.to_string());
|
assert_eq!(entry_json["deliveries"], unsafe_value.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stream_value_uses_xlen_for_its_total_size() {
|
||||||
|
let entries = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||||
|
bulk("1714470000000-0"),
|
||||||
|
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
|
||||||
|
])]);
|
||||||
|
let mut con =
|
||||||
|
FakeRedisConnection::new(vec![bulk("stream"), RedisRawValue::Int(-1), RedisRawValue::Int(177), entries]);
|
||||||
|
|
||||||
|
let value = super::get_value(&mut con, b"orders").await.unwrap();
|
||||||
|
|
||||||
|
let RedisValueData::Stream { entries, total, next_cursor } = &value.data else {
|
||||||
|
panic!("expected a Stream value");
|
||||||
|
};
|
||||||
|
assert_eq!(*total, Some(177));
|
||||||
|
assert_eq!(entries.len(), 1);
|
||||||
|
assert!(next_cursor.is_none());
|
||||||
|
assert_eq!(super::redis_search_value_size(&value), 177);
|
||||||
|
assert_eq!(con.command_count("TYPE"), 1);
|
||||||
|
assert_eq!(con.command_count("TTL"), 1);
|
||||||
|
assert_eq!(con.command_count("XLEN"), 1);
|
||||||
|
assert_eq!(con.command_count("XRANGE"), 1);
|
||||||
|
assert_eq!(con.commands.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stream_value_still_loads_when_xlen_is_not_permitted() {
|
||||||
|
let entries = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||||
|
bulk("1714470000000-0"),
|
||||||
|
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
|
||||||
|
])]);
|
||||||
|
let xlen_denied = || {
|
||||||
|
redis::make_extension_error(
|
||||||
|
"NOPERM".to_string(),
|
||||||
|
Some("this user has no permissions to run the 'xlen' command".to_string()),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let mut con = FakeRedisConnection::with_results(vec![
|
||||||
|
Ok(bulk("stream")),
|
||||||
|
Ok(RedisRawValue::Int(-1)),
|
||||||
|
Err(xlen_denied()),
|
||||||
|
Ok(RedisRawValue::Int(-1)),
|
||||||
|
Err(xlen_denied()),
|
||||||
|
Ok(entries),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let value = super::get_value(&mut con, b"orders").await.unwrap();
|
||||||
|
|
||||||
|
let RedisValueData::Stream { entries, total, .. } = &value.data else {
|
||||||
|
panic!("expected a Stream value");
|
||||||
|
};
|
||||||
|
assert_eq!(entries.len(), 1);
|
||||||
|
assert_eq!(*total, None);
|
||||||
|
assert_eq!(super::redis_search_value_size(&value), 1);
|
||||||
|
assert!(serde_json::to_value(&value).unwrap()["data"].get("total").is_none());
|
||||||
|
assert_eq!(con.command_count("XLEN"), 2);
|
||||||
|
assert_eq!(con.command_count("XRANGE"), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stream_entry_first_page_exposes_a_cursor_when_more_entries_exist() {
|
||||||
|
let entries = RedisRawValue::Array(
|
||||||
|
(0..=50)
|
||||||
|
.map(|index| {
|
||||||
|
RedisRawValue::Array(vec![
|
||||||
|
bulk(&format!("1714470000000-{index}")),
|
||||||
|
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
let mut con = FakeRedisConnection::new(vec![entries]);
|
||||||
|
|
||||||
|
let page = super::get_stream_entries_page(&mut con, b"orders", None).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(page.entries.len(), 50);
|
||||||
|
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-0"));
|
||||||
|
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-49"));
|
||||||
|
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-49"));
|
||||||
|
assert_eq!(con.command_count("XRANGE"), 1);
|
||||||
|
assert!(con.commands[0].contains("\r\n-\r\n"));
|
||||||
|
assert!(con.commands[0].contains("\r\n51\r\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stream_entry_pagination_reads_all_entries_across_sequential_pages() {
|
||||||
|
let stream_entries = |start: u64, end: u64| {
|
||||||
|
RedisRawValue::Array(
|
||||||
|
(start..=end)
|
||||||
|
.map(|index| {
|
||||||
|
RedisRawValue::Array(vec![
|
||||||
|
bulk(&format!("1714470000000-{index}")),
|
||||||
|
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let mut con =
|
||||||
|
FakeRedisConnection::new(vec![stream_entries(0, 50), stream_entries(49, 100), stream_entries(99, 120)]);
|
||||||
|
|
||||||
|
let first = super::get_stream_entries_page(&mut con, b"orders", None).await.unwrap();
|
||||||
|
let second = super::get_stream_entries_page(&mut con, b"orders", first.next_cursor.as_deref()).await.unwrap();
|
||||||
|
let third = super::get_stream_entries_page(&mut con, b"orders", second.next_cursor.as_deref()).await.unwrap();
|
||||||
|
|
||||||
|
let ids = first
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.chain(&second.entries)
|
||||||
|
.chain(&third.entries)
|
||||||
|
.map(|entry| entry.id.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let expected = (0..=120).map(|index| format!("1714470000000-{index}")).collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(first.next_cursor.as_deref(), Some("1714470000000-49"));
|
||||||
|
assert_eq!(second.next_cursor.as_deref(), Some("1714470000000-99"));
|
||||||
|
assert_eq!(third.next_cursor, None);
|
||||||
|
assert_eq!(ids, expected);
|
||||||
|
assert_eq!(con.command_count("XRANGE"), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stream_entry_pagination_uses_a_lookahead_and_skips_the_duplicate_cursor() {
|
||||||
|
let entries = RedisRawValue::Array(
|
||||||
|
(17..=68)
|
||||||
|
.map(|index| {
|
||||||
|
RedisRawValue::Array(vec![
|
||||||
|
bulk(&format!("1714470000000-{index}")),
|
||||||
|
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
let mut con = FakeRedisConnection::new(vec![entries]);
|
||||||
|
|
||||||
|
let page = super::get_stream_entries_page(&mut con, b"orders", Some("1714470000000-17")).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(page.entries.len(), 50);
|
||||||
|
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-18"));
|
||||||
|
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-67"));
|
||||||
|
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-67"));
|
||||||
|
assert_eq!(con.command_count("XRANGE"), 1);
|
||||||
|
assert!(con.commands[0].contains("\r\n1714470000000-17\r\n"));
|
||||||
|
assert!(!con.commands[0].contains("\r\n(1714470000000-17\r\n"));
|
||||||
|
assert!(con.commands[0].contains("\r\n52\r\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stream_entry_pagination_keeps_the_first_entry_when_the_cursor_was_trimmed() {
|
||||||
|
let entries = RedisRawValue::Array(
|
||||||
|
(18..=68)
|
||||||
|
.map(|index| {
|
||||||
|
RedisRawValue::Array(vec![
|
||||||
|
bulk(&format!("1714470000000-{index}")),
|
||||||
|
RedisRawValue::Array(vec![bulk("event"), bulk("login")]),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
let mut con = FakeRedisConnection::new(vec![entries]);
|
||||||
|
|
||||||
|
let page = super::get_stream_entries_page(&mut con, b"orders", Some("1714470000000-17")).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(page.entries.len(), 50);
|
||||||
|
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-18"));
|
||||||
|
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-67"));
|
||||||
|
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-67"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn stream_monitoring_commands_are_read_only_and_pending_pagination_supports_redis_5() {
|
async fn stream_monitoring_commands_are_read_only_and_pending_pagination_supports_redis_5() {
|
||||||
let groups = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
let groups = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use crate::connection::{AppState, PoolKind};
|
use crate::connection::{AppState, PoolKind};
|
||||||
use crate::db::redis_driver::{
|
use crate::db::redis_driver::{
|
||||||
self, RedisCollectionPage, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult,
|
self, RedisCollectionPage, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult,
|
||||||
RedisStreamConsumer, RedisStreamGroup, RedisStreamPendingPage, RedisValue,
|
RedisStreamConsumer, RedisStreamGroup, RedisStreamPage, RedisStreamPendingPage, RedisValue,
|
||||||
};
|
};
|
||||||
|
|
||||||
async fn ensure_redis_pool(state: &AppState, connection_id: &str) -> Result<(), String> {
|
async fn ensure_redis_pool(state: &AppState, connection_id: &str) -> Result<(), String> {
|
||||||
@@ -137,6 +137,36 @@ pub async fn redis_get_value_in_db_core(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn redis_stream_entries_in_db_core(
|
||||||
|
state: &AppState,
|
||||||
|
connection_id: &str,
|
||||||
|
db: u32,
|
||||||
|
key_raw: &str,
|
||||||
|
cursor: Option<&str>,
|
||||||
|
) -> Result<RedisStreamPage, String> {
|
||||||
|
ensure_redis_pool(state, connection_id).await?;
|
||||||
|
let connections = state.connections.read().await;
|
||||||
|
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||||
|
match pool {
|
||||||
|
PoolKind::Redis(redis) => {
|
||||||
|
let key = redis_driver::redis_key_raw_to_bytes(key_raw)?;
|
||||||
|
match redis {
|
||||||
|
RedisConnection::Direct(con) => {
|
||||||
|
let mut con = con.lock().await;
|
||||||
|
redis_driver::select_db(&mut *con, db).await?;
|
||||||
|
redis_driver::get_stream_entries_page(&mut *con, &key, cursor).await
|
||||||
|
}
|
||||||
|
RedisConnection::Cluster(cluster) => {
|
||||||
|
redis_driver::ensure_cluster_db(db)?;
|
||||||
|
let mut con = redis_driver::cluster_key_connection(cluster, &key).await?;
|
||||||
|
redis_driver::get_stream_entries_page(&mut con, &key, cursor).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => Err("Not a Redis connection".to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn redis_stream_groups_in_db_core(
|
pub async fn redis_stream_groups_in_db_core(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
connection_id: &str,
|
connection_id: &str,
|
||||||
|
|||||||
@@ -493,6 +493,7 @@ async fn main() {
|
|||||||
.route("/redis/scan-keys-batch", post(routes::redis::scan_keys_batch))
|
.route("/redis/scan-keys-batch", post(routes::redis::scan_keys_batch))
|
||||||
.route("/redis/scan-values", post(routes::redis::scan_values))
|
.route("/redis/scan-values", post(routes::redis::scan_values))
|
||||||
.route("/redis/get-value", post(routes::redis::get_value))
|
.route("/redis/get-value", post(routes::redis::get_value))
|
||||||
|
.route("/redis/get-stream-entries", post(routes::redis::get_stream_entries))
|
||||||
.route("/redis/get-stream-groups", post(routes::redis::get_stream_groups))
|
.route("/redis/get-stream-groups", post(routes::redis::get_stream_groups))
|
||||||
.route("/redis/get-stream-consumers", post(routes::redis::get_stream_consumers))
|
.route("/redis/get-stream-consumers", post(routes::redis::get_stream_consumers))
|
||||||
.route("/redis/get-stream-pending", post(routes::redis::get_stream_pending))
|
.route("/redis/get-stream-pending", post(routes::redis::get_stream_pending))
|
||||||
|
|||||||
@@ -76,6 +76,15 @@ pub struct RedisKeyRequest {
|
|||||||
pub key_raw: String,
|
pub key_raw: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct RedisStreamEntriesRequest {
|
||||||
|
pub connection_id: String,
|
||||||
|
pub db: u32,
|
||||||
|
pub key_raw: String,
|
||||||
|
pub cursor: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct RedisStreamGroupRequest {
|
pub struct RedisStreamGroupRequest {
|
||||||
@@ -322,6 +331,22 @@ pub async fn get_value(
|
|||||||
Ok(Json(serde_json::to_value(result).map_err(|e| AppError::from(e.to_string()))?))
|
Ok(Json(serde_json::to_value(result).map_err(|e| AppError::from(e.to_string()))?))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn get_stream_entries(
|
||||||
|
State(state): State<Arc<WebState>>,
|
||||||
|
Json(req): Json<RedisStreamEntriesRequest>,
|
||||||
|
) -> Result<Json<serde_json::Value>, AppError> {
|
||||||
|
let result = dbx_core::redis_ops::redis_stream_entries_in_db_core(
|
||||||
|
&state.app,
|
||||||
|
&req.connection_id,
|
||||||
|
req.db,
|
||||||
|
&req.key_raw,
|
||||||
|
req.cursor.as_deref(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(AppError::from)?;
|
||||||
|
Ok(Json(serde_json::to_value(result).map_err(|e| AppError::from(e.to_string()))?))
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get_stream_groups(
|
pub async fn get_stream_groups(
|
||||||
State(state): State<Arc<WebState>>,
|
State(state): State<Arc<WebState>>,
|
||||||
Json(req): Json<RedisKeyRequest>,
|
Json(req): Json<RedisKeyRequest>,
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ use tauri::State;
|
|||||||
use crate::commands::connection::{ensure_connection_writable, AppState};
|
use crate::commands::connection::{ensure_connection_writable, AppState};
|
||||||
use dbx_core::db::redis_driver::{
|
use dbx_core::db::redis_driver::{
|
||||||
classify_command, parse_command_argv, RedisCollectionPage, RedisCommandResult, RedisCommandSafety,
|
classify_command, parse_command_argv, RedisCollectionPage, RedisCommandResult, RedisCommandSafety,
|
||||||
RedisDatabaseInfo, RedisScanResult, RedisStreamConsumer, RedisStreamGroup, RedisStreamPendingPage, RedisValue,
|
RedisDatabaseInfo, RedisScanResult, RedisStreamConsumer, RedisStreamGroup, RedisStreamPage, RedisStreamPendingPage,
|
||||||
|
RedisValue,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
@@ -86,6 +87,17 @@ pub async fn redis_get_value(
|
|||||||
dbx_core::redis_ops::redis_get_value_in_db_core(&state, &connection_id, db, &key_raw).await
|
dbx_core::redis_ops::redis_get_value_in_db_core(&state, &connection_id, db, &key_raw).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tauri::command]
|
||||||
|
pub async fn redis_get_stream_entries(
|
||||||
|
state: State<'_, Arc<AppState>>,
|
||||||
|
connection_id: String,
|
||||||
|
db: u32,
|
||||||
|
key_raw: String,
|
||||||
|
cursor: Option<String>,
|
||||||
|
) -> Result<RedisStreamPage, String> {
|
||||||
|
dbx_core::redis_ops::redis_stream_entries_in_db_core(&state, &connection_id, db, &key_raw, cursor.as_deref()).await
|
||||||
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
pub async fn redis_get_stream_groups(
|
pub async fn redis_get_stream_groups(
|
||||||
state: State<'_, Arc<AppState>>,
|
state: State<'_, Arc<AppState>>,
|
||||||
|
|||||||
@@ -1765,6 +1765,7 @@ pub fn run() {
|
|||||||
commands::redis_cmd::redis_scan_keys_batch,
|
commands::redis_cmd::redis_scan_keys_batch,
|
||||||
commands::redis_cmd::redis_scan_values,
|
commands::redis_cmd::redis_scan_values,
|
||||||
commands::redis_cmd::redis_get_value,
|
commands::redis_cmd::redis_get_value,
|
||||||
|
commands::redis_cmd::redis_get_stream_entries,
|
||||||
commands::redis_cmd::redis_get_stream_groups,
|
commands::redis_cmd::redis_get_stream_groups,
|
||||||
commands::redis_cmd::redis_get_stream_consumers,
|
commands::redis_cmd::redis_get_stream_consumers,
|
||||||
commands::redis_cmd::redis_get_stream_pending,
|
commands::redis_cmd::redis_get_stream_pending,
|
||||||
|
|||||||
Reference in new issue
Block a user