fix(redis): honor configured value scan page size
This commit is contained in:
2 files changed
+84
-38
No files matched your search
@@ -5,6 +5,7 @@ import { createI18n } from "vue-i18n";
|
|||||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { RedisKeyInfo } from "@/lib/backend/api";
|
import type { RedisKeyInfo } from "@/lib/backend/api";
|
||||||
import { defaultRedisKeyGrouping, type RedisKeyGrouping } from "@/lib/redis/redisKeyGrouping";
|
import { defaultRedisKeyGrouping, type RedisKeyGrouping } from "@/lib/redis/redisKeyGrouping";
|
||||||
|
import { REDIS_SCAN_PAGE_SIZE_OPTIONS } from "@/lib/redis/redisKeyPattern";
|
||||||
|
|
||||||
const grouping = ref<RedisKeyGrouping>();
|
const grouping = ref<RedisKeyGrouping>();
|
||||||
vi.mock("@/lib/redis/redisKeyViewScheduler", () => ({ createRedisKeyViewYield: () => () => new Promise<void>((resolve) => requestAnimationFrame(() => resolve())) }));
|
vi.mock("@/lib/redis/redisKeyViewScheduler", () => ({ createRedisKeyViewYield: () => () => new Promise<void>((resolve) => requestAnimationFrame(() => resolve())) }));
|
||||||
@@ -368,7 +369,7 @@ function resetApiMocks() {
|
|||||||
mocks.queryResultMaxRowsEnabled = true;
|
mocks.queryResultMaxRowsEnabled = true;
|
||||||
mocks.queryResultMaxRows = 5000;
|
mocks.queryResultMaxRows = 5000;
|
||||||
mocks.redisScanKeysBatch.mockResolvedValue({ cursor: 0, keys: [], total_keys: 0 });
|
mocks.redisScanKeysBatch.mockResolvedValue({ cursor: 0, keys: [], total_keys: 0 });
|
||||||
mocks.redisScanValues.mockResolvedValue({ cursor: 0, keys: [], total_keys: 0 });
|
mocks.redisScanValues.mockReset().mockResolvedValue({ cursor: 0, keys: [], total_keys: 0 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// A short, un-collapsed tree (a real DB has millions of keys folded into a
|
// A short, un-collapsed tree (a real DB has millions of keys folded into a
|
||||||
@@ -673,25 +674,85 @@ function clickFetchAll(host: HTMLElement) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("RedisKeyBrowser bounded value search (issue #7779)", () => {
|
describe("RedisKeyBrowser bounded value search (issue #7779)", () => {
|
||||||
it.each([
|
describe.each([
|
||||||
["value", false],
|
["value", false],
|
||||||
["all", true],
|
["all", true],
|
||||||
] as const)("loads only one initial %s page and clamps COUNT", async (mode, searchBoth) => {
|
] as const)("%s search", (mode, searchBoth) => {
|
||||||
mocks.redisScanPageSize = 5000;
|
it.each(REDIS_SCAN_PAGE_SIZE_OPTIONS)("uses configured COUNT %i and stops after each explicit action on empty pages", async (pageSize) => {
|
||||||
mocks.redisScanValues.mockResolvedValueOnce({ cursor: 41, keys: [], total_keys: 5_000_000 }).mockResolvedValue({ cursor: 0, keys: [keyInfo("unexpected")], total_keys: 0 });
|
stubNonOverflowingViewport();
|
||||||
|
mocks.redisScanPageSize = pageSize;
|
||||||
|
mocks.redisScanValues
|
||||||
|
.mockResolvedValueOnce({ cursor: 41, keys: [], total_keys: 5_000_000 })
|
||||||
|
.mockResolvedValueOnce({ cursor: 73, keys: [], total_keys: 0 })
|
||||||
|
.mockResolvedValue({ cursor: 0, keys: [keyInfo("unexpected")], total_keys: 0 });
|
||||||
|
|
||||||
const host = mountBrowser();
|
const host = mountBrowser();
|
||||||
await settleThoroughly();
|
await settleThoroughly();
|
||||||
await searchByValue(host, mode);
|
await searchByValue(host, mode);
|
||||||
|
await settleThoroughly(10);
|
||||||
|
|
||||||
expect(mocks.redisScanValues).toHaveBeenCalledTimes(1);
|
expect(mocks.redisScanValues).toHaveBeenCalledTimes(1);
|
||||||
expect(mocks.redisScanValues).toHaveBeenLastCalledWith("connection", 0, 0, "*", "needle", 100, searchBoth);
|
expect(mocks.redisScanValues).toHaveBeenLastCalledWith("connection", 0, 0, "*", "needle", pageSize, searchBoth);
|
||||||
expect(host.textContent).toContain("redis.loadMoreKeys");
|
expect(host.textContent).toContain("redis.loadMoreKeys");
|
||||||
|
|
||||||
|
clickLoadMore(host);
|
||||||
|
await settleThoroughly(10);
|
||||||
|
|
||||||
|
expect(mocks.redisScanValues).toHaveBeenCalledTimes(2);
|
||||||
|
expect(mocks.redisScanValues).toHaveBeenLastCalledWith("connection", 0, 41, "*", "needle", pageSize, searchBoth);
|
||||||
|
expect(host.textContent).toContain("redis.loadMoreKeys");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(REDIS_SCAN_PAGE_SIZE_OPTIONS)("uses configured COUNT %i throughout explicit Fetch all", async (pageSize) => {
|
||||||
|
mocks.redisScanPageSize = pageSize;
|
||||||
|
mocks.redisScanValues.mockImplementation((_connectionId: string, _db: number, cursor: number) => {
|
||||||
|
if (cursor === 0) return Promise.resolve({ cursor: 11, keys: [], total_keys: 5_000_000 });
|
||||||
|
if (cursor === 11) return Promise.resolve({ cursor: 12, keys: [], total_keys: 0 });
|
||||||
|
return Promise.resolve({ cursor: 0, keys: [keyInfo("last")], total_keys: 0 });
|
||||||
|
});
|
||||||
|
|
||||||
|
const host = mountBrowser();
|
||||||
|
await settleThoroughly();
|
||||||
|
await searchByValue(host, mode);
|
||||||
|
expect(mocks.redisScanValues).toHaveBeenCalledTimes(1);
|
||||||
|
clickFetchAll(host);
|
||||||
|
await settleThoroughly();
|
||||||
|
|
||||||
|
expect(mocks.redisScanValues.mock.calls).toEqual([0, 11, 12].map((cursor) => ["connection", 0, cursor, "*", "needle", pageSize, searchBoth]));
|
||||||
|
expect(host.textContent).toContain("last");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops Fetch all after the in-flight configured page and retains its prior cursor", async () => {
|
||||||
|
mocks.redisScanPageSize = 5000;
|
||||||
|
const pending = deferred<{ cursor: number; keys: RedisKeyInfo[]; total_keys: number }>();
|
||||||
|
mocks.redisScanValues
|
||||||
|
.mockResolvedValueOnce({ cursor: 41, keys: [], total_keys: 5_000_000 })
|
||||||
|
.mockReturnValueOnce(pending.promise)
|
||||||
|
.mockResolvedValue({ cursor: 0, keys: [keyInfo("resumed")], total_keys: 0 });
|
||||||
|
|
||||||
|
const host = mountBrowser();
|
||||||
|
await settleThoroughly();
|
||||||
|
await searchByValue(host, mode);
|
||||||
|
clickFetchAll(host);
|
||||||
|
await settleThoroughly();
|
||||||
|
expect(mocks.redisScanValues).toHaveBeenCalledTimes(2);
|
||||||
|
const stop = Array.from(host.querySelectorAll<HTMLButtonElement>("button")).find((button) => button.textContent?.includes("redis.stopFetchAll") && !button.disabled);
|
||||||
|
expect(stop, "stop fetch all button").toBeDefined();
|
||||||
|
stop!.click();
|
||||||
|
pending.resolve({ cursor: 73, keys: [], total_keys: 0 });
|
||||||
|
await settleThoroughly(10);
|
||||||
|
|
||||||
|
expect(mocks.redisScanValues).toHaveBeenCalledTimes(2);
|
||||||
|
clickLoadMore(host);
|
||||||
|
await settleThoroughly();
|
||||||
|
expect(mocks.redisScanValues.mock.calls).toEqual([0, 41, 41].map((cursor) => ["connection", 0, cursor, "*", "needle", 5000, searchBoth]));
|
||||||
|
expect(host.textContent).toContain("resumed");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("advances exactly one page per Load more or real scroll and preserves the opaque cursor", async () => {
|
it.each(["value", "all"] as const)("advances exactly one %s page per Load more or real scroll and preserves the opaque cursor", async (mode) => {
|
||||||
stubNonOverflowingViewport();
|
stubNonOverflowingViewport();
|
||||||
mocks.redisScanPageSize = 1000;
|
mocks.redisScanPageSize = 5000;
|
||||||
mocks.redisScanValues.mockImplementation((_connectionId: string, _db: number, cursor: number) => {
|
mocks.redisScanValues.mockImplementation((_connectionId: string, _db: number, cursor: number) => {
|
||||||
if (cursor === 0) return Promise.resolve({ cursor: 41, keys: [keyInfo("match")], total_keys: 5_000_000 });
|
if (cursor === 0) return Promise.resolve({ cursor: 41, keys: [keyInfo("match")], total_keys: 5_000_000 });
|
||||||
if (cursor === 41) return Promise.resolve({ cursor: 73, keys: [keyInfo("match")], total_keys: 0 });
|
if (cursor === 41) return Promise.resolve({ cursor: 73, keys: [keyInfo("match")], total_keys: 0 });
|
||||||
@@ -700,7 +761,7 @@ describe("RedisKeyBrowser bounded value search (issue #7779)", () => {
|
|||||||
|
|
||||||
const host = mountBrowser();
|
const host = mountBrowser();
|
||||||
await settleThoroughly();
|
await settleThoroughly();
|
||||||
await searchByValue(host);
|
await searchByValue(host, mode);
|
||||||
expect(mocks.redisScanValues).toHaveBeenCalledTimes(1);
|
expect(mocks.redisScanValues).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
host.querySelector(".redis-key-scroller")?.dispatchEvent(new Event("resize"));
|
host.querySelector(".redis-key-scroller")?.dispatchEvent(new Event("resize"));
|
||||||
@@ -714,29 +775,13 @@ describe("RedisKeyBrowser bounded value search (issue #7779)", () => {
|
|||||||
|
|
||||||
host.querySelector(".redis-key-scroller")?.dispatchEvent(new Event("scroll"));
|
host.querySelector(".redis-key-scroller")?.dispatchEvent(new Event("scroll"));
|
||||||
await vi.waitFor(() => expect(mocks.redisScanValues).toHaveBeenCalledTimes(3));
|
await vi.waitFor(() => expect(mocks.redisScanValues).toHaveBeenCalledTimes(3));
|
||||||
|
await settleThoroughly(10);
|
||||||
|
expect(mocks.redisScanValues).toHaveBeenCalledTimes(3);
|
||||||
expect(mocks.redisScanValues.mock.calls[2]?.[2]).toBe(73);
|
expect(mocks.redisScanValues.mock.calls[2]?.[2]).toBe(73);
|
||||||
expect(mocks.redisScanValues.mock.calls.every((call) => (call[5] as number) <= 100)).toBe(true);
|
expect(mocks.redisScanValues.mock.calls).toEqual([0, 41, 73].map((cursor) => ["connection", 0, cursor, "*", "needle", 5000, mode === "all"]));
|
||||||
expect(host.textContent).toContain("sparse");
|
expect(host.textContent).toContain("sparse");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps explicit Fetch all interruptible by clamping every value page to COUNT 100", async () => {
|
|
||||||
mocks.redisScanPageSize = 10_000;
|
|
||||||
mocks.redisScanValues.mockImplementation((_connectionId: string, _db: number, cursor: number) => {
|
|
||||||
if (cursor === 0) return Promise.resolve({ cursor: 11, keys: [], total_keys: 5_000_000 });
|
|
||||||
if (cursor === 11) return Promise.resolve({ cursor: 12, keys: [], total_keys: 0 });
|
|
||||||
return Promise.resolve({ cursor: 0, keys: [keyInfo("last")], total_keys: 0 });
|
|
||||||
});
|
|
||||||
|
|
||||||
const host = mountBrowser();
|
|
||||||
await settleThoroughly();
|
|
||||||
await searchByValue(host);
|
|
||||||
clickFetchAll(host);
|
|
||||||
await settleThoroughly();
|
|
||||||
|
|
||||||
expect(mocks.redisScanValues.mock.calls.map((call) => call[2])).toEqual([0, 11, 12]);
|
|
||||||
expect(mocks.redisScanValues.mock.calls.every((call) => call[5] === 100)).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not retry a rejected value continuation", async () => {
|
it("does not retry a rejected value continuation", async () => {
|
||||||
stubNonOverflowingViewport();
|
stubNonOverflowingViewport();
|
||||||
mocks.redisScanValues.mockResolvedValueOnce({ cursor: 31, keys: [keyInfo("match")], total_keys: 5_000_000 }).mockRejectedValue(new Error("value scan failed"));
|
mocks.redisScanValues.mockResolvedValueOnce({ cursor: 31, keys: [keyInfo("match")], total_keys: 5_000_000 }).mockRejectedValue(new Error("value scan failed"));
|
||||||
@@ -756,7 +801,7 @@ describe("RedisKeyBrowser bounded value search (issue #7779)", () => {
|
|||||||
|
|
||||||
describe("RedisKeyBrowser KeepAlive empty scan pages (issue #7779)", () => {
|
describe("RedisKeyBrowser KeepAlive empty scan pages (issue #7779)", () => {
|
||||||
it.each(["value", "all"] as const)("retains an empty %s scan cursor across tab switches", async (mode) => {
|
it.each(["value", "all"] as const)("retains an empty %s scan cursor across tab switches", async (mode) => {
|
||||||
mocks.infiniteScroll = false;
|
mocks.redisScanPageSize = 5000;
|
||||||
mocks.redisScanValues.mockImplementation((_connectionId: string, _db: number, cursor: number) => {
|
mocks.redisScanValues.mockImplementation((_connectionId: string, _db: number, cursor: number) => {
|
||||||
if (cursor === 0) return Promise.resolve({ cursor: 41, keys: [], total_keys: 5_000_000 });
|
if (cursor === 0) return Promise.resolve({ cursor: 41, keys: [], total_keys: 5_000_000 });
|
||||||
if (cursor === 41) return Promise.resolve({ cursor: 73, keys: [], total_keys: 0 });
|
if (cursor === 41) return Promise.resolve({ cursor: 73, keys: [], total_keys: 0 });
|
||||||
@@ -776,6 +821,7 @@ describe("RedisKeyBrowser KeepAlive empty scan pages (issue #7779)", () => {
|
|||||||
clickLoadMore(browser.host);
|
clickLoadMore(browser.host);
|
||||||
await settleThoroughly();
|
await settleThoroughly();
|
||||||
expect(mocks.redisScanValues.mock.calls.map((call) => call[2])).toEqual([0, 41, 73]);
|
expect(mocks.redisScanValues.mock.calls.map((call) => call[2])).toEqual([0, 41, 73]);
|
||||||
|
expect(mocks.redisScanValues.mock.calls.every((call) => call[5] === 5000)).toBe(true);
|
||||||
expect(browser.host.textContent).toContain("match");
|
expect(browser.host.textContent).toContain("match");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -231,7 +231,6 @@ interface ScanIterationBudget {
|
|||||||
// per-call budget (see `fetchScanPage`); only the automatic follow-up check
|
// per-call budget (see `fetchScanPage`); only the automatic follow-up check
|
||||||
// they hand off to afterward is constrained by this shared budget.
|
// they hand off to afterward is constrained by this shared budget.
|
||||||
const AUTO_LOAD_TOTAL_SCAN_ITERATIONS = 50;
|
const AUTO_LOAD_TOTAL_SCAN_ITERATIONS = 50;
|
||||||
const REDIS_VALUE_SCAN_COUNT = 100;
|
|
||||||
let autoLoadBudget: ScanIterationBudget = { remaining: AUTO_LOAD_TOTAL_SCAN_ITERATIONS };
|
let autoLoadBudget: ScanIterationBudget = { remaining: AUTO_LOAD_TOTAL_SCAN_ITERATIONS };
|
||||||
let redisBrowserIsActive = true;
|
let redisBrowserIsActive = true;
|
||||||
let reloadKeysOnActivation = false;
|
let reloadKeysOnActivation = false;
|
||||||
@@ -840,7 +839,7 @@ function isCurrentScanOperation(requestId: number, operationId?: number): boolea
|
|||||||
async function fetchScanPage(requestId = searchRequestId, operationId?: number, iterationBudget?: ScanIterationBudget): Promise<RedisScanResult> {
|
async function fetchScanPage(requestId = searchRequestId, operationId?: number, iterationBudget?: ScanIterationBudget): Promise<RedisScanResult> {
|
||||||
const pageSize = redisScanPageSize.value;
|
const pageSize = redisScanPageSize.value;
|
||||||
if (isValueSearchMode.value) {
|
if (isValueSearchMode.value) {
|
||||||
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, Math.min(pageSize, REDIS_VALUE_SCAN_COUNT), searchMode.value === "all");
|
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, pageSize, searchMode.value === "all");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keep each backend call small so a changed search can cancel between calls.
|
// Keep each backend call small so a changed search can cancel between calls.
|
||||||
@@ -886,11 +885,12 @@ async function fetchScanPage(requestId = searchRequestId, operationId?: number,
|
|||||||
/// Batch-scan variant that performs multiple SCAN iterations server-side.
|
/// Batch-scan variant that performs multiple SCAN iterations server-side.
|
||||||
/// Dramatically reduces frontend↔backend roundtrips for bulk loading.
|
/// Dramatically reduces frontend↔backend roundtrips for bulk loading.
|
||||||
async function fetchScanBatchPage(maxIterations: number, options: { count?: number; includeTypes?: boolean } = {}): Promise<RedisScanResult> {
|
async function fetchScanBatchPage(maxIterations: number, options: { count?: number; includeTypes?: boolean } = {}): Promise<RedisScanResult> {
|
||||||
const pageSize = options.count ?? redisScanPageSize.value;
|
|
||||||
// Value search cannot be batched because each key requires a GET.
|
// Value search cannot be batched because each key requires a GET.
|
||||||
|
// Keep its configured COUNT independent of the key-only bulk override.
|
||||||
if (isValueSearchMode.value) {
|
if (isValueSearchMode.value) {
|
||||||
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, Math.min(pageSize, REDIS_VALUE_SCAN_COUNT), searchMode.value === "all");
|
return api.redisScanValues(props.connectionId, props.db, scanCursor.value, "*", valueQuery.value, redisScanPageSize.value, searchMode.value === "all");
|
||||||
}
|
}
|
||||||
|
const pageSize = options.count ?? redisScanPageSize.value;
|
||||||
return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations, options.includeTypes ?? false);
|
return api.redisScanKeysBatch(props.connectionId, props.db, scanCursor.value, effectivePattern.value, pageSize, maxIterations, options.includeTypes ?? false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user