feat(metadata): persist and lazy-load table structure metadata

This commit is contained in:
yanguibao1997 authored and GitHub committed 2026-07-31 16:27:37 +08:00
1 parent 584848f732
commit 7be11f54c9
15 files changed
+747 -89

No files matched your search

@@ -0,0 +1,100 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
getTableDisplayDdl: vi.fn(),
saveSchemaCache: vi.fn(),
loadSchemaCache: vi.fn(),
deleteSchemaCachePrefix: vi.fn(),
persisted: new Map<string, unknown>(),
}));
vi.mock("@/lib/backend/api", () => mocks);
import { invalidateObjectDdl, invalidateObjectDdlCache, loadObjectDdl, objectDdlCacheKey } from "@/lib/metadata/objectDdlCache";
const request = { connectionId: "c1", database: "app", schema: "public", tableName: "users", catalog: "analytics" } as const;
describe("objectDdlCache", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.persisted.clear();
mocks.loadSchemaCache.mockImplementation(async (cacheKey: string) => mocks.persisted.get(cacheKey) ?? null);
mocks.saveSchemaCache.mockImplementation(async (cacheKey: string, payload: unknown) => {
mocks.persisted.set(cacheKey, payload);
});
mocks.deleteSchemaCachePrefix.mockImplementation(async (prefix: string) => {
for (const cacheKey of mocks.persisted.keys()) {
if (cacheKey === prefix || cacheKey.startsWith(prefix)) mocks.persisted.delete(cacheKey);
}
});
});
it("returns persisted DDL without querying the database", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), ddl: "CREATE TABLE users (id int)" });
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "CREATE TABLE users (id int)", cacheStatus: "disk" });
expect(mocks.getTableDisplayDdl).not.toHaveBeenCalled();
});
it("persists a remote cache miss", async () => {
mocks.getTableDisplayDdl.mockResolvedValue("CREATE TABLE users (id bigint)");
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "CREATE TABLE users (id bigint)", cacheStatus: "remote" });
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(objectDdlCacheKey(request), expect.objectContaining({ version: 1, ddl: "CREATE TABLE users (id bigint)" }));
});
it("deduplicates concurrent remote loads", async () => {
let release: (ddl: string) => void = () => {};
mocks.getTableDisplayDdl.mockReturnValue(
new Promise<string>((resolve) => {
release = resolve;
}),
);
const first = loadObjectDdl(request);
const second = loadObjectDdl(request);
await vi.waitFor(() => expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(1));
release("CREATE TABLE users (id int)");
await expect(Promise.all([first, second])).resolves.toHaveLength(2);
});
it("force refresh bypasses disk and overwrites it", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), ddl: "old ddl" });
mocks.getTableDisplayDdl.mockResolvedValue("new ddl");
await expect(loadObjectDdl(request, { force: true })).resolves.toEqual({ ddl: "new ddl", cacheStatus: "remote" });
expect(mocks.loadSchemaCache).not.toHaveBeenCalled();
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(objectDdlCacheKey(request), expect.objectContaining({ ddl: "new ddl" }));
});
it("deletes the exact persisted entry", async () => {
await invalidateObjectDdl(request);
expect(mocks.deleteSchemaCachePrefix).toHaveBeenCalledWith(objectDdlCacheKey(request));
});
it("reloads from the database after table-level persisted cache invalidation", async () => {
mocks.getTableDisplayDdl.mockResolvedValueOnce("old ddl").mockResolvedValueOnce("new ddl");
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "old ddl", cacheStatus: "remote" });
await invalidateObjectDdlCache({ connectionId: request.connectionId, database: request.database, schema: request.schema, tableName: request.tableName });
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "new ddl", cacheStatus: "remote" });
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(2);
});
it("does not persist an in-flight result across invalidation", async () => {
let release: (ddl: string) => void = () => {};
mocks.getTableDisplayDdl.mockReturnValue(
new Promise<string>((resolve) => {
release = resolve;
}),
);
const load = loadObjectDdl(request);
await vi.waitFor(() => expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(1));
await invalidateObjectDdlCache({ connectionId: request.connectionId, database: request.database, schema: request.schema });
release("stale ddl");
await expect(load).resolves.toEqual({ ddl: "stale ddl", cacheStatus: "remote" });
expect(mocks.saveSchemaCache).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,63 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
saveSchemaCache: vi.fn(),
loadSchemaCache: vi.fn(),
deleteSchemaCachePrefix: vi.fn(),
persisted: new Map<string, unknown>(),
}));
vi.mock("@/lib/backend/api", () => mocks);
import { invalidateObjectMetadataCache, loadObjectMetadataFacet } from "@/lib/metadata/objectMetadataCache";
const request = { connectionId: "c1", database: "app", schema: "public", tableName: "users", catalog: "analytics" } as const;
describe("objectMetadataCache", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.persisted.clear();
mocks.loadSchemaCache.mockImplementation(async (cacheKey: string) => mocks.persisted.get(cacheKey) ?? null);
mocks.saveSchemaCache.mockImplementation(async (cacheKey: string, payload: unknown) => {
mocks.persisted.set(cacheKey, payload);
});
mocks.deleteSchemaCachePrefix.mockImplementation(async (prefix: string) => {
for (const cacheKey of mocks.persisted.keys()) {
if (cacheKey === prefix || cacheKey.startsWith(prefix)) mocks.persisted.delete(cacheKey);
}
});
});
it("reads a facet from disk without invoking the loader", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), value: [{ name: "id" }] });
const loader = vi.fn().mockResolvedValue([{ name: "remote" }]);
await expect(loadObjectMetadataFacet(request, "columns", loader)).resolves.toEqual({ value: [{ name: "id" }], cacheStatus: "disk" });
expect(loader).not.toHaveBeenCalled();
});
it("persists a remote facet and force bypasses disk", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), value: ["old"] });
const loader = vi.fn().mockResolvedValue(["new"]);
await expect(loadObjectMetadataFacet(request, "indexes", loader, { force: true })).resolves.toEqual({ value: ["new"], cacheStatus: "remote" });
expect(loader).toHaveBeenCalledTimes(1);
expect(mocks.loadSchemaCache).not.toHaveBeenCalled();
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(expect.stringContaining("object-meta:v1:c1:app:public:users:analytics:indexes:"), expect.objectContaining({ value: ["new"] }));
});
it("invalidates only the requested object's facets", async () => {
await invalidateObjectMetadataCache(request);
expect(mocks.deleteSchemaCachePrefix).toHaveBeenCalledWith("object-meta:v1:c1:app:public:users:");
});
it("reloads a persisted facet after table-level invalidation", async () => {
const initialLoader = vi.fn().mockResolvedValue(["old"]);
const refreshedLoader = vi.fn().mockResolvedValue(["new"]);
await expect(loadObjectMetadataFacet(request, "indexes", initialLoader)).resolves.toEqual({ value: ["old"], cacheStatus: "remote" });
await invalidateObjectMetadataCache({ connectionId: request.connectionId, database: request.database, schema: request.schema, tableName: request.tableName });
await expect(loadObjectMetadataFacet(request, "indexes", refreshedLoader)).resolves.toEqual({ value: ["new"], cacheStatus: "remote" });
expect(refreshedLoader).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,165 @@
import * as api from "@/lib/backend/api";
import type { ObjectSourceKind } from "@/types/database";
import type { MetadataCacheInvalidation } from "./metadataResultCache";
import { invalidateObjectMetadataCache } from "./objectMetadataCache";
const OBJECT_DDL_CACHE_PREFIX = "object-ddl:v1";
const MAX_PERSISTED_DDL_CHARS = 5 * 1024 * 1024;
interface ObjectDdlCacheEnvelope {
version: 1;
cachedAt: string;
ddl: string;
}
interface InFlightDdlLoad {
force: boolean;
invalidated: boolean;
promise: Promise<string>;
}
export interface ObjectDdlRequest {
connectionId: string;
database: string;
schema: string;
tableName: string;
objectType?: ObjectSourceKind;
catalog?: string;
}
export interface ObjectDdlLoadResult {
ddl: string;
cacheStatus: "disk" | "remote";
}
const remoteLoads = new Map<string, InFlightDdlLoad>();
const pendingInvalidations = new Map<string, Promise<void>>();
async function loadSchemaCacheSafe<T>(cacheKey: string): Promise<T | null> {
try {
return await api.loadSchemaCache<T>(cacheKey);
} catch {
return null;
}
}
async function saveSchemaCacheSafe(cacheKey: string, payload: unknown): Promise<void> {
try {
await api.saveSchemaCache(cacheKey, payload);
} catch {
// Cache persistence is best effort and must not block DDL rendering.
}
}
async function deleteSchemaCachePrefixSafe(prefix: string): Promise<void> {
try {
await api.deleteSchemaCachePrefix(prefix);
} catch {
// Cache invalidation is best effort when running with a reduced backend.
}
}
function cacheSegment(value: string | undefined): string {
return encodeURIComponent(value ?? "");
}
export function objectDdlCacheKey(request: ObjectDdlRequest): string {
return `${[OBJECT_DDL_CACHE_PREFIX, cacheSegment(request.connectionId), cacheSegment(request.database), cacheSegment(request.schema), cacheSegment(request.tableName), cacheSegment(request.catalog), cacheSegment(request.objectType ?? "TABLE")].join(":")}:`;
}
function invalidationPrefix(match: MetadataCacheInvalidation): string {
const parts = [OBJECT_DDL_CACHE_PREFIX];
if (!match.connectionId) return `${OBJECT_DDL_CACHE_PREFIX}:`;
parts.push(cacheSegment(match.connectionId ?? undefined));
if (!match.database) return `${parts.join(":")}:`;
parts.push(cacheSegment(match.database ?? undefined));
if (!match.schema) return `${parts.join(":")}:`;
parts.push(cacheSegment(match.schema ?? undefined));
if (!match.tableName) return `${parts.join(":")}:`;
parts.push(cacheSegment(match.tableName));
return `${parts.join(":")}:`;
}
function decodeCachedDdl(payload: unknown): string | null {
if (!payload || typeof payload !== "object") return null;
const envelope = payload as Partial<ObjectDdlCacheEnvelope>;
if (envelope.version !== 1 || typeof envelope.cachedAt !== "string" || !Number.isFinite(Date.parse(envelope.cachedAt)) || typeof envelope.ddl !== "string") return null;
return envelope.ddl;
}
async function waitForPendingInvalidations(cacheKey: string): Promise<void> {
const pending = [...pendingInvalidations.entries()].filter(([prefix]) => cacheKey.startsWith(prefix)).map(([, promise]) => promise);
if (pending.length) await Promise.all(pending);
}
async function loadRemoteDdl(request: ObjectDdlRequest, cacheKey: string, force: boolean): Promise<string> {
const existing = remoteLoads.get(cacheKey);
if (existing && (!force || existing.force)) return existing.promise;
if (existing) {
existing.invalidated = true;
remoteLoads.delete(cacheKey);
}
const entry: InFlightDdlLoad = { force, invalidated: false, promise: Promise.resolve("") };
entry.promise = api
.getTableDisplayDdl(request.connectionId, request.database, request.schema, request.tableName, request.objectType, request.catalog)
.then(async (ddl) => {
if (!entry.invalidated && ddl.length <= MAX_PERSISTED_DDL_CHARS) {
const envelope: ObjectDdlCacheEnvelope = { version: 1, cachedAt: new Date().toISOString(), ddl };
await saveSchemaCacheSafe(cacheKey, envelope);
}
return ddl;
})
.finally(() => {
if (remoteLoads.get(cacheKey) === entry) remoteLoads.delete(cacheKey);
});
remoteLoads.set(cacheKey, entry);
return entry.promise;
}
export async function loadObjectDdl(request: ObjectDdlRequest, options?: { force?: boolean }): Promise<ObjectDdlLoadResult> {
const cacheKey = objectDdlCacheKey(request);
await waitForPendingInvalidations(cacheKey);
if (!options?.force) {
const cached = decodeCachedDdl(await loadSchemaCacheSafe<unknown>(cacheKey));
if (cached !== null) return { ddl: cached, cacheStatus: "disk" };
}
return { ddl: await loadRemoteDdl(request, cacheKey, options?.force === true), cacheStatus: "remote" };
}
export async function invalidateObjectDdlCache(match: MetadataCacheInvalidation): Promise<void> {
const prefix = invalidationPrefix(match);
for (const [cacheKey, entry] of remoteLoads) {
if (!cacheKey.startsWith(prefix)) continue;
entry.invalidated = true;
remoteLoads.delete(cacheKey);
}
const existing = pendingInvalidations.get(prefix);
if (existing) return existing;
const deletion = deleteSchemaCachePrefixSafe(prefix).finally(() => {
if (pendingInvalidations.get(prefix) === deletion) pendingInvalidations.delete(prefix);
});
pendingInvalidations.set(prefix, deletion);
await Promise.all([deletion, invalidateObjectMetadataCache(match)]);
}
export async function invalidateObjectDdl(request: ObjectDdlRequest): Promise<void> {
const cacheKey = objectDdlCacheKey(request);
const entry = remoteLoads.get(cacheKey);
if (entry) {
entry.invalidated = true;
remoteLoads.delete(cacheKey);
}
await Promise.all([
deleteSchemaCachePrefixSafe(cacheKey),
invalidateObjectMetadataCache({
connectionId: request.connectionId,
database: request.database,
schema: request.schema,
tableName: request.tableName,
}),
]);
}
@@ -0,0 +1,130 @@
import * as api from "@/lib/backend/api";
import type { MetadataCacheInvalidation } from "./metadataResultCache";
import type { ObjectDdlRequest } from "./objectDdlCache";
const OBJECT_METADATA_CACHE_PREFIX = "object-meta:v1";
const MAX_PERSISTED_METADATA_CHARS = 5 * 1024 * 1024;
interface ObjectMetadataCacheEnvelope<T> {
version: 1;
cachedAt: string;
value: T;
}
interface InFlightLoad<T> {
force: boolean;
invalidated: boolean;
promise: Promise<T>;
}
const inFlightLoads = new Map<string, InFlightLoad<unknown>>();
const pendingInvalidations = new Map<string, Promise<void>>();
async function loadSchemaCacheSafe<T>(cacheKey: string): Promise<T | null> {
try {
return await api.loadSchemaCache<T>(cacheKey);
} catch {
return null;
}
}
async function saveSchemaCacheSafe(cacheKey: string, payload: unknown): Promise<void> {
try {
await api.saveSchemaCache(cacheKey, payload);
} catch {
// Cache persistence is best effort and must not block metadata rendering.
}
}
async function deleteSchemaCachePrefixSafe(prefix: string): Promise<void> {
try {
await api.deleteSchemaCachePrefix(prefix);
} catch {
// Cache invalidation is best effort when running with a reduced backend.
}
}
export type ObjectMetadataFacet = "columns" | "indexes" | "foreign-keys" | "triggers" | "comment";
function cacheSegment(value: string | undefined): string {
return encodeURIComponent(value ?? "");
}
function facetKey(request: ObjectDdlRequest, facet: ObjectMetadataFacet): string {
return `${[OBJECT_METADATA_CACHE_PREFIX, cacheSegment(request.connectionId), cacheSegment(request.database), cacheSegment(request.schema), cacheSegment(request.tableName), cacheSegment(request.catalog), facet].join(":")}:`;
}
function invalidationPrefix(match: MetadataCacheInvalidation): string {
const parts = [OBJECT_METADATA_CACHE_PREFIX];
if (!match.connectionId) return `${OBJECT_METADATA_CACHE_PREFIX}:`;
parts.push(cacheSegment(match.connectionId));
if (!match.database) return `${parts.join(":")}:`;
parts.push(cacheSegment(match.database));
if (!match.schema) return `${parts.join(":")}:`;
parts.push(cacheSegment(match.schema));
if (!match.tableName) return `${parts.join(":")}:`;
parts.push(cacheSegment(match.tableName));
return `${parts.join(":")}:`;
}
function decodeEnvelope<T>(payload: unknown): T | null {
if (!payload || typeof payload !== "object") return null;
const envelope = payload as Partial<ObjectMetadataCacheEnvelope<T>>;
if (envelope.version !== 1 || typeof envelope.cachedAt !== "string" || !Number.isFinite(Date.parse(envelope.cachedAt)) || !("value" in envelope)) return null;
return envelope.value === undefined ? null : envelope.value;
}
async function waitForPendingInvalidations(cacheKey: string): Promise<void> {
const pending = [...pendingInvalidations.entries()].filter(([prefix]) => cacheKey.startsWith(prefix)).map(([, promise]) => promise);
if (pending.length) await Promise.all(pending);
}
export async function loadObjectMetadataFacet<T>(request: ObjectDdlRequest, facet: ObjectMetadataFacet, loader: () => Promise<T>, options?: { force?: boolean }): Promise<{ value: T; cacheStatus: "disk" | "remote" }> {
const cacheKey = facetKey(request, facet);
await waitForPendingInvalidations(cacheKey);
if (!options?.force) {
const cached = decodeEnvelope<T>(await loadSchemaCacheSafe<unknown>(cacheKey));
if (cached !== null) return { value: cached, cacheStatus: "disk" };
}
const existing = inFlightLoads.get(cacheKey);
if (existing && (!options?.force || existing.force)) return { value: (await existing.promise) as T, cacheStatus: "remote" };
if (existing) {
existing.invalidated = true;
inFlightLoads.delete(cacheKey);
}
const entry: InFlightLoad<T> = { force: options?.force === true, invalidated: false, promise: Promise.resolve(undefined as T) };
entry.promise = loader()
.then(async (value) => {
const serialized = JSON.stringify(value);
if (!entry.invalidated && serialized.length <= MAX_PERSISTED_METADATA_CHARS) {
const envelope: ObjectMetadataCacheEnvelope<T> = { version: 1, cachedAt: new Date().toISOString(), value };
await saveSchemaCacheSafe(cacheKey, envelope);
}
return value;
})
.finally(() => {
if (inFlightLoads.get(cacheKey) === entry) inFlightLoads.delete(cacheKey);
});
inFlightLoads.set(cacheKey, entry as InFlightLoad<unknown>);
return { value: await entry.promise, cacheStatus: "remote" };
}
export async function invalidateObjectMetadataCache(match: MetadataCacheInvalidation): Promise<void> {
const prefix = invalidationPrefix(match);
for (const [cacheKey, entry] of inFlightLoads) {
if (!cacheKey.startsWith(prefix)) continue;
entry.invalidated = true;
inFlightLoads.delete(cacheKey);
}
const existing = pendingInvalidations.get(prefix);
if (existing) return existing;
const deletion = deleteSchemaCachePrefixSafe(prefix).finally(() => {
if (pendingInvalidations.get(prefix) === deletion) pendingInvalidations.delete(prefix);
});
pendingInvalidations.set(prefix, deletion);
return deletion;
}