feat(metadata): optimize agent metadata loading
This commit is contained in:
1 parent
8883a6b3b7
commit
e74782f651
65 files changed
+6192
-571
No files matched your search
@@ -0,0 +1,145 @@
|
||||
import { metadataScopeKey, type MetadataScopeInput } from "./metadataLoadScope";
|
||||
|
||||
export type MetadataLoadCacheStatus = "none" | "hit" | "miss" | "stale" | "refresh";
|
||||
|
||||
export interface MetadataLoadTraceDetails {
|
||||
cacheStatus?: MetadataLoadCacheStatus;
|
||||
resultCount?: number;
|
||||
deduped?: boolean;
|
||||
force?: boolean;
|
||||
stale?: boolean;
|
||||
superseded?: boolean;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
export interface MetadataLoadTraceEvent extends MetadataLoadTraceDetails {
|
||||
event: string;
|
||||
traceId: string;
|
||||
scopeKey: string;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
export type MetadataLoadTraceLogger = (event: MetadataLoadTraceEvent) => void;
|
||||
|
||||
export interface MetadataLoadTrace {
|
||||
traceId: string;
|
||||
scopeKey: string;
|
||||
startedAt: number;
|
||||
elapsedMs: () => number;
|
||||
event: (event: string, details?: MetadataLoadTraceDetails) => MetadataLoadTraceEvent;
|
||||
}
|
||||
|
||||
export interface MetadataLoadCoordinatorEvent {
|
||||
event: "start" | "dedupe" | "done" | "error";
|
||||
key: string;
|
||||
kind: string;
|
||||
active: number;
|
||||
force: boolean;
|
||||
}
|
||||
|
||||
export type MetadataLoadCoordinatorLogger = (event: MetadataLoadCoordinatorEvent) => void;
|
||||
|
||||
export interface MetadataLoadCoordinatorOptions {
|
||||
force?: boolean;
|
||||
kind?: string;
|
||||
}
|
||||
|
||||
interface InFlightLoad<T> {
|
||||
kind: string;
|
||||
promise: Promise<T>;
|
||||
}
|
||||
|
||||
function defaultTraceId() {
|
||||
return Math.random().toString(16).slice(2, 10);
|
||||
}
|
||||
|
||||
function defaultNow() {
|
||||
return typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
|
||||
}
|
||||
|
||||
export function createMetadataLoadTrace(
|
||||
scope: MetadataScopeInput | string,
|
||||
options?: {
|
||||
traceId?: string;
|
||||
now?: () => number;
|
||||
},
|
||||
): MetadataLoadTrace {
|
||||
const now = options?.now ?? defaultNow;
|
||||
const startedAt = now();
|
||||
const scopeKey = typeof scope === "string" ? scope : metadataScopeKey(scope);
|
||||
const traceId = options?.traceId ?? defaultTraceId();
|
||||
const elapsedMs = () => Math.max(0, Math.round(now() - startedAt));
|
||||
return {
|
||||
traceId,
|
||||
scopeKey,
|
||||
startedAt,
|
||||
elapsedMs,
|
||||
event: (event, details = {}) => ({
|
||||
event,
|
||||
traceId,
|
||||
scopeKey,
|
||||
elapsedMs: elapsedMs(),
|
||||
...details,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function logMetadataLoadTrace(logger: MetadataLoadTraceLogger | undefined, trace: MetadataLoadTrace, event: string, details?: MetadataLoadTraceDetails) {
|
||||
logger?.(trace.event(event, details));
|
||||
}
|
||||
|
||||
export class MetadataLoadCoordinator {
|
||||
private readonly inFlight = new Map<string, InFlightLoad<unknown>>();
|
||||
|
||||
constructor(private readonly logger?: MetadataLoadCoordinatorLogger) {}
|
||||
|
||||
run<T>(scope: MetadataScopeInput | string, load: () => Promise<T>, options?: MetadataLoadCoordinatorOptions): Promise<T> {
|
||||
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
|
||||
const kind = options?.kind ?? (typeof scope === "string" ? "metadata" : scope.kind);
|
||||
const force = options?.force === true;
|
||||
const existing = force ? undefined : this.inFlight.get(key);
|
||||
if (existing) {
|
||||
this.logger?.({ event: "dedupe", key, kind: existing.kind, active: this.inFlight.size, force: false });
|
||||
return existing.promise as Promise<T>;
|
||||
}
|
||||
|
||||
this.logger?.({ event: "start", key, kind, active: this.inFlight.size + 1, force });
|
||||
const promise = Promise.resolve()
|
||||
.then(load)
|
||||
.then(
|
||||
(value) => {
|
||||
this.logger?.({ event: "done", key, kind, active: this.inFlight.size, force });
|
||||
return value;
|
||||
},
|
||||
(error) => {
|
||||
this.logger?.({ event: "error", key, kind, active: this.inFlight.size, force });
|
||||
throw error;
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
const current = this.inFlight.get(key);
|
||||
if (current?.promise === promise) {
|
||||
this.inFlight.delete(key);
|
||||
}
|
||||
});
|
||||
|
||||
if (!force) {
|
||||
this.inFlight.set(key, { kind, promise });
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
has(scope: MetadataScopeInput | string): boolean {
|
||||
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
|
||||
return this.inFlight.has(key);
|
||||
}
|
||||
|
||||
clear(scope?: MetadataScopeInput | string) {
|
||||
if (scope === undefined) {
|
||||
this.inFlight.clear();
|
||||
return;
|
||||
}
|
||||
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
|
||||
this.inFlight.delete(key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
export type MetadataScopeValue = string | number | boolean | null | undefined;
|
||||
|
||||
export interface MetadataScopeInput {
|
||||
kind: string;
|
||||
connectionId?: string | null;
|
||||
database?: string | null;
|
||||
schema?: string | null;
|
||||
nodeKind?: string | null;
|
||||
tableName?: string | null;
|
||||
tableType?: string | null;
|
||||
objectTypes?: readonly (string | null | undefined)[] | null;
|
||||
searchFilter?: string | null;
|
||||
limit?: number | null;
|
||||
offset?: number | null;
|
||||
sidebarDisplayMode?: string | null;
|
||||
driverProfile?: string | null;
|
||||
extra?: Record<string, MetadataScopeValue | readonly MetadataScopeValue[]> | null;
|
||||
}
|
||||
|
||||
function normalizeString(value: string | null | undefined): string | undefined {
|
||||
if (value == null) return undefined;
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeSearchFilter(value: string | null | undefined): string | undefined {
|
||||
const normalized = value?.trim();
|
||||
return normalized ? normalized : undefined;
|
||||
}
|
||||
|
||||
function normalizeNumber(value: number | null | undefined): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function normalizeObjectTypes(values: readonly (string | null | undefined)[] | null | undefined): string[] | undefined {
|
||||
if (!values?.length) return undefined;
|
||||
const normalized = values
|
||||
.filter((value): value is string => typeof value === "string" && value.trim().length > 0)
|
||||
.map((value) => value.trim().toUpperCase().replace(/\s+/g, "_"))
|
||||
.sort();
|
||||
return normalized.length > 0 ? [...new Set(normalized)] : undefined;
|
||||
}
|
||||
|
||||
function stableEntries(record: Record<string, unknown>): [string, unknown][] {
|
||||
return Object.entries(record)
|
||||
.filter(([, value]) => value !== undefined)
|
||||
.sort(([left], [right]) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function normalizeExtra(extra: MetadataScopeInput["extra"]): Record<string, unknown> | undefined {
|
||||
if (!extra) return undefined;
|
||||
const entries = stableEntries(extra)
|
||||
.map(([key, value]) => {
|
||||
if (Array.isArray(value)) {
|
||||
return [key, value.filter((item) => item !== undefined)] as const;
|
||||
}
|
||||
return [key, value] as const;
|
||||
})
|
||||
.filter(([, value]) => {
|
||||
if (Array.isArray(value)) return value.length > 0;
|
||||
return value !== undefined;
|
||||
});
|
||||
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
||||
}
|
||||
|
||||
export function metadataScopeParts(input: MetadataScopeInput): Record<string, unknown> {
|
||||
return {
|
||||
kind: input.kind,
|
||||
connectionId: normalizeString(input.connectionId),
|
||||
database: normalizeString(input.database),
|
||||
schema: normalizeString(input.schema),
|
||||
nodeKind: normalizeString(input.nodeKind),
|
||||
tableName: normalizeString(input.tableName),
|
||||
tableType: normalizeString(input.tableType),
|
||||
objectTypes: normalizeObjectTypes(input.objectTypes),
|
||||
searchFilter: normalizeSearchFilter(input.searchFilter),
|
||||
limit: normalizeNumber(input.limit),
|
||||
offset: normalizeNumber(input.offset),
|
||||
sidebarDisplayMode: normalizeString(input.sidebarDisplayMode),
|
||||
driverProfile: normalizeString(input.driverProfile),
|
||||
extra: normalizeExtra(input.extra),
|
||||
};
|
||||
}
|
||||
|
||||
export function metadataScopeKey(input: MetadataScopeInput): string {
|
||||
return JSON.stringify(stableEntries(metadataScopeParts(input)));
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import { metadataScopeKey, metadataScopeParts, type MetadataScopeInput } from "./metadataLoadScope";
|
||||
|
||||
export interface MetadataCacheHit<T> {
|
||||
value: T;
|
||||
cachedAt: number;
|
||||
ageMs: number;
|
||||
stale: boolean;
|
||||
}
|
||||
|
||||
export type MetadataCacheInvalidation = Partial<Omit<MetadataScopeInput, "extra">>;
|
||||
|
||||
interface MetadataCacheEntry<T> {
|
||||
scope: ReturnType<typeof metadataScopeParts>;
|
||||
value: T;
|
||||
cachedAt: number;
|
||||
}
|
||||
|
||||
export interface MetadataResultCacheOptions {
|
||||
ttlMs: number;
|
||||
maxEntries: number;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
function normalizeInvalidation(match: MetadataCacheInvalidation): Partial<ReturnType<typeof metadataScopeParts>> {
|
||||
const normalized = metadataScopeParts({ ...match, kind: match.kind ?? "__metadata_cache_match__" });
|
||||
const result: Partial<ReturnType<typeof metadataScopeParts>> = {};
|
||||
for (const key of Object.keys(match) as (keyof MetadataCacheInvalidation)[]) {
|
||||
const value = normalized[key as keyof typeof normalized];
|
||||
if (value !== undefined) {
|
||||
(result as Record<string, unknown>)[key] = value;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function valuesEqual(left: unknown, right: unknown): boolean {
|
||||
if (Array.isArray(left) || Array.isArray(right) || (left && typeof left === "object") || (right && typeof right === "object")) {
|
||||
return JSON.stringify(left) === JSON.stringify(right);
|
||||
}
|
||||
return left === right;
|
||||
}
|
||||
|
||||
export class MetadataResultCache<T> {
|
||||
private readonly entries = new Map<string, MetadataCacheEntry<T>>();
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(private readonly options: MetadataResultCacheOptions) {
|
||||
this.now = options.now ?? Date.now;
|
||||
}
|
||||
|
||||
get(scope: MetadataScopeInput, options?: { allowStale?: boolean }): MetadataCacheHit<T> | undefined {
|
||||
const key = metadataScopeKey(scope);
|
||||
const entry = this.entries.get(key);
|
||||
if (!entry) return undefined;
|
||||
const ageMs = Math.max(0, this.now() - entry.cachedAt);
|
||||
const stale = ageMs > this.options.ttlMs;
|
||||
if (stale && !options?.allowStale) return undefined;
|
||||
|
||||
this.entries.delete(key);
|
||||
this.entries.set(key, entry);
|
||||
return { value: entry.value, cachedAt: entry.cachedAt, ageMs, stale };
|
||||
}
|
||||
|
||||
set(scope: MetadataScopeInput, value: T): void {
|
||||
const key = metadataScopeKey(scope);
|
||||
this.entries.delete(key);
|
||||
this.entries.set(key, { scope: metadataScopeParts(scope), value, cachedAt: this.now() });
|
||||
this.evictOldest();
|
||||
}
|
||||
|
||||
invalidate(match: MetadataCacheInvalidation): number {
|
||||
const normalized = normalizeInvalidation(match);
|
||||
let removed = 0;
|
||||
for (const [key, entry] of this.entries) {
|
||||
const matches = Object.entries(normalized).every(([field, value]) => valuesEqual(entry.scope[field as keyof typeof entry.scope], value));
|
||||
if (matches) {
|
||||
this.entries.delete(key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.entries.clear();
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
private evictOldest(): void {
|
||||
while (this.entries.size > this.options.maxEntries) {
|
||||
const oldest = this.entries.keys().next().value;
|
||||
if (oldest === undefined) return;
|
||||
this.entries.delete(oldest);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { ColumnInfo, DatabaseType, IndexInfo, QueryTab } from "@/types/database";
|
||||
import * as api from "@/lib/backend/api";
|
||||
import { editableRowIdentifierColumns } from "@/lib/table/tableEditing";
|
||||
import { createMetadataLoadTrace, logMetadataLoadTrace, MetadataLoadCoordinator, type MetadataLoadCacheStatus, type MetadataLoadTraceLogger } from "./metadataLoadCoordinator";
|
||||
import type { MetadataScopeInput } from "./metadataLoadScope";
|
||||
import { MetadataResultCache, type MetadataCacheInvalidation } from "./metadataResultCache";
|
||||
|
||||
export const TABLE_METADATA_CACHE_TTL_MS = 30_000;
|
||||
const TABLE_METADATA_CACHE_MAX_ENTRIES = 120;
|
||||
|
||||
export interface TableMetadata {
|
||||
schema?: string;
|
||||
tableName: string;
|
||||
tableType?: string;
|
||||
columns: ColumnInfo[];
|
||||
indexes: IndexInfo[];
|
||||
primaryKeys: string[];
|
||||
cachedAt: number;
|
||||
}
|
||||
|
||||
export interface TableMetadataRequest {
|
||||
connectionId: string;
|
||||
database: string;
|
||||
schema?: string;
|
||||
tableName: string;
|
||||
tableType?: string;
|
||||
databaseType: DatabaseType | string;
|
||||
driverProfile?: string;
|
||||
force?: boolean;
|
||||
traceLogger?: MetadataLoadTraceLogger;
|
||||
}
|
||||
|
||||
export interface TableMetadataLoadResult {
|
||||
metadata: TableMetadata;
|
||||
cacheStatus: MetadataLoadCacheStatus;
|
||||
ageMs: number;
|
||||
}
|
||||
|
||||
const tableMetadataCache = new MetadataResultCache<TableMetadata>({
|
||||
ttlMs: TABLE_METADATA_CACHE_TTL_MS,
|
||||
maxEntries: TABLE_METADATA_CACHE_MAX_ENTRIES,
|
||||
});
|
||||
|
||||
const tableMetadataCoordinator = new MetadataLoadCoordinator((event) => {
|
||||
console.debug("[DBX][metadata-load:table-coordinator]", event);
|
||||
});
|
||||
|
||||
export function tableMetadataScope(request: Pick<TableMetadataRequest, "connectionId" | "database" | "schema" | "tableName" | "tableType" | "driverProfile" | "databaseType">): MetadataScopeInput {
|
||||
return {
|
||||
kind: "table-metadata",
|
||||
connectionId: request.connectionId,
|
||||
database: request.database,
|
||||
schema: request.schema ?? "",
|
||||
tableName: request.tableName,
|
||||
tableType: request.tableType,
|
||||
driverProfile: request.driverProfile || request.databaseType,
|
||||
};
|
||||
}
|
||||
|
||||
export function getCachedTableMetadata(request: Pick<TableMetadataRequest, "connectionId" | "database" | "schema" | "tableName" | "tableType" | "driverProfile" | "databaseType">): TableMetadataLoadResult | undefined {
|
||||
const hit = tableMetadataCache.get(tableMetadataScope(request));
|
||||
if (!hit) return undefined;
|
||||
return { metadata: hit.value, cacheStatus: hit.stale ? "stale" : "hit", ageMs: hit.ageMs };
|
||||
}
|
||||
|
||||
export function tableMetadataToDataTabMeta(metadata: TableMetadata, schema = metadata.schema): NonNullable<QueryTab["tableMeta"]> {
|
||||
return {
|
||||
schema,
|
||||
tableName: metadata.tableName,
|
||||
tableType: metadata.tableType,
|
||||
columns: metadata.columns,
|
||||
primaryKeys: metadata.primaryKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadTableMetadata(request: TableMetadataRequest): Promise<TableMetadataLoadResult> {
|
||||
const scope = tableMetadataScope(request);
|
||||
const trace = createMetadataLoadTrace(scope);
|
||||
if (!request.force) {
|
||||
const cached = tableMetadataCache.get(scope);
|
||||
if (cached) {
|
||||
logMetadataLoadTrace(request.traceLogger, trace, "cache-hit", {
|
||||
cacheStatus: cached.stale ? "stale" : "hit",
|
||||
resultCount: cached.value.columns.length,
|
||||
stale: cached.stale,
|
||||
});
|
||||
return { metadata: cached.value, cacheStatus: cached.stale ? "stale" : "hit", ageMs: cached.ageMs };
|
||||
}
|
||||
}
|
||||
|
||||
logMetadataLoadTrace(request.traceLogger, trace, "cache-miss", { cacheStatus: request.force ? "refresh" : "miss", force: request.force === true });
|
||||
const metadata = await tableMetadataCoordinator.run(
|
||||
scope,
|
||||
async () => {
|
||||
const columns = await api.getColumns(request.connectionId, request.database, request.schema ?? "", request.tableName);
|
||||
const indexes = await api.listIndexes(request.connectionId, request.database, request.schema ?? "", request.tableName).catch((): IndexInfo[] => []);
|
||||
const primaryKeys = editableRowIdentifierColumns(request.databaseType as DatabaseType, columns, indexes, request.tableType);
|
||||
return {
|
||||
schema: request.schema || undefined,
|
||||
tableName: request.tableName,
|
||||
tableType: request.tableType,
|
||||
columns,
|
||||
indexes,
|
||||
primaryKeys,
|
||||
cachedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
{ force: request.force, kind: scope.kind },
|
||||
);
|
||||
|
||||
tableMetadataCache.set(scope, metadata);
|
||||
logMetadataLoadTrace(request.traceLogger, trace, "done", {
|
||||
cacheStatus: request.force ? "refresh" : "miss",
|
||||
resultCount: metadata.columns.length,
|
||||
force: request.force === true,
|
||||
});
|
||||
return { metadata, cacheStatus: request.force ? "refresh" : "miss", ageMs: 0 };
|
||||
}
|
||||
|
||||
export function invalidateTableMetadataCache(match: MetadataCacheInvalidation): number {
|
||||
return tableMetadataCache.invalidate(match);
|
||||
}
|
||||
|
||||
export function clearTableMetadataCache(): void {
|
||||
tableMetadataCache.clear();
|
||||
}
|
||||
Reference in new issue
Block a user