feat(metadata): optimize agent metadata loading

This commit is contained in:
t8y2 committed 2026-07-05 12:09:24 +08:00
1 parent 8883a6b3b7
commit e74782f651
65 files changed
+6192 -571

No files matched your search

+4 -1
View File
@@ -518,13 +518,16 @@ export async function getTableComment(_connectionId: string, _database: string,
throw new Error("Table comment lookup is not available in the web backend");
}
export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[]): Promise<ObjectInfo[]> {
export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[], filter?: string, limit?: number, offset?: number): Promise<ObjectInfo[]> {
return get(
`/api/schema/objects?${qs({
connection_id: connectionId,
database,
schema,
object_types: objectTypes?.join(","),
filter,
limit,
offset,
})}`,
);
}
+2 -2
View File
@@ -610,8 +610,8 @@ export async function getTableComment(connectionId: string, database: string, sc
return invoke("get_table_comment", { connectionId, database, schema, table });
}
export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[]): Promise<ObjectInfo[]> {
return invoke("list_objects", { connectionId, database, schema, objectTypes });
export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[], filter?: string, limit?: number, offset?: number): Promise<ObjectInfo[]> {
return invoke("list_objects", { connectionId, database, schema, objectTypes, filter, limit, offset });
}
export async function listObjectStatistics(connectionId: string, database: string, schema: string): Promise<ObjectStatistics[]> {
@@ -0,0 +1,145 @@
import { metadataScopeKey, type MetadataScopeInput } from "./metadataLoadScope";
export type MetadataLoadCacheStatus = "none" | "hit" | "miss" | "stale" | "refresh";
export interface MetadataLoadTraceDetails {
cacheStatus?: MetadataLoadCacheStatus;
resultCount?: number;
deduped?: boolean;
force?: boolean;
stale?: boolean;
superseded?: boolean;
error?: unknown;
}
export interface MetadataLoadTraceEvent extends MetadataLoadTraceDetails {
event: string;
traceId: string;
scopeKey: string;
elapsedMs: number;
}
export type MetadataLoadTraceLogger = (event: MetadataLoadTraceEvent) => void;
export interface MetadataLoadTrace {
traceId: string;
scopeKey: string;
startedAt: number;
elapsedMs: () => number;
event: (event: string, details?: MetadataLoadTraceDetails) => MetadataLoadTraceEvent;
}
export interface MetadataLoadCoordinatorEvent {
event: "start" | "dedupe" | "done" | "error";
key: string;
kind: string;
active: number;
force: boolean;
}
export type MetadataLoadCoordinatorLogger = (event: MetadataLoadCoordinatorEvent) => void;
export interface MetadataLoadCoordinatorOptions {
force?: boolean;
kind?: string;
}
interface InFlightLoad<T> {
kind: string;
promise: Promise<T>;
}
function defaultTraceId() {
return Math.random().toString(16).slice(2, 10);
}
function defaultNow() {
return typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
}
export function createMetadataLoadTrace(
scope: MetadataScopeInput | string,
options?: {
traceId?: string;
now?: () => number;
},
): MetadataLoadTrace {
const now = options?.now ?? defaultNow;
const startedAt = now();
const scopeKey = typeof scope === "string" ? scope : metadataScopeKey(scope);
const traceId = options?.traceId ?? defaultTraceId();
const elapsedMs = () => Math.max(0, Math.round(now() - startedAt));
return {
traceId,
scopeKey,
startedAt,
elapsedMs,
event: (event, details = {}) => ({
event,
traceId,
scopeKey,
elapsedMs: elapsedMs(),
...details,
}),
};
}
export function logMetadataLoadTrace(logger: MetadataLoadTraceLogger | undefined, trace: MetadataLoadTrace, event: string, details?: MetadataLoadTraceDetails) {
logger?.(trace.event(event, details));
}
export class MetadataLoadCoordinator {
private readonly inFlight = new Map<string, InFlightLoad<unknown>>();
constructor(private readonly logger?: MetadataLoadCoordinatorLogger) {}
run<T>(scope: MetadataScopeInput | string, load: () => Promise<T>, options?: MetadataLoadCoordinatorOptions): Promise<T> {
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
const kind = options?.kind ?? (typeof scope === "string" ? "metadata" : scope.kind);
const force = options?.force === true;
const existing = force ? undefined : this.inFlight.get(key);
if (existing) {
this.logger?.({ event: "dedupe", key, kind: existing.kind, active: this.inFlight.size, force: false });
return existing.promise as Promise<T>;
}
this.logger?.({ event: "start", key, kind, active: this.inFlight.size + 1, force });
const promise = Promise.resolve()
.then(load)
.then(
(value) => {
this.logger?.({ event: "done", key, kind, active: this.inFlight.size, force });
return value;
},
(error) => {
this.logger?.({ event: "error", key, kind, active: this.inFlight.size, force });
throw error;
},
)
.finally(() => {
const current = this.inFlight.get(key);
if (current?.promise === promise) {
this.inFlight.delete(key);
}
});
if (!force) {
this.inFlight.set(key, { kind, promise });
}
return promise;
}
has(scope: MetadataScopeInput | string): boolean {
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
return this.inFlight.has(key);
}
clear(scope?: MetadataScopeInput | string) {
if (scope === undefined) {
this.inFlight.clear();
return;
}
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
this.inFlight.delete(key);
}
}
@@ -0,0 +1,86 @@
export type MetadataScopeValue = string | number | boolean | null | undefined;
export interface MetadataScopeInput {
kind: string;
connectionId?: string | null;
database?: string | null;
schema?: string | null;
nodeKind?: string | null;
tableName?: string | null;
tableType?: string | null;
objectTypes?: readonly (string | null | undefined)[] | null;
searchFilter?: string | null;
limit?: number | null;
offset?: number | null;
sidebarDisplayMode?: string | null;
driverProfile?: string | null;
extra?: Record<string, MetadataScopeValue | readonly MetadataScopeValue[]> | null;
}
function normalizeString(value: string | null | undefined): string | undefined {
if (value == null) return undefined;
return value;
}
function normalizeSearchFilter(value: string | null | undefined): string | undefined {
const normalized = value?.trim();
return normalized ? normalized : undefined;
}
function normalizeNumber(value: number | null | undefined): number | undefined {
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
}
function normalizeObjectTypes(values: readonly (string | null | undefined)[] | null | undefined): string[] | undefined {
if (!values?.length) return undefined;
const normalized = values
.filter((value): value is string => typeof value === "string" && value.trim().length > 0)
.map((value) => value.trim().toUpperCase().replace(/\s+/g, "_"))
.sort();
return normalized.length > 0 ? [...new Set(normalized)] : undefined;
}
function stableEntries(record: Record<string, unknown>): [string, unknown][] {
return Object.entries(record)
.filter(([, value]) => value !== undefined)
.sort(([left], [right]) => left.localeCompare(right));
}
function normalizeExtra(extra: MetadataScopeInput["extra"]): Record<string, unknown> | undefined {
if (!extra) return undefined;
const entries = stableEntries(extra)
.map(([key, value]) => {
if (Array.isArray(value)) {
return [key, value.filter((item) => item !== undefined)] as const;
}
return [key, value] as const;
})
.filter(([, value]) => {
if (Array.isArray(value)) return value.length > 0;
return value !== undefined;
});
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
}
export function metadataScopeParts(input: MetadataScopeInput): Record<string, unknown> {
return {
kind: input.kind,
connectionId: normalizeString(input.connectionId),
database: normalizeString(input.database),
schema: normalizeString(input.schema),
nodeKind: normalizeString(input.nodeKind),
tableName: normalizeString(input.tableName),
tableType: normalizeString(input.tableType),
objectTypes: normalizeObjectTypes(input.objectTypes),
searchFilter: normalizeSearchFilter(input.searchFilter),
limit: normalizeNumber(input.limit),
offset: normalizeNumber(input.offset),
sidebarDisplayMode: normalizeString(input.sidebarDisplayMode),
driverProfile: normalizeString(input.driverProfile),
extra: normalizeExtra(input.extra),
};
}
export function metadataScopeKey(input: MetadataScopeInput): string {
return JSON.stringify(stableEntries(metadataScopeParts(input)));
}
@@ -0,0 +1,99 @@
import { metadataScopeKey, metadataScopeParts, type MetadataScopeInput } from "./metadataLoadScope";
export interface MetadataCacheHit<T> {
value: T;
cachedAt: number;
ageMs: number;
stale: boolean;
}
export type MetadataCacheInvalidation = Partial<Omit<MetadataScopeInput, "extra">>;
interface MetadataCacheEntry<T> {
scope: ReturnType<typeof metadataScopeParts>;
value: T;
cachedAt: number;
}
export interface MetadataResultCacheOptions {
ttlMs: number;
maxEntries: number;
now?: () => number;
}
function normalizeInvalidation(match: MetadataCacheInvalidation): Partial<ReturnType<typeof metadataScopeParts>> {
const normalized = metadataScopeParts({ ...match, kind: match.kind ?? "__metadata_cache_match__" });
const result: Partial<ReturnType<typeof metadataScopeParts>> = {};
for (const key of Object.keys(match) as (keyof MetadataCacheInvalidation)[]) {
const value = normalized[key as keyof typeof normalized];
if (value !== undefined) {
(result as Record<string, unknown>)[key] = value;
}
}
return result;
}
function valuesEqual(left: unknown, right: unknown): boolean {
if (Array.isArray(left) || Array.isArray(right) || (left && typeof left === "object") || (right && typeof right === "object")) {
return JSON.stringify(left) === JSON.stringify(right);
}
return left === right;
}
export class MetadataResultCache<T> {
private readonly entries = new Map<string, MetadataCacheEntry<T>>();
private readonly now: () => number;
constructor(private readonly options: MetadataResultCacheOptions) {
this.now = options.now ?? Date.now;
}
get(scope: MetadataScopeInput, options?: { allowStale?: boolean }): MetadataCacheHit<T> | undefined {
const key = metadataScopeKey(scope);
const entry = this.entries.get(key);
if (!entry) return undefined;
const ageMs = Math.max(0, this.now() - entry.cachedAt);
const stale = ageMs > this.options.ttlMs;
if (stale && !options?.allowStale) return undefined;
this.entries.delete(key);
this.entries.set(key, entry);
return { value: entry.value, cachedAt: entry.cachedAt, ageMs, stale };
}
set(scope: MetadataScopeInput, value: T): void {
const key = metadataScopeKey(scope);
this.entries.delete(key);
this.entries.set(key, { scope: metadataScopeParts(scope), value, cachedAt: this.now() });
this.evictOldest();
}
invalidate(match: MetadataCacheInvalidation): number {
const normalized = normalizeInvalidation(match);
let removed = 0;
for (const [key, entry] of this.entries) {
const matches = Object.entries(normalized).every(([field, value]) => valuesEqual(entry.scope[field as keyof typeof entry.scope], value));
if (matches) {
this.entries.delete(key);
removed++;
}
}
return removed;
}
clear(): void {
this.entries.clear();
}
get size(): number {
return this.entries.size;
}
private evictOldest(): void {
while (this.entries.size > this.options.maxEntries) {
const oldest = this.entries.keys().next().value;
if (oldest === undefined) return;
this.entries.delete(oldest);
}
}
}
@@ -0,0 +1,126 @@
import type { ColumnInfo, DatabaseType, IndexInfo, QueryTab } from "@/types/database";
import * as api from "@/lib/backend/api";
import { editableRowIdentifierColumns } from "@/lib/table/tableEditing";
import { createMetadataLoadTrace, logMetadataLoadTrace, MetadataLoadCoordinator, type MetadataLoadCacheStatus, type MetadataLoadTraceLogger } from "./metadataLoadCoordinator";
import type { MetadataScopeInput } from "./metadataLoadScope";
import { MetadataResultCache, type MetadataCacheInvalidation } from "./metadataResultCache";
export const TABLE_METADATA_CACHE_TTL_MS = 30_000;
const TABLE_METADATA_CACHE_MAX_ENTRIES = 120;
export interface TableMetadata {
schema?: string;
tableName: string;
tableType?: string;
columns: ColumnInfo[];
indexes: IndexInfo[];
primaryKeys: string[];
cachedAt: number;
}
export interface TableMetadataRequest {
connectionId: string;
database: string;
schema?: string;
tableName: string;
tableType?: string;
databaseType: DatabaseType | string;
driverProfile?: string;
force?: boolean;
traceLogger?: MetadataLoadTraceLogger;
}
export interface TableMetadataLoadResult {
metadata: TableMetadata;
cacheStatus: MetadataLoadCacheStatus;
ageMs: number;
}
const tableMetadataCache = new MetadataResultCache<TableMetadata>({
ttlMs: TABLE_METADATA_CACHE_TTL_MS,
maxEntries: TABLE_METADATA_CACHE_MAX_ENTRIES,
});
const tableMetadataCoordinator = new MetadataLoadCoordinator((event) => {
console.debug("[DBX][metadata-load:table-coordinator]", event);
});
export function tableMetadataScope(request: Pick<TableMetadataRequest, "connectionId" | "database" | "schema" | "tableName" | "tableType" | "driverProfile" | "databaseType">): MetadataScopeInput {
return {
kind: "table-metadata",
connectionId: request.connectionId,
database: request.database,
schema: request.schema ?? "",
tableName: request.tableName,
tableType: request.tableType,
driverProfile: request.driverProfile || request.databaseType,
};
}
export function getCachedTableMetadata(request: Pick<TableMetadataRequest, "connectionId" | "database" | "schema" | "tableName" | "tableType" | "driverProfile" | "databaseType">): TableMetadataLoadResult | undefined {
const hit = tableMetadataCache.get(tableMetadataScope(request));
if (!hit) return undefined;
return { metadata: hit.value, cacheStatus: hit.stale ? "stale" : "hit", ageMs: hit.ageMs };
}
export function tableMetadataToDataTabMeta(metadata: TableMetadata, schema = metadata.schema): NonNullable<QueryTab["tableMeta"]> {
return {
schema,
tableName: metadata.tableName,
tableType: metadata.tableType,
columns: metadata.columns,
primaryKeys: metadata.primaryKeys,
};
}
export async function loadTableMetadata(request: TableMetadataRequest): Promise<TableMetadataLoadResult> {
const scope = tableMetadataScope(request);
const trace = createMetadataLoadTrace(scope);
if (!request.force) {
const cached = tableMetadataCache.get(scope);
if (cached) {
logMetadataLoadTrace(request.traceLogger, trace, "cache-hit", {
cacheStatus: cached.stale ? "stale" : "hit",
resultCount: cached.value.columns.length,
stale: cached.stale,
});
return { metadata: cached.value, cacheStatus: cached.stale ? "stale" : "hit", ageMs: cached.ageMs };
}
}
logMetadataLoadTrace(request.traceLogger, trace, "cache-miss", { cacheStatus: request.force ? "refresh" : "miss", force: request.force === true });
const metadata = await tableMetadataCoordinator.run(
scope,
async () => {
const columns = await api.getColumns(request.connectionId, request.database, request.schema ?? "", request.tableName);
const indexes = await api.listIndexes(request.connectionId, request.database, request.schema ?? "", request.tableName).catch((): IndexInfo[] => []);
const primaryKeys = editableRowIdentifierColumns(request.databaseType as DatabaseType, columns, indexes, request.tableType);
return {
schema: request.schema || undefined,
tableName: request.tableName,
tableType: request.tableType,
columns,
indexes,
primaryKeys,
cachedAt: Date.now(),
};
},
{ force: request.force, kind: scope.kind },
);
tableMetadataCache.set(scope, metadata);
logMetadataLoadTrace(request.traceLogger, trace, "done", {
cacheStatus: request.force ? "refresh" : "miss",
resultCount: metadata.columns.length,
force: request.force === true,
});
return { metadata, cacheStatus: request.force ? "refresh" : "miss", ageMs: 0 };
}
export function invalidateTableMetadataCache(match: MetadataCacheInvalidation): number {
return tableMetadataCache.invalidate(match);
}
export function clearTableMetadataCache(): void {
tableMetadataCache.clear();
}
@@ -2,7 +2,7 @@ import type { FlatTreeNode } from "@/composables/useFlatTree";
import type { TreeNode, TreeNodeType } from "@/types/database";
const simpleObjectParentTypes = new Set<TreeNodeType>(["database", "schema", "linked-server-schema"]);
const simpleObjectChildTypes = new Set<TreeNodeType>(["table", "view", "materialized_view", "procedure", "function", "sequence", "package", "package-body", "load-more"]);
const tableSearchableChildTypes = new Set<TreeNodeType>(["table", "view", "materialized_view", "load-more"]);
export function isSidebarTableSearchControlNode(node: TreeNode): boolean {
return node.type === "table-search-control";
@@ -12,18 +12,19 @@ function tableSearchControlId(parentId: string): string {
return `${parentId}:__table_search`;
}
function parentHasDirectTableList(node: TreeNode): boolean {
return !!node.children?.some((child) => simpleObjectChildTypes.has(child.type));
function parentHasSearchableTableList(node: TreeNode): boolean {
// Routine-only schemas should not show a table-specific search box.
return !!node.children?.some((child) => tableSearchableChildTypes.has(child.type));
}
function shouldInsertTableSearchControl(item: FlatTreeNode, sidebarObjectDisplay: "simple" | "grouped", activeQueries: Readonly<Record<string, string | undefined>>): boolean {
const node = item.node;
if (!node.isExpanded) return false;
if (sidebarObjectDisplay === "grouped") {
return node.type === "group-tables";
return node.type === "group-tables" && (parentHasSearchableTableList(node) || !!activeQueries[node.id]?.trim());
}
if (!simpleObjectParentTypes.has(node.type)) return false;
return parentHasDirectTableList(node) || !!activeQueries[node.id]?.trim();
return parentHasSearchableTableList(node) || !!activeQueries[node.id]?.trim();
}
function buildTableSearchControlNode(parent: TreeNode): TreeNode {