feat(mongodb): improve document quick filters

This commit is contained in:
co1dsand authored and GitHub committed 2026-07-17 18:51:33 +08:00
1 parent c417e54698
commit 681da22c33
32 files changed
+1639 -116

No files matched your search

@@ -18,6 +18,19 @@ export type DocumentFilterRule = {
elasticsearchQueryType?: ElasticsearchQueryType;
};
export type DocumentFieldPathKind = "scalar" | "object" | "array" | "array-object" | "mixed";
export type DocumentFieldPathNode = {
key: string;
path: string;
label: string;
displayPath: string;
kind: DocumentFieldPathKind;
selectable: boolean;
sampleValue?: unknown;
children: DocumentFieldPathNode[];
};
export type DocumentStoreQueryPreviewOptions = {
collection: string;
filterJson?: string;
@@ -119,6 +132,152 @@ export function documentFilterModeNeedsValue(mode: DocumentFilterMode): boolean
return mode !== "is-null" && mode !== "is-not-null";
}
type DocumentFieldPathAccumulatorNode = {
key: string;
path: string;
kind: DocumentFieldPathKind;
sampleValue?: unknown;
children: DocumentFieldPathAccumulatorNode[];
childByKey: Map<string, DocumentFieldPathAccumulatorNode>;
};
export function documentFieldPathOptionsFromDocuments(documents: readonly Record<string, unknown>[]): string[] {
return flattenDocumentFieldPathTree(documentFieldPathTreeFromDocuments(documents)).map((node) => node.path);
}
export function documentFieldPathTreeFromDocuments(documents: readonly Record<string, unknown>[]): DocumentFieldPathNode[] {
if (documents.length === 0) return [];
const rootNodes: DocumentFieldPathAccumulatorNode[] = [];
const rootByKey = new Map<string, DocumentFieldPathAccumulatorNode>();
ensureDocumentFieldPathNode(rootNodes, rootByKey, "_id", "_id", "scalar");
for (const doc of documents) {
for (const [key, value] of Object.entries(doc)) {
if (key === "_id") continue;
collectDocumentFieldPathNode(rootNodes, rootByKey, key, value);
}
}
return finalizeDocumentFieldPathNodes(rootNodes);
}
export function flattenDocumentFieldPathTree(nodes: readonly DocumentFieldPathNode[]): DocumentFieldPathNode[] {
const flattened: DocumentFieldPathNode[] = [];
for (const node of nodes) {
flattened.push(node);
flattened.push(...flattenDocumentFieldPathTree(node.children));
}
return flattened;
}
export function searchDocumentFieldPathTree(nodes: readonly DocumentFieldPathNode[], query: string): DocumentFieldPathNode[] {
const normalizedQuery = query.trim().toLowerCase();
if (!normalizedQuery) return flattenDocumentFieldPathTree(nodes);
return flattenDocumentFieldPathTree(nodes).filter((node) => {
return node.path.toLowerCase().includes(normalizedQuery) || node.displayPath.toLowerCase().includes(normalizedQuery) || node.label.toLowerCase().includes(normalizedQuery);
});
}
export function arrayObjectAncestorPathForDocumentField(nodes: readonly DocumentFieldPathNode[], path: string): string | null {
for (const node of nodes) {
if (node.path === path) return null;
if (path.startsWith(`${node.path}.`)) {
if (node.kind === "array-object") return node.path;
return arrayObjectAncestorPathForDocumentField(node.children, path);
}
}
return null;
}
function collectDocumentFieldPathNode(nodes: DocumentFieldPathAccumulatorNode[], byKey: Map<string, DocumentFieldPathAccumulatorNode>, path: string, value: unknown, depth = 0): void {
const key = path.split(".").pop() || path;
const node = ensureDocumentFieldPathNode(nodes, byKey, key, path, documentFieldPathKindFromValue(value));
if (node.sampleValue === undefined) node.sampleValue = value;
if (depth >= 6) return;
if (isBsonExtendedJsonWrapper(value)) return;
if (Array.isArray(value)) {
collectArrayDocumentFieldPathNodes(node, value, depth + 1);
return;
}
if (isPlainRecord(value)) collectNestedDocumentFieldPathNodes(node, value, depth + 1);
}
function collectArrayDocumentFieldPathNodes(parent: DocumentFieldPathAccumulatorNode, values: readonly unknown[], depth: number): void {
if (depth > 6) return;
for (const value of values) {
if (Array.isArray(value)) collectArrayDocumentFieldPathNodes(parent, value, depth + 1);
else if (isPlainRecord(value) && !isBsonExtendedJsonWrapper(value)) collectNestedDocumentFieldPathNodes(parent, value, depth);
}
}
function collectNestedDocumentFieldPathNodes(parent: DocumentFieldPathAccumulatorNode, value: Record<string, unknown>, depth: number): void {
for (const [key, nestedValue] of Object.entries(value)) {
collectDocumentFieldPathNode(parent.children, parent.childByKey, `${parent.path}.${key}`, nestedValue, depth);
}
}
function ensureDocumentFieldPathNode(nodes: DocumentFieldPathAccumulatorNode[], byKey: Map<string, DocumentFieldPathAccumulatorNode>, key: string, path: string, kind: DocumentFieldPathKind): DocumentFieldPathAccumulatorNode {
const existing = byKey.get(key);
if (existing) {
existing.kind = mergeDocumentFieldPathKind(existing.kind, kind);
return existing;
}
const node: DocumentFieldPathAccumulatorNode = {
key,
path,
kind,
children: [],
childByKey: new Map(),
};
byKey.set(key, node);
nodes.push(node);
return node;
}
function documentFieldPathKindFromValue(value: unknown): DocumentFieldPathKind {
if (isBsonExtendedJsonWrapper(value)) return "scalar";
if (Array.isArray(value)) return arrayContainsPlainRecord(value) ? "array-object" : "array";
if (isPlainRecord(value)) return "object";
return "scalar";
}
const BSON_EXTENDED_JSON_SINGLE_KEY_WRAPPERS = new Set(["$oid", "$numberInt", "$numberLong", "$numberDouble", "$numberDecimal", "$date", "$timestamp", "$binary", "$regularExpression", "$code", "$symbol", "$undefined", "$minKey", "$maxKey", "$dbPointer"]);
function isBsonExtendedJsonWrapper(value: unknown): value is Record<string, unknown> {
if (!isPlainRecord(value)) return false;
const keys = Object.keys(value);
if (keys.length === 0) return false;
if (keys.length === 1) return BSON_EXTENDED_JSON_SINGLE_KEY_WRAPPERS.has(keys[0]);
return (keys.length === 2 && keys.includes("$binary") && keys.includes("$type")) || (keys.length === 2 && keys.includes("$code") && keys.includes("$scope")) || (keys.length === 2 && keys.includes("$dbPointer") && keys.includes("$ref"));
}
function arrayContainsPlainRecord(values: readonly unknown[]): boolean {
return values.some((value) => (isPlainRecord(value) && !isBsonExtendedJsonWrapper(value)) || (Array.isArray(value) && arrayContainsPlainRecord(value)));
}
function mergeDocumentFieldPathKind(current: DocumentFieldPathKind, next: DocumentFieldPathKind): DocumentFieldPathKind {
if (current === next) return current;
if (current === "mixed" || next === "mixed") return "mixed";
if ((current === "array-object" && next === "array") || (current === "array" && next === "array-object")) return "array-object";
return "mixed";
}
function finalizeDocumentFieldPathNodes(nodes: readonly DocumentFieldPathAccumulatorNode[], parentDisplaySegments: readonly string[] = []): DocumentFieldPathNode[] {
return nodes.map((node) => {
const label = node.kind === "array" || node.kind === "array-object" ? `${node.key}[]` : node.key;
const displaySegments = [...parentDisplaySegments, label];
return {
key: node.key,
path: node.path,
label,
displayPath: displaySegments.join(" > "),
kind: node.kind,
selectable: true,
sampleValue: node.sampleValue,
children: finalizeDocumentFieldPathNodes(node.children, displaySegments),
};
});
}
export function elasticsearchQueryTypeNeedsValue(queryType: ElasticsearchQueryType | undefined): boolean {
return queryType !== "exists";
}
@@ -183,6 +342,7 @@ export function elasticsearchStructuredFilter(query: Record<string, unknown> | n
type DocumentFilterParseOptions = {
kind?: DocumentStoreKind;
sampleValue?: unknown;
};
export function buildDocumentFilterCondition(rule: DocumentFilterRule, options: DocumentFilterParseOptions = {}): Record<string, unknown> | null {
@@ -196,9 +356,9 @@ export function buildDocumentFilterCondition(rule: DocumentFilterRule, options:
case "not-equals":
return { [rule.fieldName]: { $ne: value } };
case "like":
return { [rule.fieldName]: { $regex: textValue, $options: "i" } };
return { [rule.fieldName]: { $regex: escapeRegexLiteral(textValue), $options: "i" } };
case "not-like":
return { [rule.fieldName]: { $not: { $regex: textValue, $options: "i" } } };
return { [rule.fieldName]: { $not: { $regex: escapeRegexLiteral(textValue), $options: "i" } } };
case "greater-than":
return { [rule.fieldName]: { $gt: value } };
case "less-than":
@@ -210,16 +370,70 @@ export function buildDocumentFilterCondition(rule: DocumentFilterRule, options:
}
}
export function combineDocumentFilterConditions(conditions: Record<string, unknown>[], rules: Pick<DocumentFilterRule, "conjunction">[]): Record<string, unknown> | null {
function escapeRegexLiteral(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
export function combineDocumentFilterConditions(conditions: Record<string, unknown>[], rules: Pick<DocumentFilterRule, "conjunction">[], arrayObjectParents: Array<string | null> = []): Record<string, unknown> | null {
if (conditions.length === 0) return null;
let result = conditions[0];
for (let i = 1; i < conditions.length; i++) {
const operator = rules[i]?.conjunction === "OR" ? "$or" : "$and";
result = { [operator]: [result, conditions[i]] };
const grouped = groupLeadingArrayObjectConditions(conditions, rules, arrayObjectParents);
let result = grouped.conditions[0];
for (let i = 1; i < grouped.conditions.length; i++) {
const operator = grouped.rules[i]?.conjunction === "OR" ? "$or" : "$and";
result = { [operator]: [result, grouped.conditions[i]] };
}
return result;
}
function groupLeadingArrayObjectConditions(conditions: Record<string, unknown>[], rules: Pick<DocumentFilterRule, "conjunction">[], arrayObjectParents: Array<string | null>): { conditions: Record<string, unknown>[]; rules: Pick<DocumentFilterRule, "conjunction">[] } {
const groupedConditions: Record<string, unknown>[] = [];
const groupedRules: Pick<DocumentFilterRule, "conjunction">[] = [];
let index = 0;
// A later AND after an OR is part of the left-associated expression produced below,
// so only the prefix before the first OR can be safely collapsed into $elemMatch.
while (index < conditions.length && (index === 0 || rules[index]?.conjunction !== "OR")) {
const parent = arrayObjectParents[index];
if (!parent) {
groupedConditions.push(conditions[index]);
groupedRules.push(rules[index]);
index++;
continue;
}
const elementConditions: Record<string, unknown>[] = [];
let end = index;
while (end < conditions.length && (end === index || rules[end]?.conjunction === "AND") && arrayObjectParents[end] === parent) {
const relative = relativeArrayObjectCondition(conditions[end], parent);
if (!relative) break;
elementConditions.push(relative);
end++;
}
if (elementConditions.length < 2) {
groupedConditions.push(conditions[index]);
groupedRules.push(rules[index]);
index++;
continue;
}
groupedConditions.push({ [parent]: { $elemMatch: { $and: elementConditions } } });
groupedRules.push(rules[index]);
index = end;
}
return {
conditions: [...groupedConditions, ...conditions.slice(index)],
rules: [...groupedRules, ...rules.slice(index)],
};
}
function relativeArrayObjectCondition(condition: Record<string, unknown>, parent: string): Record<string, unknown> | null {
const entries = Object.entries(condition);
if (entries.length !== 1) return null;
const [field, value] = entries[0];
const prefix = `${parent}.`;
return field.startsWith(prefix) ? { [field.slice(prefix.length)]: value } : null;
}
const MAX_SAFE_BIGINT = 9007199254740991n;
const MIN_BSON_INT64 = -9223372036854775808n;
const MAX_BSON_INT64 = 9223372036854775807n;
@@ -333,16 +547,51 @@ export function currentDocumentSortJson(input: string): string | undefined {
return Object.keys(sort).length ? JSON.stringify(sort) : undefined;
}
export function formatDocumentQueryInput(input: string, kind?: DocumentStoreKind): string {
const parsed = parseDocumentFilterInput(input, { kind });
return JSON.stringify(parsed, null, 2);
}
function parseDocumentFilterValue(raw: string, options: DocumentFilterParseOptions = {}): unknown {
const trimmed = raw.trim();
if (!trimmed) return "";
try {
return parseJsonPreservingLargeIntegers(trimmed, options);
} catch {
return trimmed;
return mongoTypedFilterValue(trimmed, options);
}
}
function mongoTypedFilterValue(raw: string, options: DocumentFilterParseOptions): unknown {
if (options.kind !== "mongodb") return raw;
const sampleValue = mongoTypedFilterSample(options.sampleValue);
if (!sampleValue) return raw;
if (typeof sampleValue.$oid === "string") return { $oid: raw };
if ("$date" in sampleValue) return { $date: raw };
if (typeof sampleValue.$numberLong === "string" && /^-?\d+$/.test(raw)) return { $numberLong: raw };
return raw;
}
function mongoTypedFilterSample(sampleValue: unknown): Record<string, unknown> | null {
if (isPlainRecord(sampleValue)) return sampleValue;
if (!Array.isArray(sampleValue)) return null;
const samples = sampleValue.filter((value) => value !== null && value !== undefined);
if (!samples.length || samples.some((value) => !isPlainRecord(value))) return null;
const records = samples as Record<string, unknown>[];
const sampleKind = mongoTypedFilterSampleKind(records[0]);
// Mixed arrays are ambiguous, so infer a BSON type only from homogeneous scalar wrappers.
if (!sampleKind || records.some((value) => mongoTypedFilterSampleKind(value) !== sampleKind)) return null;
return records[0];
}
function mongoTypedFilterSampleKind(sampleValue: Record<string, unknown>): "$oid" | "$date" | "$numberLong" | null {
if (typeof sampleValue.$oid === "string") return "$oid";
if ("$date" in sampleValue) return "$date";
if (typeof sampleValue.$numberLong === "string") return "$numberLong";
return null;
}
export function elasticsearchSearchBodyFromDocumentQuery(options: Pick<DocumentStoreQueryPreviewOptions, "filterJson" | "sortJson" | "skip" | "limit">): Record<string, unknown> {
const body: Record<string, unknown> = {
from: options.skip,
@@ -176,12 +176,32 @@ function detailColumnType(typeByColumn: ReadonlyMap<string, string> | undefined,
return resultColumnTypes?.[columnIndex]?.trim() ?? "";
}
export function dataGridRowDetailJson(detail: DataGridRowDetail): string {
export function dataGridRowDetailJson(detail: DataGridRowDetail, originalDocument?: unknown): string {
if (originalDocument !== undefined) return JSON.stringify(jsonDetailDisplayValue(originalDocument), null, 2);
const row: Record<string, CellValue> = {};
detail.fields.forEach((field) => {
row[field.column] = field.value;
});
return JSON.stringify(row, null, 2);
return JSON.stringify(jsonDetailDisplayValue(row), null, 2);
}
export function jsonDetailDisplayValue(value: unknown): unknown {
if (typeof value === "string") {
const trimmed = value.trim();
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
try {
return jsonDetailDisplayValue(JSON.parse(trimmed));
} catch {
return value;
}
}
return value;
}
if (Array.isArray(value)) return value.map(jsonDetailDisplayValue);
if (value && typeof value === "object") {
return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([key, item]) => [key, jsonDetailDisplayValue(item)]));
}
return value;
}
export function dataGridRowDetailTsv(detail: DataGridRowDetail): string {
@@ -0,0 +1,98 @@
export interface DocumentTextMatch {
start: number;
end: number;
}
type JsonTokenRange = DocumentTextMatch & { className: string };
const JSON_TOKEN_PATTERN = /("(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(?:true|false|null)\b|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/g;
export function findDocumentTextMatches(text: string, query: string): DocumentTextMatch[] {
if (!query) return [];
const haystack = text.toLowerCase();
const needle = query.toLowerCase();
const matches: DocumentTextMatch[] = [];
let start = 0;
while (start <= haystack.length - needle.length) {
const index = haystack.indexOf(needle, start);
if (index < 0) break;
matches.push({ start: index, end: index + needle.length });
start = index + Math.max(needle.length, 1);
}
return matches;
}
export function renderDocumentJsonHtml(json: string, query = "", activeMatchIndex = 0): string {
const tokenRanges = jsonTokenRanges(json);
const searchRanges = findDocumentTextMatches(json, query);
const boundaries = new Set<number>([0, json.length]);
for (const range of tokenRanges) {
boundaries.add(range.start);
boundaries.add(range.end);
}
for (const range of searchRanges) {
boundaries.add(range.start);
boundaries.add(range.end);
}
const points = [...boundaries].sort((left, right) => left - right);
let tokenIndex = 0;
let searchIndex = 0;
let html = "";
for (let index = 0; index < points.length - 1; index += 1) {
const start = points[index];
const end = points[index + 1];
if (end <= start) continue;
while (tokenRanges[tokenIndex]?.end <= start) tokenIndex += 1;
while (searchRanges[searchIndex]?.end <= start) searchIndex += 1;
const token = tokenRanges[tokenIndex];
const search = searchRanges[searchIndex];
let segment = escapeHtml(json.slice(start, end));
if (token && token.start <= start && token.end >= end) {
segment = `<span class="${token.className}">${segment}</span>`;
}
if (search && search.start <= start && search.end >= end) {
const activeClass = searchIndex === activeMatchIndex ? " document-search-match-active" : "";
const activeAttribute = searchIndex === activeMatchIndex ? ' data-document-search-active="true"' : "";
segment = `<mark class="document-search-match${activeClass}" data-document-search-match="${searchIndex}"${activeAttribute}>${segment}</mark>`;
}
html += segment;
}
return html;
}
function jsonTokenRanges(json: string): JsonTokenRange[] {
const ranges: JsonTokenRange[] = [];
const pattern = new RegExp(JSON_TOKEN_PATTERN.source, "g");
let match: RegExpExecArray | null;
while ((match = pattern.exec(json))) {
ranges.push({
start: match.index,
end: match.index + match[0].length,
className: jsonTokenClass(match[0]),
});
}
return ranges;
}
function jsonTokenClass(token: string): string {
if (token.startsWith('"')) return token.endsWith(":") ? "json-key" : "json-string";
if (token === "true" || token === "false") return "json-boolean";
if (token === "null") return "json-null";
return "json-number";
}
function escapeHtml(text: string): string {
return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
}
@@ -1,6 +1,7 @@
export type MongoInputValue = string | number | boolean | null;
const MONGO_SHELL_DATE_PATTERN = /^(?:ISODate|new Date)\(\s*(["'])(.+)\1\s*\)$/;
const MONGO_SHELL_NUMBER_LONG_PATTERN = /^NumberLong\(\s*(["'])(-?\d+)\1\s*\)$/;
const LEGACY_MONGO_DATE_DISPLAY_PATTERN = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d{1,3}))?$/;
const MONGO_OBJECT_ID_PATTERN = /^[a-fA-F0-9]{24}$/;
const MONGO_INTEGER_PATTERN = /^-?\d+$/;
@@ -24,6 +25,8 @@ export function parseMongoDocumentInputValue(raw: MongoInputValue): unknown {
const shellDate = mongoShellDateToExtendedJson(trimmed);
if (shellDate !== trimmed) return shellDate;
const shellNumberLong = mongoShellNumberLongToExtendedJson(trimmed);
if (shellNumberLong !== trimmed) return shellNumberLong;
const legacyDate = legacyMongoDateDisplayToExtendedJson(trimmed);
if (legacyDate) return legacyDate;
@@ -42,6 +45,14 @@ export function parseMongoDocumentInputValue(raw: MongoInputValue): unknown {
return raw;
}
export function mongoDocumentDisplayValue(value: unknown): unknown {
if (value && typeof value === "object" && !Array.isArray(value)) {
const object = value as Record<string, unknown>;
if (Object.keys(object).length === 1 && typeof object.$numberLong === "string") return `NumberLong(${JSON.stringify(object.$numberLong)})`;
}
return value;
}
function parseMongoExistingFieldInputValue(raw: Exclude<MongoInputValue, null>, originalValue: unknown): unknown {
// Objects and arrays are serialized into grid text too, so the raw document
// is the only reliable way to distinguish them from JSON-shaped BSON strings.
@@ -63,6 +74,11 @@ function legacyMongoDateDisplayToExtendedJson(value: string): { $date: string }
return { $date: `${date}T${time}.${millis.padEnd(3, "0")}Z` };
}
function mongoShellNumberLongToExtendedJson(value: string): unknown {
const match = value.match(MONGO_SHELL_NUMBER_LONG_PATTERN);
return match ? { $numberLong: match[2] } : value;
}
export function buildMongoUpdateDocument(changes: Map<number, MongoInputValue>, columns: string[], originalDocument?: unknown): Record<string, unknown> {
const setFields: Record<string, unknown> = {};
const unsetFields: Record<string, unknown> = {};