mirror of
https://github.com/t8y2/dbx.git
synced 2026-10-21 06:00:39 +00:00
fix(mongodb): apply substring filters on non-string columns
This commit is contained in:
Loc Nguyen
@@ -73,6 +73,68 @@ describe("document store structured filters", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("matches substrings that are not valid values of the column's own type", () => {
|
||||
// Regression for #10056: the typed parse ran before the operator was known, so searching a
|
||||
// JSON/number/date/boolean column for text threw and the Apply click did nothing.
|
||||
const samples: Array<[string, unknown]> = [
|
||||
["json object", { a: 1, b: "abc" }],
|
||||
["number", 42],
|
||||
["int64", { $numberLong: "42" }],
|
||||
["date", { $date: "2026-01-01T00:00:00Z" }],
|
||||
["boolean", true],
|
||||
];
|
||||
for (const [label, sampleValue] of samples) {
|
||||
for (const mode of ["like", "not-like", "begins-with", "ends-with"] as const) {
|
||||
const rule = { id: mode, fieldName: "MessageText", mode, rawValue: "abc", conjunction: "AND" as const };
|
||||
expect(() => buildDocumentFilterCondition(rule, { kind: "mongodb", sampleValue }), `${mode} on ${label}`).not.toThrow();
|
||||
expect(buildDocumentFilterCondition(rule, { kind: "mongodb", sampleValue }), `${mode} on ${label}`).not.toBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
// Non-string columns are matched through the $convert coercion form: plain $regex only
|
||||
// matches fields that hold a string, so it silently filtered such columns to 0 rows.
|
||||
expect(buildDocumentFilterCondition({ id: "contains", fieldName: "MessageText", mode: "like", rawValue: "abc", conjunction: "AND" }, { kind: "mongodb", sampleValue: { a: 1 } })).toEqual({
|
||||
$expr: {
|
||||
$regexMatch: {
|
||||
input: { $convert: { input: "$MessageText", to: "string", onError: "", onNull: "" } },
|
||||
regex: "abc",
|
||||
options: "i",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(buildDocumentFilterCondition({ id: "starts", fieldName: "MessageText", mode: "begins-with", rawValue: "abc", conjunction: "AND" }, { kind: "mongodb", sampleValue: { $date: "2026-01-01T00:00:00Z" } })).toEqual({
|
||||
$expr: {
|
||||
$regexMatch: {
|
||||
input: { $convert: { input: "$MessageText", to: "string", onError: "", onNull: "" } },
|
||||
regex: "^abc",
|
||||
options: "i",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(buildDocumentFilterCondition({ id: "not-contains", fieldName: "MessageText", mode: "not-like", rawValue: "abc", conjunction: "AND" }, { kind: "mongodb", sampleValue: true })).toEqual({
|
||||
$expr: {
|
||||
$not: [
|
||||
{
|
||||
$regexMatch: {
|
||||
input: { $convert: { input: "$MessageText", to: "string", onError: "", onNull: "" } },
|
||||
regex: "abc",
|
||||
options: "i",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
// Solr keeps the plain $regex path even for non-string samples: its driver translates
|
||||
// anchored $regex into fq clauses but cannot translate $expr.
|
||||
expect(buildDocumentFilterCondition({ id: "starts", fieldName: "MessageText", mode: "begins-with", rawValue: "abc", conjunction: "AND" }, { kind: "solr", sampleValue: 42 })).toEqual({
|
||||
MessageText: { $regex: "^abc", $options: "i" },
|
||||
});
|
||||
|
||||
// Operators that genuinely need a typed value still reject text that is not one.
|
||||
expect(() => buildDocumentFilterCondition({ id: "eq", fieldName: "age", mode: "equals", rawValue: "abc", conjunction: "AND" }, { kind: "mongodb", sampleValue: 28 })).toThrow(/number/);
|
||||
expect(() => buildDocumentFilterCondition({ id: "gt", fieldName: "age", mode: "greater-than", rawValue: "abc", conjunction: "AND" }, { kind: "mongodb", sampleValue: 28 })).toThrow(/number/);
|
||||
});
|
||||
|
||||
it("keeps the MongoDB _id sample for automatic type inference", () => {
|
||||
const tree = documentFieldPathTreeFromDocuments([{ _id: "001", name: "Alice" }]);
|
||||
|
||||
|
||||
@@ -542,40 +542,45 @@ export function buildDocumentFilterCondition(rule: DocumentFilterRule, options:
|
||||
// Range and list predicates parse their raw input inside their own case, so the
|
||||
// single-value parse must not run for them (it would reject "18, 30" as a number).
|
||||
const parsesSingleValue = documentFilterModeNeedsValue(rule.mode) && !documentFilterModeUsesList(rule.mode) && !documentFilterModeUsesRange(rule.mode);
|
||||
const value = parsesSingleValue ? parseDocumentFilterValue(rule.rawValue, { ...options, valueType: rule.valueType }) : null;
|
||||
// Substring predicates match against the text the user typed, so the typed parse stays lazy:
|
||||
// coercing "abc" to the column's own type (number, date, boolean, or a JSON object) throws,
|
||||
// and that error reached the filter bar as an Apply click that appeared to do nothing.
|
||||
const typedValue = () => (parsesSingleValue ? parseDocumentFilterValue(rule.rawValue, { ...options, valueType: rule.valueType }) : null);
|
||||
const textValue = parsesSingleValue ? String(parseDocumentFilterValue(rule.rawValue)) : "";
|
||||
switch (rule.mode) {
|
||||
case "equals":
|
||||
return { [rule.fieldName]: value };
|
||||
return { [rule.fieldName]: typedValue() };
|
||||
case "not-equals":
|
||||
return { [rule.fieldName]: { $ne: value } };
|
||||
return { [rule.fieldName]: { $ne: typedValue() } };
|
||||
case "like":
|
||||
if (options.kind === "dynamodb") return { [rule.fieldName]: { $contains: value } };
|
||||
if (options.kind === "mongodb" && mongoFilterValueIsNumeric(rule.valueType, options.sampleValue)) {
|
||||
return mongoNumericContainsCondition(rule.fieldName, textValue);
|
||||
if (options.kind === "dynamodb") return { [rule.fieldName]: { $contains: typedValue() } };
|
||||
if (options.kind === "mongodb" && mongoFilterValueNeedsTextCoercion(rule.valueType, options.sampleValue)) {
|
||||
return mongoTextCoercionContainsCondition(rule.fieldName, textValue);
|
||||
}
|
||||
return { [rule.fieldName]: { $regex: escapeRegexLiteral(textValue), $options: "i" } };
|
||||
case "not-like":
|
||||
if (options.kind === "dynamodb") return { [rule.fieldName]: { $notContains: value } };
|
||||
if (options.kind === "mongodb" && mongoFilterValueIsNumeric(rule.valueType, options.sampleValue)) {
|
||||
return mongoNumericContainsCondition(rule.fieldName, textValue, true);
|
||||
if (options.kind === "dynamodb") return { [rule.fieldName]: { $notContains: typedValue() } };
|
||||
if (options.kind === "mongodb" && mongoFilterValueNeedsTextCoercion(rule.valueType, options.sampleValue)) {
|
||||
return mongoTextCoercionContainsCondition(rule.fieldName, textValue, true);
|
||||
}
|
||||
return { [rule.fieldName]: { $not: { $regex: escapeRegexLiteral(textValue), $options: "i" } } };
|
||||
case "begins-with":
|
||||
case "ends-with": {
|
||||
if (!mongoDocumentFilterKind(options)) return null;
|
||||
const pattern = `${rule.mode === "begins-with" ? "^" : ""}${escapeRegexLiteral(textValue)}${rule.mode === "ends-with" ? "$" : ""}`;
|
||||
if (mongoFilterValueIsNumeric(rule.valueType, options.sampleValue)) return mongoNumericRegexCondition(rule.fieldName, pattern);
|
||||
// The coercion form is MongoDB-only: Solr shares this filter shape but translates
|
||||
// anchored $regex into fq wildcards and cannot translate $expr.
|
||||
if (options.kind === "mongodb" && mongoFilterValueNeedsTextCoercion(rule.valueType, options.sampleValue)) return mongoTextCoercionRegexCondition(rule.fieldName, pattern);
|
||||
return { [rule.fieldName]: { $regex: pattern, $options: "i" } };
|
||||
}
|
||||
case "greater-than":
|
||||
return { [rule.fieldName]: { $gt: value } };
|
||||
return { [rule.fieldName]: { $gt: typedValue() } };
|
||||
case "greater-than-or-equal":
|
||||
return { [rule.fieldName]: { $gte: value } };
|
||||
return { [rule.fieldName]: { $gte: typedValue() } };
|
||||
case "less-than":
|
||||
return { [rule.fieldName]: { $lt: value } };
|
||||
return { [rule.fieldName]: { $lt: typedValue() } };
|
||||
case "less-than-or-equal":
|
||||
return { [rule.fieldName]: { $lte: value } };
|
||||
return { [rule.fieldName]: { $lte: typedValue() } };
|
||||
case "in":
|
||||
case "not-in": {
|
||||
if (!mongoDocumentFilterKind(options)) return null;
|
||||
@@ -660,19 +665,21 @@ function unquoteSingleQuotedListToken(token: string): string {
|
||||
return token;
|
||||
}
|
||||
|
||||
function mongoFilterValueIsNumeric(valueType?: DocumentFilterValueType, sampleValue?: unknown): boolean {
|
||||
if (valueType === "number" || valueType === "int32" || valueType === "int64" || valueType === "decimal128") return true;
|
||||
if (typeof sampleValue === "number") return true;
|
||||
if (!isPlainRecord(sampleValue)) return false;
|
||||
const keys = Object.keys(sampleValue);
|
||||
return keys.length === 1 && ["$numberInt", "$numberLong", "$numberDouble", "$numberDecimal"].includes(keys[0]);
|
||||
/** Substring filters need the text-coercion form whenever the column does not hold plain
|
||||
* strings: MongoDB `$regex` only matches a field whose value is itself a string (or a string
|
||||
* element of an array), so numbers, dates, booleans, ObjectIds and embedded documents would
|
||||
* otherwise silently match nothing. */
|
||||
function mongoFilterValueNeedsTextCoercion(valueType?: DocumentFilterValueType, sampleValue?: unknown): boolean {
|
||||
if (valueType && valueType !== "auto" && valueType !== "string") return true;
|
||||
const inferred = inferMongoFilterValueType(sampleValue);
|
||||
return inferred !== null && inferred !== "string";
|
||||
}
|
||||
|
||||
function mongoNumericContainsCondition(fieldName: string, textValue: string, negate = false): Record<string, unknown> {
|
||||
return mongoNumericRegexCondition(fieldName, escapeRegexLiteral(textValue), negate);
|
||||
function mongoTextCoercionContainsCondition(fieldName: string, textValue: string, negate = false): Record<string, unknown> {
|
||||
return mongoTextCoercionRegexCondition(fieldName, escapeRegexLiteral(textValue), negate);
|
||||
}
|
||||
|
||||
function mongoNumericRegexCondition(fieldName: string, regex: string, negate = false): Record<string, unknown> {
|
||||
function mongoTextCoercionRegexCondition(fieldName: string, regex: string, negate = false): Record<string, unknown> {
|
||||
const regexMatch = { $regexMatch: { input: { $convert: { input: `$${fieldName}`, to: "string", onError: "", onNull: "" } }, regex, options: "i" } };
|
||||
return negate ? { $expr: { $not: [regexMatch] } } : { $expr: regexMatch };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user