fix(mongodb): support shell regex literals

This commit is contained in:
Guoyu Su
2026-09-02 20:36:59 +08:00
committed by GitHub
parent d6c4457756
commit d92e119464
4 changed files with 638 additions and 8 deletions
+24
View File
@@ -2655,6 +2655,11 @@ fn json_filter_value_to_bson(value: &serde_json::Value, field_name: Option<&str>
}
serde_json::Value::Object(obj) => {
if obj.len() == 1 {
if obj.contains_key("$regularExpression") {
if let Ok(Some(value)) = parse_extended_json_value(obj) {
return value;
}
}
if let Some(serde_json::Value::String(hex)) = obj.get("$oid") {
if let Ok(oid) = ObjectId::parse_str(hex) {
return Bson::ObjectId(oid);
@@ -3968,6 +3973,25 @@ mod tests {
assert!(matches!(doc.get("count"), Some(Bson::Int64(42))));
}
#[test]
fn json_filter_to_document_parses_extended_json_regex() {
let value = serde_json::json!({
"packagingRatio": {
"$regularExpression": {
"pattern": "^[^::]*盒",
"options": "im",
}
}
});
let doc = json_filter_to_document(&value).unwrap();
assert!(matches!(
doc.get("packagingRatio"),
Some(Bson::RegularExpression(regex))
if regex.pattern == "^[^::]*盒" && regex.options == "im"
));
}
#[test]
fn json_object_to_document_extended_json_keeps_shell_date_strings_literal() {
let value = serde_json::json!({
+358 -7
View File
@@ -847,17 +847,228 @@ fn parse_explain_verbosity(args: &[String]) -> Result<String, String> {
}
fn normalized_json(input: &str) -> Result<String, String> {
let transformed = transform_shell_constructors(input.trim())?;
let transformed = transform_shell_regex_literals(input.trim())?;
let transformed = transform_shell_constructors(&transformed)?;
let value: Value =
json5::from_str(&transformed).map_err(|error| format!("Invalid MongoDB JSON argument: {error}"))?;
serde_json::to_string(&value).map_err(|error| error.to_string())
}
struct ShellRegexLiteral {
end: usize,
pattern: String,
options: String,
}
fn shell_regex_literal_at(input: &str, index: usize) -> Result<Option<ShellRegexLiteral>, String> {
if !input[index..].starts_with('/') || !is_shell_regex_value_position(input, index) {
return Ok(None);
}
read_shell_regex_literal(input, index).map(Some)
}
fn read_shell_regex_literal(input: &str, index: usize) -> Result<ShellRegexLiteral, String> {
let mut cursor = index + 1;
let mut pattern = String::new();
let mut escaped = false;
let mut in_character_class = false;
let mut closed = false;
while cursor < input.len() {
let current = input[cursor..].chars().next().ok_or("Invalid MongoDB regex literal.")?;
if matches!(current, '\n' | '\r' | '\u{2028}' | '\u{2029}') {
return Err("MongoDB regex literals cannot contain an unescaped line break.".to_string());
}
cursor += current.len_utf8();
if escaped {
pattern.push(current);
escaped = false;
continue;
}
if current == '\\' {
pattern.push(current);
escaped = true;
continue;
}
if current == '[' {
in_character_class = true;
} else if current == ']' && in_character_class {
in_character_class = false;
} else if current == '/' && !in_character_class {
closed = true;
break;
}
pattern.push(current);
}
if !closed {
return Err("Unclosed MongoDB regex literal.".to_string());
}
let mut options = Vec::new();
while cursor < input.len() {
let option = input[cursor..].chars().next().ok_or("Invalid MongoDB regex literal.")?;
if !option.is_ascii_alphabetic() {
break;
}
// JS-only regex flags (d/g/v/y) have no server-side meaning for a
// stored regex literal — MongoDB's $regex has no global modifier — so
// drop them instead of failing the whole command.
if matches!(option, 'd' | 'g' | 'v' | 'y') {
cursor += option.len_utf8();
continue;
}
if !matches!(option, 'i' | 'm' | 's' | 'u') || options.contains(&option) {
return Err(format!("Unsupported or duplicate MongoDB regex option: {option}"));
}
options.push(option);
cursor += option.len_utf8();
}
options.sort_unstable();
Ok(ShellRegexLiteral { end: cursor, pattern, options: options.into_iter().collect() })
}
fn transform_shell_regex_literals(input: &str) -> Result<String, String> {
let mut output = String::with_capacity(input.len());
let mut index = 0;
while index < input.len() {
let rest = &input[index..];
let ch = rest.chars().next().ok_or("Invalid MongoDB argument.")?;
if matches!(ch, '"' | '\'') {
let start = index;
index += ch.len_utf8();
let mut escaped = false;
while index < input.len() {
let current = input[index..].chars().next().ok_or("Invalid MongoDB argument.")?;
index += current.len_utf8();
if escaped {
escaped = false;
} else if current == '\\' {
escaped = true;
} else if current == ch {
break;
}
}
output.push_str(&input[start..index]);
continue;
}
if rest.starts_with("//") {
let end = rest.find(['\n', '\r']).map(|offset| index + offset).unwrap_or(input.len());
output.push_str(&input[index..end]);
index = end;
continue;
}
if rest.starts_with("/*") {
let end = rest.find("*/").map(|offset| index + offset + 2).unwrap_or(input.len());
output.push_str(&input[index..end]);
index = end;
continue;
}
if ch != '/' || !is_shell_regex_value_position(input, index) {
output.push(ch);
index += ch.len_utf8();
continue;
}
let literal = shell_regex_literal_at(input, index)?.ok_or("Invalid MongoDB regex literal.")?;
output.push_str(
&serde_json::to_string(&serde_json::json!({
"$regularExpression": {
"pattern": literal.pattern,
"options": literal.options,
}
}))
.map_err(|error| error.to_string())?,
);
index = literal.end;
}
Ok(output)
}
fn is_shell_regex_value_position(input: &str, index: usize) -> bool {
let mut previous_significant = None;
let mut cursor = 0;
while cursor < index {
let rest = &input[cursor..];
let current = rest.chars().next().expect("cursor is on a character boundary");
if matches!(current, '"' | '\'') {
let quote = current;
cursor += current.len_utf8();
let mut escaped = false;
while cursor < index {
let character = input[cursor..].chars().next().expect("cursor is on a character boundary");
cursor += character.len_utf8();
if escaped {
escaped = false;
} else if character == '\\' {
escaped = true;
} else if character == quote {
break;
}
}
previous_significant = Some('\0');
continue;
}
if rest.starts_with("//") {
cursor = rest.find(['\n', '\r']).map(|offset| cursor + offset).unwrap_or(index).min(index);
continue;
}
if rest.starts_with("/*") {
cursor = rest.find("*/").map(|offset| cursor + offset + 2).unwrap_or(index).min(index);
continue;
}
if current == '/' && is_shell_regex_value_prefix(previous_significant) {
if let Ok(literal) = read_shell_regex_literal(input, cursor) {
if literal.end <= index {
previous_significant = Some('\0');
cursor = literal.end;
continue;
}
}
}
if !current.is_whitespace() {
previous_significant = Some(current);
}
cursor += current.len_utf8();
}
is_shell_regex_value_prefix(previous_significant)
}
fn is_shell_regex_value_prefix(previous_significant: Option<char>) -> bool {
previous_significant.is_none_or(|character| matches!(character, ':' | '[' | ',' | '('))
}
fn transform_shell_constructors(input: &str) -> Result<String, String> {
let mut output = String::with_capacity(input.len());
let mut index = 0;
while index < input.len() {
let rest = &input[index..];
let ch = rest.chars().next().ok_or("Invalid MongoDB argument.")?;
if matches!(ch, '"' | '\'') {
let start = index;
index += ch.len_utf8();
let mut escaped = false;
while index < input.len() {
let current = input[index..].chars().next().ok_or("Invalid MongoDB argument.")?;
index += current.len_utf8();
if escaped {
escaped = false;
} else if current == '\\' {
escaped = true;
} else if current == ch {
break;
}
}
output.push_str(&input[start..index]);
continue;
}
let constructor = if rest.starts_with("ObjectId(") {
Some("ObjectId(")
} else if rest.starts_with("ISODate(") {
@@ -866,7 +1077,6 @@ fn transform_shell_constructors(input: &str) -> Result<String, String> {
None
};
let Some(constructor) = constructor else {
let ch = rest.chars().next().ok_or("Invalid MongoDB argument.")?;
output.push(ch);
index += ch.len_utf8();
continue;
@@ -944,14 +1154,15 @@ fn aggregate_writes(pipeline: &str) -> bool {
}
fn matching_paren(source: &str, open: usize) -> Option<usize> {
let bytes = source.as_bytes();
let mut depth = 0;
let mut quote = None;
let mut escape = false;
for (index, byte) in bytes.iter().enumerate().skip(open) {
let ch = *byte as char;
let mut index = open;
while index < source.len() {
let ch = source[index..].chars().next()?;
if escape {
escape = false;
index += ch.len_utf8();
continue;
}
if quote.is_some() {
@@ -960,10 +1171,16 @@ fn matching_paren(source: &str, open: usize) -> Option<usize> {
} else if Some(ch) == quote {
quote = None;
}
index += ch.len_utf8();
continue;
}
if ch == '\'' || ch == '"' || ch == '`' {
quote = Some(ch);
} else if ch == '/' && is_shell_regex_value_position(source, index) {
if let Ok(Some(literal)) = shell_regex_literal_at(source, index) {
index = literal.end;
continue;
}
} else if ch == '(' {
depth += 1;
} else if ch == ')' {
@@ -972,6 +1189,7 @@ fn matching_paren(source: &str, open: usize) -> Option<usize> {
return Some(index);
}
}
index += ch.len_utf8();
}
None
}
@@ -985,10 +1203,12 @@ fn split_top_level(source: &str) -> Vec<String> {
let mut depth = 0;
let mut quote = None;
let mut escape = false;
for (index, byte) in source.as_bytes().iter().enumerate() {
let ch = *byte as char;
let mut index = 0;
while index < source.len() {
let ch = source[index..].chars().next().expect("index is on a character boundary");
if escape {
escape = false;
index += ch.len_utf8();
continue;
}
if quote.is_some() {
@@ -997,10 +1217,16 @@ fn split_top_level(source: &str) -> Vec<String> {
} else if Some(ch) == quote {
quote = None;
}
index += ch.len_utf8();
continue;
}
if ch == '\'' || ch == '"' || ch == '`' {
quote = Some(ch);
} else if ch == '/' && is_shell_regex_value_position(source, index) {
if let Ok(Some(literal)) = shell_regex_literal_at(source, index) {
index = literal.end;
continue;
}
} else if matches!(ch, '(' | '[' | '{') {
depth += 1;
} else if matches!(ch, ')' | ']' | '}') {
@@ -1009,6 +1235,7 @@ fn split_top_level(source: &str) -> Vec<String> {
result.push(source[start..index].trim().to_string());
start = index + 1;
}
index += ch.len_utf8();
}
result.push(source[start..].trim().to_string());
result
@@ -1231,6 +1458,130 @@ mod tests {
assert!(matches!(update, MongoCommand::Update { many: true, options: Some(_), .. }));
}
#[test]
fn parses_mongo_shell_regex_literals_in_update_filters() {
let command = parse(
r#"db.code_info.updateMany(
{ level: "SECOND_LAYER", position: "B", packagingRatio: /^[^::]*盒/, type: { $ne: "PACK_IN" } },
{ $set: { type: "PACK_IN" }, $currentDate: { lastUpdateTime: true } }
)"#,
)
.unwrap();
let MongoCommand::Update { collection, filter, many, .. } = command else {
panic!("expected update command");
};
assert_eq!(collection, "code_info");
assert!(many);
assert_eq!(
serde_json::from_str::<Value>(&filter).unwrap(),
serde_json::json!({
"level": "SECOND_LAYER",
"position": "B",
"packagingRatio": {
"$regularExpression": {
"pattern": "^[^::]*盒",
"options": "",
}
},
"type": { "$ne": "PACK_IN" },
})
);
}
#[test]
fn drops_js_only_regex_literal_flags() {
let command = parse(r#"db.items.find({ value: /abc/gi })"#).unwrap();
let MongoCommand::Find { filter, .. } = command else {
panic!("expected find command");
};
assert_eq!(
serde_json::from_str::<Value>(&filter).unwrap(),
serde_json::json!({
"value": {
"$regularExpression": {
"pattern": "abc",
"options": "i",
}
}
})
);
}
#[test]
fn ignores_delimiters_inside_mongo_shell_regex_literals() {
let command = parse(r#"db.items.updateMany({ value: /a\),b/ }, { $set: { matched: true } })"#).unwrap();
let MongoCommand::Update { filter, many, .. } = command else {
panic!("expected update command");
};
assert!(many);
assert_eq!(
serde_json::from_str::<Value>(&filter).unwrap(),
serde_json::json!({
"value": {
"$regularExpression": {
"pattern": r#"a\),b"#,
"options": "",
}
}
})
);
}
#[test]
fn normalizes_regex_literal_edges_without_touching_strings_or_comments() {
let normalized = normalized_json(
r#"{
pattern: /a\/b[/::]/mi,
constructorText: "ObjectId('literal')",
url: "https://example.com/a/b",
// slash comments stay comments
active: true,
}"#,
)
.unwrap();
assert_eq!(
serde_json::from_str::<Value>(&normalized).unwrap(),
serde_json::json!({
"pattern": {
"$regularExpression": {
"pattern": r#"a\/b[/::]"#,
"options": "im",
}
},
"constructorText": "ObjectId('literal')",
"url": "https://example.com/a/b",
"active": true,
})
);
for source in ["{pattern: /unterminated}", "{pattern: /value/ii}", "{pattern: /value/q}"] {
assert!(normalized_json(source).is_err(), "{source}");
}
}
#[test]
fn normalizes_regex_literals_after_comments() {
let normalized = normalized_json(
r#"{
block: /* explain the pattern */ /block/i,
line: // explain the next pattern
/line/m,
}"#,
)
.unwrap();
assert_eq!(
serde_json::from_str::<Value>(&normalized).unwrap(),
serde_json::json!({
"block": { "$regularExpression": { "pattern": "block", "options": "i" } },
"line": { "$regularExpression": { "pattern": "line", "options": "m" } },
})
);
}
#[test]
fn parses_legacy_update_with_single_and_multi_semantics() {
assert_eq!(
+82
View File
@@ -35,6 +35,7 @@ import {
} from "../../apps/desktop/src/lib/mongo/mongoShellCommand.ts";
import type { MongoWriteCommand } from "../../apps/desktop/src/lib/mongo/mongoShellCommand.ts";
import { buildMongoUpdateDocument as buildMongoDocumentUpdate, formatMongoShellLiteral as formatMongoDocumentShellLiteral } from "../../apps/desktop/src/lib/mongo/mongoDocumentValues.ts";
import { normalizeJsonArgument } from "../mongo-shell/src/json.ts";
test("parseMongoFindCommand parses db collection find with an empty JSON filter", () => {
assert.deepEqual(parseMongoFindCommand("db.users.find({})"), {
@@ -596,6 +597,87 @@ test("parseMongoWriteCommand accepts updateMany arrayFilters options", () => {
);
});
test("parseMongoWriteCommand accepts Mongo shell regex literals", () => {
const command = parseMongoWriteCommand(`db.code_info.updateMany(
{ level: "SECOND_LAYER", position: "B", packagingRatio: /^[^::]*盒/, type: { $ne: "PACK_IN" } },
{ $set: { type: "PACK_IN" }, $currentDate: { lastUpdateTime: true } }
)`);
assert.equal(command?.kind, "update");
if (command?.kind !== "update") return;
assert.equal(command.collection, "code_info");
assert.equal(command.many, true);
assert.deepEqual(JSON.parse(command.filter), {
level: "SECOND_LAYER",
position: "B",
packagingRatio: { $regularExpression: { pattern: "^[^::]*盒", options: "" } },
type: { $ne: "PACK_IN" },
});
});
test("parseMongoWriteCommand ignores delimiters inside Mongo shell regex literals", () => {
const command = parseMongoWriteCommand(String.raw`db.items.updateMany({ value: /a\),b/ }, { $set: { matched: true } })`);
assert.equal(command?.kind, "update");
if (command?.kind !== "update") return;
assert.deepEqual(JSON.parse(command.filter), {
value: { $regularExpression: { pattern: "a\\),b", options: "" } },
});
});
test("normalizeJsonArgument handles regex escapes, character classes, flags, strings, and comments", () => {
const normalized = normalizeJsonArgument(`{
pattern: /a\\/b[/::]/mi,
url: "https://example.com/a/b",
note: "// remains text",
// slash comments remain comments
active: true
}`);
assert.ok(normalized);
assert.deepEqual(JSON.parse(normalized), {
pattern: { $regularExpression: { pattern: "a\\/b[/::]", options: "im" } },
url: "https://example.com/a/b",
note: "// remains text",
active: true,
});
});
test("normalizeJsonArgument accepts a regex literal after comments", () => {
const normalized = normalizeJsonArgument(`{
block: /* explain the pattern */ /block/i,
line: // explain the next pattern
/line/m
}`);
assert.ok(normalized);
assert.deepEqual(JSON.parse(normalized), {
block: { $regularExpression: { pattern: "block", options: "i" } },
line: { $regularExpression: { pattern: "line", options: "m" } },
});
});
test("normalizeJsonArgument drops JS-only regex literal flags", () => {
const normalized = normalizeJsonArgument(`{
global: /value/g,
sticky: /value/yd,
verbose: /value/givs
}`);
assert.ok(normalized);
assert.deepEqual(JSON.parse(normalized), {
global: { $regularExpression: { pattern: "value", options: "" } },
sticky: { $regularExpression: { pattern: "value", options: "" } },
verbose: { $regularExpression: { pattern: "value", options: "is" } },
});
});
test("normalizeJsonArgument rejects malformed Mongo shell regex literals", () => {
for (const source of ["{pattern: /unterminated}", "{pattern: /value/ii}", "{pattern: /value/q}"]) {
assert.equal(normalizeJsonArgument(source), null, source);
}
});
test("parseMongoWriteCommand parses createIndex with optional options", () => {
assert.deepEqual(parseMongoWriteCommand("db.users.createIndex({email: 1}, {unique: true, name: 'users_email_unique'})"), {
kind: "createIndex",
+174 -1
View File
@@ -7,7 +7,9 @@
export function normalizeJsonArgument(value: string): string | null {
const trimmed = value.trim();
if (!trimmed) return "{}";
const withoutComments = stripMongoJsonComments(trimmed).trim();
const withRegexLiterals = replaceMongoRegexLiterals(trimmed);
if (!withRegexLiterals) return null;
const withoutComments = stripMongoJsonComments(withRegexLiterals).trim();
if (!withoutComments) return "{}";
const withoutEjsonDeserialize = replaceMongoEjsonDeserialize(withoutComments);
// Rewrite mongo shell constructors that are not valid JSON into extended JSON
@@ -22,6 +24,160 @@ export function normalizeJsonArgument(value: string): string | null {
}
}
const MONGO_REGEX_LITERAL_OPTIONS = new Set(["i", "m", "s", "u"]);
// JS-only regex flags with no server-side meaning for a stored regex literal
// (MongoDB's $regex has no global modifier) are dropped instead of failing
// the whole command.
const MONGO_REGEX_LITERAL_IGNORED_OPTIONS = new Set(["d", "g", "v", "y"]);
interface MongoRegexLiteral {
end: number;
pattern: string;
options: string;
}
function mongoRegexLiteralAt(source: string, index: number): MongoRegexLiteral | null {
if (source[index] !== "/" || !isMongoRegexValuePosition(source, index)) return null;
return readMongoRegexLiteral(source, index);
}
function readMongoRegexLiteral(source: string, index: number): MongoRegexLiteral | null {
let cursor = index + 1;
let pattern = "";
let escaped = false;
let inCharacterClass = false;
let closed = false;
while (cursor < source.length) {
const current = source[cursor] ?? "";
if (current === "\n" || current === "\r" || current === "\u2028" || current === "\u2029") return null;
if (escaped) {
pattern += current;
escaped = false;
cursor += 1;
continue;
}
if (current === "\\") {
pattern += current;
escaped = true;
cursor += 1;
continue;
}
if (current === "[") inCharacterClass = true;
else if (current === "]" && inCharacterClass) inCharacterClass = false;
else if (current === "/" && !inCharacterClass) {
closed = true;
cursor += 1;
break;
}
pattern += current;
cursor += 1;
}
if (!closed) return null;
const options: string[] = [];
while (/[A-Za-z]/.test(source[cursor] ?? "")) {
const option = source[cursor] ?? "";
cursor += 1;
if (MONGO_REGEX_LITERAL_IGNORED_OPTIONS.has(option)) continue;
if (!MONGO_REGEX_LITERAL_OPTIONS.has(option) || options.includes(option)) return null;
options.push(option);
}
options.sort();
return { end: cursor, pattern, options: options.join("") };
}
function replaceMongoRegexLiterals(source: string): string | null {
let result = "";
let index = 0;
while (index < source.length) {
const char = source[index] ?? "";
if (char === '"' || char === "'") {
const start = index;
const quote = char;
index += 1;
let escaped = false;
while (index < source.length) {
const current = source[index] ?? "";
index += 1;
if (escaped) escaped = false;
else if (current === "\\") escaped = true;
else if (current === quote) break;
}
result += source.slice(start, index);
continue;
}
const commentEnd = mongoCommentEndAt(source, index);
if (commentEnd !== null) {
result += source.slice(index, commentEnd);
index = commentEnd;
continue;
}
if (char !== "/" || !isMongoRegexValuePosition(source, index)) {
result += char;
index += 1;
continue;
}
const literal = mongoRegexLiteralAt(source, index);
if (!literal) return null;
result += JSON.stringify({ $regularExpression: { pattern: literal.pattern, options: literal.options } });
index = literal.end;
}
return result;
}
function isMongoRegexValuePosition(source: string, index: number): boolean {
let previousSignificant: string | null = null;
let cursor = 0;
while (cursor < index) {
const char = source[cursor] ?? "";
if (char === '"' || char === "'") {
const quote = char;
cursor += 1;
let escaped = false;
while (cursor < index) {
const current = source[cursor] ?? "";
cursor += 1;
if (escaped) escaped = false;
else if (current === "\\") escaped = true;
else if (current === quote) break;
}
previousSignificant = "value";
continue;
}
const commentEnd = mongoCommentEndAt(source, cursor);
if (commentEnd !== null) {
cursor = Math.min(commentEnd, index);
continue;
}
if (char === "/" && isMongoRegexValuePrefix(previousSignificant)) {
const literal = readMongoRegexLiteral(source, cursor);
if (literal && literal.end <= index) {
previousSignificant = "value";
cursor = literal.end;
continue;
}
}
if (!/\s/.test(char)) previousSignificant = char;
cursor += 1;
}
return isMongoRegexValuePrefix(previousSignificant);
}
function isMongoRegexValuePrefix(previousSignificant: string | null): boolean {
return previousSignificant === null || previousSignificant === ":" || previousSignificant === "[" || previousSignificant === "," || previousSignificant === "(";
}
/** Object-shaped shell arg (options documents, etc.). */
export function parseMongoObjectArgument(arg: string | undefined): string | null {
if (!arg?.trim()) return null;
@@ -82,6 +238,12 @@ export function splitTopLevel(source: string): string[] {
continue;
}
const regexLiteral = mongoRegexLiteralAt(source, i);
if (regexLiteral) {
i = regexLiteral.end - 1;
continue;
}
if (char === '"' || char === "'") quote = char;
else if (char === "{" || char === "[" || char === "(") depth += 1;
else if (char === "}" || char === "]" || char === ")") depth -= 1;
@@ -116,6 +278,12 @@ export function findMatchingParen(source: string, openIndex: number): number {
continue;
}
const regexLiteral = mongoRegexLiteralAt(source, i);
if (regexLiteral) {
i = regexLiteral.end - 1;
continue;
}
if (char === '"' || char === "'") quote = char;
else if (char === "(") depth += 1;
else if (char === ")") {
@@ -145,6 +313,11 @@ export function hasUnclosedMongoDelimiters(source: string): boolean {
i = commentEnd - 1;
continue;
}
const regexLiteral = mongoRegexLiteralAt(source, i);
if (regexLiteral) {
i = regexLiteral.end - 1;
continue;
}
if (char === '"' || char === "'") {
quote = char;
continue;