fix(mongodb): use dedicated count for legacy collections

This commit is contained in:
Guoyu Su
2026-07-13 13:21:50 +08:00
committed by GitHub
parent 03bc87c77a
commit a99a34b4ab
28 changed files with 916 additions and 19 deletions
@@ -43,6 +43,7 @@ public final class AgentProtocol {
* MongoDB read path that returns documents as relaxed Extended JSON for transfer.
*/
public static final String MONGO_METHOD_FIND_DOCUMENTS_EXTENDED_JSON = "find_documents_extended_json";
public static final String MONGO_METHOD_COUNT_DOCUMENTS = "count_documents";
public static final String MONGO_METHOD_SERVER_VERSION = "server_version";
public static final String MONGO_METHOD_CREATE_INDEX = "create_index";
public static final String MONGO_METHOD_DROP_INDEXES = "drop_indexes";
@@ -124,6 +125,7 @@ public final class AgentProtocol {
MONGO_METHOD_LIST_COLLECTIONS,
MONGO_METHOD_FIND_DOCUMENTS,
MONGO_METHOD_FIND_DOCUMENTS_EXTENDED_JSON,
MONGO_METHOD_COUNT_DOCUMENTS,
MONGO_METHOD_SERVER_VERSION,
MONGO_METHOD_CREATE_INDEX,
MONGO_METHOD_DROP_INDEXES,
@@ -69,6 +69,7 @@
"list_collections",
"find_documents",
"find_documents_extended_json",
"count_documents",
"server_version",
"create_index",
"drop_indexes",
@@ -429,6 +429,29 @@ public final class MongoAgent {
return result;
}
private static Object countDocuments(JsonObject params) {
MongoClient c = requireClient();
String database = params.get("database").getAsString();
String collection = params.get("collection").getAsString();
Document filterDoc = documentOrNull(params, "filter");
if (filterDoc == null) {
filterDoc = new Document();
}
boolean accurate = !params.has("accurate") || params.get("accurate").getAsBoolean();
if (accurate) {
return c.getDatabase(database).getCollection(collection).countDocuments(filterDoc);
}
// MongoDB 3.4 count() needs the legacy command to avoid the slow countDocuments path.
Document result = c.getDatabase(database).runCommand(new Document("count", collection).append("query", filterDoc));
Object n = result.get("n");
if (n instanceof Number number) {
return number.longValue();
}
return 0L;
}
private static Object serverVersion(JsonObject params) {
MongoClient c = requireClient();
String database = defaultString(stringOrNull(params, "database"), "admin");
@@ -846,6 +869,7 @@ public final class MongoAgent {
case AgentProtocol.METHOD_LIST_INDEXES -> listIndexes(params);
case AgentProtocol.MONGO_METHOD_FIND_DOCUMENTS -> findDocuments(params);
case AgentProtocol.MONGO_METHOD_FIND_DOCUMENTS_EXTENDED_JSON -> findDocumentsExtendedJson(params);
case AgentProtocol.MONGO_METHOD_COUNT_DOCUMENTS -> countDocuments(params);
case AgentProtocol.MONGO_METHOD_SERVER_VERSION -> serverVersion(params);
case AgentProtocol.MONGO_METHOD_CREATE_INDEX -> createIndex(params);
case AgentProtocol.MONGO_METHOD_DROP_INDEXES -> dropIndexes(params);
@@ -110,6 +110,18 @@ class MongoAgentTest {
assertFalse(json.getAsJsonObject("error").get("message").getAsString().contains("Unknown method"));
}
@Test
void countDocumentsMethodIsRecognizedOverJsonRpc() {
String response = MongoAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":15,\"method\":\"count_documents\","
+ "\"params\":{\"database\":\"app\",\"collection\":\"orders\",\"filter\":\"{}\"}}");
JsonObject json = JsonParser.parseString(response).getAsJsonObject();
assertEquals(15, json.get("id").getAsInt());
assertEquals("Not connected", json.getAsJsonObject("error").get("message").getAsString());
assertFalse(json.getAsJsonObject("error").get("message").getAsString().contains("Unknown method"));
}
@Test
void parsesOptionalDocumentParameters() {
JsonObject params = new JsonObject();
+1
View File
@@ -432,6 +432,7 @@ export const mongoDropDatabase = forward("mongoDropDatabase");
export const mongoDropCollection = forward("mongoDropCollection");
export const documentFindDocuments = forward("documentFindDocuments");
export const mongoFindDocuments = forward("mongoFindDocuments");
export const mongoCountDocuments = forward("mongoCountDocuments");
export const mongoServerVersion = forward("mongoServerVersion");
export const mongoAggregateDocuments = forward("mongoAggregateDocuments");
export const mongoCollectionStats = forward("mongoCollectionStats");
+4
View File
@@ -2009,6 +2009,10 @@ export async function documentFindDocuments(connectionId: string, database: stri
return post("/api/document-store/find-documents", { connectionId, database, collection, skip, limit, filter, projection, sort, executionId });
}
export async function mongoCountDocuments(connectionId: string, database: string, collection: string, filter?: string, mode?: "accurate" | "legacy", executionId?: string): Promise<number> {
return post("/api/mongo/count-documents", { connectionId, database, collection, filter, mode, executionId });
}
export async function documentListGridFsFiles(connectionId: string, database: string, bucket: string, filter?: string, sort?: string): Promise<MongoGridFsFileInfo[]> {
return post("/api/document-store/list-gridfs-files", { connectionId, database, bucket, filter, sort });
}
+4
View File
@@ -1792,6 +1792,10 @@ export async function documentFindDocuments(connectionId: string, database: stri
return invoke("document_find_documents", { connectionId, database, collection, skip, limit, filter, projection, sort, executionId });
}
export async function mongoCountDocuments(connectionId: string, database: string, collection: string, filter?: string, mode?: "accurate" | "legacy", executionId?: string): Promise<number> {
return invoke("mongo_count_documents", { connectionId, database, collection, filter, mode, executionId });
}
export async function documentListGridFsFiles(connectionId: string, database: string, bucket: string, filter?: string, sort?: string): Promise<MongoGridFsFileInfo[]> {
return invoke("document_list_gridfs_files", { connectionId, database, bucket, filter, sort });
}
@@ -12,6 +12,7 @@ export interface MongoFindCommand {
export interface MongoCountDocumentsCommand {
collection: string;
filter: string;
mode: "accurate" | "legacy";
}
export interface MongoAggregateCommand {
@@ -145,8 +146,6 @@ export function applyMongoFindSort(input: string, column: string, direction: "as
export function parseMongoCountDocumentsCommand(input: string): MongoCountDocumentsCommand | null {
const source = input.trim().replace(/;$/, "").trim();
// Accept deprecated Mongo shell count helpers for old server workflows, but
// keep DBX's internal execution mapped to the countDocuments result shape.
return parseCollectionCountCommand(source, "countDocuments") ?? parseCollectionCountCommand(source, "count") ?? parseFindCountCommand(source);
}
@@ -166,6 +165,7 @@ function parseCollectionCountCommand(source: string, method: "countDocuments" |
return {
collection: target.collection,
filter,
mode: method === "countDocuments" ? "accurate" : "legacy",
};
}
@@ -188,6 +188,7 @@ function parseFindCountCommand(source: string): MongoCountDocumentsCommand | nul
return {
collection: target.collection,
filter,
mode: "legacy",
};
}
+3 -3
View File
@@ -2645,14 +2645,14 @@ export const useQueryStore = defineStore("query", () => {
}
case "countDocuments": {
console.info("[DBX][executeTabSql:mongo-count:start]", { traceId, collection: mongoCommand.collection, database: currentDatabase });
const result = await api.mongoFindDocuments(tab.connectionId, currentDatabase, mongoCommand.collection, 0, 1, mongoCommand.filter, undefined, undefined, executionId);
allResults.push(markQueryResultRowsRaw(annotateMongoResult(mongoCountToQueryResult(result.total, performance.now() - commandStartedAt))));
const total = await api.mongoCountDocuments(tab.connectionId, currentDatabase, mongoCommand.collection, mongoCommand.filter, mongoCommand.mode, executionId);
allResults.push(markQueryResultRowsRaw(annotateMongoResult(mongoCountToQueryResult(total, performance.now() - commandStartedAt))));
mongoEditTarget = undefined;
console.info("[DBX][executeTabSql:mongo-count:done]", {
traceId,
collection: mongoCommand.collection,
database: currentDatabase,
total: result.total,
total,
elapsed: elapsed(),
});
break;
@@ -69,6 +69,7 @@
"list_collections",
"find_documents",
"find_documents_extended_json",
"count_documents",
"server_version",
"create_index",
"drop_indexes",
+12 -1
View File
@@ -252,6 +252,7 @@ pub enum MongoAgentMethod {
ListCollections,
FindDocuments,
FindDocumentsExtendedJson,
CountDocuments,
ServerVersion,
CreateIndex,
DropIndexes,
@@ -264,11 +265,12 @@ pub enum MongoAgentMethod {
}
impl MongoAgentMethod {
pub const ALL: [Self; 13] = [
pub const ALL: [Self; 14] = [
Self::ListDatabases,
Self::ListCollections,
Self::FindDocuments,
Self::FindDocumentsExtendedJson,
Self::CountDocuments,
Self::ServerVersion,
Self::CreateIndex,
Self::DropIndexes,
@@ -286,6 +288,7 @@ impl MongoAgentMethod {
Self::ListCollections => "list_collections",
Self::FindDocuments => "find_documents",
Self::FindDocumentsExtendedJson => "find_documents_extended_json",
Self::CountDocuments => "count_documents",
Self::ServerVersion => "server_version",
Self::CreateIndex => "create_index",
Self::DropIndexes => "drop_indexes",
@@ -999,6 +1002,13 @@ impl AgentDriverClient {
self.call_mongo_method(MongoAgentMethod::FindDocumentsExtendedJson, params).await
}
pub async fn mongo_count_documents<T: DeserializeOwned + Send + 'static>(
&mut self,
params: Value,
) -> Result<T, String> {
self.call_mongo_method(MongoAgentMethod::CountDocuments, params).await
}
pub async fn mongo_server_version<T: DeserializeOwned + Send + 'static>(
&mut self,
database: &str,
@@ -1727,6 +1737,7 @@ mod tests {
assert_eq!(MongoAgentMethod::ListCollections.as_str(), "list_collections");
assert_eq!(MongoAgentMethod::FindDocuments.as_str(), "find_documents");
assert_eq!(MongoAgentMethod::FindDocumentsExtendedJson.as_str(), "find_documents_extended_json");
assert_eq!(MongoAgentMethod::CountDocuments.as_str(), "count_documents");
assert_eq!(MongoAgentMethod::ServerVersion.as_str(), "server_version");
assert_eq!(MongoAgentMethod::CreateIndex.as_str(), "create_index");
assert_eq!(MongoAgentMethod::DropIndexes.as_str(), "drop_indexes");
+25
View File
@@ -592,6 +592,31 @@ pub async fn find_documents(
Ok(MongoDocumentResult { documents, total })
}
pub async fn count_documents(
client: &Client,
database: &str,
collection: &str,
filter: Option<&str>,
accurate: bool,
) -> Result<u64, String> {
let col = client.database(database).collection::<Document>(collection);
let filter_doc: Document = match filter {
Some(f) if !f.trim().is_empty() => {
let json: serde_json::Value = serde_json::from_str(f).map_err(|e| format!("Invalid filter JSON: {e}"))?;
json_filter_to_document(&json)?
}
_ => doc! {},
};
if !accurate && filter_doc.is_empty() {
// Legacy count() permits the metadata-backed fast path; countDocuments() must scan accurately.
col.estimated_document_count().await.map_err(|e| e.to_string())
} else {
col.count_documents(filter_doc).await.map_err(|e| e.to_string())
}
}
/// Find MongoDB documents as relaxed Extended JSON for MongoDB transfer paths.
#[allow(clippy::too_many_arguments)]
pub async fn find_documents_extended_json(
+44
View File
@@ -121,6 +121,50 @@ pub async fn mongo_find_documents_core(
.await
}
pub async fn mongo_count_documents_core(
state: &AppState,
connection_id: &str,
database: &str,
collection: &str,
filter: Option<&str>,
mode: Option<&str>,
) -> Result<u64, String> {
let accurate = mode != Some("legacy");
ensure_document_pool(state, connection_id).await?;
let connections = state.connections.read().await;
match connections.get(connection_id).ok_or("Not found")? {
PoolKind::MongoDb(client) => {
mongo_driver::count_documents(client, database, collection, filter, accurate).await
}
PoolKind::Agent(client) => {
let mut client = client.lock().await;
let params = serde_json::json!({
"database": database,
"collection": collection,
"filter": filter,
"accurate": accurate,
});
match client.mongo_count_documents(params.clone()).await {
Ok(total) => Ok(total),
Err(error) if is_unknown_agent_method_error(&error, "count_documents") => {
let result: MongoDocumentResult = client
.mongo_find_documents(serde_json::json!({
"database": database,
"collection": collection,
"skip": 0,
"limit": 1,
"filter": filter,
}))
.await?;
Ok(result.total)
}
Err(error) => Err(error),
}
}
_ => Err("Not a MongoDB connection".to_string()),
}
}
/// Read MongoDB documents as relaxed Extended JSON for MongoDB transfer paths.
#[allow(clippy::too_many_arguments)]
pub async fn mongo_find_documents_extended_json_core(
+1
View File
@@ -474,6 +474,7 @@ async fn main() {
.route("/document-store/update-document", post(routes::document_store::update_document))
.route("/document-store/delete-document", post(routes::document_store::delete_document))
.route("/mongo/find-documents", post(routes::mongo::find_documents))
.route("/mongo/count-documents", post(routes::mongo::count_documents))
.route("/mongo/server-version", post(routes::mongo::server_version))
.route("/mongo/collection-stats", post(routes::mongo::collection_stats))
.route("/mongo/aggregate-documents", post(routes::mongo::aggregate_documents))
+31
View File
@@ -78,6 +78,17 @@ pub struct MongoFindRequest {
pub execution_id: Option<String>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoCountRequest {
pub connection_id: String,
pub database: String,
pub collection: String,
pub filter: Option<String>,
pub mode: Option<String>,
pub execution_id: Option<String>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoServerVersionRequest {
@@ -286,6 +297,26 @@ pub async fn find_documents(
Ok(Json(serde_json::to_value(result).map_err(|e| AppError(e.to_string()))?))
}
pub async fn count_documents(
State(state): State<Arc<WebState>>,
Json(req): Json<MongoCountRequest>,
) -> Result<Json<u64>, AppError> {
let result = run_cancellable(
&state,
req.execution_id,
dbx_core::mongo_ops::mongo_count_documents_core(
&state.app,
&req.connection_id,
&req.database,
&req.collection,
req.filter.as_deref(),
req.mode.as_deref(),
),
)
.await?;
Ok(Json(result))
}
pub async fn server_version(
State(state): State<Arc<WebState>>,
Json(req): Json<MongoServerVersionRequest>,
+1
View File
@@ -23,6 +23,7 @@
"build:checked": "pnpm typecheck && pnpm build",
"check": "node scripts/run-check.mjs",
"bench:redis-key-search": "node scripts/bench/redis-key-search.mjs",
"bench:mongodb-count": "node scripts/bench/mongodb-count.mjs",
"lint": "oxlint --vue-plugin apps/desktop/src",
"fmt": "oxfmt \"apps/desktop/src/**/*.{ts,vue}\"",
"test": "vitest run",
+5
View File
@@ -252,6 +252,7 @@ test("parseMongoCountDocumentsCommand parses db collection countDocuments", () =
assert.deepEqual(parseMongoCountDocumentsCommand("db.products.countDocuments({})"), {
collection: "products",
filter: "{}",
mode: "accurate",
});
});
@@ -259,20 +260,24 @@ test("parseMongoCountDocumentsCommand parses legacy count helpers", () => {
assert.deepEqual(parseMongoCountDocumentsCommand("db.products.count({ active: true })"), {
collection: "products",
filter: '{ "active": true }',
mode: "legacy",
});
assert.deepEqual(parseMongoCountDocumentsCommand('db.getCollection("audit.logs").count()'), {
collection: "audit.logs",
filter: "{}",
mode: "legacy",
});
assert.deepEqual(parseMongoCountDocumentsCommand("db.products.find({ active: true }).count()"), {
collection: "products",
filter: '{ "active": true }',
mode: "legacy",
});
assert.equal(parseMongoFindCommand("db.products.find({ active: true }).count()"), null);
assert.deepEqual(parseMongoCommand("db.products.find({ active: true }).count()")?.command, {
kind: "countDocuments",
collection: "products",
filter: '{ "active": true }',
mode: "legacy",
});
});
+52
View File
@@ -3000,6 +3000,58 @@ test("replacing one paginated SQL result preserves the grouped refresh SQL", asy
}
});
test("mongo count execution uses the dedicated count endpoint", async () => {
const restoreStorage = installMemoryStorage();
setActivePinia(createPinia());
const connectionStore = useConnectionStore();
const store = useQueryStore();
const originalFetch = globalThis.fetch;
let countBody: any;
connectionStore.addEphemeralConnection({
...conn("mongo-1"),
db_type: "mongodb",
port: 27017,
});
globalThis.fetch = withConnectionHealthMock(async (input, init) => {
const url = String(input);
if (url === "/api/query/prepare-pagination-plan") {
const body = JSON.parse(String(init?.body ?? "{}"));
return new Response(JSON.stringify({ sqlToExecute: body.options.sql, useAgentResultSession: false }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "/api/mongo/count-documents") {
countBody = JSON.parse(String(init?.body ?? "{}"));
return new Response(JSON.stringify(21606536), { status: 200, headers: { "Content-Type": "application/json" } });
}
return new Response("unexpected request", { status: 500 });
});
try {
const tabId = store.createTab("mongo-1", "dbx_issue_2959", "Query", "query", "");
await store.executeTabSql(tabId, "db.large_count.count()");
const tab = store.tabs.find((item) => item.id === tabId);
assert.deepEqual(countBody, {
connectionId: "mongo-1",
database: "dbx_issue_2959",
collection: "large_count",
filter: "{}",
mode: "legacy",
executionId: countBody.executionId,
});
assert.equal(typeof countBody.executionId, "string");
assert.deepEqual(tab?.result?.columns, ["count"]);
assert.deepEqual(tab?.result?.rows, [[21606536]]);
} finally {
globalThis.fetch = originalFetch;
restoreStorage();
}
});
test("mongo createIndex execution uses the dedicated create-index endpoint", async () => {
const restoreStorage = installMemoryStorage();
setActivePinia(createPinia());
+16 -6
View File
@@ -881,8 +881,8 @@ export async function executeQuery(config: ConnectionConfig, sql: string, option
}
const count = parseMongoCountDocumentsCommand(sql);
if (count) {
const result = await withTimeout(mongoFindDocuments(config, count.collection, 0, 1, count.filter), resolveTimeoutMs(options));
return { columns: ["count"], rows: [{ count: result.total }], row_count: 1 };
const total = await withTimeout(mongoCountDocuments(config, count.collection, count.filter, count.mode), resolveTimeoutMs(options));
return { columns: ["count"], rows: [{ count: total }], row_count: 1 };
}
const find = parseMongoFindCommand(sql);
if (find) {
@@ -1125,6 +1125,17 @@ async function mongoFindDocuments(config: ConnectionConfig, collection: string,
});
}
async function mongoCountDocuments(config: ConnectionConfig, collection: string, filter: string, mode: MongoCountDocumentsCommand["mode"]): Promise<number> {
return bridgeDataRequest<number>("/data/mongo/count-documents", {
connection_id: config.id,
connection_name: config.name,
database: config.database || "",
collection,
filter,
mode,
});
}
async function mongoServerVersion(config: ConnectionConfig): Promise<string> {
return bridgeDataRequest<string>("/data/mongo/server-version", {
connection_id: config.id,
@@ -1296,6 +1307,7 @@ interface MongoFindCommand {
interface MongoCountDocumentsCommand {
collection: string;
filter: string;
mode: "accurate" | "legacy";
}
interface MongoAggregateCommand {
@@ -1364,8 +1376,6 @@ export function parseMongoVersionCommand(input: string): boolean {
export function parseMongoCountDocumentsCommand(input: string): MongoCountDocumentsCommand | null {
const source = input.trim().replace(/;$/, "").trim();
// Accept deprecated Mongo shell count helpers for old server workflows, but
// keep DBX's internal execution mapped to the countDocuments result shape.
return parseCollectionCountCommand(source, "countDocuments") ?? parseCollectionCountCommand(source, "count") ?? parseFindCountCommand(source);
}
@@ -1378,7 +1388,7 @@ function parseCollectionCountCommand(source: string, method: "countDocuments" |
const args = splitTopLevel(source.slice(openIndex + 1, closeIndex));
if (args.length > 1 && args.slice(1).some((arg) => arg.trim())) return null;
const filter = normalizeJsonArgument(args[0] || "{}");
return filter ? { collection: target.collection, filter } : null;
return filter ? { collection: target.collection, filter, mode: method === "countDocuments" ? "accurate" : "legacy" } : null;
}
function parseFindCountCommand(source: string): MongoCountDocumentsCommand | null {
@@ -1392,7 +1402,7 @@ function parseFindCountCommand(source: string): MongoCountDocumentsCommand | nul
const findArgs = splitTopLevel(source.slice(findOpenIndex + 1, findCloseIndex));
if (findArgs.length > 2 && findArgs.slice(2).some((arg) => arg.trim())) return null;
const filter = normalizeJsonArgument(findArgs[0] || "{}");
return filter ? { collection: target.collection, filter } : null;
return filter ? { collection: target.collection, filter, mode: "legacy" } : null;
}
export function parseMongoAggregateCommand(input: string): MongoAggregateCommand | null {
+4 -5
View File
@@ -219,19 +219,18 @@ export async function executeQuery(config: ConnectionConfig, sql: string, option
}
const count = parseMongoCountDocumentsCommand(sql);
if (count) {
const res = await apiFetch("/api/mongo/find-documents", {
const res = await apiFetch("/api/mongo/count-documents", {
method: "POST",
body: JSON.stringify({
connectionId: config.id,
database: config.database || "",
collection: count.collection,
skip: 0,
limit: 1,
filter: count.filter,
mode: count.mode,
}),
});
const result = (await res.json()) as { documents: unknown[]; total: number };
return { columns: ["count"], rows: [{ count: result.total }], row_count: 1 };
const total = (await res.json()) as number;
return { columns: ["count"], rows: [{ count: total }], row_count: 1 };
}
const find = parseMongoFindCommand(sql);
if (find) {
+71
View File
@@ -0,0 +1,71 @@
import assert from "node:assert/strict";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { createServer } from "node:http";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "vitest";
import type { ConnectionConfig } from "../src/connections.js";
import { executeQuery } from "../src/database.js";
const mongoConfig: ConnectionConfig = {
id: "mongo-bridge",
name: "mongo-bridge",
db_type: "mongodb",
host: "127.0.0.1",
port: 27017,
username: "",
password: "",
database: "app",
ssh_enabled: false,
ssl: false,
};
test("direct backend routes MongoDB count commands through the count bridge", async () => {
const tempDir = await mkdtemp(join(tmpdir(), "dbx-node-core-count-"));
const previousDataDir = process.env.DBX_DATA_DIR;
let requestBody: unknown;
const server = createServer((req, res) => {
assert.equal(req.url, "/data/mongo/count-documents");
let body = "";
req.setEncoding("utf8");
req.on("data", (chunk) => {
body += chunk;
});
req.on("end", () => {
requestBody = JSON.parse(body);
res.writeHead(200, { "content-type": "application/json" });
res.end("42");
});
});
try {
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string") throw new Error("expected TCP bridge address");
process.env.DBX_DATA_DIR = tempDir;
await writeFile(join(tempDir, "mcp-bridge-port"), String(address.port));
const result = await executeQuery(mongoConfig, "db.projects.count({ active: true })");
assert.deepEqual(requestBody, {
connection_id: "mongo-bridge",
connection_name: "mongo-bridge",
database: "app",
collection: "projects",
filter: '{ "active": true }',
mode: "legacy",
});
assert.deepEqual(result, {
columns: ["count"],
rows: [{ count: 42 }],
row_count: 1,
});
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
if (previousDataDir === undefined) delete process.env.DBX_DATA_DIR;
else process.env.DBX_DATA_DIR = previousDataDir;
await rm(tempDir, { recursive: true, force: true });
}
});
+4
View File
@@ -93,6 +93,7 @@ test("parseMongoCountDocumentsCommand accepts shell-style count commands", () =>
assert.deepEqual(parseMongoCountDocumentsCommand('db.projects.countDocuments({"active":true})'), {
collection: "projects",
filter: '{"active":true}',
mode: "accurate",
});
});
@@ -100,14 +101,17 @@ test("parseMongoCountDocumentsCommand accepts legacy count helpers", () => {
assert.deepEqual(parseMongoCountDocumentsCommand("db.projects.count({ active: true })"), {
collection: "projects",
filter: '{ "active": true }',
mode: "legacy",
});
assert.deepEqual(parseMongoCountDocumentsCommand('db.getCollection("audit.logs").count()'), {
collection: "audit.logs",
filter: "{}",
mode: "legacy",
});
assert.deepEqual(parseMongoCountDocumentsCommand("db.projects.find({ active: true }).count()"), {
collection: "projects",
filter: '{ "active": true }',
mode: "legacy",
});
assert.equal(parseMongoFindCommand("db.projects.find({ active: true }).count()"), null);
});
+59 -1
View File
@@ -1,7 +1,7 @@
import assert from "node:assert/strict";
import { afterEach, test } from "vitest";
import { loadConnections, resetWebAuthForTests } from "../src/web-backend.js";
import { executeQuery, loadConnections, resetWebAuthForTests } from "../src/web-backend.js";
const originalFetch = globalThis.fetch;
const originalWebUrl = process.env.DBX_WEB_URL;
@@ -123,3 +123,61 @@ test("web backend still allows DBX Web instances with password auth disabled", a
assert.deepEqual(await loadConnections(), []);
});
test("web backend routes MongoDB count commands through count-documents", async () => {
process.env.DBX_WEB_URL = "http://127.0.0.1:4224";
delete process.env.DBX_WEB_PASSWORD;
const calls: Array<{ url: string; init?: RequestInit }> = [];
globalThis.fetch = (async (input, init) => {
const url = String(input);
calls.push({ url, init });
if (url.endsWith("/api/auth/check")) {
return jsonResponse({ authenticated: true, required: false, setup_required: false });
}
if (url.endsWith("/api/connection/connect")) {
return jsonResponse({ ok: true });
}
if (url.endsWith("/api/mongo/count-documents")) {
assert.equal(init?.method, "POST");
assert.deepEqual(JSON.parse(String(init?.body)), {
connectionId: "mongo-web",
database: "app",
collection: "projects",
filter: '{"active":true}',
mode: "accurate",
});
return jsonResponse(42);
}
throw new Error(`unexpected request: ${url}`);
}) as typeof fetch;
const result = await executeQuery(
{
id: "mongo-web",
name: "mongo-web",
db_type: "mongodb",
host: "127.0.0.1",
port: 27017,
username: "",
password: "",
database: "app",
ssh_enabled: false,
ssl: false,
},
'db.projects.countDocuments({"active":true})',
);
assert.deepEqual(result, {
columns: ["count"],
rows: [{ count: 42 }],
row_count: 1,
});
assert.deepEqual(
calls.map((call) => call.url),
[
"http://127.0.0.1:4224/api/auth/check",
"http://127.0.0.1:4224/api/connection/connect",
"http://127.0.0.1:4224/api/mongo/count-documents",
],
);
});
+464
View File
@@ -0,0 +1,464 @@
#!/usr/bin/env node
import { spawn } from "node:child_process";
import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { performance } from "node:perf_hooks";
const DEFAULTS = {
apiBase: "http://127.0.0.1:4224/api",
container: "dbx-issue-2959-mongo",
image: "mongo:3.4",
host: "127.0.0.1",
port: 12959,
database: "dbx_issue_2959",
collection: "large_count",
expectedCount: 21_606_536,
iterations: 5,
warmups: 1,
seed: false,
forceSeed: false,
seedBatchSize: 10_000,
dbxDataDir: "",
json: false,
};
function parseArgs(argv) {
const options = { ...DEFAULTS };
for (const arg of argv) {
if (arg === "--") continue;
const [rawKey, rawValue] = arg.split("=", 2);
const key = rawKey.replace(/^--/, "");
const value = rawValue ?? "true";
switch (key) {
case "api-base":
options.apiBase = value;
break;
case "container":
options.container = value;
break;
case "image":
options.image = value;
break;
case "host":
options.host = value;
break;
case "port":
options.port = Number.parseInt(value, 10);
break;
case "database":
options.database = value;
break;
case "collection":
options.collection = value;
break;
case "expected-count":
options.expectedCount = Number.parseInt(value, 10);
break;
case "iterations":
options.iterations = Number.parseInt(value, 10);
break;
case "warmups":
options.warmups = Number.parseInt(value, 10);
break;
case "seed":
options.seed = value !== "false";
break;
case "force-seed":
options.forceSeed = value !== "false";
break;
case "seed-batch-size":
options.seedBatchSize = Number.parseInt(value, 10);
break;
case "dbx-data-dir":
options.dbxDataDir = value;
break;
case "json":
options.json = value !== "false";
break;
case "help":
printHelp();
process.exit(0);
default:
throw new Error(`Unknown option: ${rawKey}`);
}
}
return options;
}
function printHelp() {
console.log(`MongoDB count benchmark
Usage:
pnpm bench:mongodb-count [options]
Options:
--api-base=http://127.0.0.1:4224/api DBX Web API base URL
--container=dbx-issue-2959-mongo MongoDB Docker container name
--image=mongo:3.4 MongoDB image used when creating the container
--port=12959 Host port for MongoDB
--database=dbx_issue_2959 Database name
--collection=large_count Collection name
--expected-count=21606536 Expected collection count
--iterations=5 Timed iterations per case
--warmups=1 Warmup iterations per case
--dbx-data-dir=/tmp/dbx-bench Copy local Mongo agent jar into this DBX data dir
--seed Seed the collection if its count does not match
--force-seed Drop and reseed even if the collection exists
--seed-batch-size=10000 Number of docs per insertMany batch
--json Print JSON only
Before running this benchmark, start DBX Web with the same data dir, for example:
DBX_DATA_DIR=/tmp/dbx-bench DBX_DISABLE_PASSWORD=1 cargo run -p dbx-web
The benchmark saves one temporary connection into that DBX data dir, so use an
isolated DBX_DATA_DIR instead of your normal desktop profile.
`);
}
function assertInteger(name, value, min = 0) {
if (!Number.isFinite(value) || Math.trunc(value) !== value || value < min) {
throw new Error(`${name} must be an integer >= ${min}, got ${value}`);
}
}
function run(command, args, options = {}) {
const startedAt = performance.now();
const child = spawn(command, args, {
cwd: options.cwd ?? process.cwd(),
env: process.env,
stdio: options.stdio ?? ["ignore", "pipe", "pipe"],
});
const stdout = [];
const stderr = [];
if (child.stdout) child.stdout.on("data", (chunk) => stdout.push(chunk));
if (child.stderr) child.stderr.on("data", (chunk) => stderr.push(chunk));
return new Promise((resolve, reject) => {
child.on("error", reject);
child.on("close", (code) => {
const result = {
code,
stdout: Buffer.concat(stdout).toString(),
stderr: Buffer.concat(stderr).toString(),
elapsedMs: performance.now() - startedAt,
};
if (code === 0) resolve(result);
else reject(new Error(`${command} ${args.join(" ")} failed with ${code}\n${result.stderr || result.stdout}`));
});
});
}
async function dockerExec(container, args) {
return run("docker", ["exec", container, ...args]);
}
async function ensureMongoContainer(options) {
try {
await run("docker", ["inspect", options.container]);
await run("docker", ["start", options.container]);
} catch {
await run("docker", [
"run",
"-d",
"--name",
options.container,
"--platform",
"linux/amd64",
"-p",
`${options.host}:${options.port}:27017`,
options.image,
"--nojournal",
"--wiredTigerCacheSizeGB",
"0.5",
]);
}
const deadline = Date.now() + 30_000;
while (Date.now() < deadline) {
try {
const out = await dockerExec(options.container, ["mongo", "--quiet", "--eval", "db.runCommand({ ping: 1 }).ok"]);
if (out.stdout.trim().endsWith("1")) return;
} catch {
await new Promise((resolve) => setTimeout(resolve, 500));
}
}
throw new Error("Timed out waiting for MongoDB container");
}
async function mongoEval(options, script) {
return dockerExec(options.container, ["mongo", options.database, "--quiet", "--eval", script]);
}
async function collectionCount(options) {
const out = await mongoEval(options, `print(db.${options.collection}.count())`);
const line = out.stdout.trim().split(/\r?\n/).at(-1) ?? "";
return Number.parseInt(line, 10);
}
async function seedMongo(options) {
const current = await collectionCount(options);
if (!options.forceSeed && current === options.expectedCount) {
return { skipped: true, count: current, elapsedMs: 0 };
}
if (!options.seed && !options.forceSeed) {
throw new Error(
`Collection count is ${current}, expected ${options.expectedCount}. Re-run with --seed to create the benchmark dataset.`,
);
}
const startedAt = performance.now();
const script = `
var collection = db.${options.collection};
collection.drop();
var expected = ${options.expectedCount};
var batchSize = ${options.seedBatchSize};
for (var start = 0; start < expected; start += batchSize) {
var docs = [];
var end = Math.min(start + batchSize, expected);
for (var i = start; i < end; i++) {
docs.push({ seq: i, bucket: i % 10, payload: "payload-" + i });
}
collection.insertMany(docs, { ordered: false });
}
print(collection.count());
`;
const out = await mongoEval(options, script);
const count = Number.parseInt(out.stdout.trim().split(/\r?\n/).at(-1) ?? "", 10);
if (count !== options.expectedCount) {
throw new Error(`Seed finished with ${count} docs, expected ${options.expectedCount}`);
}
return { skipped: false, count, elapsedMs: performance.now() - startedAt };
}
function localMongoAgentJar() {
return resolve("agents", "drivers", "mongodb", "build", "libs", "dbx-agent-mongodb.jar");
}
function syncMongoAgentJar(options) {
if (!options.dbxDataDir) return null;
const source = localMongoAgentJar();
if (!existsSync(source)) {
throw new Error(`MongoDB agent jar not found: ${source}. Run ./agents/gradlew -p agents :mongodb:shadowJar first.`);
}
const dest = resolve(options.dbxDataDir, "agents", "drivers", "mongodb", "agent.jar");
mkdirSync(dirname(dest), { recursive: true });
copyFileSync(source, dest);
ensureAgentState(options.dbxDataDir);
return dest;
}
function ensureAgentState(dbxDataDir) {
const statePath = resolve(dbxDataDir, "agents", "state.json");
const now = new Date().toISOString();
let state = {};
if (existsSync(statePath)) {
state = JSON.parse(readFileSync(statePath, "utf8"));
}
state.jre_versions ??= {};
state.installed_drivers ??= {};
state.installed_drivers.mongodb = {
version: "0.1.0-local",
installed_at: state.installed_drivers.mongodb?.installed_at ?? now,
jre: "21",
};
state.java_runtime = {
...(state.java_runtime ?? {}),
mode: "system",
};
mkdirSync(dirname(statePath), { recursive: true });
writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`);
}
async function postJsonText(url, body) {
const response = await fetch(url, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
});
const text = await response.text();
return { ok: response.ok, status: response.status, text, json: parseJsonMaybe(text) };
}
function parseJsonMaybe(text) {
try {
return JSON.parse(text);
} catch {
return undefined;
}
}
async function ensureDbxConnection(options) {
const connectionId = `bench-mongodb-count-${options.port}`;
const config = {
id: connectionId,
name: `Bench MongoDB Count ${options.port}`,
db_type: "mongodb",
driver_profile: "mongodb-legacy",
host: options.host,
port: options.port,
username: "",
password: "",
database: options.database,
ssl: false,
connect_timeout_secs: 10,
query_timeout_secs: 30,
};
const save = await postJsonText(`${options.apiBase}/connection/save`, { configs: [config] });
if (!save.ok) throw new Error(`/connection/save failed with ${save.status}\n${save.text}`);
const connect = await postJsonText(`${options.apiBase}/connection/connect`, { config });
if (!connect.ok) throw new Error(`/connection/connect failed with ${connect.status}\n${connect.text}`);
return connectionId;
}
async function measureCase(label, iterations, warmups, fn) {
for (let index = 0; index < warmups; index += 1) {
await fn();
}
const samples = [];
let lastValue;
let lastPayloadBytes = 0;
for (let index = 0; index < iterations; index += 1) {
const startedAt = performance.now();
const result = await fn();
samples.push(performance.now() - startedAt);
lastValue = result.value;
lastPayloadBytes = result.payloadBytes ?? 0;
}
samples.sort((a, b) => a - b);
const sum = samples.reduce((total, value) => total + value, 0);
return {
label,
value: lastValue,
samples,
minMs: samples[0],
p50Ms: percentile(samples, 0.5),
p90Ms: percentile(samples, 0.9),
maxMs: samples.at(-1),
avgMs: sum / samples.length,
payloadBytes: lastPayloadBytes,
};
}
function percentile(sortedSamples, p) {
if (sortedSamples.length === 0) return 0;
const index = Math.min(sortedSamples.length - 1, Math.ceil(sortedSamples.length * p) - 1);
return sortedSamples[index];
}
async function measureNativeMongoCount(options) {
const script = `var r = db.runCommand({ count: "${options.collection}", query: {} }); print(r.n);`;
const out = await mongoEval(options, script);
return { value: Number.parseInt(out.stdout.trim().split(/\r?\n/).at(-1) ?? "", 10) };
}
async function measureDbxFindTotal(options, connectionId) {
const response = await postJsonText(`${options.apiBase}/document-store/find-documents`, {
connectionId,
database: options.database,
collection: options.collection,
skip: 0,
limit: 1,
filter: "{}",
});
if (!response.ok) throw new Error(`/document-store/find-documents failed with ${response.status}\n${response.text}`);
return { value: response.json?.total, payloadBytes: Buffer.byteLength(response.text) };
}
async function measureDbxDedicatedCount(options, connectionId) {
const response = await postJsonText(`${options.apiBase}/mongo/count-documents`, {
connectionId,
database: options.database,
collection: options.collection,
filter: "{}",
mode: "legacy",
});
if (!response.ok) {
return { value: `unavailable (${response.status})`, payloadBytes: Buffer.byteLength(response.text) };
}
return { value: response.json, payloadBytes: Buffer.byteLength(response.text) };
}
function formatMs(ms) {
return `${Math.round(ms)}ms`;
}
function formatBytes(bytes) {
if (!bytes) return "0 B";
const units = ["B", "KB", "MB", "GB"];
let value = bytes;
let unit = 0;
while (value >= 1024 && unit < units.length - 1) {
value /= 1024;
unit += 1;
}
return `${value.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`;
}
function printReport(result) {
console.log("# MongoDB count benchmark");
console.log("");
console.log(`- API: ${result.config.apiBase}`);
console.log(`- MongoDB: ${result.config.container} on ${result.config.host}:${result.config.port}`);
console.log(`- Collection: ${result.config.database}.${result.config.collection}`);
console.log(`- Expected count: ${result.config.expectedCount}`);
console.log(`- Iterations: ${result.config.iterations} timed, ${result.config.warmups} warmup`);
console.log(`- Seed: ${result.seed.skipped ? "reused existing dataset" : `loaded in ${formatMs(result.seed.elapsedMs)}`}`);
if (result.agentJar) console.log(`- Synced agent jar: ${result.agentJar}`);
console.log("");
console.log("| Case | Value | Payload | Min | P50 | P90 | Max | Avg |");
console.log("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |");
for (const row of result.measurements) {
console.log(
`| ${row.label} | ${row.value} | ${formatBytes(row.payloadBytes)} | ${formatMs(row.minMs)} | ${formatMs(row.p50Ms)} | ${formatMs(row.p90Ms)} | ${formatMs(row.maxMs)} | ${formatMs(row.avgMs)} |`,
);
}
}
async function main() {
const options = parseArgs(process.argv.slice(2));
assertInteger("port", options.port, 1);
assertInteger("expected-count", options.expectedCount, 1);
assertInteger("iterations", options.iterations, 1);
assertInteger("warmups", options.warmups, 0);
assertInteger("seed-batch-size", options.seedBatchSize, 1);
if (!options.json) console.log("Preparing MongoDB count benchmark...");
await ensureMongoContainer(options);
const seed = await seedMongo(options);
const agentJar = syncMongoAgentJar(options);
if (!options.json) console.log("Connecting DBX Web API...");
const connectionId = await ensureDbxConnection(options);
const measurements = [
await measureCase("mongo runCommand count", options.iterations, options.warmups, () => measureNativeMongoCount(options)),
await measureCase("DBX find-documents total", options.iterations, options.warmups, () => measureDbxFindTotal(options, connectionId)),
await measureCase("DBX count-documents", options.iterations, options.warmups, () => measureDbxDedicatedCount(options, connectionId)),
];
const result = {
config: {
apiBase: options.apiBase,
container: options.container,
host: options.host,
port: options.port,
database: options.database,
collection: options.collection,
expectedCount: options.expectedCount,
iterations: options.iterations,
warmups: options.warmups,
},
seed,
agentJar,
measurements,
generatedAt: new Date().toISOString(),
};
if (options.json) console.log(JSON.stringify(result, null, 2));
else printReport(result);
}
main().catch((error) => {
console.error(error instanceof Error ? error.stack || error.message : String(error));
process.exit(1);
});
+5 -1
View File
@@ -6,7 +6,11 @@ use crate::commands::connection::{ensure_connection_writable, AppState};
use dbx_core::db::mongo_driver::MongoDocumentResult;
use dbx_core::document_ops::CollectionInfo;
async fn run_cancellable<T, F>(state: &Arc<AppState>, execution_id: Option<String>, future: F) -> Result<T, String>
pub(crate) async fn run_cancellable<T, F>(
state: &Arc<AppState>,
execution_id: Option<String>,
future: F,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
+40
View File
@@ -62,6 +62,16 @@ struct MongoFindDocumentsRequest {
sort: Option<String>,
}
#[derive(Deserialize)]
struct MongoCountDocumentsRequest {
connection_name: String,
connection_id: Option<String>,
database: Option<String>,
collection: String,
filter: Option<String>,
mode: Option<String>,
}
#[derive(Deserialize)]
struct MongoServerVersionRequest {
connection_name: String,
@@ -214,6 +224,8 @@ pub fn start(app_handle: AppHandle, state: Arc<AppState>, data_dir: PathBuf) {
handle_describe_table_data(&st, body, &mut stream).await;
} else if first_line.starts_with("POST /data/mongo/list-collections") {
handle_mongo_list_collections_data(&st, body, &mut stream).await;
} else if first_line.starts_with("POST /data/mongo/count-documents") {
handle_mongo_count_documents_data(&st, body, &mut stream).await;
} else if first_line.starts_with("POST /data/mongo/find-documents") {
handle_mongo_find_documents_data(&st, body, &mut stream).await;
} else if first_line.starts_with("POST /data/mongo/server-version") {
@@ -518,6 +530,34 @@ async fn handle_mongo_find_documents_data(state: &Arc<AppState>, body: &str, str
}
}
async fn handle_mongo_count_documents_data(state: &Arc<AppState>, body: &str, stream: &mut tokio::net::TcpStream) {
let req: MongoCountDocumentsRequest = match serde_json::from_str(body) {
Ok(r) => r,
Err(_) => {
respond_error(stream, "400 Bad Request", "Invalid JSON").await;
return;
}
};
let Some((pool_key, database, _connection_id)) =
resolve_mongo_pool_key(state, req.connection_id.as_deref(), &req.connection_name, req.database, stream).await
else {
return;
};
match dbx_core::mongo_ops::mongo_count_documents_core(
state,
&pool_key,
&database,
&req.collection,
req.filter.as_deref(),
req.mode.as_deref(),
)
.await
{
Ok(total) => respond_json(stream, &total).await,
Err(e) => respond_error(stream, "500 Internal Server Error", &e).await,
}
}
async fn handle_mongo_server_version_data(state: &Arc<AppState>, body: &str, stream: &mut tokio::net::TcpStream) {
let req: MongoServerVersionRequest = match serde_json::from_str(body) {
Ok(r) => r,
+26
View File
@@ -110,6 +110,32 @@ pub async fn mongo_find_documents(
.await
}
#[tauri::command]
pub async fn mongo_count_documents(
state: State<'_, Arc<AppState>>,
connection_id: String,
database: String,
collection: String,
filter: Option<String>,
mode: Option<String>,
execution_id: Option<String>,
) -> Result<u64, String> {
let app = state.inner().clone();
crate::commands::document_cmd::run_cancellable(
&app,
execution_id,
dbx_core::mongo_ops::mongo_count_documents_core(
&app,
&connection_id,
&database,
&collection,
filter.as_deref(),
mode.as_deref(),
),
)
.await
}
#[tauri::command]
pub async fn mongo_server_version(
state: State<'_, Arc<AppState>>,
+1
View File
@@ -1128,6 +1128,7 @@ pub fn run() {
commands::document_cmd::document_upload_gridfs_file,
commands::document_cmd::document_delete_gridfs_file,
commands::mongo_cmd::mongo_find_documents,
commands::mongo_cmd::mongo_count_documents,
commands::mongo_cmd::mongo_server_version,
commands::mongo_cmd::mongo_collection_stats,
commands::mongo_cmd::mongo_aggregate_documents,