feat(vector): add ChromaDB support

This commit is contained in:
lexmin0412 authored and GitHub committed 2026-06-26 01:41:36 +08:00
1 parent 485188322b
commit c2376494d3
43 files changed
+437 -81

No files matched your search

+7 -2
View File
@@ -832,11 +832,12 @@ impl AppState {
db::elasticsearch_driver::test_connection(&mut client, connect_timeout).await?;
PoolKind::Elasticsearch(client)
}
DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate => {
DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate | DatabaseType::ChromaDb => {
let kind = match db_config.db_type {
DatabaseType::Qdrant => db::vector_driver::VectorDbKind::Qdrant,
DatabaseType::Milvus => db::vector_driver::VectorDbKind::Milvus,
DatabaseType::Weaviate => db::vector_driver::VectorDbKind::Weaviate,
DatabaseType::ChromaDb => db::vector_driver::VectorDbKind::ChromaDb,
_ => unreachable!(),
};
let client = db::vector_driver::VectorClient::new(
@@ -2140,7 +2141,11 @@ fn base_pool_key_for(
|| (include_elasticsearch_single_pool
&& matches!(
db_type,
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate
DatabaseType::Elasticsearch
| DatabaseType::Qdrant
| DatabaseType::Milvus
| DatabaseType::Weaviate
| DatabaseType::ChromaDb
));
is_single && (!database_capabilities::is_agent_type(db_type) || shares_database_pool_with_connection(db_type))
});
+190 -17
View File
@@ -27,11 +27,19 @@ const PATH_SEGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
const QUERY_VALUE_ENCODE_SET: &AsciiSet = &PATH_SEGMENT_ENCODE_SET.add(b'&').add(b'=').add(b'+');
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CollectionInfo {
pub name: String,
pub id: String,
pub dimension: Option<u32>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VectorDbKind {
Qdrant,
Milvus,
Weaviate,
ChromaDb,
}
impl VectorDbKind {
@@ -40,6 +48,7 @@ impl VectorDbKind {
VectorDbKind::Qdrant => "Qdrant",
VectorDbKind::Milvus => "Milvus",
VectorDbKind::Weaviate => "Weaviate",
VectorDbKind::ChromaDb => "ChromaDB",
}
}
}
@@ -57,6 +66,7 @@ enum VectorAuth {
Basic(String, String),
Bearer(String),
ApiKey(String),
ChromaToken(String),
}
impl VectorClient {
@@ -96,6 +106,7 @@ impl VectorClient {
Some(VectorAuth::Basic(user, pass)) => req.basic_auth(user, Some(pass)),
Some(VectorAuth::Bearer(token)) => req.bearer_auth(token),
Some(VectorAuth::ApiKey(token)) => req.header("api-key", token),
Some(VectorAuth::ChromaToken(token)) => req.header("x-chroma-token", token),
None => req,
}
}
@@ -114,6 +125,8 @@ fn vector_auth(kind: VectorDbKind, username: Option<&str>, password: Option<&str
VectorDbKind::Milvus => None,
VectorDbKind::Weaviate if !password.is_empty() => Some(VectorAuth::Bearer(password.to_string())),
VectorDbKind::Weaviate => None,
VectorDbKind::ChromaDb if !password.is_empty() => Some(VectorAuth::ChromaToken(password.to_string())),
VectorDbKind::ChromaDb => None,
}
}
@@ -123,11 +136,13 @@ pub async fn test_connection(client: &VectorClient, timeout: Duration) -> Result
VectorDbKind::Qdrant => "/collections",
VectorDbKind::Milvus => "/v2/vectordb/collections/list",
VectorDbKind::Weaviate => "/v1/meta",
VectorDbKind::ChromaDb => "/api/v2/heartbeat",
};
let request = match client.kind {
VectorDbKind::Qdrant => client.get(path),
VectorDbKind::Milvus => client.post(path).json(&serde_json::json!({ "dbName": "default" })),
VectorDbKind::Weaviate => client.get(path),
VectorDbKind::ChromaDb => client.get(path),
};
let resp = with_connection_timeout(label, timeout, async {
request.send().await.map_err(|e| format!("{label} connection failed: {}", format_reqwest_error(&e)))
@@ -136,39 +151,49 @@ pub async fn test_connection(client: &VectorClient, timeout: Duration) -> Result
ensure_success(label, resp).await.map(|_| ())
}
pub async fn list_collections(client: &VectorClient) -> Result<Vec<String>, String> {
pub async fn list_collections(client: &VectorClient) -> Result<Vec<CollectionInfo>, String> {
match client.kind {
VectorDbKind::Qdrant => list_qdrant_collections(client).await,
VectorDbKind::Milvus => list_milvus_collections(client).await,
VectorDbKind::Weaviate => list_weaviate_collections(client).await,
VectorDbKind::ChromaDb => list_chroma_collections(client).await,
}
}
async fn list_qdrant_collections(client: &VectorClient) -> Result<Vec<String>, String> {
async fn list_qdrant_collections(client: &VectorClient) -> Result<Vec<CollectionInfo>, String> {
let body = send_json(client.get("/collections"), "Qdrant").await?;
let mut names: Vec<String> = body
let mut infos: Vec<CollectionInfo> = body
.pointer("/result/collections")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|item| item.get("name").and_then(Value::as_str).map(str::to_string))
.filter_map(|item| {
let name = item.get("name").and_then(Value::as_str)?;
Some(CollectionInfo { name: name.to_string(), id: name.to_string(), dimension: None })
})
.collect();
names.sort();
Ok(names)
infos.sort_by(|a, b| a.name.cmp(&b.name));
Ok(infos)
}
async fn list_milvus_collections(client: &VectorClient) -> Result<Vec<String>, String> {
async fn list_milvus_collections(client: &VectorClient) -> Result<Vec<CollectionInfo>, String> {
let body = send_json(
client.post("/v2/vectordb/collections/list").json(&serde_json::json!({ "dbName": "default" })),
"Milvus",
)
.await?;
let mut names: Vec<String> = match body.get("data") {
Some(Value::Array(items)) => items.iter().filter_map(collection_name_from_milvus_item).collect(),
let mut infos: Vec<CollectionInfo> = match body.get("data") {
Some(Value::Array(items)) => items
.iter()
.filter_map(|item| {
let name = collection_name_from_milvus_item(item)?;
Some(CollectionInfo { name: name.clone(), id: name, dimension: None })
})
.collect(),
_ => Vec::new(),
};
names.sort();
Ok(names)
infos.sort_by(|a, b| a.name.cmp(&b.name));
Ok(infos)
}
fn collection_name_from_milvus_item(item: &Value) -> Option<String> {
@@ -178,11 +203,56 @@ fn collection_name_from_milvus_item(item: &Value) -> Option<String> {
.or_else(|| item.get("name").and_then(Value::as_str).map(str::to_string))
}
async fn list_weaviate_collections(client: &VectorClient) -> Result<Vec<String>, String> {
async fn list_weaviate_collections(client: &VectorClient) -> Result<Vec<CollectionInfo>, String> {
let body = send_json(client.get("/v1/schema"), "Weaviate").await?;
let mut names = weaviate_collection_names_from_schema(&body);
names.sort();
Ok(names)
let mut infos: Vec<CollectionInfo> = weaviate_collection_names_from_schema(&body)
.into_iter()
.map(|name| CollectionInfo { name: name.clone(), id: name, dimension: None })
.collect();
infos.sort_by(|a, b| a.name.cmp(&b.name));
Ok(infos)
}
async fn list_chroma_collections(client: &VectorClient) -> Result<Vec<CollectionInfo>, String> {
let body =
send_json(client.get("/api/v2/tenants/default_tenant/databases/default_database/collections"), "ChromaDB")
.await?;
let mut infos: Vec<CollectionInfo> = body
.as_array()
.into_iter()
.flatten()
.filter_map(|item| {
let name = item.get("name").and_then(Value::as_str)?;
let id = item.get("id").and_then(Value::as_str)?;
let dimension = item.get("dimension").and_then(|v| v.as_u64()).map(|d| d as u32);
Some(CollectionInfo { name: name.to_string(), id: id.to_string(), dimension })
})
.collect();
infos.sort_by(|a, b| a.name.cmp(&b.name));
Ok(infos)
}
fn chroma_get_response_to_rows(body: &Value) -> Vec<Value> {
let ids = body.get("ids").and_then(Value::as_array).cloned().unwrap_or_default();
let docs = body.get("documents").and_then(Value::as_array).cloned().unwrap_or_default();
let metas = body.get("metadatas").and_then(Value::as_array).cloned().unwrap_or_default();
ids.into_iter()
.enumerate()
.map(|(i, id_val)| {
let mut row = serde_json::Map::new();
row.insert("id".to_string(), id_val);
if let Some(doc) = docs.get(i) {
row.insert("document".to_string(), doc.clone());
}
if let Some(Value::Object(meta_obj)) = metas.get(i) {
for (k, v) in meta_obj {
row.insert(k.clone(), v.clone());
}
}
Value::Object(row)
})
.collect()
}
fn weaviate_collection_names_from_schema(body: &Value) -> Vec<String> {
@@ -200,6 +270,45 @@ pub async fn find_documents(
skip: u64,
limit: i64,
) -> Result<crate::db::mongo_driver::MongoDocumentResult, String> {
if client.kind == VectorDbKind::ChromaDb {
let start = std::time::Instant::now();
let url = format!(
"{}/api/v2/tenants/default_tenant/databases/default_database/collections/{}/get",
client.base_url,
path_segment(collection),
);
let resp = client
.with_auth(client.http.post(&url))
.json(&serde_json::json!({
"limit": limit.max(1) as u64,
"offset": skip,
"include": ["documents", "metadatas"],
}))
.send()
.await
.map_err(|e| format!("ChromaDB request failed: {}", format_reqwest_error(&e)))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
return Err(format!("ChromaDB error ({status}): {body}"));
}
let body: Value = resp.json().await.unwrap_or(Value::Null);
let rows = chroma_get_response_to_rows(&body);
let result = values_to_query_result(rows, start);
let documents = result
.rows
.into_iter()
.map(|row| {
let mut map = serde_json::Map::new();
for (idx, col) in result.columns.iter().enumerate() {
map.insert(col.clone(), row.get(idx).cloned().unwrap_or(Value::Null));
}
Value::Object(map)
})
.collect();
return Ok(crate::db::mongo_driver::MongoDocumentResult { documents, total: result.affected_rows });
}
let query = match client.kind {
VectorDbKind::Qdrant => format!(
"POST /collections/{}/points/scroll\n{}",
@@ -225,6 +334,7 @@ pub async fn find_documents(
VectorDbKind::Weaviate => {
format!("GET /v1/objects?class={}&limit={}&offset={}", query_value(collection), limit.max(1), skip)
}
VectorDbKind::ChromaDb => unreachable!("ChromaDB handled above"),
};
let result = execute_rest_query(client, &query).await?;
let documents = result
@@ -301,6 +411,12 @@ fn default_collection_query(client: &VectorClient, collection: &str) -> Result<r
"outputFields": ["*"],
}))),
VectorDbKind::Weaviate => Ok(client.get(&format!("/v1/objects?class={}&limit=100", query_value(collection)))),
VectorDbKind::ChromaDb => Ok(client
.post(&format!(
"/api/v2/tenants/default_tenant/databases/default_database/collections/{}/get",
path_segment(collection)
))
.json(&serde_json::json!({"limit": 100, "include": ["documents", "metadatas"]}))),
}
}
@@ -343,6 +459,11 @@ fn json_to_query_result(status: u16, body: Value, start: Instant) -> QueryResult
if let Some(Value::Array(items)) = body.pointer("/result/collections").cloned() {
return values_to_query_result(items, start);
}
if body.get("ids").and_then(Value::as_array).is_some() && body.get("documents").and_then(Value::as_array).is_some()
{
let rows = chroma_get_response_to_rows(&body);
return values_to_query_result(rows, start);
}
QueryResult {
columns: vec!["status".to_string(), "response".to_string()],
column_types: Vec::new(),
@@ -428,8 +549,8 @@ fn format_reqwest_error(err: &reqwest::Error) -> String {
#[cfg(test)]
mod tests {
use super::{
starts_with_http_method, values_to_query_result, vector_auth, weaviate_collection_names_from_schema,
VectorAuth, VectorDbKind,
chroma_get_response_to_rows, starts_with_http_method, values_to_query_result, vector_auth,
weaviate_collection_names_from_schema, CollectionInfo, VectorAuth, VectorDbKind,
};
use serde_json::json;
use std::time::Instant;
@@ -479,4 +600,56 @@ mod tests {
Some(VectorAuth::Bearer("token".to_string()))
);
}
#[test]
fn chroma_db_uses_x_chroma_token_header() {
assert_eq!(
vector_auth(VectorDbKind::ChromaDb, None, Some("my-key")),
Some(VectorAuth::ChromaToken("my-key".to_string()))
);
}
#[test]
fn chroma_db_no_auth_when_no_password() {
assert_eq!(vector_auth(VectorDbKind::ChromaDb, None, None), None);
}
#[test]
fn parses_chroma_collection_list() {
let body = json!([
{"id": "uuid-123", "name": "my_collection", "dimension": 384},
{"id": "uuid-456", "name": "other", "dimension": 768}
]);
let infos: Vec<CollectionInfo> = body
.as_array()
.unwrap()
.iter()
.filter_map(|item| {
let name = item.get("name").and_then(|v| v.as_str())?;
let id = item.get("id").and_then(|v| v.as_str())?;
Some(CollectionInfo { name: name.to_string(), id: id.to_string(), dimension: None })
})
.collect();
assert_eq!(infos.len(), 2);
assert_eq!(infos[0].name, "my_collection");
assert_eq!(infos[0].id, "uuid-123");
assert_eq!(infos[1].name, "other");
assert_eq!(infos[1].id, "uuid-456");
}
#[test]
fn converts_chroma_column_major_to_rows() {
let body = json!({
"ids": ["id1", "id2"],
"documents": ["hello world", "test doc"],
"metadatas": [{"source": "test"}, {"source": "demo"}]
});
let rows = chroma_get_response_to_rows(&body);
assert_eq!(rows.len(), 2);
assert_eq!(rows[0]["id"], json!("id1"));
assert_eq!(rows[0]["document"], json!("hello world"));
assert_eq!(rows[0]["source"], json!("test"));
assert_eq!(rows[1]["id"], json!("id2"));
assert_eq!(rows[1]["source"], json!("demo"));
}
}
+12 -2
View File
@@ -263,6 +263,8 @@ pub enum DatabaseType {
Milvus,
#[serde(rename = "weaviate")]
Weaviate,
#[serde(rename = "chromadb")]
ChromaDb,
Doris,
#[serde(rename = "starrocks")]
StarRocks,
@@ -743,7 +745,11 @@ impl ConnectionConfig {
format!("mongodb://{host}:{port}{db_part}{suffix}")
}
DatabaseType::Oracle => format!("oracle://{host}:{port}{db_part}"),
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate => {
DatabaseType::Elasticsearch
| DatabaseType::Qdrant
| DatabaseType::Milvus
| DatabaseType::Weaviate
| DatabaseType::ChromaDb => {
let scheme = if self.ssl { "https" } else { "http" };
format!("{scheme}://{host}:{port}")
}
@@ -882,7 +888,11 @@ impl ConnectionConfig {
DatabaseType::Oracle => {
format!("oracle://{}:{}@{host}:{port}{db_part}", username, password)
}
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate => {
DatabaseType::Elasticsearch
| DatabaseType::Qdrant
| DatabaseType::Milvus
| DatabaseType::Weaviate
| DatabaseType::ChromaDb => {
let scheme = if self.ssl { "https" } else { "http" };
format!("{scheme}://{host}:{port}")
}
+14 -5
View File
@@ -60,20 +60,29 @@ fn mongo_list_databases_unauthorized(error: &str) -> bool {
lower.contains("not authorized") && lower.contains("listdatabases")
}
use crate::db::vector_driver::CollectionInfo;
pub async fn mongo_list_collections_core(
state: &AppState,
connection_id: &str,
database: &str,
) -> Result<Vec<String>, String> {
) -> Result<Vec<CollectionInfo>, String> {
ensure_document_pool(state, connection_id).await?;
let connections = state.connections.read().await;
match connections.get(connection_id).ok_or("Not found")? {
PoolKind::MongoDb(client) => mongo_driver::list_collections(client, database).await.map(sort_names),
PoolKind::Elasticsearch(client) => elasticsearch_driver::list_indices(client).await.map(sort_names),
PoolKind::VectorDb(client) => vector_driver::list_collections(client).await.map(sort_names),
PoolKind::MongoDb(client) => {
let names = sort_names(mongo_driver::list_collections(client, database).await?);
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
}
PoolKind::Elasticsearch(client) => {
let names = sort_names(elasticsearch_driver::list_indices(client).await?);
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
}
PoolKind::VectorDb(client) => vector_driver::list_collections(client).await,
PoolKind::Agent(client) => {
let mut client = client.lock().await;
client.mongo_list_collections(database).await.map(sort_names)
let names = sort_names(client.mongo_list_collections(database).await?);
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
}
_ => Err("Not a MongoDB/Elasticsearch/vector connection".to_string()),
}
+1
View File
@@ -734,6 +734,7 @@ fn should_discard_pool_after_query_timeout(db_type: Option<DatabaseType>) -> boo
| DatabaseType::Qdrant
| DatabaseType::Milvus
| DatabaseType::Weaviate
| DatabaseType::ChromaDb
| DatabaseType::InfluxDb
)
}
@@ -90,6 +90,7 @@ pub fn supports_sql_query(database_type: DatabaseType) -> bool {
| DatabaseType::Qdrant
| DatabaseType::Milvus
| DatabaseType::Weaviate
| DatabaseType::ChromaDb
| DatabaseType::InfluxDb
| DatabaseType::Neo4j
| DatabaseType::Etcd
+1 -1
View File
@@ -1057,7 +1057,7 @@ async fn list_tables_once(
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types)),
PoolKind::VectorDb(client) => db::vector_driver::list_collections(client)
.await
.map(|names| collection_names_to_tables(names, "COLLECTION"))
.map(|infos| collection_names_to_tables(infos.into_iter().map(|i| i.name).collect(), "COLLECTION"))
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types)),
_ => Ok(vec![]),
}
@@ -59,7 +59,7 @@ pub(in crate::schema) async fn list_tables(
db::elasticsearch_driver::list_indices(client).await.map(|names| collection_names_to_tables(names, "INDEX"))
}
PoolKind::VectorDb(client) => {
db::vector_driver::list_collections(client).await.map(|names| collection_names_to_tables(names, "COLLECTION"))
db::vector_driver::list_collections(client).await.map(|infos| collection_names_to_tables(infos.into_iter().map(|i| i.name).collect(), "COLLECTION"))
}
_ => Ok(vec![]),
}