feat(vector): add weaviate support

This commit is contained in:
t8y2 committed 2026-06-25 00:31:27 +08:00
1 parent 09cdf07aab
commit bf9ef4365b
31 files changed
+180 -28

No files matched your search

+6 -2
View File
@@ -831,10 +831,11 @@ impl AppState {
db::elasticsearch_driver::test_connection(&mut client, connect_timeout).await?;
PoolKind::Elasticsearch(client)
}
DatabaseType::Qdrant | DatabaseType::Milvus => {
DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate => {
let kind = match db_config.db_type {
DatabaseType::Qdrant => db::vector_driver::VectorDbKind::Qdrant,
DatabaseType::Milvus => db::vector_driver::VectorDbKind::Milvus,
DatabaseType::Weaviate => db::vector_driver::VectorDbKind::Weaviate,
_ => unreachable!(),
};
let client = db::vector_driver::VectorClient::new(
@@ -2124,7 +2125,10 @@ fn base_pool_key_for(
let is_single_connection_pool = db_type.as_ref().is_some_and(|db_type| {
let is_single = database_capabilities::is_single_connection_pool(db_type)
|| (include_elasticsearch_single_pool
&& matches!(db_type, DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus));
&& matches!(
db_type,
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate
));
is_single && (!database_capabilities::is_agent_type(db_type) || shares_database_pool_with_connection(db_type))
});
+76 -2
View File
@@ -25,10 +25,13 @@ const PATH_SEGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
.add(b'|')
.add(b'}');
const QUERY_VALUE_ENCODE_SET: &AsciiSet = &PATH_SEGMENT_ENCODE_SET.add(b'&').add(b'=').add(b'+');
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VectorDbKind {
Qdrant,
Milvus,
Weaviate,
}
impl VectorDbKind {
@@ -36,6 +39,7 @@ impl VectorDbKind {
match self {
VectorDbKind::Qdrant => "Qdrant",
VectorDbKind::Milvus => "Milvus",
VectorDbKind::Weaviate => "Weaviate",
}
}
}
@@ -48,7 +52,7 @@ pub struct VectorClient {
auth: Option<VectorAuth>,
}
#[derive(Clone)]
#[derive(Clone, Debug, PartialEq, Eq)]
enum VectorAuth {
Basic(String, String),
Bearer(String),
@@ -108,6 +112,8 @@ fn vector_auth(kind: VectorDbKind, username: Option<&str>, password: Option<&str
VectorDbKind::Qdrant => None,
VectorDbKind::Milvus if !username.is_empty() => Some(VectorAuth::Bearer(format!("{username}:{password}"))),
VectorDbKind::Milvus => None,
VectorDbKind::Weaviate if !password.is_empty() => Some(VectorAuth::Bearer(password.to_string())),
VectorDbKind::Weaviate => None,
}
}
@@ -116,10 +122,12 @@ pub async fn test_connection(client: &VectorClient, timeout: Duration) -> Result
let path = match client.kind {
VectorDbKind::Qdrant => "/collections",
VectorDbKind::Milvus => "/v2/vectordb/collections/list",
VectorDbKind::Weaviate => "/v1/meta",
};
let request = match client.kind {
VectorDbKind::Qdrant => client.get(path),
VectorDbKind::Milvus => client.post(path).json(&serde_json::json!({ "dbName": "default" })),
VectorDbKind::Weaviate => client.get(path),
};
let resp = with_connection_timeout(label, timeout, async {
request.send().await.map_err(|e| format!("{label} connection failed: {}", format_reqwest_error(&e)))
@@ -132,6 +140,7 @@ pub async fn list_collections(client: &VectorClient) -> Result<Vec<String>, Stri
match client.kind {
VectorDbKind::Qdrant => list_qdrant_collections(client).await,
VectorDbKind::Milvus => list_milvus_collections(client).await,
VectorDbKind::Weaviate => list_weaviate_collections(client).await,
}
}
@@ -169,6 +178,22 @@ fn collection_name_from_milvus_item(item: &Value) -> Option<String> {
.or_else(|| item.get("name").and_then(Value::as_str).map(str::to_string))
}
async fn list_weaviate_collections(client: &VectorClient) -> Result<Vec<String>, String> {
let body = send_json(client.get("/v1/schema"), "Weaviate").await?;
let mut names = weaviate_collection_names_from_schema(&body);
names.sort();
Ok(names)
}
fn weaviate_collection_names_from_schema(body: &Value) -> Vec<String> {
body.get("classes")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|item| item.get("class").and_then(Value::as_str).map(str::to_string))
.collect()
}
pub async fn find_documents(
client: &VectorClient,
collection: &str,
@@ -197,6 +222,9 @@ pub async fn find_documents(
"outputFields": ["*"],
})
),
VectorDbKind::Weaviate => {
format!("GET /v1/objects?class={}&limit={}&offset={}", query_value(collection), limit.max(1), skip)
}
};
let result = execute_rest_query(client, &query).await?;
let documents = result
@@ -272,6 +300,7 @@ fn default_collection_query(client: &VectorClient, collection: &str) -> Result<r
"limit": 100,
"outputFields": ["*"],
}))),
VectorDbKind::Weaviate => Ok(client.get(&format!("/v1/objects?class={}&limit=100", query_value(collection)))),
}
}
@@ -283,6 +312,10 @@ fn path_segment(value: &str) -> String {
utf8_percent_encode(value, PATH_SEGMENT_ENCODE_SET).to_string()
}
fn query_value(value: &str) -> String {
utf8_percent_encode(value, QUERY_VALUE_ENCODE_SET).to_string()
}
async fn send_json(req: reqwest::RequestBuilder, label: &str) -> Result<Value, String> {
let resp = req.send().await.map_err(|e| format!("{label} request failed: {e}"))?;
let resp = ensure_success(label, resp).await?;
@@ -304,6 +337,9 @@ fn json_to_query_result(status: u16, body: Value, start: Instant) -> QueryResult
if let Some(Value::Array(items)) = rows_value {
return values_to_query_result(items, start);
}
if let Some(Value::Array(items)) = body.get("objects").cloned() {
return values_to_query_result(items, start);
}
if let Some(Value::Array(items)) = body.pointer("/result/collections").cloned() {
return values_to_query_result(items, start);
}
@@ -361,6 +397,11 @@ fn normalize_row_object(value: Value) -> Map<String, Value> {
object.entry(key).or_insert(value);
}
}
if let Some(Value::Object(properties)) = object.remove("properties") {
for (key, value) in properties {
object.entry(key).or_insert(value);
}
}
object
}
other => {
@@ -386,7 +427,10 @@ fn format_reqwest_error(err: &reqwest::Error) -> String {
#[cfg(test)]
mod tests {
use super::{starts_with_http_method, values_to_query_result};
use super::{
starts_with_http_method, values_to_query_result, vector_auth, weaviate_collection_names_from_schema,
VectorAuth, VectorDbKind,
};
use serde_json::json;
use std::time::Instant;
@@ -405,4 +449,34 @@ mod tests {
assert!(result.columns.contains(&"score".to_string()));
assert!(result.columns.contains(&"title".to_string()));
}
#[test]
fn extracts_weaviate_schema_class_names() {
let names = weaviate_collection_names_from_schema(&json!({
"classes": [
{ "class": "Article" },
{ "class": "Product" }
]
}));
assert_eq!(names, vec!["Article".to_string(), "Product".to_string()]);
}
#[test]
fn flattens_weaviate_properties_columns() {
let result = values_to_query_result(
vec![json!({"id": "abc", "class": "Article", "properties": {"title": "hello"}})],
Instant::now(),
);
assert!(result.columns.contains(&"id".to_string()));
assert!(result.columns.contains(&"class".to_string()));
assert!(result.columns.contains(&"title".to_string()));
}
#[test]
fn uses_bearer_auth_for_weaviate_tokens_even_with_username() {
assert_eq!(
vector_auth(VectorDbKind::Weaviate, Some("user"), Some("token")),
Some(VectorAuth::Bearer("token".to_string()))
);
}
}
+4 -2
View File
@@ -258,6 +258,8 @@ pub enum DatabaseType {
Qdrant,
#[serde(rename = "milvus")]
Milvus,
#[serde(rename = "weaviate")]
Weaviate,
Doris,
#[serde(rename = "starrocks")]
StarRocks,
@@ -733,7 +735,7 @@ impl ConnectionConfig {
format!("mongodb://{host}:{port}{db_part}{suffix}")
}
DatabaseType::Oracle => format!("oracle://{host}:{port}{db_part}"),
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus => {
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate => {
let scheme = if self.ssl { "https" } else { "http" };
format!("{scheme}://{host}:{port}")
}
@@ -869,7 +871,7 @@ impl ConnectionConfig {
DatabaseType::Oracle => {
format!("oracle://{}:{}@{host}:{port}{db_part}", username, password)
}
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus => {
DatabaseType::Elasticsearch | DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate => {
let scheme = if self.ssl { "https" } else { "http" };
format!("{scheme}://{host}:{port}")
}
+1
View File
@@ -733,6 +733,7 @@ fn should_discard_pool_after_query_timeout(db_type: Option<DatabaseType>) -> boo
| DatabaseType::Elasticsearch
| DatabaseType::Qdrant
| DatabaseType::Milvus
| DatabaseType::Weaviate
| DatabaseType::InfluxDb
)
}
@@ -89,6 +89,7 @@ pub fn supports_sql_query(database_type: DatabaseType) -> bool {
| DatabaseType::Elasticsearch
| DatabaseType::Qdrant
| DatabaseType::Milvus
| DatabaseType::Weaviate
| DatabaseType::InfluxDb
| DatabaseType::Neo4j
| DatabaseType::Etcd