feat(hbase): add HBase REST support
This commit is contained in:
48 files changed
+2455
-19
No files matched your search
@@ -355,6 +355,35 @@
|
||||
"driverManagement": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"dbType": "hbase",
|
||||
"label": "Apache HBase",
|
||||
"runtimeMode": "native",
|
||||
"mcpMode": "bridge",
|
||||
"singleConnectionPool": true,
|
||||
"metadataConnectionScoped": false,
|
||||
"skipTcpProbe": false,
|
||||
"defaultPort": 8080,
|
||||
"supportLevel": "operate",
|
||||
"capabilities": {
|
||||
"queryExecution": false,
|
||||
"metadataBrowse": true,
|
||||
"objectBrowser": false,
|
||||
"objectSource": false,
|
||||
"schemaSearch": false,
|
||||
"diagram": false,
|
||||
"tableDataEdit": true,
|
||||
"tableStructureEdit": false,
|
||||
"tableImport": false,
|
||||
"dataTransfer": false,
|
||||
"sqlFileExecution": false,
|
||||
"databaseCreate": false,
|
||||
"fieldLineage": false,
|
||||
"sqlExplain": false,
|
||||
"userAdmin": false,
|
||||
"driverManagement": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"dbType": "qdrant",
|
||||
"label": "Qdrant",
|
||||
|
||||
@@ -96,6 +96,7 @@ pub enum PoolKind {
|
||||
ClickHouse(db::clickhouse_driver::ChClient),
|
||||
SqlServer(Arc<tokio::sync::Mutex<db::sqlserver::SqlServerClient>>),
|
||||
Elasticsearch(db::elasticsearch_driver::EsClient),
|
||||
HBase(db::hbase_driver::HBaseClient),
|
||||
VectorDb(db::vector_driver::VectorClient),
|
||||
InfluxDb(db::influxdb_driver::InfluxdbClient),
|
||||
Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>),
|
||||
@@ -116,6 +117,7 @@ enum ConnectionDatabaseInfoSource {
|
||||
Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>),
|
||||
ExternalDriver { config: Arc<ConnectionConfig>, session: Arc<PluginDriverSession> },
|
||||
NativeMysql(db::mysql::MySqlPool),
|
||||
NativeHBase(db::hbase_driver::HBaseClient),
|
||||
}
|
||||
|
||||
/// Held connection for a manual transaction session
|
||||
@@ -1396,6 +1398,17 @@ impl AppState {
|
||||
db::elasticsearch_driver::test_connection(&mut client, connect_timeout).await?;
|
||||
PoolKind::Elasticsearch(client)
|
||||
}
|
||||
DatabaseType::Hbase => {
|
||||
let client = db::hbase_driver::HBaseClient::new(
|
||||
&url,
|
||||
Some(&db_config.username),
|
||||
Some(&db_config.password),
|
||||
false,
|
||||
connect_timeout,
|
||||
)?;
|
||||
db::hbase_driver::test_connection(&client, connect_timeout).await?;
|
||||
PoolKind::HBase(client)
|
||||
}
|
||||
DatabaseType::Qdrant | DatabaseType::Milvus | DatabaseType::Weaviate | DatabaseType::ChromaDb => {
|
||||
let kind = match db_config.db_type {
|
||||
DatabaseType::Qdrant => db::vector_driver::VectorDbKind::Qdrant,
|
||||
@@ -2225,6 +2238,18 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
}
|
||||
PoolKind::HBase(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let timeout = crate::db::connection_timeout();
|
||||
match db::hbase_driver::test_connection(&client, timeout).await {
|
||||
Ok(_) => false,
|
||||
Err(err) => {
|
||||
log::warn!("HBase connection pool '{pool_key}' is stale: {err}");
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
PoolKind::VectorDb(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
@@ -2789,6 +2814,7 @@ impl AppState {
|
||||
})
|
||||
}
|
||||
Some(PoolKind::Mysql(pool, _)) => Some(ConnectionDatabaseInfoSource::NativeMysql(pool.clone())),
|
||||
Some(PoolKind::HBase(client)) => Some(ConnectionDatabaseInfoSource::NativeHBase(client.clone())),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
@@ -2811,6 +2837,9 @@ impl AppState {
|
||||
Some(ConnectionDatabaseInfoSource::NativeMysql(pool)) => {
|
||||
db::mysql::database_connection_info(&pool, db::mysql::protocol_product_name(&config)).await.map(Some)
|
||||
}
|
||||
Some(ConnectionDatabaseInfoSource::NativeHBase(client)) => {
|
||||
db::hbase_driver::database_connection_info(&client).await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
@@ -2985,6 +3014,13 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
}
|
||||
PoolKind::HBase(client) => match db::hbase_driver::test_connection(client, timeout).await {
|
||||
Ok(_) => true,
|
||||
Err(e) => {
|
||||
log::warn!("HBase connection pool '{key}' is unhealthy: {e}");
|
||||
false
|
||||
}
|
||||
},
|
||||
PoolKind::VectorDb(client) => match db::vector_driver::test_connection(client, timeout).await {
|
||||
Ok(()) => true,
|
||||
Err(e) => {
|
||||
@@ -3226,6 +3262,7 @@ enum KeepaliveTarget {
|
||||
ClickHouse(db::clickhouse_driver::ChClient),
|
||||
SqlServer(Arc<tokio::sync::Mutex<db::sqlserver::SqlServerClient>>),
|
||||
Elasticsearch(db::elasticsearch_driver::EsClient),
|
||||
HBase(db::hbase_driver::HBaseClient),
|
||||
VectorDb(db::vector_driver::VectorClient),
|
||||
InfluxDb(db::influxdb_driver::InfluxdbClient),
|
||||
Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>),
|
||||
@@ -3244,6 +3281,7 @@ fn keepalive_target_from_pool(pool: &PoolKind, config: &ConnectionConfig) -> Opt
|
||||
PoolKind::ClickHouse(client) => Some(KeepaliveTarget::ClickHouse(client.clone())),
|
||||
PoolKind::SqlServer(client) => Some(KeepaliveTarget::SqlServer(client.clone())),
|
||||
PoolKind::Elasticsearch(client) => Some(KeepaliveTarget::Elasticsearch(client.clone())),
|
||||
PoolKind::HBase(client) => Some(KeepaliveTarget::HBase(client.clone())),
|
||||
PoolKind::VectorDb(client) => Some(KeepaliveTarget::VectorDb(client.clone())),
|
||||
PoolKind::InfluxDb(client) => Some(KeepaliveTarget::InfluxDb(client.clone())),
|
||||
PoolKind::Agent(client) => Some(KeepaliveTarget::Agent(client.clone())),
|
||||
@@ -3274,6 +3312,7 @@ async fn ping_keepalive_target(target: &mut KeepaliveTarget, timeout: Duration)
|
||||
db::sqlserver::test_connection(&mut client).await
|
||||
}
|
||||
KeepaliveTarget::Elasticsearch(client) => db::elasticsearch_driver::test_connection(client, timeout).await,
|
||||
KeepaliveTarget::HBase(client) => db::hbase_driver::test_connection(client, timeout).await.map(|_| ()),
|
||||
KeepaliveTarget::VectorDb(client) => db::vector_driver::test_connection(client, timeout).await,
|
||||
KeepaliveTarget::InfluxDb(client) => db::influxdb_driver::test_connection(client, timeout).await,
|
||||
KeepaliveTarget::Agent(client) => {
|
||||
@@ -3466,6 +3505,7 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
|
||||
PoolKind::ClickHouse(client) => PoolKind::ClickHouse(client.clone()),
|
||||
PoolKind::SqlServer(client) => PoolKind::SqlServer(client.clone()),
|
||||
PoolKind::Elasticsearch(client) => PoolKind::Elasticsearch(client.clone()),
|
||||
PoolKind::HBase(client) => PoolKind::HBase(client.clone()),
|
||||
PoolKind::VectorDb(client) => PoolKind::VectorDb(client.clone()),
|
||||
PoolKind::InfluxDb(client) => PoolKind::InfluxDb(client.clone()),
|
||||
PoolKind::Agent(client) => PoolKind::Agent(client.clone()),
|
||||
@@ -3519,6 +3559,9 @@ pub async fn close_pool_kind(pool: PoolKind) {
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
drop(client);
|
||||
}
|
||||
PoolKind::HBase(client) => {
|
||||
drop(client);
|
||||
}
|
||||
PoolKind::VectorDb(client) => {
|
||||
drop(client);
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ pub fn is_single_connection_pool(db_type: &DatabaseType) -> bool {
|
||||
| DatabaseType::Turso
|
||||
| DatabaseType::CloudflareD1
|
||||
| DatabaseType::MongoDb
|
||||
| DatabaseType::Hbase
|
||||
| DatabaseType::Oracle
|
||||
| DatabaseType::Dameng
|
||||
| DatabaseType::Kingbase
|
||||
|
||||
@@ -0,0 +1,902 @@
|
||||
use base64::{
|
||||
engine::general_purpose::{STANDARD as BASE64, URL_SAFE_NO_PAD as BASE64_URL},
|
||||
Engine as _,
|
||||
};
|
||||
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
|
||||
use reqwest::{Client as HttpClient, RequestBuilder, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::time::Duration;
|
||||
|
||||
use super::{http_client_builder, with_connection_timeout};
|
||||
use crate::models::connection::DatabaseConnectionInfo;
|
||||
use crate::types::{ColumnInfo, DatabaseInfo, TableInfo};
|
||||
|
||||
const PATH_SEGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
|
||||
.add(b' ')
|
||||
.add(b'"')
|
||||
.add(b'#')
|
||||
.add(b'%')
|
||||
.add(b'/')
|
||||
.add(b'<')
|
||||
.add(b'>')
|
||||
.add(b'?')
|
||||
.add(b'[')
|
||||
.add(b'\\')
|
||||
.add(b']')
|
||||
.add(b'^')
|
||||
.add(b'`')
|
||||
.add(b'{')
|
||||
.add(b'|')
|
||||
.add(b'}');
|
||||
|
||||
const HBASE_JSON: &str = "application/json";
|
||||
const HBASE_KEY_ENCODING_HEADER: &str = "Encoding";
|
||||
const HBASE_KEY_ENCODING_BASE64: &str = "base64";
|
||||
const DEFAULT_NAMESPACE: &str = "default";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HBaseClient {
|
||||
http: HttpClient,
|
||||
base_url: String,
|
||||
auth: Option<(String, String)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBaseVersionInfo {
|
||||
pub version: String,
|
||||
pub rest_version: Option<String>,
|
||||
pub server: Option<String>,
|
||||
pub jvm: Option<String>,
|
||||
pub os: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBaseColumnFamily {
|
||||
pub name: String,
|
||||
pub properties: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBaseTableSchema {
|
||||
pub name: String,
|
||||
pub column_families: Vec<HBaseColumnFamily>,
|
||||
pub properties: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBaseCell {
|
||||
pub column: String,
|
||||
pub value: String,
|
||||
pub value_encoding: String,
|
||||
pub value_base64: String,
|
||||
pub timestamp: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBaseRow {
|
||||
pub row_key: String,
|
||||
pub row_key_encoding: String,
|
||||
pub row_key_base64: String,
|
||||
pub cells: Vec<HBaseCell>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBaseScanResult {
|
||||
pub rows: Vec<HBaseRow>,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBaseCellInput {
|
||||
pub column: String,
|
||||
pub value: String,
|
||||
#[serde(default)]
|
||||
pub value_encoding: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HBasePutRowInput {
|
||||
pub row_key: String,
|
||||
#[serde(default)]
|
||||
pub row_key_encoding: Option<String>,
|
||||
pub cells: Vec<HBaseCellInput>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RestNamespaces {
|
||||
#[serde(rename = "Namespace", default)]
|
||||
namespaces: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RestTableList {
|
||||
#[serde(rename = "table", default)]
|
||||
tables: Vec<RestTableName>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RestTableName {
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RestRows {
|
||||
#[serde(rename = "Row", default)]
|
||||
rows: Vec<RestRow>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RestRow {
|
||||
key: String,
|
||||
#[serde(rename = "Cell", default)]
|
||||
cells: Vec<RestCell>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RestCell {
|
||||
column: String,
|
||||
#[serde(default)]
|
||||
timestamp: Option<u64>,
|
||||
#[serde(rename = "$", default)]
|
||||
value: String,
|
||||
}
|
||||
|
||||
impl HBaseClient {
|
||||
pub fn new(
|
||||
url: &str,
|
||||
username: Option<&str>,
|
||||
password: Option<&str>,
|
||||
accept_invalid_certs: bool,
|
||||
timeout: Duration,
|
||||
) -> Result<Self, String> {
|
||||
let base_url = url.trim().trim_end_matches('/').to_string();
|
||||
reqwest::Url::parse(&base_url).map_err(|error| format!("Invalid HBase REST URL: {error}"))?;
|
||||
let auth = username
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(|value| (value.to_string(), password.unwrap_or_default().to_string()));
|
||||
let http = http_client_builder(timeout)
|
||||
.danger_accept_invalid_certs(accept_invalid_certs)
|
||||
.build()
|
||||
.map_err(|error| format!("Failed to create HBase HTTP client: {error}"))?;
|
||||
Ok(Self { http, base_url, auth })
|
||||
}
|
||||
|
||||
fn request(&self, method: reqwest::Method, path: &str) -> RequestBuilder {
|
||||
let request = self.http.request(method, format!("{}{}", self.base_url, path));
|
||||
self.with_auth(request)
|
||||
}
|
||||
|
||||
fn scanner_request(&self, method: reqwest::Method, location: &str) -> Result<RequestBuilder, String> {
|
||||
let path = scanner_resource_path(location)?;
|
||||
Ok(self.request(method, &path))
|
||||
}
|
||||
|
||||
fn with_auth(&self, request: RequestBuilder) -> RequestBuilder {
|
||||
match &self.auth {
|
||||
Some((username, password)) => request.basic_auth(username, Some(password)),
|
||||
None => request,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn test_connection(client: &HBaseClient, timeout: Duration) -> Result<HBaseVersionInfo, String> {
|
||||
with_connection_timeout("HBase", timeout, async { version_info(client).await }).await
|
||||
}
|
||||
|
||||
pub async fn version_info(client: &HBaseClient) -> Result<HBaseVersionInfo, String> {
|
||||
let body = send_json(client.request(reqwest::Method::GET, "/version"), "read server version").await?;
|
||||
Ok(HBaseVersionInfo {
|
||||
version: json_string(&body, "Version").unwrap_or_else(|| "unknown".to_string()),
|
||||
rest_version: json_string(&body, "REST"),
|
||||
server: json_string(&body, "Server"),
|
||||
jvm: json_string(&body, "JVM"),
|
||||
os: json_string(&body, "OS"),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn database_connection_info(client: &HBaseClient) -> Result<Option<DatabaseConnectionInfo>, String> {
|
||||
let version = version_info(client).await?;
|
||||
Ok(Some(DatabaseConnectionInfo {
|
||||
product_name: Some("Apache HBase".to_string()),
|
||||
product_version: Some(version.version),
|
||||
server_comment: version.rest_version.map(|rest| format!("HBase REST {rest}")),
|
||||
driver_name: Some("HBase REST API".to_string()),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn list_namespaces(client: &HBaseClient) -> Result<Vec<DatabaseInfo>, String> {
|
||||
let response = client
|
||||
.request(reqwest::Method::GET, "/namespaces")
|
||||
.header(reqwest::header::ACCEPT, HBASE_JSON)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("list namespaces", error))?;
|
||||
let body: RestNamespaces = parse_success_json(response, "list namespaces").await?;
|
||||
let mut namespaces = body.namespaces;
|
||||
namespaces.sort_by_key(|name| name.to_ascii_lowercase());
|
||||
Ok(namespaces.into_iter().map(|name| DatabaseInfo { name }).collect())
|
||||
}
|
||||
|
||||
pub async fn list_tables(client: &HBaseClient, namespace: &str) -> Result<Vec<TableInfo>, String> {
|
||||
let response = client
|
||||
.request(reqwest::Method::GET, "/")
|
||||
.header(reqwest::header::ACCEPT, HBASE_JSON)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("list tables", error))?;
|
||||
let body: RestTableList = parse_success_json(response, "list tables").await?;
|
||||
let mut names: Vec<String> =
|
||||
body.tables.into_iter().filter_map(|table| table_name_in_namespace(&table.name, namespace)).collect();
|
||||
names.sort_by_key(|name| name.to_ascii_lowercase());
|
||||
Ok(names
|
||||
.into_iter()
|
||||
.map(|name| TableInfo {
|
||||
name,
|
||||
table_type: "HBASE_TABLE".to_string(),
|
||||
comment: None,
|
||||
parent_schema: None,
|
||||
parent_name: None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn get_table_schema(client: &HBaseClient, namespace: &str, table: &str) -> Result<HBaseTableSchema, String> {
|
||||
let qualified = qualified_table_name(namespace, table)?;
|
||||
let path = format!("/{}/schema", path_segment(&qualified));
|
||||
let body = send_json(client.request(reqwest::Method::GET, &path), "read table schema").await?;
|
||||
parse_table_schema(body)
|
||||
}
|
||||
|
||||
pub async fn get_columns(client: &HBaseClient, namespace: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
let schema = get_table_schema(client, namespace, table).await?;
|
||||
Ok(schema
|
||||
.column_families
|
||||
.into_iter()
|
||||
.map(|family| ColumnInfo {
|
||||
name: family.name,
|
||||
data_type: "COLUMN_FAMILY".to_string(),
|
||||
is_nullable: true,
|
||||
extra: family.properties.get("VERSIONS").map(|versions| format!("versions={versions}")),
|
||||
comment: family.properties.get("COMPRESSION").map(|compression| format!("compression={compression}")),
|
||||
..Default::default()
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn scan_rows(
|
||||
client: &HBaseClient,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
row_key_prefix: Option<&str>,
|
||||
limit: usize,
|
||||
) -> Result<HBaseScanResult, String> {
|
||||
let qualified = qualified_table_name(namespace, table)?;
|
||||
let limit = limit.clamp(1, 1000);
|
||||
let prefix = row_key_prefix.map(str::as_bytes).filter(|value| !value.is_empty());
|
||||
let mut scanner = Map::new();
|
||||
scanner.insert("batch".to_string(), Value::from((limit.saturating_mul(10)).clamp(10, 1000) as u64));
|
||||
scanner.insert("maxVersions".to_string(), Value::from(1));
|
||||
if let Some(prefix) = prefix {
|
||||
scanner.insert("startRow".to_string(), Value::String(BASE64.encode(prefix)));
|
||||
if let Some(end) = prefix_range_end(prefix) {
|
||||
scanner.insert("endRow".to_string(), Value::String(BASE64.encode(end)));
|
||||
}
|
||||
}
|
||||
|
||||
let create_path = format!("/{}/scanner", path_segment(&qualified));
|
||||
let response = client
|
||||
.request(reqwest::Method::PUT, &create_path)
|
||||
.header(reqwest::header::CONTENT_TYPE, HBASE_JSON)
|
||||
.json(&Value::Object(scanner))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("create scanner", error))?;
|
||||
let response = ensure_success(response, "create scanner").await?;
|
||||
let scanner_url = response
|
||||
.headers()
|
||||
.get(reqwest::header::LOCATION)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| "HBase REST did not return a scanner location".to_string())?;
|
||||
|
||||
let scan_result = read_scanner(client, &scanner_url, limit).await;
|
||||
let close_result = close_scanner(client, &scanner_url).await;
|
||||
match (scan_result, close_result) {
|
||||
(Ok(result), Ok(())) => Ok(result),
|
||||
(Ok(_), Err(error)) => Err(error),
|
||||
(Err(error), _) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_row(
|
||||
client: &HBaseClient,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
row_key: &str,
|
||||
row_key_encoding: Option<&str>,
|
||||
) -> Result<Option<HBaseRow>, String> {
|
||||
let qualified = qualified_table_name(namespace, table)?;
|
||||
let row_key_bytes = decode_input(row_key, row_key_encoding, "row key")?;
|
||||
if row_key_bytes.is_empty() {
|
||||
return Err("HBase row key cannot be empty".to_string());
|
||||
}
|
||||
let path = encoded_row_path(&qualified, &row_key_bytes);
|
||||
let response = client
|
||||
.request(reqwest::Method::GET, &path)
|
||||
.header(HBASE_KEY_ENCODING_HEADER, HBASE_KEY_ENCODING_BASE64)
|
||||
.header(reqwest::header::ACCEPT, HBASE_JSON)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("read row", error))?;
|
||||
if response.status() == StatusCode::NOT_FOUND {
|
||||
return Ok(None);
|
||||
}
|
||||
let body: RestRows = parse_success_json(response, "read row").await?;
|
||||
let mut rows = decode_rest_rows(body.rows)?;
|
||||
Ok(rows.pop())
|
||||
}
|
||||
|
||||
pub async fn put_row(
|
||||
client: &HBaseClient,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
input: &HBasePutRowInput,
|
||||
) -> Result<(), String> {
|
||||
let qualified = qualified_table_name(namespace, table)?;
|
||||
let row_key = decode_input(&input.row_key, input.row_key_encoding.as_deref(), "row key")?;
|
||||
if row_key.is_empty() {
|
||||
return Err("HBase row key cannot be empty".to_string());
|
||||
}
|
||||
if input.cells.is_empty() {
|
||||
return Err("At least one HBase cell is required".to_string());
|
||||
}
|
||||
let mut cells = Vec::with_capacity(input.cells.len());
|
||||
for cell in &input.cells {
|
||||
validate_column_name(&cell.column)?;
|
||||
let value = decode_input(&cell.value, cell.value_encoding.as_deref(), "cell value")?;
|
||||
cells.push(serde_json::json!({
|
||||
"column": BASE64.encode(cell.column.as_bytes()),
|
||||
"$": BASE64.encode(value),
|
||||
}));
|
||||
}
|
||||
let body = serde_json::json!({
|
||||
"Row": [{
|
||||
"key": BASE64.encode(&row_key),
|
||||
"Cell": cells,
|
||||
}]
|
||||
});
|
||||
let path = encoded_row_path(&qualified, &row_key);
|
||||
let response = client
|
||||
.request(reqwest::Method::PUT, &path)
|
||||
.header(HBASE_KEY_ENCODING_HEADER, HBASE_KEY_ENCODING_BASE64)
|
||||
.header(reqwest::header::CONTENT_TYPE, HBASE_JSON)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("write row", error))?;
|
||||
ensure_success(response, "write row").await.map(|_| ())
|
||||
}
|
||||
|
||||
pub async fn delete_row(
|
||||
client: &HBaseClient,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
row_key: &str,
|
||||
row_key_encoding: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
let qualified = qualified_table_name(namespace, table)?;
|
||||
let row_key = decode_input(row_key, row_key_encoding, "row key")?;
|
||||
if row_key.is_empty() {
|
||||
return Err("HBase row key cannot be empty".to_string());
|
||||
}
|
||||
let path = encoded_row_path(&qualified, &row_key);
|
||||
let response = client
|
||||
.request(reqwest::Method::DELETE, &path)
|
||||
.header(HBASE_KEY_ENCODING_HEADER, HBASE_KEY_ENCODING_BASE64)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("delete row", error))?;
|
||||
ensure_success(response, "delete row").await.map(|_| ())
|
||||
}
|
||||
|
||||
pub async fn create_table(
|
||||
client: &HBaseClient,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
column_families: &[String],
|
||||
) -> Result<(), String> {
|
||||
let qualified = qualified_table_name(namespace, table)?;
|
||||
if column_families.is_empty() {
|
||||
return Err("At least one HBase column family is required".to_string());
|
||||
}
|
||||
let mut families = Vec::with_capacity(column_families.len());
|
||||
for family in column_families {
|
||||
validate_family_name(family)?;
|
||||
families.push(serde_json::json!({ "name": family.trim() }));
|
||||
}
|
||||
let path = format!("/{}/schema", path_segment(&qualified));
|
||||
let response = client
|
||||
.request(reqwest::Method::PUT, &path)
|
||||
.header(reqwest::header::CONTENT_TYPE, HBASE_JSON)
|
||||
.json(&serde_json::json!({ "ColumnSchema": families }))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("create table", error))?;
|
||||
ensure_success(response, "create table").await.map(|_| ())
|
||||
}
|
||||
|
||||
pub async fn delete_table(client: &HBaseClient, namespace: &str, table: &str) -> Result<(), String> {
|
||||
let qualified = qualified_table_name(namespace, table)?;
|
||||
let path = format!("/{}/schema", path_segment(&qualified));
|
||||
let response = client
|
||||
.request(reqwest::Method::DELETE, &path)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("delete table", error))?;
|
||||
ensure_success(response, "delete table").await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn read_scanner(client: &HBaseClient, scanner_url: &str, limit: usize) -> Result<HBaseScanResult, String> {
|
||||
let mut order = Vec::<String>::new();
|
||||
let mut rows = HashMap::<String, RestRow>::new();
|
||||
let mut exhausted = false;
|
||||
|
||||
while rows.len() <= limit {
|
||||
let response = client
|
||||
.scanner_request(reqwest::Method::GET, scanner_url)?
|
||||
.header(reqwest::header::ACCEPT, HBASE_JSON)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("read scanner", error))?;
|
||||
if response.status() == StatusCode::NO_CONTENT {
|
||||
exhausted = true;
|
||||
break;
|
||||
}
|
||||
let body: RestRows = parse_success_json(response, "read scanner").await?;
|
||||
if body.rows.is_empty() {
|
||||
exhausted = true;
|
||||
break;
|
||||
}
|
||||
for row in body.rows {
|
||||
let key = row.key.clone();
|
||||
if let Some(existing) = rows.get_mut(&key) {
|
||||
existing.cells.extend(row.cells);
|
||||
} else {
|
||||
order.push(key.clone());
|
||||
rows.insert(key, row);
|
||||
}
|
||||
}
|
||||
if rows.len() > limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let truncated = !exhausted && rows.len() >= limit;
|
||||
let mut ordered_rows = Vec::with_capacity(order.len().min(limit));
|
||||
for key in order.into_iter().take(limit) {
|
||||
if let Some(row) = rows.remove(&key) {
|
||||
ordered_rows.push(row);
|
||||
}
|
||||
}
|
||||
Ok(HBaseScanResult { rows: decode_rest_rows(ordered_rows)?, truncated })
|
||||
}
|
||||
|
||||
async fn close_scanner(client: &HBaseClient, scanner_url: &str) -> Result<(), String> {
|
||||
let response = client
|
||||
.scanner_request(reqwest::Method::DELETE, scanner_url)?
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error("close scanner", error))?;
|
||||
if response.status() == StatusCode::NOT_FOUND {
|
||||
return Ok(());
|
||||
}
|
||||
ensure_success(response, "close scanner").await.map(|_| ())
|
||||
}
|
||||
|
||||
fn decode_rest_rows(rows: Vec<RestRow>) -> Result<Vec<HBaseRow>, String> {
|
||||
rows.into_iter()
|
||||
.map(|row| {
|
||||
let row_key_bytes =
|
||||
BASE64.decode(&row.key).map_err(|error| format!("Invalid HBase row key Base64: {error}"))?;
|
||||
let (row_key, row_key_encoding) = display_bytes(&row_key_bytes);
|
||||
let mut cells = row
|
||||
.cells
|
||||
.into_iter()
|
||||
.map(|cell| {
|
||||
let column_bytes =
|
||||
BASE64.decode(&cell.column).map_err(|error| format!("Invalid HBase column Base64: {error}"))?;
|
||||
let column = String::from_utf8(column_bytes)
|
||||
.map_err(|_| "HBase returned a non-UTF-8 column identifier".to_string())?;
|
||||
let value_bytes =
|
||||
BASE64.decode(&cell.value).map_err(|error| format!("Invalid HBase cell Base64: {error}"))?;
|
||||
let value_base64 = BASE64.encode(&value_bytes);
|
||||
let (value, value_encoding) = display_bytes(&value_bytes);
|
||||
Ok(HBaseCell { column, value, value_encoding, value_base64, timestamp: cell.timestamp })
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?;
|
||||
cells.sort_by(|left, right| left.column.cmp(&right.column));
|
||||
Ok(HBaseRow { row_key, row_key_encoding, row_key_base64: BASE64.encode(row_key_bytes), cells })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_table_schema(body: Value) -> Result<HBaseTableSchema, String> {
|
||||
let object = body.as_object().ok_or_else(|| "Invalid HBase table schema response".to_string())?;
|
||||
let name = object.get("name").and_then(Value::as_str).unwrap_or_default().to_string();
|
||||
let mut column_families = Vec::new();
|
||||
for raw_family in object.get("ColumnSchema").and_then(Value::as_array).into_iter().flatten() {
|
||||
let family = raw_family.as_object().ok_or_else(|| "Invalid HBase column family response".to_string())?;
|
||||
let family_name = family.get("name").and_then(Value::as_str).unwrap_or_default().to_string();
|
||||
if family_name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
column_families.push(HBaseColumnFamily { name: family_name, properties: string_properties(family, &["name"]) });
|
||||
}
|
||||
column_families.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
Ok(HBaseTableSchema { name, column_families, properties: string_properties(object, &["name", "ColumnSchema"]) })
|
||||
}
|
||||
|
||||
fn string_properties(object: &Map<String, Value>, excluded: &[&str]) -> BTreeMap<String, String> {
|
||||
object
|
||||
.iter()
|
||||
.filter(|(key, _)| !excluded.contains(&key.as_str()))
|
||||
.filter_map(|(key, value)| match value {
|
||||
Value::String(value) => Some((key.clone(), value.clone())),
|
||||
Value::Bool(value) => Some((key.clone(), value.to_string())),
|
||||
Value::Number(value) => Some((key.clone(), value.to_string())),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn json_string(value: &Value, key: &str) -> Option<String> {
|
||||
value.get(key).and_then(Value::as_str).map(str::to_string)
|
||||
}
|
||||
|
||||
async fn send_json(request: RequestBuilder, action: &str) -> Result<Value, String> {
|
||||
let response = request
|
||||
.header(reqwest::header::ACCEPT, HBASE_JSON)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format_request_error(action, error))?;
|
||||
parse_success_json(response, action).await
|
||||
}
|
||||
|
||||
async fn parse_success_json<T: serde::de::DeserializeOwned>(response: Response, action: &str) -> Result<T, String> {
|
||||
let response = ensure_success(response, action).await?;
|
||||
response.json::<T>().await.map_err(|error| format!("HBase failed to parse {action} response: {error}"))
|
||||
}
|
||||
|
||||
async fn ensure_success(response: Response, action: &str) -> Result<Response, String> {
|
||||
let status = response.status();
|
||||
if status.is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
let detail = body.trim();
|
||||
if detail.is_empty() {
|
||||
Err(format!("HBase {action} failed ({status})"))
|
||||
} else {
|
||||
Err(format!("HBase {action} failed ({status}): {detail}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn format_request_error(action: &str, error: reqwest::Error) -> String {
|
||||
format!("HBase {action} request failed: {error}")
|
||||
}
|
||||
|
||||
fn qualified_table_name(namespace: &str, table: &str) -> Result<String, String> {
|
||||
let namespace = namespace.trim();
|
||||
let table = table.trim();
|
||||
if table.is_empty() {
|
||||
return Err("HBase table name cannot be empty".to_string());
|
||||
}
|
||||
if table.contains(':') {
|
||||
return Ok(table.to_string());
|
||||
}
|
||||
if namespace.is_empty() || namespace == DEFAULT_NAMESPACE {
|
||||
Ok(table.to_string())
|
||||
} else {
|
||||
Ok(format!("{namespace}:{table}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn table_name_in_namespace(qualified: &str, namespace: &str) -> Option<String> {
|
||||
let namespace = namespace.trim();
|
||||
match qualified.split_once(':') {
|
||||
Some((table_namespace, table)) if table_namespace == namespace => Some(table.to_string()),
|
||||
Some(_) => None,
|
||||
None if namespace.is_empty() || namespace == DEFAULT_NAMESPACE => Some(qualified.to_string()),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_family_name(value: &str) -> Result<(), String> {
|
||||
let value = value.trim();
|
||||
if value.is_empty() {
|
||||
return Err("HBase column family name cannot be empty".to_string());
|
||||
}
|
||||
if value.contains(':') {
|
||||
return Err(format!("HBase column family name cannot contain ':': {value}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_column_name(value: &str) -> Result<(), String> {
|
||||
let Some((family, qualifier)) = value.split_once(':') else {
|
||||
return Err(format!("HBase column must use family:qualifier syntax: {value}"));
|
||||
};
|
||||
validate_family_name(family)?;
|
||||
if qualifier.is_empty() {
|
||||
return Err(format!("HBase column qualifier cannot be empty: {value}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_input(value: &str, encoding: Option<&str>, label: &str) -> Result<Vec<u8>, String> {
|
||||
let encoding = encoding.unwrap_or("utf8").trim();
|
||||
if encoding.eq_ignore_ascii_case("base64") {
|
||||
BASE64.decode(value.trim()).map_err(|error| format!("Invalid Base64 {label}: {error}"))
|
||||
} else if encoding.eq_ignore_ascii_case("utf8") || encoding.is_empty() {
|
||||
Ok(value.as_bytes().to_vec())
|
||||
} else {
|
||||
Err(format!("Unsupported HBase {label} encoding: {encoding}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn display_bytes(bytes: &[u8]) -> (String, String) {
|
||||
match std::str::from_utf8(bytes) {
|
||||
Ok(text) if text.chars().all(|ch| !ch.is_control() || matches!(ch, '\n' | '\r' | '\t')) => {
|
||||
(text.to_string(), "utf8".to_string())
|
||||
}
|
||||
_ => (format!("base64:{}", BASE64.encode(bytes)), "base64".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn prefix_range_end(prefix: &[u8]) -> Option<Vec<u8>> {
|
||||
let mut end = prefix.to_vec();
|
||||
for index in (0..end.len()).rev() {
|
||||
if end[index] != u8::MAX {
|
||||
end[index] += 1;
|
||||
end.truncate(index + 1);
|
||||
return Some(end);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn path_segment(value: &str) -> String {
|
||||
utf8_percent_encode(value, PATH_SEGMENT_ENCODE_SET).to_string()
|
||||
}
|
||||
|
||||
fn encoded_row_path(qualified_table: &str, row_key: &[u8]) -> String {
|
||||
format!("/{}/{}", path_segment(qualified_table), BASE64_URL.encode(row_key))
|
||||
}
|
||||
|
||||
fn scanner_resource_path(location: &str) -> Result<String, String> {
|
||||
let location = location.trim();
|
||||
if location.is_empty() {
|
||||
return Err("HBase REST returned an empty scanner location".to_string());
|
||||
}
|
||||
if let Ok(url) = reqwest::Url::parse(location) {
|
||||
let mut path = url.path().to_string();
|
||||
if let Some(query) = url.query() {
|
||||
path.push('?');
|
||||
path.push_str(query);
|
||||
}
|
||||
return Ok(path);
|
||||
}
|
||||
Ok(if location.starts_with('/') { location.to_string() } else { format!("/{location}") })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[test]
|
||||
fn namespace_table_names_are_filtered_and_unqualified() {
|
||||
assert_eq!(table_name_in_namespace("orders", "default"), Some("orders".to_string()));
|
||||
assert_eq!(table_name_in_namespace("sales:orders", "sales"), Some("orders".to_string()));
|
||||
assert_eq!(table_name_in_namespace("sales:orders", "default"), None);
|
||||
assert_eq!(qualified_table_name("sales", "orders").unwrap(), "sales:orders");
|
||||
assert_eq!(qualified_table_name("default", "orders").unwrap(), "orders");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_range_end_handles_carry_and_unbounded_prefix() {
|
||||
assert_eq!(prefix_range_end(b"customer#00"), Some(b"customer#01".to_vec()));
|
||||
assert_eq!(prefix_range_end(&[0x01, 0xff]), Some(vec![0x02]));
|
||||
assert_eq!(prefix_range_end(&[0xff, 0xff]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binary_values_keep_base64_representation() {
|
||||
assert_eq!(display_bytes(b"hello"), ("hello".to_string(), "utf8".to_string()));
|
||||
assert_eq!(display_bytes(&[0, 1, 2]), ("base64:AAEC".to_string(), "base64".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn row_paths_use_url_safe_base64_for_arbitrary_keys() {
|
||||
assert_eq!(encoded_row_path("demo", b"customer#001"), "/demo/Y3VzdG9tZXIjMDAx");
|
||||
assert_eq!(encoded_row_path("demo", &[0, 1, 2, 0xfb, 0xff]), "/demo/AAEC-_8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_schema_properties_are_preserved() {
|
||||
let schema = parse_table_schema(serde_json::json!({
|
||||
"name": "demo",
|
||||
"ColumnSchema": [{"name": "cf", "VERSIONS": "3", "COMPRESSION": "SNAPPY"}],
|
||||
"IS_META": "false"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(schema.name, "demo");
|
||||
assert_eq!(schema.column_families[0].name, "cf");
|
||||
assert_eq!(schema.column_families[0].properties.get("VERSIONS").map(String::as_str), Some("3"));
|
||||
assert_eq!(schema.properties.get("IS_META").map(String::as_str), Some("false"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_location_reuses_the_configured_gateway_origin() {
|
||||
assert_eq!(
|
||||
scanner_resource_path("http://hbase.internal:8080/dbx/scanner/123?token=abc").unwrap(),
|
||||
"/dbx/scanner/123?token=abc"
|
||||
);
|
||||
assert_eq!(scanner_resource_path("/dbx/scanner/123").unwrap(), "/dbx/scanner/123");
|
||||
assert_eq!(scanner_resource_path("dbx/scanner/123").unwrap(), "/dbx/scanner/123");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_TEST_HBASE_URL and a writable HBase REST gateway"]
|
||||
async fn live_hbase_rest_crud_and_scan() {
|
||||
let url = std::env::var("DBX_TEST_HBASE_URL").expect("DBX_TEST_HBASE_URL is required");
|
||||
let username = std::env::var("DBX_TEST_HBASE_USERNAME").ok();
|
||||
let password = std::env::var("DBX_TEST_HBASE_PASSWORD").ok();
|
||||
let client = HBaseClient::new(&url, username.as_deref(), password.as_deref(), false, Duration::from_secs(5))
|
||||
.expect("create HBase client");
|
||||
let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis();
|
||||
let table = format!("dbx_hbase_live_{suffix}");
|
||||
let namespace = "default";
|
||||
|
||||
let version = test_connection(&client, Duration::from_secs(5)).await.expect("test connection");
|
||||
assert!(!version.version.is_empty());
|
||||
let namespaces = list_namespaces(&client).await.expect("list namespaces");
|
||||
assert!(namespaces.iter().any(|item| item.name == namespace));
|
||||
|
||||
create_table(&client, namespace, &table, &["profile".to_string(), "metrics".to_string()])
|
||||
.await
|
||||
.expect("create table");
|
||||
let test_result: Result<(), String> = async {
|
||||
let tables = list_tables(&client, namespace).await?;
|
||||
if !tables.iter().any(|item| item.name == table) {
|
||||
return Err("created table is missing from the table list".to_string());
|
||||
}
|
||||
let schema = get_table_schema(&client, namespace, &table).await?;
|
||||
if schema.column_families.iter().map(|family| family.name.as_str()).collect::<Vec<_>>()
|
||||
!= vec!["metrics", "profile"]
|
||||
{
|
||||
return Err(format!("unexpected column families: {:?}", schema.column_families));
|
||||
}
|
||||
|
||||
for (key, name, score) in
|
||||
[("customer#001", "Alice", "10"), ("customer#002", "Bob", "20"), ("customer#003", "Carol", "30")]
|
||||
{
|
||||
put_row(
|
||||
&client,
|
||||
namespace,
|
||||
&table,
|
||||
&HBasePutRowInput {
|
||||
row_key: key.to_string(),
|
||||
row_key_encoding: Some("utf8".to_string()),
|
||||
cells: vec![
|
||||
HBaseCellInput {
|
||||
column: "profile:name".to_string(),
|
||||
value: name.to_string(),
|
||||
value_encoding: Some("utf8".to_string()),
|
||||
},
|
||||
HBaseCellInput {
|
||||
column: "metrics:score".to_string(),
|
||||
value: score.to_string(),
|
||||
value_encoding: Some("utf8".to_string()),
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let limited = scan_rows(&client, namespace, &table, Some("customer#00"), 2).await?;
|
||||
if limited.rows.len() != 2 || !limited.truncated {
|
||||
return Err(format!("unexpected limited scan result: {limited:?}"));
|
||||
}
|
||||
let all = scan_rows(&client, namespace, &table, Some("customer#00"), 10).await?;
|
||||
if all.rows.iter().map(|row| row.row_key.as_str()).collect::<Vec<_>>()
|
||||
!= vec!["customer#001", "customer#002", "customer#003"]
|
||||
|| all.truncated
|
||||
{
|
||||
return Err(format!("unexpected full scan result: {all:?}"));
|
||||
}
|
||||
|
||||
let row = get_row(&client, namespace, &table, "customer#002", Some("utf8"))
|
||||
.await?
|
||||
.ok_or_else(|| "customer#002 is missing".to_string())?;
|
||||
if !row.cells.iter().any(|cell| cell.column == "profile:name" && cell.value == "Bob") {
|
||||
return Err(format!("unexpected customer#002 row: {row:?}"));
|
||||
}
|
||||
put_row(
|
||||
&client,
|
||||
namespace,
|
||||
&table,
|
||||
&HBasePutRowInput {
|
||||
row_key: "customer#002".to_string(),
|
||||
row_key_encoding: Some("utf8".to_string()),
|
||||
cells: vec![HBaseCellInput {
|
||||
column: "metrics:score".to_string(),
|
||||
value: "25".to_string(),
|
||||
value_encoding: Some("utf8".to_string()),
|
||||
}],
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let updated = get_row(&client, namespace, &table, "customer#002", Some("utf8"))
|
||||
.await?
|
||||
.ok_or_else(|| "updated customer#002 is missing".to_string())?;
|
||||
if !updated.cells.iter().any(|cell| cell.column == "metrics:score" && cell.value == "25") {
|
||||
return Err(format!("updated cell is missing: {updated:?}"));
|
||||
}
|
||||
|
||||
put_row(
|
||||
&client,
|
||||
namespace,
|
||||
&table,
|
||||
&HBasePutRowInput {
|
||||
row_key: "AAEC".to_string(),
|
||||
row_key_encoding: Some("base64".to_string()),
|
||||
cells: vec![HBaseCellInput {
|
||||
column: "profile:binary".to_string(),
|
||||
value: "/wA=".to_string(),
|
||||
value_encoding: Some("base64".to_string()),
|
||||
}],
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let binary = get_row(&client, namespace, &table, "AAEC", Some("base64"))
|
||||
.await?
|
||||
.ok_or_else(|| "binary row is missing".to_string())?;
|
||||
if binary.row_key != "base64:AAEC"
|
||||
|| !binary.cells.iter().any(|cell| {
|
||||
cell.column == "profile:binary" && cell.value == "base64:/wA=" && cell.value_base64 == "/wA="
|
||||
})
|
||||
{
|
||||
return Err(format!("unexpected binary row: {binary:?}"));
|
||||
}
|
||||
|
||||
delete_row(&client, namespace, &table, "customer#003", Some("utf8")).await?;
|
||||
if get_row(&client, namespace, &table, "customer#003", Some("utf8")).await?.is_some() {
|
||||
return Err("customer#003 still exists after deletion".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
let cleanup_result = delete_table(&client, namespace, &table).await;
|
||||
if let Err(error) = test_result {
|
||||
panic!("live HBase test failed: {error:?}; cleanup result: {cleanup_result:?}");
|
||||
}
|
||||
cleanup_result.expect("delete temporary table");
|
||||
let tables = list_tables(&client, namespace).await.expect("list tables after cleanup");
|
||||
assert!(!tables.iter().any(|item| item.name == table));
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ pub mod duckdb_worker_runtime;
|
||||
pub mod elasticsearch_driver;
|
||||
pub mod elasticsearch_sql;
|
||||
pub mod file_validator;
|
||||
pub mod hbase_driver;
|
||||
pub mod http_tunnel;
|
||||
pub mod influxdb_driver;
|
||||
pub mod manticoresearch;
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::hbase_driver::{self, HBasePutRowInput, HBaseRow, HBaseScanResult, HBaseTableSchema};
|
||||
|
||||
async fn client(state: &AppState, connection_id: &str) -> Result<hbase_driver::HBaseClient, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(&pool_key) {
|
||||
Some(PoolKind::HBase(client)) => Ok(client.clone()),
|
||||
_ => Err("Not an HBase connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_table_schema_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
) -> Result<HBaseTableSchema, String> {
|
||||
hbase_driver::get_table_schema(&client(state, connection_id).await?, namespace, table).await
|
||||
}
|
||||
|
||||
pub async fn scan_rows_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
row_key_prefix: Option<&str>,
|
||||
limit: usize,
|
||||
) -> Result<HBaseScanResult, String> {
|
||||
hbase_driver::scan_rows(&client(state, connection_id).await?, namespace, table, row_key_prefix, limit).await
|
||||
}
|
||||
|
||||
pub async fn get_row_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
row_key: &str,
|
||||
row_key_encoding: Option<&str>,
|
||||
) -> Result<Option<HBaseRow>, String> {
|
||||
hbase_driver::get_row(&client(state, connection_id).await?, namespace, table, row_key, row_key_encoding).await
|
||||
}
|
||||
|
||||
pub async fn put_row_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
input: &HBasePutRowInput,
|
||||
) -> Result<(), String> {
|
||||
hbase_driver::put_row(&client(state, connection_id).await?, namespace, table, input).await
|
||||
}
|
||||
|
||||
pub async fn delete_row_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
row_key: &str,
|
||||
row_key_encoding: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
hbase_driver::delete_row(&client(state, connection_id).await?, namespace, table, row_key, row_key_encoding).await
|
||||
}
|
||||
|
||||
pub async fn create_table_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
column_families: &[String],
|
||||
) -> Result<(), String> {
|
||||
hbase_driver::create_table(&client(state, connection_id).await?, namespace, table, column_families).await
|
||||
}
|
||||
|
||||
pub async fn delete_table_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
) -> Result<(), String> {
|
||||
hbase_driver::delete_table(&client(state, connection_id).await?, namespace, table).await
|
||||
}
|
||||
@@ -28,6 +28,7 @@ pub mod db_admin_sql;
|
||||
pub mod document_ops;
|
||||
pub mod driver_runtime;
|
||||
pub mod external;
|
||||
pub mod hbase_ops;
|
||||
pub mod history;
|
||||
pub mod jdbc;
|
||||
pub mod models;
|
||||
|
||||
@@ -455,6 +455,7 @@ pub enum DatabaseType {
|
||||
Oracle,
|
||||
#[serde(rename = "elasticsearch")]
|
||||
Elasticsearch,
|
||||
Hbase,
|
||||
#[serde(rename = "qdrant")]
|
||||
Qdrant,
|
||||
#[serde(rename = "milvus")]
|
||||
@@ -1001,6 +1002,7 @@ impl ConnectionConfig {
|
||||
}
|
||||
DatabaseType::Oracle => format!("oracle://{host}:{port}{db_part}"),
|
||||
DatabaseType::Elasticsearch
|
||||
| DatabaseType::Hbase
|
||||
| DatabaseType::Qdrant
|
||||
| DatabaseType::Milvus
|
||||
| DatabaseType::Weaviate
|
||||
@@ -1149,6 +1151,7 @@ impl ConnectionConfig {
|
||||
format!("oracle://{}:{}@{host}:{port}{db_part}", username, password)
|
||||
}
|
||||
DatabaseType::Elasticsearch
|
||||
| DatabaseType::Hbase
|
||||
| DatabaseType::Qdrant
|
||||
| DatabaseType::Milvus
|
||||
| DatabaseType::Weaviate
|
||||
@@ -2762,6 +2765,20 @@ mod tests {
|
||||
assert_eq!(config.connection_url(), "https://10.1.2.3:8443");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hbase_rest_url_uses_http_or_https_without_embedding_credentials() {
|
||||
let mut config = mysql_config("hbase-user", "secret", None);
|
||||
config.db_type = DatabaseType::Hbase;
|
||||
config.port = 8080;
|
||||
|
||||
assert_eq!(config.connection_url(), "http://10.1.2.3:8080");
|
||||
assert_eq!(config.redacted_connection_url(), "http://10.1.2.3:8080");
|
||||
|
||||
config.ssl = true;
|
||||
assert_eq!(config.connection_url(), "https://10.1.2.3:8080");
|
||||
assert_eq!(config.redacted_connection_url(), "https://10.1.2.3:8080");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clickhouse_host_may_include_http_scheme() {
|
||||
let mut config = mysql_config("default", "", None);
|
||||
|
||||
@@ -1696,6 +1696,7 @@ pub async fn do_execute(
|
||||
.await
|
||||
.map(|result| truncate_result_with_max_rows(result, max_rows))
|
||||
}
|
||||
PoolKind::HBase(_) => Err("SQL execution is not supported for HBase connections".to_string()),
|
||||
};
|
||||
result.map(normalize_query_result_for_js)
|
||||
}
|
||||
@@ -2592,7 +2593,7 @@ pub async fn execute_statements_in_transaction_on_pool(
|
||||
| PoolKind::Turso(_)
|
||||
| PoolKind::SqlServer(_)
|
||||
| PoolKind::Agent(_) => TxPath::Explicit,
|
||||
PoolKind::MessageQueue | PoolKind::Nacos => TxPath::None,
|
||||
PoolKind::MessageQueue | PoolKind::Nacos | PoolKind::HBase(_) => TxPath::None,
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::DuckDb(_)
|
||||
| PoolKind::DuckDbWorker(_)
|
||||
|
||||
@@ -1124,6 +1124,7 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
|
||||
PoolKind::Postgres(p) => db::postgres::list_databases(p).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::list_databases(p).await,
|
||||
PoolKind::Rqlite(client) => db::rqlite_driver::list_databases(client).await,
|
||||
PoolKind::HBase(client) => db::hbase_driver::list_namespaces(client).await,
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
PoolKind::DuckDb(con) => {
|
||||
let con = con.lock().map_err(|e| e.to_string())?;
|
||||
@@ -2473,6 +2474,9 @@ async fn list_tables_once(
|
||||
.await
|
||||
.map(|names| collection_names_to_tables(names, "INDEX"))
|
||||
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types, table_name_filter)),
|
||||
PoolKind::HBase(client) => db::hbase_driver::list_tables(client, database)
|
||||
.await
|
||||
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types, table_name_filter)),
|
||||
PoolKind::VectorDb(client) => db::vector_driver::list_collections(client)
|
||||
.await
|
||||
.map(|infos| collection_names_to_tables(infos.into_iter().map(|i| i.name).collect(), "COLLECTION"))
|
||||
@@ -5215,6 +5219,9 @@ pub async fn get_columns_core_for_session(
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
db::elasticsearch_driver::get_columns(client, table).await.map(deduplicate_column_infos)
|
||||
}
|
||||
PoolKind::HBase(client) => {
|
||||
db::hbase_driver::get_columns(client, database, table).await.map(deduplicate_column_infos)
|
||||
}
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
})
|
||||
|
||||
@@ -15,6 +15,7 @@ pub(in crate::schema) async fn list_databases(
|
||||
PoolKind::Postgres(p) => db::postgres::list_databases(p).await,
|
||||
PoolKind::Sqlite(p) => db::sqlite::list_databases(p).await,
|
||||
PoolKind::Rqlite(client) => db::rqlite_driver::list_databases(client).await,
|
||||
PoolKind::HBase(client) => db::hbase_driver::list_namespaces(client).await,
|
||||
PoolKind::Turso(client) => db::turso_driver::list_databases(client).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
@@ -60,6 +61,7 @@ pub(in crate::schema) async fn list_tables(
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
db::elasticsearch_driver::list_indices(client).await.map(|names| collection_names_to_tables(names, "INDEX"))
|
||||
}
|
||||
PoolKind::HBase(client) => db::hbase_driver::list_tables(client, database).await,
|
||||
PoolKind::VectorDb(client) => db::vector_driver::list_collections(client)
|
||||
.await
|
||||
.map(|infos| collection_names_to_tables(infos.into_iter().map(|i| i.name).collect(), "COLLECTION")),
|
||||
@@ -149,6 +151,7 @@ pub(in crate::schema) async fn get_columns(
|
||||
PoolKind::Rqlite(client) => db::rqlite_driver::get_columns(client, schema, table).await,
|
||||
PoolKind::Turso(client) => db::turso_driver::get_columns(client, schema, table).await,
|
||||
PoolKind::Elasticsearch(client) => db::elasticsearch_driver::get_columns(client, table).await,
|
||||
PoolKind::HBase(client) => db::hbase_driver::get_columns(client, database, table).await,
|
||||
PoolKind::VectorDb(_) => Ok(vec![]),
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user