fix(influxdb): support table export query dispatch and pagination

Closes #8022
This commit is contained in:
t8y2 committed 2026-09-04 02:35:01 +08:00
1 parent 0a511072a7
commit 48410b70b4
2 files changed
+254 -5

No files matched your search

+153 -5
View File
@@ -351,6 +351,11 @@ fn table_page_sql(
offset: u64,
batch_size: usize,
) -> String {
let default_order_columns = if sql_context.database_type == DatabaseType::InfluxDb {
primary_keys.iter().find(|column| column.eq_ignore_ascii_case("time")).map(std::slice::from_ref).unwrap_or(&[])
} else {
primary_keys
};
let sql = if use_keyset {
keyset_pagination_sql_with_identifier_quote(
col_names,
@@ -372,7 +377,7 @@ fn table_page_sql(
batch_size,
request.where_input.as_deref(),
request.order_by.as_deref(),
primary_keys,
default_order_columns,
request.identifier_quote.as_deref(),
)
};
@@ -1472,8 +1477,10 @@ async fn export_table_data_core_inner(
// When no PK is available, falls back to offset-based pagination.
let has_custom_filter_or_order = request.where_input.as_ref().is_some_and(|value| !value.trim().is_empty())
|| request.order_by.as_ref().is_some_and(|value| !value.trim().is_empty());
let use_keyset =
!has_custom_filter_or_order && !primary_keys.is_empty() && primary_keys.iter().all(|pk| col_names.contains(pk));
let use_keyset = db_type != DatabaseType::InfluxDb
&& !has_custom_filter_or_order
&& !primary_keys.is_empty()
&& primary_keys.iter().all(|pk| col_names.contains(pk));
// PK column indices within result rows (for extracting last-row values)
let pk_indices: Vec<usize> = if use_keyset {
@@ -2156,13 +2163,11 @@ async fn export_table_data_core_inner(
mod tests {
use super::*;
use crate::database_export::{clear_export_cancelled, set_export_cancelled};
#[cfg(unix)]
use crate::models::connection::ConnectionConfig;
#[cfg(unix)]
use crate::plugins::{
InstalledPlugin, PluginDriverManifest, PluginDriverSession, PluginManifest, PluginRuntimeEnv,
};
#[cfg(unix)]
use crate::storage::Storage;
use crate::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
use serde_json::json;
@@ -2310,6 +2315,149 @@ mod tests {
let _ = std::fs::remove_dir_all(fixture.dir);
}
async fn read_table_export_http_request(socket: &mut tokio::net::TcpStream) -> String {
use tokio::io::AsyncReadExt;
let mut request = Vec::new();
loop {
let mut chunk = [0_u8; 2048];
let bytes_read = socket.read(&mut chunk).await.unwrap();
assert!(bytes_read > 0, "request ended before headers were complete");
request.extend_from_slice(&chunk[..bytes_read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
return String::from_utf8(request).unwrap();
}
}
}
async fn write_table_export_http_json(socket: &mut tokio::net::TcpStream, status: &str, body: &str) {
use tokio::io::AsyncWriteExt;
let response = format!(
"HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
socket.write_all(response.as_bytes()).await.unwrap();
}
fn table_export_query_param(target: &str, name: &str) -> Option<String> {
target.split_once('?')?.1.split('&').find_map(|pair| {
let (key, value) = pair.split_once('=')?;
(key == name).then(|| percent_encoding::percent_decode_str(value).decode_utf8_lossy().into_owned())
})
}
async fn spawn_influxdb_table_export_server() -> (String, tokio::task::JoinHandle<Vec<String>>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let mut queries = Vec::new();
loop {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_table_export_http_request(&mut socket).await;
let target = request.lines().next().unwrap().split_whitespace().nth(1).unwrap();
let query = table_export_query_param(target, "q").expect("InfluxDB request must include q");
if query != "SHOW DATABASES" {
assert_eq!(table_export_query_param(target, "db").as_deref(), Some("dbx_issue_8022"));
}
let final_page = query.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2");
let body = match query.as_str() {
"SHOW DATABASES" => {
r#"{"results":[{"statement_id":0,"series":[{"name":"databases","columns":["name"],"values":[["dbx_issue_8022"]]}]}]}"#
}
"SHOW TAG KEYS FROM \"weather\"" => {
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["tagKey"],"values":[["location"]]}]}]}"#
}
"SHOW FIELD KEYS FROM \"weather\"" => {
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["fieldKey","fieldType"],"values":[["temperature","float"]]}]}]}"#
}
value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 0") => {
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:00:00Z","us-midwest",82]]}]}]}"#
}
value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 1") => {
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:01:00Z","us-midwest",83]]}]}]}"#
}
value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2") => {
r#"{"results":[{"statement_id":0}]}"#
}
unexpected => panic!("unexpected InfluxDB export query: {unexpected}"),
};
queries.push(query);
write_table_export_http_json(&mut socket, "200 OK", body).await;
if final_page {
break;
}
}
queries
});
(format!("http://{address}"), server)
}
#[tokio::test]
async fn influxdb_table_export_exports_paginated_rows() {
let (base_url, server) = spawn_influxdb_table_export_server().await;
let port = base_url.rsplit_once(':').unwrap().1.parse::<u16>().unwrap();
let dir = std::env::temp_dir().join(format!("dbx-influxdb-table-export-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let config: ConnectionConfig = serde_json::from_value(json!({
"id": "conn-influxdb-export",
"name": "InfluxDB export",
"db_type": "influxdb",
"host": "127.0.0.1",
"port": port,
"username": "",
"password": "",
"database": "dbx_issue_8022",
"query_timeout_secs": 30
}))
.unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
state.configs.write().await.insert(config.id.clone(), config);
let export_id = format!("export-{}", uuid::Uuid::new_v4());
let output = dir.join("weather.csv");
let request = TableExportRequest {
export_id,
connection_id: "conn-influxdb-export".to_string(),
database: "dbx_issue_8022".to_string(),
schema: None,
identifier_quote: None,
table_name: "weather".to_string(),
file_path: output.to_string_lossy().into_owned(),
format: "csv".to_string(),
csv_quote_mode: CsvQuoteMode::All,
columns: None,
column_types: None,
primary_keys: None,
where_input: None,
order_by: None,
skip_count: true,
batch_size: Some(1),
row_limit: None,
date_time_format: None,
numeric_column_right_align: false,
column_comments: None,
auto_filter: None,
};
export_table_data_core(&state, &request, |_| {}).await.unwrap();
let queries = tokio::time::timeout(std::time::Duration::from_secs(5), server)
.await
.expect("InfluxDB export server did not receive the final page")
.unwrap();
let csv = std::fs::read_to_string(&output).unwrap_or_default();
let _ = std::fs::remove_dir_all(&dir);
let first_page = queries.iter().find(|query| query.contains("LIMIT 1 OFFSET 0")).unwrap();
assert!(first_page.contains("ORDER BY \"time\""));
assert!(!first_page.contains("ORDER BY \"time\", \"location\""));
assert!(csv.contains("location"));
assert!(csv.contains("temperature"));
assert!(csv.contains("us-midwest"));
assert!(csv.contains("82"));
assert!(csv.contains("83"));
}
/// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer.
fn read_zip_entry(bytes: &[u8], path: &str) -> String {
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive");
+101
View File
@@ -5168,6 +5168,16 @@ async fn execute_on_pool_once(
let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
db::clickhouse_driver::execute_query_with_max_rows(&client, &database, sql, max_rows).await
}
PoolKind::InfluxDb(client) => {
let client = client.clone();
let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
db::influxdb_driver::execute_query(&client, &database, sql).await
}
PoolKind::InfluxDb3(client) => {
let client = client.clone();
let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
db::influxdb3_driver::execute_query(&client, &database, sql, max_rows).await
}
PoolKind::SqlServer(client) => {
let client = client.clone();
let mut client = client.lock().await;
@@ -8665,6 +8675,97 @@ mod tests {
(AppState::new(storage), dir)
}
async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle<String>) {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = Vec::new();
let header_end = loop {
let mut chunk = [0_u8; 2048];
let bytes_read = socket.read(&mut chunk).await.unwrap();
assert!(bytes_read > 0, "request ended before headers were complete");
request.extend_from_slice(&chunk[..bytes_read]);
if let Some(index) = request.windows(4).position(|window| window == b"\r\n\r\n") {
break index + 4;
}
};
let headers = String::from_utf8(request[..header_end].to_vec()).unwrap();
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length").then(|| value.trim().parse::<usize>().unwrap())
})
.unwrap_or(0);
while request.len() < header_end + content_length {
let mut chunk = [0_u8; 2048];
let bytes_read = socket.read(&mut chunk).await.unwrap();
assert!(bytes_read > 0, "request ended before body was complete");
request.extend_from_slice(&chunk[..bytes_read]);
}
let body = r#"[{"time":"2026-09-03T00:00:00Z","value":42}]"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
socket.write_all(response.as_bytes()).await.unwrap();
String::from_utf8(request).unwrap()
});
(format!("http://{address}"), server)
}
#[tokio::test]
async fn transfer_read_dispatches_influxdb3_with_database_and_row_limit() {
let (base_url, server) = spawn_influxdb3_transfer_server().await;
let (state, dir) = test_app_state().await;
let config: ConnectionConfig = serde_json::from_value(json!({
"id": "influxdb3-transfer",
"name": "InfluxDB 3 transfer",
"db_type": "influxdb3",
"host": "127.0.0.1",
"port": 8181,
"username": "",
"password": "",
"database": "metrics"
}))
.unwrap();
let client = db::influxdb3_driver::Influxdb3Client::new_for_config(
&base_url,
&config,
std::time::Duration::from_secs(5),
)
.unwrap();
state.configs.write().await.insert(config.id.clone(), config);
state
.update_connection_pools(|connections| {
connections.insert("influxdb3-transfer:metrics".to_string(), PoolKind::InfluxDb3(client));
})
.await;
let result = execute_read_on_pool_with_max_rows(
&state,
"influxdb3-transfer:metrics",
"SELECT value FROM weather",
Some(1),
)
.await
.unwrap();
let request = tokio::time::timeout(std::time::Duration::from_secs(5), server)
.await
.expect("InfluxDB 3 transfer server did not receive a request")
.unwrap();
let body = request.split_once("\r\n\r\n").unwrap().1;
let payload: serde_json::Value = serde_json::from_str(body).unwrap();
let _ = std::fs::remove_dir_all(dir);
assert_eq!(payload["db"], json!("metrics"));
assert_eq!(payload["q"], json!("SELECT value FROM weather LIMIT 2"));
assert_eq!(result.affected_rows, 1);
}
#[tokio::test]
async fn postgres_transfer_metadata_routes_agent_pools() {
let (state, dir) = test_app_state().await;