fix(influxdb): support table export query dispatch and pagination
Closes #8022
This commit is contained in:
2 files changed
+254
-5
No files matched your search
@@ -351,6 +351,11 @@ fn table_page_sql(
|
||||
offset: u64,
|
||||
batch_size: usize,
|
||||
) -> String {
|
||||
let default_order_columns = if sql_context.database_type == DatabaseType::InfluxDb {
|
||||
primary_keys.iter().find(|column| column.eq_ignore_ascii_case("time")).map(std::slice::from_ref).unwrap_or(&[])
|
||||
} else {
|
||||
primary_keys
|
||||
};
|
||||
let sql = if use_keyset {
|
||||
keyset_pagination_sql_with_identifier_quote(
|
||||
col_names,
|
||||
@@ -372,7 +377,7 @@ fn table_page_sql(
|
||||
batch_size,
|
||||
request.where_input.as_deref(),
|
||||
request.order_by.as_deref(),
|
||||
primary_keys,
|
||||
default_order_columns,
|
||||
request.identifier_quote.as_deref(),
|
||||
)
|
||||
};
|
||||
@@ -1472,8 +1477,10 @@ async fn export_table_data_core_inner(
|
||||
// When no PK is available, falls back to offset-based pagination.
|
||||
let has_custom_filter_or_order = request.where_input.as_ref().is_some_and(|value| !value.trim().is_empty())
|
||||
|| request.order_by.as_ref().is_some_and(|value| !value.trim().is_empty());
|
||||
let use_keyset =
|
||||
!has_custom_filter_or_order && !primary_keys.is_empty() && primary_keys.iter().all(|pk| col_names.contains(pk));
|
||||
let use_keyset = db_type != DatabaseType::InfluxDb
|
||||
&& !has_custom_filter_or_order
|
||||
&& !primary_keys.is_empty()
|
||||
&& primary_keys.iter().all(|pk| col_names.contains(pk));
|
||||
|
||||
// PK column indices within result rows (for extracting last-row values)
|
||||
let pk_indices: Vec<usize> = if use_keyset {
|
||||
@@ -2156,13 +2163,11 @@ async fn export_table_data_core_inner(
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::database_export::{clear_export_cancelled, set_export_cancelled};
|
||||
#[cfg(unix)]
|
||||
use crate::models::connection::ConnectionConfig;
|
||||
#[cfg(unix)]
|
||||
use crate::plugins::{
|
||||
InstalledPlugin, PluginDriverManifest, PluginDriverSession, PluginManifest, PluginRuntimeEnv,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::storage::Storage;
|
||||
use crate::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
|
||||
use serde_json::json;
|
||||
@@ -2310,6 +2315,149 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(fixture.dir);
|
||||
}
|
||||
|
||||
async fn read_table_export_http_request(socket: &mut tokio::net::TcpStream) -> String {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let mut request = Vec::new();
|
||||
loop {
|
||||
let mut chunk = [0_u8; 2048];
|
||||
let bytes_read = socket.read(&mut chunk).await.unwrap();
|
||||
assert!(bytes_read > 0, "request ended before headers were complete");
|
||||
request.extend_from_slice(&chunk[..bytes_read]);
|
||||
if request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
return String::from_utf8(request).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_table_export_http_json(socket: &mut tokio::net::TcpStream, status: &str, body: &str) {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
socket.write_all(response.as_bytes()).await.unwrap();
|
||||
}
|
||||
|
||||
fn table_export_query_param(target: &str, name: &str) -> Option<String> {
|
||||
target.split_once('?')?.1.split('&').find_map(|pair| {
|
||||
let (key, value) = pair.split_once('=')?;
|
||||
(key == name).then(|| percent_encoding::percent_decode_str(value).decode_utf8_lossy().into_owned())
|
||||
})
|
||||
}
|
||||
|
||||
async fn spawn_influxdb_table_export_server() -> (String, tokio::task::JoinHandle<Vec<String>>) {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut queries = Vec::new();
|
||||
loop {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let request = read_table_export_http_request(&mut socket).await;
|
||||
let target = request.lines().next().unwrap().split_whitespace().nth(1).unwrap();
|
||||
let query = table_export_query_param(target, "q").expect("InfluxDB request must include q");
|
||||
if query != "SHOW DATABASES" {
|
||||
assert_eq!(table_export_query_param(target, "db").as_deref(), Some("dbx_issue_8022"));
|
||||
}
|
||||
let final_page = query.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2");
|
||||
let body = match query.as_str() {
|
||||
"SHOW DATABASES" => {
|
||||
r#"{"results":[{"statement_id":0,"series":[{"name":"databases","columns":["name"],"values":[["dbx_issue_8022"]]}]}]}"#
|
||||
}
|
||||
"SHOW TAG KEYS FROM \"weather\"" => {
|
||||
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["tagKey"],"values":[["location"]]}]}]}"#
|
||||
}
|
||||
"SHOW FIELD KEYS FROM \"weather\"" => {
|
||||
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["fieldKey","fieldType"],"values":[["temperature","float"]]}]}]}"#
|
||||
}
|
||||
value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 0") => {
|
||||
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:00:00Z","us-midwest",82]]}]}]}"#
|
||||
}
|
||||
value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 1") => {
|
||||
r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:01:00Z","us-midwest",83]]}]}]}"#
|
||||
}
|
||||
value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2") => {
|
||||
r#"{"results":[{"statement_id":0}]}"#
|
||||
}
|
||||
unexpected => panic!("unexpected InfluxDB export query: {unexpected}"),
|
||||
};
|
||||
queries.push(query);
|
||||
write_table_export_http_json(&mut socket, "200 OK", body).await;
|
||||
if final_page {
|
||||
break;
|
||||
}
|
||||
}
|
||||
queries
|
||||
});
|
||||
(format!("http://{address}"), server)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn influxdb_table_export_exports_paginated_rows() {
|
||||
let (base_url, server) = spawn_influxdb_table_export_server().await;
|
||||
let port = base_url.rsplit_once(':').unwrap().1.parse::<u16>().unwrap();
|
||||
let dir = std::env::temp_dir().join(format!("dbx-influxdb-table-export-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let config: ConnectionConfig = serde_json::from_value(json!({
|
||||
"id": "conn-influxdb-export",
|
||||
"name": "InfluxDB export",
|
||||
"db_type": "influxdb",
|
||||
"host": "127.0.0.1",
|
||||
"port": port,
|
||||
"username": "",
|
||||
"password": "",
|
||||
"database": "dbx_issue_8022",
|
||||
"query_timeout_secs": 30
|
||||
}))
|
||||
.unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
let export_id = format!("export-{}", uuid::Uuid::new_v4());
|
||||
let output = dir.join("weather.csv");
|
||||
let request = TableExportRequest {
|
||||
export_id,
|
||||
connection_id: "conn-influxdb-export".to_string(),
|
||||
database: "dbx_issue_8022".to_string(),
|
||||
schema: None,
|
||||
identifier_quote: None,
|
||||
table_name: "weather".to_string(),
|
||||
file_path: output.to_string_lossy().into_owned(),
|
||||
format: "csv".to_string(),
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
where_input: None,
|
||||
order_by: None,
|
||||
skip_count: true,
|
||||
batch_size: Some(1),
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
};
|
||||
|
||||
export_table_data_core(&state, &request, |_| {}).await.unwrap();
|
||||
let queries = tokio::time::timeout(std::time::Duration::from_secs(5), server)
|
||||
.await
|
||||
.expect("InfluxDB export server did not receive the final page")
|
||||
.unwrap();
|
||||
let csv = std::fs::read_to_string(&output).unwrap_or_default();
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
|
||||
let first_page = queries.iter().find(|query| query.contains("LIMIT 1 OFFSET 0")).unwrap();
|
||||
assert!(first_page.contains("ORDER BY \"time\""));
|
||||
assert!(!first_page.contains("ORDER BY \"time\", \"location\""));
|
||||
assert!(csv.contains("location"));
|
||||
assert!(csv.contains("temperature"));
|
||||
assert!(csv.contains("us-midwest"));
|
||||
assert!(csv.contains("82"));
|
||||
assert!(csv.contains("83"));
|
||||
}
|
||||
|
||||
/// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer.
|
||||
fn read_zip_entry(bytes: &[u8], path: &str) -> String {
|
||||
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive");
|
||||
|
||||
@@ -5168,6 +5168,16 @@ async fn execute_on_pool_once(
|
||||
let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
|
||||
db::clickhouse_driver::execute_query_with_max_rows(&client, &database, sql, max_rows).await
|
||||
}
|
||||
PoolKind::InfluxDb(client) => {
|
||||
let client = client.clone();
|
||||
let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
|
||||
db::influxdb_driver::execute_query(&client, &database, sql).await
|
||||
}
|
||||
PoolKind::InfluxDb3(client) => {
|
||||
let client = client.clone();
|
||||
let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
|
||||
db::influxdb3_driver::execute_query(&client, &database, sql, max_rows).await
|
||||
}
|
||||
PoolKind::SqlServer(client) => {
|
||||
let client = client.clone();
|
||||
let mut client = client.lock().await;
|
||||
@@ -8665,6 +8675,97 @@ mod tests {
|
||||
(AppState::new(storage), dir)
|
||||
}
|
||||
|
||||
async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle<String>) {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let header_end = loop {
|
||||
let mut chunk = [0_u8; 2048];
|
||||
let bytes_read = socket.read(&mut chunk).await.unwrap();
|
||||
assert!(bytes_read > 0, "request ended before headers were complete");
|
||||
request.extend_from_slice(&chunk[..bytes_read]);
|
||||
if let Some(index) = request.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break index + 4;
|
||||
}
|
||||
};
|
||||
let headers = String::from_utf8(request[..header_end].to_vec()).unwrap();
|
||||
let content_length = headers
|
||||
.lines()
|
||||
.find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
name.eq_ignore_ascii_case("content-length").then(|| value.trim().parse::<usize>().unwrap())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
while request.len() < header_end + content_length {
|
||||
let mut chunk = [0_u8; 2048];
|
||||
let bytes_read = socket.read(&mut chunk).await.unwrap();
|
||||
assert!(bytes_read > 0, "request ended before body was complete");
|
||||
request.extend_from_slice(&chunk[..bytes_read]);
|
||||
}
|
||||
let body = r#"[{"time":"2026-09-03T00:00:00Z","value":42}]"#;
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
socket.write_all(response.as_bytes()).await.unwrap();
|
||||
String::from_utf8(request).unwrap()
|
||||
});
|
||||
(format!("http://{address}"), server)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transfer_read_dispatches_influxdb3_with_database_and_row_limit() {
|
||||
let (base_url, server) = spawn_influxdb3_transfer_server().await;
|
||||
let (state, dir) = test_app_state().await;
|
||||
let config: ConnectionConfig = serde_json::from_value(json!({
|
||||
"id": "influxdb3-transfer",
|
||||
"name": "InfluxDB 3 transfer",
|
||||
"db_type": "influxdb3",
|
||||
"host": "127.0.0.1",
|
||||
"port": 8181,
|
||||
"username": "",
|
||||
"password": "",
|
||||
"database": "metrics"
|
||||
}))
|
||||
.unwrap();
|
||||
let client = db::influxdb3_driver::Influxdb3Client::new_for_config(
|
||||
&base_url,
|
||||
&config,
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.unwrap();
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
state
|
||||
.update_connection_pools(|connections| {
|
||||
connections.insert("influxdb3-transfer:metrics".to_string(), PoolKind::InfluxDb3(client));
|
||||
})
|
||||
.await;
|
||||
|
||||
let result = execute_read_on_pool_with_max_rows(
|
||||
&state,
|
||||
"influxdb3-transfer:metrics",
|
||||
"SELECT value FROM weather",
|
||||
Some(1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let request = tokio::time::timeout(std::time::Duration::from_secs(5), server)
|
||||
.await
|
||||
.expect("InfluxDB 3 transfer server did not receive a request")
|
||||
.unwrap();
|
||||
let body = request.split_once("\r\n\r\n").unwrap().1;
|
||||
let payload: serde_json::Value = serde_json::from_str(body).unwrap();
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
|
||||
assert_eq!(payload["db"], json!("metrics"));
|
||||
assert_eq!(payload["q"], json!("SELECT value FROM weather LIMIT 2"));
|
||||
assert_eq!(result.affected_rows, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_transfer_metadata_routes_agent_pools() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
|
||||
Reference in new issue
Block a user