diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index d5c8d4114a..6e0cbf9b5d 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -351,6 +351,11 @@ fn table_page_sql( offset: u64, batch_size: usize, ) -> String { + let default_order_columns = if sql_context.database_type == DatabaseType::InfluxDb { + primary_keys.iter().find(|column| column.eq_ignore_ascii_case("time")).map(std::slice::from_ref).unwrap_or(&[]) + } else { + primary_keys + }; let sql = if use_keyset { keyset_pagination_sql_with_identifier_quote( col_names, @@ -372,7 +377,7 @@ fn table_page_sql( batch_size, request.where_input.as_deref(), request.order_by.as_deref(), - primary_keys, + default_order_columns, request.identifier_quote.as_deref(), ) }; @@ -1472,8 +1477,10 @@ async fn export_table_data_core_inner( // When no PK is available, falls back to offset-based pagination. let has_custom_filter_or_order = request.where_input.as_ref().is_some_and(|value| !value.trim().is_empty()) || request.order_by.as_ref().is_some_and(|value| !value.trim().is_empty()); - let use_keyset = - !has_custom_filter_or_order && !primary_keys.is_empty() && primary_keys.iter().all(|pk| col_names.contains(pk)); + let use_keyset = db_type != DatabaseType::InfluxDb + && !has_custom_filter_or_order + && !primary_keys.is_empty() + && primary_keys.iter().all(|pk| col_names.contains(pk)); // PK column indices within result rows (for extracting last-row values) let pk_indices: Vec = if use_keyset { @@ -2156,13 +2163,11 @@ async fn export_table_data_core_inner( mod tests { use super::*; use crate::database_export::{clear_export_cancelled, set_export_cancelled}; - #[cfg(unix)] use crate::models::connection::ConnectionConfig; #[cfg(unix)] use crate::plugins::{ InstalledPlugin, PluginDriverManifest, PluginDriverSession, PluginManifest, PluginRuntimeEnv, }; - #[cfg(unix)] use crate::storage::Storage; use crate::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData}; use serde_json::json; @@ -2310,6 +2315,149 @@ mod tests { let _ = std::fs::remove_dir_all(fixture.dir); } + async fn read_table_export_http_request(socket: &mut tokio::net::TcpStream) -> String { + use tokio::io::AsyncReadExt; + + let mut request = Vec::new(); + loop { + let mut chunk = [0_u8; 2048]; + let bytes_read = socket.read(&mut chunk).await.unwrap(); + assert!(bytes_read > 0, "request ended before headers were complete"); + request.extend_from_slice(&chunk[..bytes_read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + return String::from_utf8(request).unwrap(); + } + } + } + + async fn write_table_export_http_json(socket: &mut tokio::net::TcpStream, status: &str, body: &str) { + use tokio::io::AsyncWriteExt; + + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.unwrap(); + } + + fn table_export_query_param(target: &str, name: &str) -> Option { + target.split_once('?')?.1.split('&').find_map(|pair| { + let (key, value) = pair.split_once('=')?; + (key == name).then(|| percent_encoding::percent_decode_str(value).decode_utf8_lossy().into_owned()) + }) + } + + async fn spawn_influxdb_table_export_server() -> (String, tokio::task::JoinHandle>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut queries = Vec::new(); + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let request = read_table_export_http_request(&mut socket).await; + let target = request.lines().next().unwrap().split_whitespace().nth(1).unwrap(); + let query = table_export_query_param(target, "q").expect("InfluxDB request must include q"); + if query != "SHOW DATABASES" { + assert_eq!(table_export_query_param(target, "db").as_deref(), Some("dbx_issue_8022")); + } + let final_page = query.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2"); + let body = match query.as_str() { + "SHOW DATABASES" => { + r#"{"results":[{"statement_id":0,"series":[{"name":"databases","columns":["name"],"values":[["dbx_issue_8022"]]}]}]}"# + } + "SHOW TAG KEYS FROM \"weather\"" => { + r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["tagKey"],"values":[["location"]]}]}]}"# + } + "SHOW FIELD KEYS FROM \"weather\"" => { + r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["fieldKey","fieldType"],"values":[["temperature","float"]]}]}]}"# + } + value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 0") => { + r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:00:00Z","us-midwest",82]]}]}]}"# + } + value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 1") => { + r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:01:00Z","us-midwest",83]]}]}]}"# + } + value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2") => { + r#"{"results":[{"statement_id":0}]}"# + } + unexpected => panic!("unexpected InfluxDB export query: {unexpected}"), + }; + queries.push(query); + write_table_export_http_json(&mut socket, "200 OK", body).await; + if final_page { + break; + } + } + queries + }); + (format!("http://{address}"), server) + } + + #[tokio::test] + async fn influxdb_table_export_exports_paginated_rows() { + let (base_url, server) = spawn_influxdb_table_export_server().await; + let port = base_url.rsplit_once(':').unwrap().1.parse::().unwrap(); + let dir = std::env::temp_dir().join(format!("dbx-influxdb-table-export-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let config: ConnectionConfig = serde_json::from_value(json!({ + "id": "conn-influxdb-export", + "name": "InfluxDB export", + "db_type": "influxdb", + "host": "127.0.0.1", + "port": port, + "username": "", + "password": "", + "database": "dbx_issue_8022", + "query_timeout_secs": 30 + })) + .unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + state.configs.write().await.insert(config.id.clone(), config); + let export_id = format!("export-{}", uuid::Uuid::new_v4()); + let output = dir.join("weather.csv"); + let request = TableExportRequest { + export_id, + connection_id: "conn-influxdb-export".to_string(), + database: "dbx_issue_8022".to_string(), + schema: None, + identifier_quote: None, + table_name: "weather".to_string(), + file_path: output.to_string_lossy().into_owned(), + format: "csv".to_string(), + csv_quote_mode: CsvQuoteMode::All, + columns: None, + column_types: None, + primary_keys: None, + where_input: None, + order_by: None, + skip_count: true, + batch_size: Some(1), + row_limit: None, + date_time_format: None, + numeric_column_right_align: false, + column_comments: None, + auto_filter: None, + }; + + export_table_data_core(&state, &request, |_| {}).await.unwrap(); + let queries = tokio::time::timeout(std::time::Duration::from_secs(5), server) + .await + .expect("InfluxDB export server did not receive the final page") + .unwrap(); + let csv = std::fs::read_to_string(&output).unwrap_or_default(); + let _ = std::fs::remove_dir_all(&dir); + + let first_page = queries.iter().find(|query| query.contains("LIMIT 1 OFFSET 0")).unwrap(); + assert!(first_page.contains("ORDER BY \"time\"")); + assert!(!first_page.contains("ORDER BY \"time\", \"location\"")); + assert!(csv.contains("location")); + assert!(csv.contains("temperature")); + assert!(csv.contains("us-midwest")); + assert!(csv.contains("82")); + assert!(csv.contains("83")); + } + /// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer. fn read_zip_entry(bytes: &[u8], path: &str) -> String { let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive"); diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 3a6b647c58..c3ef608c3e 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -5168,6 +5168,16 @@ async fn execute_on_pool_once( let database = database_from_pool_key(pool_key).unwrap_or("default").to_string(); db::clickhouse_driver::execute_query_with_max_rows(&client, &database, sql, max_rows).await } + PoolKind::InfluxDb(client) => { + let client = client.clone(); + let database = database_from_pool_key(pool_key).unwrap_or("default").to_string(); + db::influxdb_driver::execute_query(&client, &database, sql).await + } + PoolKind::InfluxDb3(client) => { + let client = client.clone(); + let database = database_from_pool_key(pool_key).unwrap_or("default").to_string(); + db::influxdb3_driver::execute_query(&client, &database, sql, max_rows).await + } PoolKind::SqlServer(client) => { let client = client.clone(); let mut client = client.lock().await; @@ -8665,6 +8675,97 @@ mod tests { (AppState::new(storage), dir) } + async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let header_end = loop { + let mut chunk = [0_u8; 2048]; + let bytes_read = socket.read(&mut chunk).await.unwrap(); + assert!(bytes_read > 0, "request ended before headers were complete"); + request.extend_from_slice(&chunk[..bytes_read]); + if let Some(index) = request.windows(4).position(|window| window == b"\r\n\r\n") { + break index + 4; + } + }; + let headers = String::from_utf8(request[..header_end].to_vec()).unwrap(); + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length").then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + while request.len() < header_end + content_length { + let mut chunk = [0_u8; 2048]; + let bytes_read = socket.read(&mut chunk).await.unwrap(); + assert!(bytes_read > 0, "request ended before body was complete"); + request.extend_from_slice(&chunk[..bytes_read]); + } + let body = r#"[{"time":"2026-09-03T00:00:00Z","value":42}]"#; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.unwrap(); + String::from_utf8(request).unwrap() + }); + (format!("http://{address}"), server) + } + + #[tokio::test] + async fn transfer_read_dispatches_influxdb3_with_database_and_row_limit() { + let (base_url, server) = spawn_influxdb3_transfer_server().await; + let (state, dir) = test_app_state().await; + let config: ConnectionConfig = serde_json::from_value(json!({ + "id": "influxdb3-transfer", + "name": "InfluxDB 3 transfer", + "db_type": "influxdb3", + "host": "127.0.0.1", + "port": 8181, + "username": "", + "password": "", + "database": "metrics" + })) + .unwrap(); + let client = db::influxdb3_driver::Influxdb3Client::new_for_config( + &base_url, + &config, + std::time::Duration::from_secs(5), + ) + .unwrap(); + state.configs.write().await.insert(config.id.clone(), config); + state + .update_connection_pools(|connections| { + connections.insert("influxdb3-transfer:metrics".to_string(), PoolKind::InfluxDb3(client)); + }) + .await; + + let result = execute_read_on_pool_with_max_rows( + &state, + "influxdb3-transfer:metrics", + "SELECT value FROM weather", + Some(1), + ) + .await + .unwrap(); + let request = tokio::time::timeout(std::time::Duration::from_secs(5), server) + .await + .expect("InfluxDB 3 transfer server did not receive a request") + .unwrap(); + let body = request.split_once("\r\n\r\n").unwrap().1; + let payload: serde_json::Value = serde_json::from_str(body).unwrap(); + let _ = std::fs::remove_dir_all(dir); + + assert_eq!(payload["db"], json!("metrics")); + assert_eq!(payload["q"], json!("SELECT value FROM weather LIMIT 2")); + assert_eq!(result.affected_rows, 1); + } + #[tokio::test] async fn postgres_transfer_metadata_routes_agent_pools() { let (state, dir) = test_app_state().await;