diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-core/src/db/mysql.rs index 1b5159422b..b47f777082 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-core/src/db/mysql.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use futures::StreamExt; use rust_decimal::Decimal; use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow}; use sqlx::{Column, Executor, Row, TypeInfo, ValueRef}; @@ -252,54 +253,70 @@ pub async fn execute_query(pool: &MySqlPool, sql: &str, bare: bool) -> Result = sqlx::raw_sql(sql).fetch_all(pool).await.map_err(|e| e.to_string())?; + let mut stream = sqlx::raw_sql(sql).fetch(&*pool); + let mut columns: Vec = vec![]; + let mut column_types: Vec = vec![]; + let mut result_rows: Vec> = Vec::new(); - let (columns, column_types) = if let Some(first) = rows.first() { - let cols: Vec = first.columns().iter().map(|c| c.name().to_string()).collect(); - let types: Vec = first.columns().iter().map(|c| c.type_info().name().to_string()).collect(); - (cols, types) - } else { - (vec![], vec![]) - }; - - let result_rows: Vec> = rows - .iter() - .map(|row| { + while let Some(row) = stream.next().await { + let row: MySqlRow = row.map_err(|e| e.to_string())?; + if columns.is_empty() { + columns = row.columns().iter().map(|c| c.name().to_string()).collect(); + column_types = row.columns().iter().map(|c| c.type_info().name().to_string()).collect(); + } + result_rows.push( (0..row.len()) - .map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) - .collect() - }) - .collect(); + .map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) + .collect(), + ); + if result_rows.len() > crate::query::MAX_ROWS { + break; + } + } + + let truncated = result_rows.len() > crate::query::MAX_ROWS; + if truncated { + result_rows.truncate(crate::query::MAX_ROWS); + } Ok(QueryResult { columns, rows: result_rows, affected_rows: 0, execution_time_ms: start.elapsed().as_millis(), - truncated: false, + truncated, }) } else { let desc = pool.describe(sql).await.map_err(|e| e.to_string())?; let columns: Vec = desc.columns().iter().map(|c| c.name().to_string()).collect(); let column_types: Vec = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(); - let rows: Vec = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?; + let mut stream = sqlx::query(sql).fetch(&*pool); + let mut result_rows: Vec> = Vec::new(); - let result_rows: Vec> = rows - .iter() - .map(|row| { + while let Some(row) = stream.next().await { + let row = row.map_err(|e| e.to_string())?; + result_rows.push( (0..row.len()) - .map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) - .collect() - }) - .collect(); + .map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) + .collect(), + ); + if result_rows.len() > crate::query::MAX_ROWS { + break; + } + } + + let truncated = result_rows.len() > crate::query::MAX_ROWS; + if truncated { + result_rows.truncate(crate::query::MAX_ROWS); + } Ok(QueryResult { columns, rows: result_rows, affected_rows: 0, execution_time_ms: start.elapsed().as_millis(), - truncated: false, + truncated, }) } } else { diff --git a/crates/dbx-core/src/db/postgres.rs b/crates/dbx-core/src/db/postgres.rs index 3e4a835945..2484db1b82 100644 --- a/crates/dbx-core/src/db/postgres.rs +++ b/crates/dbx-core/src/db/postgres.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use futures::StreamExt; use percent_encoding::percent_decode_str; use rust_decimal::Decimal; use sqlx::postgres::{PgPool, PgPoolOptions, PgRow}; @@ -260,37 +261,44 @@ pub async fn execute_query(pool: &PgPool, sql: &str) -> Result = sqlx::query(sql).persistent(false).fetch_all(pool).await.map_err(|e| e.to_string())?; + let mut stream = sqlx::query(sql).persistent(false).fetch(pool); + let mut columns: Vec = vec![]; + let mut column_types: Vec = vec![]; + let mut result_rows: Vec> = Vec::new(); - let (columns, column_types): (Vec, Vec) = if let Some(first) = rows.first() { - let cols = first.columns(); - ( - cols.iter().map(|c| c.name().to_string()).collect(), - cols.iter().map(|c| c.type_info().name().to_string()).collect(), - ) - } else { - let desc = pool.describe(sql).await.map_err(|e| e.to_string())?; - ( - desc.columns().iter().map(|c| c.name().to_string()).collect(), - desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(), - ) - }; - - let result_rows: Vec> = rows - .iter() - .map(|row| { + while let Some(row) = stream.next().await { + let row = row.map_err(|e| e.to_string())?; + if columns.is_empty() { + let cols = row.columns(); + columns = cols.iter().map(|c| c.name().to_string()).collect(); + column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect(); + } + result_rows.push( (0..row.len()) - .map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) - .collect() - }) - .collect(); + .map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) + .collect(), + ); + if result_rows.len() > crate::query::MAX_ROWS { + break; + } + } + + if columns.is_empty() { + let desc = pool.describe(sql).await.map_err(|e| e.to_string())?; + columns = desc.columns().iter().map(|c| c.name().to_string()).collect(); + } + + let truncated = result_rows.len() > crate::query::MAX_ROWS; + if truncated { + result_rows.truncate(crate::query::MAX_ROWS); + } Ok(QueryResult { columns, rows: result_rows, affected_rows: 0, execution_time_ms: start.elapsed().as_millis(), - truncated: false, + truncated, }) } else { let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?; @@ -319,38 +327,45 @@ pub async fn execute_query_with_schema(pool: &PgPool, schema: &str, sql: &str) - || trimmed.starts_with("WITH") || trimmed.starts_with("TABLE") { - let rows: Vec = - sqlx::query(sql).persistent(false).fetch_all(&mut *conn).await.map_err(|e| e.to_string())?; + let mut stream = sqlx::query(sql).persistent(false).fetch(&mut *conn); + let mut columns: Vec = vec![]; + let mut column_types: Vec = vec![]; + let mut result_rows: Vec> = Vec::new(); - let (columns, column_types): (Vec, Vec) = if let Some(first) = rows.first() { - let cols = first.columns(); - ( - cols.iter().map(|c| c.name().to_string()).collect(), - cols.iter().map(|c| c.type_info().name().to_string()).collect(), - ) - } else { - let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?; - ( - desc.columns().iter().map(|c| c.name().to_string()).collect(), - desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(), - ) - }; - - let result_rows: Vec> = rows - .iter() - .map(|row| { + while let Some(row) = stream.next().await { + let row = row.map_err(|e| e.to_string())?; + if columns.is_empty() { + let cols = row.columns(); + columns = cols.iter().map(|c| c.name().to_string()).collect(); + column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect(); + } + result_rows.push( (0..row.len()) - .map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) - .collect() - }) - .collect(); + .map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) + .collect(), + ); + if result_rows.len() > crate::query::MAX_ROWS { + break; + } + } + drop(stream); + + if columns.is_empty() { + let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?; + columns = desc.columns().iter().map(|c| c.name().to_string()).collect(); + } + + let truncated = result_rows.len() > crate::query::MAX_ROWS; + if truncated { + result_rows.truncate(crate::query::MAX_ROWS); + } Ok(QueryResult { columns, rows: result_rows, affected_rows: 0, execution_time_ms: start.elapsed().as_millis(), - truncated: false, + truncated, }) } else { let result = sqlx::query(sql).execute(&mut *conn).await.map_err(|e| e.to_string())?; diff --git a/crates/dbx-core/src/db/sqlite.rs b/crates/dbx-core/src/db/sqlite.rs index 99a4cdb382..622c556245 100644 --- a/crates/dbx-core/src/db/sqlite.rs +++ b/crates/dbx-core/src/db/sqlite.rs @@ -1,3 +1,4 @@ +use futures::StreamExt; use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow}; use sqlx::{Column, Executor, Row}; use std::time::{Duration, Instant}; @@ -171,11 +172,12 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result = desc.columns().iter().map(|c| c.name().to_string()).collect(); - let rows: Vec = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?; + let mut stream = sqlx::query(sql).fetch(pool); + let mut result_rows: Vec> = Vec::new(); - let result_rows: Vec> = rows - .iter() - .map(|row| { + while let Some(row) = stream.next().await { + let row = row.map_err(|e| e.to_string())?; + result_rows.push( (0..row.len()) .map(|i| { row.try_get::(i) @@ -191,16 +193,24 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result(i).map(serde_json::Value::Bool)) .unwrap_or(serde_json::Value::Null) }) - .collect() - }) - .collect(); + .collect(), + ); + if result_rows.len() > crate::query::MAX_ROWS { + break; + } + } + + let truncated = result_rows.len() > crate::query::MAX_ROWS; + if truncated { + result_rows.truncate(crate::query::MAX_ROWS); + } Ok(QueryResult { columns, rows: result_rows, affected_rows: 0, execution_time_ms: start.elapsed().as_millis(), - truncated: false, + truncated, }) } else { let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?; diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query.rs index 6378abd76d..b72cea3a44 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query.rs @@ -158,24 +158,22 @@ pub async fn do_execute( let p = p.clone(); let bare = *mode == crate::connection::MysqlMode::Bare; drop(connections); - wait_for_query(cancel_token, db::mysql::execute_query(&p, sql, bare)).await.map(truncate_result) + wait_for_query(cancel_token, db::mysql::execute_query(&p, sql, bare)).await } PoolKind::Postgres(p) => { let p = p.clone(); let schema = schema.map(|s| s.to_string()); drop(connections); if let Some(schema) = schema { - wait_for_query(cancel_token, db::postgres::execute_query_with_schema(&p, &schema, sql)) - .await - .map(truncate_result) + wait_for_query(cancel_token, db::postgres::execute_query_with_schema(&p, &schema, sql)).await } else { - wait_for_query(cancel_token, db::postgres::execute_query(&p, sql)).await.map(truncate_result) + wait_for_query(cancel_token, db::postgres::execute_query(&p, sql)).await } } PoolKind::Sqlite(p) => { let p = p.clone(); drop(connections); - wait_for_query(cancel_token, db::sqlite::execute_query(&p, sql)).await.map(truncate_result) + wait_for_query(cancel_token, db::sqlite::execute_query(&p, sql)).await } PoolKind::ClickHouse(client) => { let client = client.clone();