perf(query): stream rows with result limit

This commit is contained in:
t8y2 committed 2026-05-09 15:57:55 +08:00
1 parent af029daf57
commit f9993dd3f8
4 files changed
+127 -87

No files matched your search

+43 -26
View File
@@ -1,4 +1,5 @@
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
use futures::StreamExt;
use rust_decimal::Decimal; use rust_decimal::Decimal;
use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow}; use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow};
use sqlx::{Column, Executor, Row, TypeInfo, ValueRef}; use sqlx::{Column, Executor, Row, TypeInfo, ValueRef};
@@ -252,54 +253,70 @@ pub async fn execute_query(pool: &MySqlPool, sql: &str, bare: bool) -> Result<Qu
|| trimmed.starts_with("EXPLAIN") || trimmed.starts_with("EXPLAIN")
{ {
if bare { if bare {
let rows: Vec<MySqlRow> = sqlx::raw_sql(sql).fetch_all(pool).await.map_err(|e| e.to_string())?; let mut stream = sqlx::raw_sql(sql).fetch(&*pool);
let mut columns: Vec<String> = vec![];
let mut column_types: Vec<String> = vec![];
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let (columns, column_types) = if let Some(first) = rows.first() { while let Some(row) = stream.next().await {
let cols: Vec<String> = first.columns().iter().map(|c| c.name().to_string()).collect(); let row: MySqlRow = row.map_err(|e| e.to_string())?;
let types: Vec<String> = first.columns().iter().map(|c| c.type_info().name().to_string()).collect(); if columns.is_empty() {
(cols, types) columns = row.columns().iter().map(|c| c.name().to_string()).collect();
} else { column_types = row.columns().iter().map(|c| c.type_info().name().to_string()).collect();
(vec![], vec![]) }
}; result_rows.push(
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
(0..row.len()) (0..row.len())
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) .map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect() .collect(),
}) );
.collect(); if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult { Ok(QueryResult {
columns, columns,
rows: result_rows, rows: result_rows,
affected_rows: 0, affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(), execution_time_ms: start.elapsed().as_millis(),
truncated: false, truncated,
}) })
} else { } else {
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?; let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect(); let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
let column_types: Vec<String> = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(); let column_types: Vec<String> = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect();
let rows: Vec<MySqlRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?; let mut stream = sqlx::query(sql).fetch(&*pool);
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let result_rows: Vec<Vec<serde_json::Value>> = rows while let Some(row) = stream.next().await {
.iter() let row = row.map_err(|e| e.to_string())?;
.map(|row| { result_rows.push(
(0..row.len()) (0..row.len())
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) .map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect() .collect(),
}) );
.collect(); if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult { Ok(QueryResult {
columns, columns,
rows: result_rows, rows: result_rows,
affected_rows: 0, affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(), execution_time_ms: start.elapsed().as_millis(),
truncated: false, truncated,
}) })
} }
} else { } else {
+62 -47
View File
@@ -1,4 +1,5 @@
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
use futures::StreamExt;
use percent_encoding::percent_decode_str; use percent_encoding::percent_decode_str;
use rust_decimal::Decimal; use rust_decimal::Decimal;
use sqlx::postgres::{PgPool, PgPoolOptions, PgRow}; use sqlx::postgres::{PgPool, PgPoolOptions, PgRow};
@@ -260,37 +261,44 @@ pub async fn execute_query(pool: &PgPool, sql: &str) -> Result<QueryResult, Stri
|| trimmed.starts_with("WITH") || trimmed.starts_with("WITH")
|| trimmed.starts_with("TABLE") || trimmed.starts_with("TABLE")
{ {
let rows: Vec<PgRow> = sqlx::query(sql).persistent(false).fetch_all(pool).await.map_err(|e| e.to_string())?; let mut stream = sqlx::query(sql).persistent(false).fetch(pool);
let mut columns: Vec<String> = vec![];
let mut column_types: Vec<String> = vec![];
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() { while let Some(row) = stream.next().await {
let cols = first.columns(); let row = row.map_err(|e| e.to_string())?;
( if columns.is_empty() {
cols.iter().map(|c| c.name().to_string()).collect(), let cols = row.columns();
cols.iter().map(|c| c.type_info().name().to_string()).collect(), columns = cols.iter().map(|c| c.name().to_string()).collect();
) column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
} else { }
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?; result_rows.push(
(
desc.columns().iter().map(|c| c.name().to_string()).collect(),
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
)
};
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
(0..row.len()) (0..row.len())
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) .map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect() .collect(),
}) );
.collect(); if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
if columns.is_empty() {
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult { Ok(QueryResult {
columns, columns,
rows: result_rows, rows: result_rows,
affected_rows: 0, affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(), execution_time_ms: start.elapsed().as_millis(),
truncated: false, truncated,
}) })
} else { } else {
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?; let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
@@ -319,38 +327,45 @@ pub async fn execute_query_with_schema(pool: &PgPool, schema: &str, sql: &str) -
|| trimmed.starts_with("WITH") || trimmed.starts_with("WITH")
|| trimmed.starts_with("TABLE") || trimmed.starts_with("TABLE")
{ {
let rows: Vec<PgRow> = let mut stream = sqlx::query(sql).persistent(false).fetch(&mut *conn);
sqlx::query(sql).persistent(false).fetch_all(&mut *conn).await.map_err(|e| e.to_string())?; let mut columns: Vec<String> = vec![];
let mut column_types: Vec<String> = vec![];
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() { while let Some(row) = stream.next().await {
let cols = first.columns(); let row = row.map_err(|e| e.to_string())?;
( if columns.is_empty() {
cols.iter().map(|c| c.name().to_string()).collect(), let cols = row.columns();
cols.iter().map(|c| c.type_info().name().to_string()).collect(), columns = cols.iter().map(|c| c.name().to_string()).collect();
) column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
} else { }
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?; result_rows.push(
(
desc.columns().iter().map(|c| c.name().to_string()).collect(),
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
)
};
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
(0..row.len()) (0..row.len())
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) .map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect() .collect(),
}) );
.collect(); if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
drop(stream);
if columns.is_empty() {
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?;
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult { Ok(QueryResult {
columns, columns,
rows: result_rows, rows: result_rows,
affected_rows: 0, affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(), execution_time_ms: start.elapsed().as_millis(),
truncated: false, truncated,
}) })
} else { } else {
let result = sqlx::query(sql).execute(&mut *conn).await.map_err(|e| e.to_string())?; let result = sqlx::query(sql).execute(&mut *conn).await.map_err(|e| e.to_string())?;
+18 -8
View File
@@ -1,3 +1,4 @@
use futures::StreamExt;
use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow}; use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow};
use sqlx::{Column, Executor, Row}; use sqlx::{Column, Executor, Row};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
@@ -171,11 +172,12 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?; let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect(); let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
let rows: Vec<SqliteRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?; let mut stream = sqlx::query(sql).fetch(pool);
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let result_rows: Vec<Vec<serde_json::Value>> = rows while let Some(row) = stream.next().await {
.iter() let row = row.map_err(|e| e.to_string())?;
.map(|row| { result_rows.push(
(0..row.len()) (0..row.len())
.map(|i| { .map(|i| {
row.try_get::<String, _>(i) row.try_get::<String, _>(i)
@@ -191,16 +193,24 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
.or_else(|_| row.try_get::<bool, _>(i).map(serde_json::Value::Bool)) .or_else(|_| row.try_get::<bool, _>(i).map(serde_json::Value::Bool))
.unwrap_or(serde_json::Value::Null) .unwrap_or(serde_json::Value::Null)
}) })
.collect() .collect(),
}) );
.collect(); if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult { Ok(QueryResult {
columns, columns,
rows: result_rows, rows: result_rows,
affected_rows: 0, affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(), execution_time_ms: start.elapsed().as_millis(),
truncated: false, truncated,
}) })
} else { } else {
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?; let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
+4 -6
View File
@@ -158,24 +158,22 @@ pub async fn do_execute(
let p = p.clone(); let p = p.clone();
let bare = *mode == crate::connection::MysqlMode::Bare; let bare = *mode == crate::connection::MysqlMode::Bare;
drop(connections); drop(connections);
wait_for_query(cancel_token, db::mysql::execute_query(&p, sql, bare)).await.map(truncate_result) wait_for_query(cancel_token, db::mysql::execute_query(&p, sql, bare)).await
} }
PoolKind::Postgres(p) => { PoolKind::Postgres(p) => {
let p = p.clone(); let p = p.clone();
let schema = schema.map(|s| s.to_string()); let schema = schema.map(|s| s.to_string());
drop(connections); drop(connections);
if let Some(schema) = schema { if let Some(schema) = schema {
wait_for_query(cancel_token, db::postgres::execute_query_with_schema(&p, &schema, sql)) wait_for_query(cancel_token, db::postgres::execute_query_with_schema(&p, &schema, sql)).await
.await
.map(truncate_result)
} else { } else {
wait_for_query(cancel_token, db::postgres::execute_query(&p, sql)).await.map(truncate_result) wait_for_query(cancel_token, db::postgres::execute_query(&p, sql)).await
} }
} }
PoolKind::Sqlite(p) => { PoolKind::Sqlite(p) => {
let p = p.clone(); let p = p.clone();
drop(connections); drop(connections);
wait_for_query(cancel_token, db::sqlite::execute_query(&p, sql)).await.map(truncate_result) wait_for_query(cancel_token, db::sqlite::execute_query(&p, sql)).await
} }
PoolKind::ClickHouse(client) => { PoolKind::ClickHouse(client) => {
let client = client.clone(); let client = client.clone();