perf(query): stream rows with result limit

This commit is contained in:
t8y2 committed 2026-05-09 15:57:55 +08:00
1 parent af029daf57
commit f9993dd3f8
4 files changed
+127 -87

No files matched your search

+43 -26
View File
@@ -1,4 +1,5 @@
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
use futures::StreamExt;
use rust_decimal::Decimal;
use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow};
use sqlx::{Column, Executor, Row, TypeInfo, ValueRef};
@@ -252,54 +253,70 @@ pub async fn execute_query(pool: &MySqlPool, sql: &str, bare: bool) -> Result<Qu
|| trimmed.starts_with("EXPLAIN")
{
if bare {
let rows: Vec<MySqlRow> = sqlx::raw_sql(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
let mut stream = sqlx::raw_sql(sql).fetch(&*pool);
let mut columns: Vec<String> = vec![];
let mut column_types: Vec<String> = vec![];
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let (columns, column_types) = if let Some(first) = rows.first() {
let cols: Vec<String> = first.columns().iter().map(|c| c.name().to_string()).collect();
let types: Vec<String> = first.columns().iter().map(|c| c.type_info().name().to_string()).collect();
(cols, types)
} else {
(vec![], vec![])
};
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
while let Some(row) = stream.next().await {
let row: MySqlRow = row.map_err(|e| e.to_string())?;
if columns.is_empty() {
columns = row.columns().iter().map(|c| c.name().to_string()).collect();
column_types = row.columns().iter().map(|c| c.type_info().name().to_string()).collect();
}
result_rows.push(
(0..row.len())
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect()
})
.collect();
.map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect(),
);
if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult {
columns,
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
truncated,
})
} else {
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
let column_types: Vec<String> = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect();
let rows: Vec<MySqlRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
let mut stream = sqlx::query(sql).fetch(&*pool);
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
while let Some(row) = stream.next().await {
let row = row.map_err(|e| e.to_string())?;
result_rows.push(
(0..row.len())
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect()
})
.collect();
.map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect(),
);
if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult {
columns,
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
truncated,
})
}
} else {
+62 -47
View File
@@ -1,4 +1,5 @@
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
use futures::StreamExt;
use percent_encoding::percent_decode_str;
use rust_decimal::Decimal;
use sqlx::postgres::{PgPool, PgPoolOptions, PgRow};
@@ -260,37 +261,44 @@ pub async fn execute_query(pool: &PgPool, sql: &str) -> Result<QueryResult, Stri
|| trimmed.starts_with("WITH")
|| trimmed.starts_with("TABLE")
{
let rows: Vec<PgRow> = sqlx::query(sql).persistent(false).fetch_all(pool).await.map_err(|e| e.to_string())?;
let mut stream = sqlx::query(sql).persistent(false).fetch(pool);
let mut columns: Vec<String> = vec![];
let mut column_types: Vec<String> = vec![];
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() {
let cols = first.columns();
(
cols.iter().map(|c| c.name().to_string()).collect(),
cols.iter().map(|c| c.type_info().name().to_string()).collect(),
)
} else {
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
(
desc.columns().iter().map(|c| c.name().to_string()).collect(),
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
)
};
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
while let Some(row) = stream.next().await {
let row = row.map_err(|e| e.to_string())?;
if columns.is_empty() {
let cols = row.columns();
columns = cols.iter().map(|c| c.name().to_string()).collect();
column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
}
result_rows.push(
(0..row.len())
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect()
})
.collect();
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect(),
);
if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
if columns.is_empty() {
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult {
columns,
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
truncated,
})
} else {
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
@@ -319,38 +327,45 @@ pub async fn execute_query_with_schema(pool: &PgPool, schema: &str, sql: &str) -
|| trimmed.starts_with("WITH")
|| trimmed.starts_with("TABLE")
{
let rows: Vec<PgRow> =
sqlx::query(sql).persistent(false).fetch_all(&mut *conn).await.map_err(|e| e.to_string())?;
let mut stream = sqlx::query(sql).persistent(false).fetch(&mut *conn);
let mut columns: Vec<String> = vec![];
let mut column_types: Vec<String> = vec![];
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() {
let cols = first.columns();
(
cols.iter().map(|c| c.name().to_string()).collect(),
cols.iter().map(|c| c.type_info().name().to_string()).collect(),
)
} else {
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?;
(
desc.columns().iter().map(|c| c.name().to_string()).collect(),
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
)
};
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
while let Some(row) = stream.next().await {
let row = row.map_err(|e| e.to_string())?;
if columns.is_empty() {
let cols = row.columns();
columns = cols.iter().map(|c| c.name().to_string()).collect();
column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
}
result_rows.push(
(0..row.len())
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect()
})
.collect();
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect(),
);
if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
drop(stream);
if columns.is_empty() {
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?;
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult {
columns,
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
truncated,
})
} else {
let result = sqlx::query(sql).execute(&mut *conn).await.map_err(|e| e.to_string())?;
+18 -8
View File
@@ -1,3 +1,4 @@
use futures::StreamExt;
use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow};
use sqlx::{Column, Executor, Row};
use std::time::{Duration, Instant};
@@ -171,11 +172,12 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
let rows: Vec<SqliteRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
let mut stream = sqlx::query(sql).fetch(pool);
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
let result_rows: Vec<Vec<serde_json::Value>> = rows
.iter()
.map(|row| {
while let Some(row) = stream.next().await {
let row = row.map_err(|e| e.to_string())?;
result_rows.push(
(0..row.len())
.map(|i| {
row.try_get::<String, _>(i)
@@ -191,16 +193,24 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
.or_else(|_| row.try_get::<bool, _>(i).map(serde_json::Value::Bool))
.unwrap_or(serde_json::Value::Null)
})
.collect()
})
.collect();
.collect(),
);
if result_rows.len() > crate::query::MAX_ROWS {
break;
}
}
let truncated = result_rows.len() > crate::query::MAX_ROWS;
if truncated {
result_rows.truncate(crate::query::MAX_ROWS);
}
Ok(QueryResult {
columns,
rows: result_rows,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
truncated,
})
} else {
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;