perf(query): stream rows with result limit
This commit is contained in:
1 parent
af029daf57
commit
f9993dd3f8
4 files changed
+127
-87
No files matched your search
@@ -1,4 +1,5 @@
|
||||
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
||||
use futures::StreamExt;
|
||||
use rust_decimal::Decimal;
|
||||
use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow};
|
||||
use sqlx::{Column, Executor, Row, TypeInfo, ValueRef};
|
||||
@@ -252,54 +253,70 @@ pub async fn execute_query(pool: &MySqlPool, sql: &str, bare: bool) -> Result<Qu
|
||||
|| trimmed.starts_with("EXPLAIN")
|
||||
{
|
||||
if bare {
|
||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
||||
let mut stream = sqlx::raw_sql(sql).fetch(&*pool);
|
||||
let mut columns: Vec<String> = vec![];
|
||||
let mut column_types: Vec<String> = vec![];
|
||||
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||
|
||||
let (columns, column_types) = if let Some(first) = rows.first() {
|
||||
let cols: Vec<String> = first.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
let types: Vec<String> = first.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
||||
(cols, types)
|
||||
} else {
|
||||
(vec![], vec![])
|
||||
};
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
while let Some(row) = stream.next().await {
|
||||
let row: MySqlRow = row.map_err(|e| e.to_string())?;
|
||||
if columns.is_empty() {
|
||||
columns = row.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
column_types = row.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
||||
}
|
||||
result_rows.push(
|
||||
(0..row.len())
|
||||
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
.map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect(),
|
||||
);
|
||||
if result_rows.len() > crate::query::MAX_ROWS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||
if truncated {
|
||||
result_rows.truncate(crate::query::MAX_ROWS);
|
||||
}
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
truncated,
|
||||
})
|
||||
} else {
|
||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
let column_types: Vec<String> = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
||||
|
||||
let rows: Vec<MySqlRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
||||
let mut stream = sqlx::query(sql).fetch(&*pool);
|
||||
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
while let Some(row) = stream.next().await {
|
||||
let row = row.map_err(|e| e.to_string())?;
|
||||
result_rows.push(
|
||||
(0..row.len())
|
||||
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
.map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect(),
|
||||
);
|
||||
if result_rows.len() > crate::query::MAX_ROWS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||
if truncated {
|
||||
result_rows.truncate(crate::query::MAX_ROWS);
|
||||
}
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
truncated,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
||||
use futures::StreamExt;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rust_decimal::Decimal;
|
||||
use sqlx::postgres::{PgPool, PgPoolOptions, PgRow};
|
||||
@@ -260,37 +261,44 @@ pub async fn execute_query(pool: &PgPool, sql: &str) -> Result<QueryResult, Stri
|
||||
|| trimmed.starts_with("WITH")
|
||||
|| trimmed.starts_with("TABLE")
|
||||
{
|
||||
let rows: Vec<PgRow> = sqlx::query(sql).persistent(false).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
||||
let mut stream = sqlx::query(sql).persistent(false).fetch(pool);
|
||||
let mut columns: Vec<String> = vec![];
|
||||
let mut column_types: Vec<String> = vec![];
|
||||
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||
|
||||
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() {
|
||||
let cols = first.columns();
|
||||
(
|
||||
cols.iter().map(|c| c.name().to_string()).collect(),
|
||||
cols.iter().map(|c| c.type_info().name().to_string()).collect(),
|
||||
)
|
||||
} else {
|
||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||
(
|
||||
desc.columns().iter().map(|c| c.name().to_string()).collect(),
|
||||
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
|
||||
)
|
||||
};
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
while let Some(row) = stream.next().await {
|
||||
let row = row.map_err(|e| e.to_string())?;
|
||||
if columns.is_empty() {
|
||||
let cols = row.columns();
|
||||
columns = cols.iter().map(|c| c.name().to_string()).collect();
|
||||
column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
|
||||
}
|
||||
result_rows.push(
|
||||
(0..row.len())
|
||||
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect(),
|
||||
);
|
||||
if result_rows.len() > crate::query::MAX_ROWS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if columns.is_empty() {
|
||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
}
|
||||
|
||||
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||
if truncated {
|
||||
result_rows.truncate(crate::query::MAX_ROWS);
|
||||
}
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
truncated,
|
||||
})
|
||||
} else {
|
||||
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
|
||||
@@ -319,38 +327,45 @@ pub async fn execute_query_with_schema(pool: &PgPool, schema: &str, sql: &str) -
|
||||
|| trimmed.starts_with("WITH")
|
||||
|| trimmed.starts_with("TABLE")
|
||||
{
|
||||
let rows: Vec<PgRow> =
|
||||
sqlx::query(sql).persistent(false).fetch_all(&mut *conn).await.map_err(|e| e.to_string())?;
|
||||
let mut stream = sqlx::query(sql).persistent(false).fetch(&mut *conn);
|
||||
let mut columns: Vec<String> = vec![];
|
||||
let mut column_types: Vec<String> = vec![];
|
||||
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||
|
||||
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() {
|
||||
let cols = first.columns();
|
||||
(
|
||||
cols.iter().map(|c| c.name().to_string()).collect(),
|
||||
cols.iter().map(|c| c.type_info().name().to_string()).collect(),
|
||||
)
|
||||
} else {
|
||||
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?;
|
||||
(
|
||||
desc.columns().iter().map(|c| c.name().to_string()).collect(),
|
||||
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
|
||||
)
|
||||
};
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
while let Some(row) = stream.next().await {
|
||||
let row = row.map_err(|e| e.to_string())?;
|
||||
if columns.is_empty() {
|
||||
let cols = row.columns();
|
||||
columns = cols.iter().map(|c| c.name().to_string()).collect();
|
||||
column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
|
||||
}
|
||||
result_rows.push(
|
||||
(0..row.len())
|
||||
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect(),
|
||||
);
|
||||
if result_rows.len() > crate::query::MAX_ROWS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
drop(stream);
|
||||
|
||||
if columns.is_empty() {
|
||||
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?;
|
||||
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
}
|
||||
|
||||
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||
if truncated {
|
||||
result_rows.truncate(crate::query::MAX_ROWS);
|
||||
}
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
truncated,
|
||||
})
|
||||
} else {
|
||||
let result = sqlx::query(sql).execute(&mut *conn).await.map_err(|e| e.to_string())?;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use futures::StreamExt;
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow};
|
||||
use sqlx::{Column, Executor, Row};
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -171,11 +172,12 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
|
||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
|
||||
let rows: Vec<SqliteRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
||||
let mut stream = sqlx::query(sql).fetch(pool);
|
||||
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||
|
||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
while let Some(row) = stream.next().await {
|
||||
let row = row.map_err(|e| e.to_string())?;
|
||||
result_rows.push(
|
||||
(0..row.len())
|
||||
.map(|i| {
|
||||
row.try_get::<String, _>(i)
|
||||
@@ -191,16 +193,24 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
|
||||
.or_else(|_| row.try_get::<bool, _>(i).map(serde_json::Value::Bool))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
.collect(),
|
||||
);
|
||||
if result_rows.len() > crate::query::MAX_ROWS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||
if truncated {
|
||||
result_rows.truncate(crate::query::MAX_ROWS);
|
||||
}
|
||||
|
||||
Ok(QueryResult {
|
||||
columns,
|
||||
rows: result_rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: start.elapsed().as_millis(),
|
||||
truncated: false,
|
||||
truncated,
|
||||
})
|
||||
} else {
|
||||
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
|
||||
|
||||
Reference in new issue
Block a user