perf(query): stream rows with result limit
This commit is contained in:
1 parent
af029daf57
commit
f9993dd3f8
4 files changed
+127
-87
No files matched your search
@@ -1,4 +1,5 @@
|
|||||||
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
||||||
|
use futures::StreamExt;
|
||||||
use rust_decimal::Decimal;
|
use rust_decimal::Decimal;
|
||||||
use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow};
|
use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow};
|
||||||
use sqlx::{Column, Executor, Row, TypeInfo, ValueRef};
|
use sqlx::{Column, Executor, Row, TypeInfo, ValueRef};
|
||||||
@@ -252,54 +253,70 @@ pub async fn execute_query(pool: &MySqlPool, sql: &str, bare: bool) -> Result<Qu
|
|||||||
|| trimmed.starts_with("EXPLAIN")
|
|| trimmed.starts_with("EXPLAIN")
|
||||||
{
|
{
|
||||||
if bare {
|
if bare {
|
||||||
let rows: Vec<MySqlRow> = sqlx::raw_sql(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
let mut stream = sqlx::raw_sql(sql).fetch(&*pool);
|
||||||
|
let mut columns: Vec<String> = vec![];
|
||||||
|
let mut column_types: Vec<String> = vec![];
|
||||||
|
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||||
|
|
||||||
let (columns, column_types) = if let Some(first) = rows.first() {
|
while let Some(row) = stream.next().await {
|
||||||
let cols: Vec<String> = first.columns().iter().map(|c| c.name().to_string()).collect();
|
let row: MySqlRow = row.map_err(|e| e.to_string())?;
|
||||||
let types: Vec<String> = first.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
if columns.is_empty() {
|
||||||
(cols, types)
|
columns = row.columns().iter().map(|c| c.name().to_string()).collect();
|
||||||
} else {
|
column_types = row.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
||||||
(vec![], vec![])
|
}
|
||||||
};
|
result_rows.push(
|
||||||
|
|
||||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
|
||||||
.iter()
|
|
||||||
.map(|row| {
|
|
||||||
(0..row.len())
|
(0..row.len())
|
||||||
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
.map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||||
.collect()
|
.collect(),
|
||||||
})
|
);
|
||||||
.collect();
|
if result_rows.len() > crate::query::MAX_ROWS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||||
|
if truncated {
|
||||||
|
result_rows.truncate(crate::query::MAX_ROWS);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(QueryResult {
|
Ok(QueryResult {
|
||||||
columns,
|
columns,
|
||||||
rows: result_rows,
|
rows: result_rows,
|
||||||
affected_rows: 0,
|
affected_rows: 0,
|
||||||
execution_time_ms: start.elapsed().as_millis(),
|
execution_time_ms: start.elapsed().as_millis(),
|
||||||
truncated: false,
|
truncated,
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||||
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||||
let column_types: Vec<String> = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
let column_types: Vec<String> = desc.columns().iter().map(|c| c.type_info().name().to_string()).collect();
|
||||||
|
|
||||||
let rows: Vec<MySqlRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
let mut stream = sqlx::query(sql).fetch(&*pool);
|
||||||
|
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||||
|
|
||||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
while let Some(row) = stream.next().await {
|
||||||
.iter()
|
let row = row.map_err(|e| e.to_string())?;
|
||||||
.map(|row| {
|
result_rows.push(
|
||||||
(0..row.len())
|
(0..row.len())
|
||||||
.map(|i| mysql_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
.map(|i| mysql_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||||
.collect()
|
.collect(),
|
||||||
})
|
);
|
||||||
.collect();
|
if result_rows.len() > crate::query::MAX_ROWS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||||
|
if truncated {
|
||||||
|
result_rows.truncate(crate::query::MAX_ROWS);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(QueryResult {
|
Ok(QueryResult {
|
||||||
columns,
|
columns,
|
||||||
rows: result_rows,
|
rows: result_rows,
|
||||||
affected_rows: 0,
|
affected_rows: 0,
|
||||||
execution_time_ms: start.elapsed().as_millis(),
|
execution_time_ms: start.elapsed().as_millis(),
|
||||||
truncated: false,
|
truncated,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
|
||||||
|
use futures::StreamExt;
|
||||||
use percent_encoding::percent_decode_str;
|
use percent_encoding::percent_decode_str;
|
||||||
use rust_decimal::Decimal;
|
use rust_decimal::Decimal;
|
||||||
use sqlx::postgres::{PgPool, PgPoolOptions, PgRow};
|
use sqlx::postgres::{PgPool, PgPoolOptions, PgRow};
|
||||||
@@ -260,37 +261,44 @@ pub async fn execute_query(pool: &PgPool, sql: &str) -> Result<QueryResult, Stri
|
|||||||
|| trimmed.starts_with("WITH")
|
|| trimmed.starts_with("WITH")
|
||||||
|| trimmed.starts_with("TABLE")
|
|| trimmed.starts_with("TABLE")
|
||||||
{
|
{
|
||||||
let rows: Vec<PgRow> = sqlx::query(sql).persistent(false).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
let mut stream = sqlx::query(sql).persistent(false).fetch(pool);
|
||||||
|
let mut columns: Vec<String> = vec![];
|
||||||
|
let mut column_types: Vec<String> = vec![];
|
||||||
|
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||||
|
|
||||||
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() {
|
while let Some(row) = stream.next().await {
|
||||||
let cols = first.columns();
|
let row = row.map_err(|e| e.to_string())?;
|
||||||
(
|
if columns.is_empty() {
|
||||||
cols.iter().map(|c| c.name().to_string()).collect(),
|
let cols = row.columns();
|
||||||
cols.iter().map(|c| c.type_info().name().to_string()).collect(),
|
columns = cols.iter().map(|c| c.name().to_string()).collect();
|
||||||
)
|
column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
|
||||||
} else {
|
}
|
||||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
result_rows.push(
|
||||||
(
|
|
||||||
desc.columns().iter().map(|c| c.name().to_string()).collect(),
|
|
||||||
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
|
|
||||||
)
|
|
||||||
};
|
|
||||||
|
|
||||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
|
||||||
.iter()
|
|
||||||
.map(|row| {
|
|
||||||
(0..row.len())
|
(0..row.len())
|
||||||
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||||
.collect()
|
.collect(),
|
||||||
})
|
);
|
||||||
.collect();
|
if result_rows.len() > crate::query::MAX_ROWS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if columns.is_empty() {
|
||||||
|
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||||
|
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||||
|
}
|
||||||
|
|
||||||
|
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||||
|
if truncated {
|
||||||
|
result_rows.truncate(crate::query::MAX_ROWS);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(QueryResult {
|
Ok(QueryResult {
|
||||||
columns,
|
columns,
|
||||||
rows: result_rows,
|
rows: result_rows,
|
||||||
affected_rows: 0,
|
affected_rows: 0,
|
||||||
execution_time_ms: start.elapsed().as_millis(),
|
execution_time_ms: start.elapsed().as_millis(),
|
||||||
truncated: false,
|
truncated,
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
|
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
|
||||||
@@ -319,38 +327,45 @@ pub async fn execute_query_with_schema(pool: &PgPool, schema: &str, sql: &str) -
|
|||||||
|| trimmed.starts_with("WITH")
|
|| trimmed.starts_with("WITH")
|
||||||
|| trimmed.starts_with("TABLE")
|
|| trimmed.starts_with("TABLE")
|
||||||
{
|
{
|
||||||
let rows: Vec<PgRow> =
|
let mut stream = sqlx::query(sql).persistent(false).fetch(&mut *conn);
|
||||||
sqlx::query(sql).persistent(false).fetch_all(&mut *conn).await.map_err(|e| e.to_string())?;
|
let mut columns: Vec<String> = vec![];
|
||||||
|
let mut column_types: Vec<String> = vec![];
|
||||||
|
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||||
|
|
||||||
let (columns, column_types): (Vec<String>, Vec<String>) = if let Some(first) = rows.first() {
|
while let Some(row) = stream.next().await {
|
||||||
let cols = first.columns();
|
let row = row.map_err(|e| e.to_string())?;
|
||||||
(
|
if columns.is_empty() {
|
||||||
cols.iter().map(|c| c.name().to_string()).collect(),
|
let cols = row.columns();
|
||||||
cols.iter().map(|c| c.type_info().name().to_string()).collect(),
|
columns = cols.iter().map(|c| c.name().to_string()).collect();
|
||||||
)
|
column_types = cols.iter().map(|c| c.type_info().name().to_string()).collect();
|
||||||
} else {
|
}
|
||||||
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?;
|
result_rows.push(
|
||||||
(
|
|
||||||
desc.columns().iter().map(|c| c.name().to_string()).collect(),
|
|
||||||
desc.columns().iter().map(|c| c.type_info().name().to_string()).collect(),
|
|
||||||
)
|
|
||||||
};
|
|
||||||
|
|
||||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
|
||||||
.iter()
|
|
||||||
.map(|row| {
|
|
||||||
(0..row.len())
|
(0..row.len())
|
||||||
.map(|i| pg_value_to_json(row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||||
.collect()
|
.collect(),
|
||||||
})
|
);
|
||||||
.collect();
|
if result_rows.len() > crate::query::MAX_ROWS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(stream);
|
||||||
|
|
||||||
|
if columns.is_empty() {
|
||||||
|
let desc = (&mut *conn).describe(sql).await.map_err(|e| e.to_string())?;
|
||||||
|
columns = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||||
|
}
|
||||||
|
|
||||||
|
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||||
|
if truncated {
|
||||||
|
result_rows.truncate(crate::query::MAX_ROWS);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(QueryResult {
|
Ok(QueryResult {
|
||||||
columns,
|
columns,
|
||||||
rows: result_rows,
|
rows: result_rows,
|
||||||
affected_rows: 0,
|
affected_rows: 0,
|
||||||
execution_time_ms: start.elapsed().as_millis(),
|
execution_time_ms: start.elapsed().as_millis(),
|
||||||
truncated: false,
|
truncated,
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
let result = sqlx::query(sql).execute(&mut *conn).await.map_err(|e| e.to_string())?;
|
let result = sqlx::query(sql).execute(&mut *conn).await.map_err(|e| e.to_string())?;
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use futures::StreamExt;
|
||||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow};
|
use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions, SqliteRow};
|
||||||
use sqlx::{Column, Executor, Row};
|
use sqlx::{Column, Executor, Row};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
@@ -171,11 +172,12 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
|
|||||||
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
let desc = pool.describe(sql).await.map_err(|e| e.to_string())?;
|
||||||
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
let columns: Vec<String> = desc.columns().iter().map(|c| c.name().to_string()).collect();
|
||||||
|
|
||||||
let rows: Vec<SqliteRow> = sqlx::query(sql).fetch_all(pool).await.map_err(|e| e.to_string())?;
|
let mut stream = sqlx::query(sql).fetch(pool);
|
||||||
|
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||||
|
|
||||||
let result_rows: Vec<Vec<serde_json::Value>> = rows
|
while let Some(row) = stream.next().await {
|
||||||
.iter()
|
let row = row.map_err(|e| e.to_string())?;
|
||||||
.map(|row| {
|
result_rows.push(
|
||||||
(0..row.len())
|
(0..row.len())
|
||||||
.map(|i| {
|
.map(|i| {
|
||||||
row.try_get::<String, _>(i)
|
row.try_get::<String, _>(i)
|
||||||
@@ -191,16 +193,24 @@ pub async fn execute_query(pool: &SqlitePool, sql: &str) -> Result<QueryResult,
|
|||||||
.or_else(|_| row.try_get::<bool, _>(i).map(serde_json::Value::Bool))
|
.or_else(|_| row.try_get::<bool, _>(i).map(serde_json::Value::Bool))
|
||||||
.unwrap_or(serde_json::Value::Null)
|
.unwrap_or(serde_json::Value::Null)
|
||||||
})
|
})
|
||||||
.collect()
|
.collect(),
|
||||||
})
|
);
|
||||||
.collect();
|
if result_rows.len() > crate::query::MAX_ROWS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let truncated = result_rows.len() > crate::query::MAX_ROWS;
|
||||||
|
if truncated {
|
||||||
|
result_rows.truncate(crate::query::MAX_ROWS);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(QueryResult {
|
Ok(QueryResult {
|
||||||
columns,
|
columns,
|
||||||
rows: result_rows,
|
rows: result_rows,
|
||||||
affected_rows: 0,
|
affected_rows: 0,
|
||||||
execution_time_ms: start.elapsed().as_millis(),
|
execution_time_ms: start.elapsed().as_millis(),
|
||||||
truncated: false,
|
truncated,
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
|
let result = sqlx::query(sql).execute(pool).await.map_err(|e| e.to_string())?;
|
||||||
|
|||||||
@@ -158,24 +158,22 @@ pub async fn do_execute(
|
|||||||
let p = p.clone();
|
let p = p.clone();
|
||||||
let bare = *mode == crate::connection::MysqlMode::Bare;
|
let bare = *mode == crate::connection::MysqlMode::Bare;
|
||||||
drop(connections);
|
drop(connections);
|
||||||
wait_for_query(cancel_token, db::mysql::execute_query(&p, sql, bare)).await.map(truncate_result)
|
wait_for_query(cancel_token, db::mysql::execute_query(&p, sql, bare)).await
|
||||||
}
|
}
|
||||||
PoolKind::Postgres(p) => {
|
PoolKind::Postgres(p) => {
|
||||||
let p = p.clone();
|
let p = p.clone();
|
||||||
let schema = schema.map(|s| s.to_string());
|
let schema = schema.map(|s| s.to_string());
|
||||||
drop(connections);
|
drop(connections);
|
||||||
if let Some(schema) = schema {
|
if let Some(schema) = schema {
|
||||||
wait_for_query(cancel_token, db::postgres::execute_query_with_schema(&p, &schema, sql))
|
wait_for_query(cancel_token, db::postgres::execute_query_with_schema(&p, &schema, sql)).await
|
||||||
.await
|
|
||||||
.map(truncate_result)
|
|
||||||
} else {
|
} else {
|
||||||
wait_for_query(cancel_token, db::postgres::execute_query(&p, sql)).await.map(truncate_result)
|
wait_for_query(cancel_token, db::postgres::execute_query(&p, sql)).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
PoolKind::Sqlite(p) => {
|
PoolKind::Sqlite(p) => {
|
||||||
let p = p.clone();
|
let p = p.clone();
|
||||||
drop(connections);
|
drop(connections);
|
||||||
wait_for_query(cancel_token, db::sqlite::execute_query(&p, sql)).await.map(truncate_result)
|
wait_for_query(cancel_token, db::sqlite::execute_query(&p, sql)).await
|
||||||
}
|
}
|
||||||
PoolKind::ClickHouse(client) => {
|
PoolKind::ClickHouse(client) => {
|
||||||
let client = client.clone();
|
let client = client.clone();
|
||||||
|
|||||||
Reference in new issue
Block a user