perf(transfer): make large transfers progress-aware with keyset paging

This commit is contained in:
HolyeTyrone authored and GitHub committed 2026-09-11 14:38:34 +08:00
1 parent 5e111145d3
commit a04713001e
10 files changed
+1531 -24

No files matched your search

+137
View File
@@ -4427,6 +4427,7 @@ async fn execute_result_set_with_text_protocol_on_conn(
result_key_columns: &[String],
diagnostic_trace_id: Option<&str>,
start: Instant,
progress_clock: Option<&crate::query::StreamProgressClock>,
) -> Result<MySqlQueryResult, String> {
let diagnostics_enabled = diagnostic_trace_id.is_some() && log::log_enabled!(log::Level::Info);
let dispatch_started_at = diagnostics_enabled.then(Instant::now);
@@ -4517,6 +4518,9 @@ async fn execute_result_set_with_text_protocol_on_conn(
}
let Some(row) = next_row else { break };
let row = row.map_err(|e| e.to_string())?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
if result_rows.len() >= row_limit {
truncated = true;
break;
@@ -4816,6 +4820,7 @@ async fn execute_result_set_with_prepared_protocol_on_conn(
result_key_columns: &[String],
diagnostic_trace_id: Option<&str>,
start: Instant,
progress_clock: Option<&crate::query::StreamProgressClock>,
) -> Result<MySqlQueryResult, String> {
let diagnostics_enabled = diagnostic_trace_id.is_some() && log::log_enabled!(log::Level::Info);
let dispatch_started_at = diagnostics_enabled.then(Instant::now);
@@ -4848,6 +4853,9 @@ async fn execute_result_set_with_prepared_protocol_on_conn(
}
let Some(row) = next_row else { break };
let row = row.map_err(|e| e.to_string())?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
if result_rows.len() >= row_limit {
truncated = true;
break;
@@ -4958,6 +4966,54 @@ where
execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, dialect).await
}
/// Progress-aware variant of [`execute_query_with_max_rows`] for long transfers.
///
/// The statement runs under an inactivity budget: the clock is reset for every
/// row MySQL delivers, so a large table that keeps streaming is never cancelled
/// merely for exceeding the timeout in total — only a genuine stall is.
///
/// Unlike the Postgres/SQL Server variants there is no returns-rows gate here:
/// a write delivers no rows, so no progress mark ever resets the clock and the
/// budget degrades to the same plain wall-clock timeout anyway.
pub(crate) async fn execute_query_with_max_rows_progress<P>(
pool: &P,
sql: &str,
bare: bool,
max_rows: Option<usize>,
dialect: MySqlQueryDialect,
progress_clock: std::sync::Arc<crate::query::StreamProgressClock>,
timeout: Option<std::time::Duration>,
) -> Result<QueryResult, String>
where
P: MySqlPoolAccess + ?Sized,
{
let mut conn = get_conn_with_health_check(pool).await?;
let timeout_error = format!("Query timed out after {} seconds", timeout.map_or(0, |timeout| timeout.as_secs()));
let clock_for_query = progress_clock.clone();
crate::query::await_stream_with_progress_timeout(
async move {
execute_query_on_conn_with_limits_progress(
&mut conn,
sql,
bare,
max_rows,
None,
&[],
dialect,
None,
Some(&clock_for_query),
)
.await
.map(|result| result.result)
},
timeout,
progress_clock,
None,
timeout_error,
)
.await
}
pub async fn stream_query_rows(
pool: &MySqlPool,
sql: &str,
@@ -5163,6 +5219,37 @@ pub async fn execute_query_on_conn_with_limits(
result_key_columns: &[String],
dialect: MySqlQueryDialect,
diagnostic_trace_id: Option<&str>,
) -> Result<MySqlQueryResult, String> {
execute_query_on_conn_with_limits_progress(
conn,
sql,
bare,
max_rows,
max_result_bytes,
result_key_columns,
dialect,
diagnostic_trace_id,
None,
)
.await
}
/// [`execute_query_on_conn_with_limits`] with an optional progress clock.
///
/// The clock is marked for every row the server delivers, so a caller wrapping
/// this in an inactivity budget (see [`crate::query::await_stream_with_progress_timeout`])
/// only times out on a genuine stall, not on a long-but-steady stream — which is
/// what lets a transfer of a large table survive the configured timeout.
pub(crate) async fn execute_query_on_conn_with_limits_progress(
conn: &mut mysql_async::Conn,
sql: &str,
bare: bool,
max_rows: Option<usize>,
max_result_bytes: Option<usize>,
result_key_columns: &[String],
dialect: MySqlQueryDialect,
diagnostic_trace_id: Option<&str>,
progress_clock: Option<&crate::query::StreamProgressClock>,
) -> Result<MySqlQueryResult, String> {
let start = Instant::now();
let row_limit = query_result_row_limit(max_rows);
@@ -5178,6 +5265,7 @@ pub async fn execute_query_on_conn_with_limits(
result_key_columns,
diagnostic_trace_id,
start,
progress_clock,
)
.await
} else {
@@ -5189,6 +5277,7 @@ pub async fn execute_query_on_conn_with_limits(
result_key_columns,
diagnostic_trace_id,
start,
progress_clock,
)
.await
{
@@ -5203,6 +5292,7 @@ pub async fn execute_query_on_conn_with_limits(
result_key_columns,
diagnostic_trace_id,
start,
progress_clock,
)
.await
}
@@ -5922,6 +6012,53 @@ mod tests {
new_row(values, columns.into())
}
#[tokio::test]
#[ignore = "requires a disposable MySQL reachable via DBX_LIVE_MYSQL_TRANSFER_{HOST,PORT,USER,PASSWORD}"]
async fn live_mysql_progress_read_survives_a_total_duration_beyond_the_timeout() {
let Ok(host) = std::env::var("DBX_LIVE_MYSQL_TRANSFER_HOST") else {
return;
};
let port = std::env::var("DBX_LIVE_MYSQL_TRANSFER_PORT").unwrap_or_else(|_| "3306".to_string());
let user = std::env::var("DBX_LIVE_MYSQL_TRANSFER_USER").unwrap_or_else(|_| "root".to_string());
let password = std::env::var("DBX_LIVE_MYSQL_TRANSFER_PASSWORD").unwrap_or_default();
let database = std::env::var("DBX_LIVE_MYSQL_TRANSFER_DATABASE").unwrap_or_else(|_| "test".to_string());
let url = format!("mysql://{user}:{password}@{host}:{port}/{database}");
let pool = connect(&url, Duration::from_secs(5)).await.unwrap();
// 20 rows produced ~50 ms apart, each carrying >8 KB so the server flushes
// it per row instead of buffering the small result set. The SLEEP sits in
// the WHERE clause so MySQL evaluates it per row — in the SELECT list the
// planner folds the constant call and the whole statement returns at once.
let sql = "WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 20) \
SELECT REPEAT('x', 20000) AS payload, n FROM seq WHERE SLEEP(0.05) = 0";
let progress_clock = std::sync::Arc::new(crate::query::StreamProgressClock::new());
let result = execute_query_with_max_rows_progress(
&pool,
sql,
false,
None,
Default::default(),
progress_clock,
Some(Duration::from_millis(200)),
)
.await;
assert!(result.is_ok(), "progress-aware read must survive a total duration beyond the timeout: {result:?}");
assert_eq!(result.unwrap().rows.len(), 20);
// Contrast: the same statement under a never-reset wall-clock budget must
// time out, proving this test actually exercises the difference.
let wall_clock = crate::query::await_stream_with_progress_timeout(
execute_query_with_max_rows(&pool, sql, false, None, Default::default()),
Some(Duration::from_millis(200)),
std::sync::Arc::new(crate::query::StreamProgressClock::new()),
None,
"Query timed out after 0 seconds".to_string(),
)
.await;
assert!(wall_clock.is_err(), "the wall-clock path must still time out");
}
#[test]
fn mysql_first_column_value_ignores_extra_result_columns() {
let row = mysql_async::from_row::<mysql_async::Row>(mysql_test_row(vec![
+74
View File
@@ -6867,6 +6867,44 @@ pub async fn execute_query_with_max_rows(
}
}
/// Progress-aware variant of [`execute_query_with_max_rows`] for long transfers.
///
/// Row-returning queries run under an *inactivity* budget: the clock is reset
/// every time PostgreSQL actually delivers a row, so a large table that keeps
/// streaming is never cancelled merely for exceeding the timeout in total. Only
/// a genuine stall (no row for the whole timeout) is reported as a timeout.
/// Statements that return no rows have no incremental progress to report, so
/// they keep the plain wall-clock path.
pub(crate) async fn execute_query_with_max_rows_progress(
pool: &Pool,
sql: &str,
max_rows: Option<usize>,
progress_clock: Arc<StreamProgressClock>,
timeout: Option<Duration>,
) -> Result<QueryResult, String> {
if !postgres_statement_returns_rows(sql) {
// DDL/DML expose no incremental progress, so keep the original wall-clock
// budget: a hung write must still be bounded by the configured timeout.
return crate::query::wait_for_query_opt(None, timeout, execute_query_with_max_rows(pool, sql, max_rows)).await;
}
let start = Instant::now();
let row_limit = query_result_row_limit(max_rows);
let timeout_error = format!("Query timed out after {} seconds", timeout.map_or(0, |timeout| timeout.as_secs()));
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let clock_for_select = progress_clock.clone();
await_stream_with_progress_timeout(
async move {
execute_select_query_with_progress(&client, sql, start, row_limit, Some(&clock_for_select), false).await
},
timeout,
progress_clock,
None,
timeout_error,
)
.await
}
pub async fn execute_query_with_max_rows_and_cancel(
pool: &Pool,
sql: &str,
@@ -11405,6 +11443,42 @@ mod tests {
assert_eq!(result, Ok(()));
}
#[tokio::test]
#[ignore = "requires DBX_LIVE_PG_TRANSFER_SOURCE_URL pointing at a disposable PostgreSQL"]
async fn live_postgres_progress_read_survives_a_total_duration_beyond_the_timeout() {
let Ok(url) = std::env::var("DBX_LIVE_PG_TRANSFER_SOURCE_URL") else {
return;
};
let pool = connect(&url, Duration::from_secs(5)).await.unwrap();
// 20 rows produced ~50 ms apart: the statement streams for ~1 s in total,
// well beyond the 200 ms budget, while never stalling that long between
// rows — the exact shape a progress-aware transfer read has to survive.
// Each row carries >8 KB so PostgreSQL flushes it immediately instead of
// buffering the whole (tiny) result set and sending it in one packet.
let sql = "SELECT pg_sleep(0.05) IS NULL AS slept, repeat('x', 20000) AS payload, n \
FROM generate_series(1, 20) AS n";
let progress_clock = Arc::new(StreamProgressClock::new());
let result =
execute_query_with_max_rows_progress(&pool, sql, None, progress_clock, Some(Duration::from_millis(200)))
.await;
assert!(result.is_ok(), "progress-aware read must survive a total duration beyond the timeout: {result:?}");
assert_eq!(result.unwrap().rows.len(), 20);
// Contrast: the same statement under a plain, never-reset wall-clock budget
// must time out, proving this test actually exercises the difference.
let wall_clock = await_stream_with_progress_timeout(
execute_query_with_max_rows(&pool, sql, None),
Some(Duration::from_millis(200)),
Arc::new(StreamProgressClock::new()),
None,
"Query timed out after 0 seconds".to_string(),
)
.await;
assert!(wall_clock.is_err(), "the wall-clock path must still time out");
}
#[tokio::test]
async fn postgres_row_query_timeout_still_fires_after_no_progress() {
let progress_clock = Arc::new(StreamProgressClock::new());
+71 -2
View File
@@ -677,6 +677,31 @@ mod tests {
assert_eq!(result.rows[0][0], serde_json::json!("Ada"));
}
#[tokio::test]
async fn progress_read_marks_the_inactivity_clock_for_a_multi_row_select() {
let pool = connect_path(":memory:").await.expect("connect in-memory SQLite");
execute_query(&pool, "CREATE TABLE progress_probe (id INTEGER PRIMARY KEY)").await.expect("create table");
for id in 1..=5 {
execute_query(&pool, &format!("INSERT INTO progress_probe (id) VALUES ({id})")).await.expect("insert row");
}
let progress_clock = std::sync::Arc::new(crate::query::StreamProgressClock::new());
assert!(!progress_clock.marked(), "a fresh clock must not report progress");
let result = execute_query_with_max_rows_progress(
&pool,
"SELECT id FROM progress_probe ORDER BY id",
None,
progress_clock.clone(),
Some(std::time::Duration::from_secs(30)),
)
.await
.expect("progress read");
assert_eq!(result.rows.len(), 5);
assert!(progress_clock.marked(), "streamed rows must record progress so the inactivity budget resets");
}
#[tokio::test]
async fn sqlite_dml_returning_preserves_result_rows() {
let pool = connect_path(":memory:").await.expect("connect in-memory SQLite");
@@ -2830,12 +2855,53 @@ pub async fn execute_query_with_max_rows(
return worker.query(&sql, max_rows).await;
}
let pool = pool.clone();
tokio::task::spawn_blocking(move || execute_query_blocking(&pool, &sql, max_rows))
tokio::task::spawn_blocking(move || execute_query_blocking(&pool, &sql, max_rows, None))
.await
.map_err(|e| e.to_string())?
}
fn execute_query_blocking(pool: &SqliteHandle, sql: &str, max_rows: Option<usize>) -> Result<QueryResult, String> {
/// Progress-aware variant of [`execute_query_with_max_rows`] for long transfers.
///
/// The clock is marked for every row the statement produces, so a caller wrapping
/// this in an inactivity budget only times out on a genuine stall, not on a long
/// but steady read. The SQLite worker (sidecar) path keeps the plain call: its RPC
/// returns the whole result at once and exposes no incremental progress.
pub(crate) async fn execute_query_with_max_rows_progress(
pool: &SqliteHandle,
sql: &str,
max_rows: Option<usize>,
progress_clock: std::sync::Arc<crate::query::StreamProgressClock>,
timeout: Option<std::time::Duration>,
) -> Result<QueryResult, String> {
let sql = normalize_sqlite_sql(sql);
if let Some(worker) = pool.worker() {
// The sidecar worker returns the whole result in one RPC and exposes no
// incremental progress, so keep the original wall-clock budget.
return crate::query::wait_for_query_opt(None, timeout, worker.query(&sql, max_rows)).await;
}
let pool = pool.clone();
let timeout_error = format!("Query timed out after {} seconds", timeout.map_or(0, |timeout| timeout.as_secs()));
let clock_for_query = progress_clock.clone();
crate::query::await_stream_with_progress_timeout(
async move {
tokio::task::spawn_blocking(move || execute_query_blocking(&pool, &sql, max_rows, Some(&clock_for_query)))
.await
.map_err(|e| e.to_string())?
},
timeout,
progress_clock,
None,
timeout_error,
)
.await
}
fn execute_query_blocking(
pool: &SqliteHandle,
sql: &str,
max_rows: Option<usize>,
progress_clock: Option<&crate::query::StreamProgressClock>,
) -> Result<QueryResult, String> {
let start = Instant::now();
let row_limit = query_result_row_limit(max_rows);
@@ -2849,6 +2915,9 @@ fn execute_query_blocking(pool: &SqliteHandle, sql: &str, max_rows: Option<usize
let mut result_rows = Vec::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
let mut values = Vec::with_capacity(columns.len());
for i in 0..columns.len() {
values.push(value_ref_to_json(
+65 -6
View File
@@ -712,6 +712,7 @@ async fn collect_first_result_limited(
result_offset: usize,
sql: &str,
query: &SqlServerUnsafeTypeQuery,
progress_clock: Option<&crate::query::StreamProgressClock>,
) -> Result<QueryResult, String> {
let row_limit = query_result_row_limit(max_rows);
let mut remaining_offset = result_offset;
@@ -724,6 +725,9 @@ async fn collect_first_result_limited(
let mut truncated = false;
while let Some(item) = stream.try_next().await.map_err(|e| e.to_string())? {
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
match item {
QueryItem::Metadata(metadata) if metadata.result_index() == 0 => {
columns = columns_from_metadata(&metadata);
@@ -3160,6 +3164,52 @@ pub async fn execute_query_with_max_rows(
client: &mut SqlServerClient,
sql: &str,
max_rows: Option<usize>,
) -> Result<QueryResult, String> {
execute_query_with_max_rows_inner(client, sql, max_rows, None).await
}
/// Progress-aware variant of [`execute_query_with_max_rows`] for long transfers.
///
/// Row-returning statements run under an inactivity budget: the clock is reset for
/// every streamed item, so a large table that keeps arriving is never cancelled
/// merely for exceeding the timeout in total — only a genuine stall is. Statements
/// without a result stream expose no incremental progress and keep the plain path.
pub(crate) async fn execute_query_with_max_rows_progress(
client: &mut SqlServerClient,
sql: &str,
max_rows: Option<usize>,
progress_clock: std::sync::Arc<crate::query::StreamProgressClock>,
timeout: Option<std::time::Duration>,
) -> Result<QueryResult, String> {
let returns_rows = starts_with_executable_sql_keyword(sql, &["SELECT", "EXEC", "WITH", "TABLE"])
|| sqlserver_dml_output_returns_rows(sql);
if !returns_rows {
// DDL/DML expose no incremental progress, so keep the original wall-clock
// budget: a hung write must still be bounded by the configured timeout.
return crate::query::wait_for_query_opt(
None,
timeout,
execute_query_with_max_rows_inner(client, sql, max_rows, None),
)
.await;
}
let timeout_error = format!("Query timed out after {} seconds", timeout.map_or(0, |timeout| timeout.as_secs()));
let clock_for_query = progress_clock.clone();
crate::query::await_stream_with_progress_timeout(
execute_query_with_max_rows_inner(client, sql, max_rows, Some(&clock_for_query)),
timeout,
progress_clock,
None,
timeout_error,
)
.await
}
async fn execute_query_with_max_rows_inner(
client: &mut SqlServerClient,
sql: &str,
max_rows: Option<usize>,
progress_clock: Option<&crate::query::StreamProgressClock>,
) -> Result<QueryResult, String> {
let start = Instant::now();
let result_offset = crate::query_result_sql::sqlserver_result_offset(sql);
@@ -3175,8 +3225,16 @@ pub async fn execute_query_with_max_rows(
};
let (result, messages) = capture_sqlserver_messages(async {
let stream = sqlserver_driver_result(client.query(query.sql.as_str(), &[])).await?;
sqlserver_driver_result(collect_first_result_limited(stream, start, max_rows, result_offset, sql, &query))
.await
sqlserver_driver_result(collect_first_result_limited(
stream,
start,
max_rows,
result_offset,
sql,
&query,
progress_clock,
))
.await
})
.await;
let mut result = query_result_with_server_messages(result?, messages);
@@ -3264,6 +3322,7 @@ pub(crate) async fn execute_batch_with_max_rows_metadata(
result_offset,
sql,
&query,
None,
))
.await
})
@@ -3358,7 +3417,8 @@ async fn execute_simple_batch_first_result_with_max_rows(
let (result, messages) = capture_sqlserver_messages(async {
let stream = sqlserver_driver_result(client.simple_query(sql)).await?;
let query = SqlServerUnsafeTypeQuery::plain(sql);
sqlserver_driver_result(collect_first_result_limited(stream, start, max_rows, result_offset, sql, &query)).await
sqlserver_driver_result(collect_first_result_limited(stream, start, max_rows, result_offset, sql, &query, None))
.await
})
.await;
let mut result = query_result_with_server_messages(result?, messages);
@@ -4327,9 +4387,8 @@ mod tests {
let first_result = source.split("async fn execute_simple_batch_first_result_with_max_rows").nth(1).unwrap();
let first_result = first_result.split("fn strip_dbx_sqlserver_row_number_column").next().unwrap();
assert!(
first_result.contains("collect_first_result_limited(stream, start, max_rows, result_offset, sql, &query)")
);
assert!(first_result
.contains("collect_first_result_limited(stream, start, max_rows, result_offset, sql, &query, None)"));
assert!(!first_result.contains("collect_result_sets_limited"));
}