fix(export): use inactivity timeout for SQL Server streams

This commit is contained in:
t8y2 committed 2026-07-19 12:47:08 +08:00
1 parent c6d238348c
commit 368351d70e
2 files changed
+323 -7

No files matched your search

+192 -7
View File
@@ -1,8 +1,9 @@
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::future::Future;
use std::io::{BufWriter, Seek, Write};
use std::sync::{
atomic::{AtomicBool, Ordering},
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
};
use std::time::{Duration, Instant};
@@ -192,6 +193,85 @@ fn query_export_timeout(timeout_secs: Option<u64>) -> Option<Duration> {
}
}
struct StreamProgressClock {
started_at: tokio::time::Instant,
last_progress_ms: AtomicU64,
}
impl StreamProgressClock {
fn new() -> Self {
Self { started_at: tokio::time::Instant::now(), last_progress_ms: AtomicU64::new(0) }
}
fn mark(&self) {
self.last_progress_ms.store(self.started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
}
fn elapsed_since_progress(&self) -> Duration {
let last_progress_ms = self.last_progress_ms.load(Ordering::Relaxed);
let elapsed_ms = self.started_at.elapsed().as_millis() as u64;
Duration::from_millis(elapsed_ms.saturating_sub(last_progress_ms))
}
}
async fn await_stream_with_progress_timeout<F, T>(
stream_future: F,
timeout: Option<Duration>,
progress_clock: Arc<StreamProgressClock>,
cancel_token: Option<&CancellationToken>,
timeout_message: String,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
let Some(timeout) = timeout else {
return match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => Err(canceled_error()),
result = stream_future => result,
}
}
None => stream_future.await,
};
};
tokio::pin!(stream_future);
loop {
// The query timeout is an inactivity budget, not a cap on total export duration.
// This keeps a stalled server bounded while allowing large local file writes to finish.
let remaining = timeout.saturating_sub(progress_clock.elapsed_since_progress());
if remaining.is_zero() {
return Err(timeout_message.clone());
}
let sleep = tokio::time::sleep(remaining);
tokio::pin!(sleep);
match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => return Err(canceled_error()),
result = &mut stream_future => return result,
_ = &mut sleep => {},
}
}
None => {
tokio::select! {
biased;
result = &mut stream_future => return result,
_ = &mut sleep => {},
}
}
}
if progress_clock.elapsed_since_progress() >= timeout {
return Err(timeout_message);
}
}
}
fn should_fetch_next_page(
use_agent_result_session: bool,
has_more: bool,
@@ -1309,6 +1389,9 @@ async fn try_export_sqlserver_query_result_stream(
None
};
let mut xlsx = None;
let progress_clock = Arc::new(StreamProgressClock::new());
let progress_clock_for_stream = progress_clock.clone();
let query_timeout = query_export_timeout(request.timeout_secs);
let mut client = match cancel_token.as_ref() {
Some(token) => {
@@ -1378,15 +1461,20 @@ async fn try_export_sqlserver_query_result_stream(
}
}
}
// Mark only after the row is fully written so local XLSX work never consumes
// the next database inactivity window.
progress_clock_for_stream.mark();
Ok(())
},
);
match query_export_timeout(request.timeout_secs) {
Some(timeout) => tokio::time::timeout(timeout, stream_future)
.await
.map_err(|_| format!("Query timed out after {} seconds", timeout.as_secs()))??,
None => stream_future.await?,
};
await_stream_with_progress_timeout(
stream_future,
query_timeout,
progress_clock,
cancel_token.as_ref(),
format!("Query timed out after {} seconds", query_timeout.map_or(0, |timeout| timeout.as_secs())),
)
.await?;
drop(client);
if rows_exported != last_progress_rows {
@@ -1593,4 +1681,101 @@ mod tests {
.await;
assert!(result.is_err());
}
#[tokio::test]
async fn sqlserver_stream_times_out_when_database_makes_no_progress() {
let progress_clock = Arc::new(StreamProgressClock::new());
let result = await_stream_with_progress_timeout(
std::future::pending::<Result<(), String>>(),
Some(Duration::from_millis(20)),
progress_clock,
None,
"query timeout".to_string(),
)
.await;
assert_eq!(result, Err("query timeout".to_string()));
}
#[tokio::test]
async fn sqlserver_stream_timeout_resets_after_each_completed_row() {
let progress_clock = Arc::new(StreamProgressClock::new());
let progress_clock_for_stream = progress_clock.clone();
let result = await_stream_with_progress_timeout(
async move {
for row in 1..=5 {
tokio::time::sleep(Duration::from_millis(50)).await;
progress_clock_for_stream.mark();
assert!(row <= 5);
}
Ok::<_, String>(5_u8)
},
Some(Duration::from_millis(150)),
progress_clock,
None,
"query timeout".to_string(),
)
.await;
assert_eq!(result, Ok(5));
}
#[tokio::test]
async fn sqlserver_stream_does_not_count_synchronous_local_writes_as_database_idle_time() {
let progress_clock = Arc::new(StreamProgressClock::new());
let progress_clock_for_stream = progress_clock.clone();
let result = await_stream_with_progress_timeout(
async move {
std::thread::sleep(Duration::from_millis(50));
progress_clock_for_stream.mark();
Ok::<_, String>(())
},
Some(Duration::from_millis(20)),
progress_clock,
None,
"query timeout".to_string(),
)
.await;
assert_eq!(result, Ok(()));
}
#[tokio::test]
async fn sqlserver_stream_timeout_zero_disables_idle_timeout() {
let progress_clock = Arc::new(StreamProgressClock::new());
let result = await_stream_with_progress_timeout(
async {
tokio::time::sleep(Duration::from_millis(20)).await;
Ok::<_, String>(())
},
None,
progress_clock,
None,
"query timeout".to_string(),
)
.await;
assert_eq!(result, Ok(()));
}
#[tokio::test]
async fn sqlserver_stream_cancellation_wins_over_idle_timeout() {
let progress_clock = Arc::new(StreamProgressClock::new());
let cancel_token = CancellationToken::new();
let cancel_token_for_task = cancel_token.clone();
let task = tokio::spawn(async move {
await_stream_with_progress_timeout(
async { std::future::pending::<Result<(), String>>().await },
Some(Duration::from_secs(1)),
progress_clock,
Some(&cancel_token_for_task),
"query timeout".to_string(),
)
.await
});
tokio::task::yield_now().await;
cancel_token.cancel();
assert_eq!(task.await.unwrap(), Err(QUERY_CANCELED.to_string()));
}
}