fix(export): use inactivity timeout for SQL Server streams
This commit is contained in:
1 parent
c6d238348c
commit
368351d70e
2 files changed
+323
-7
No files matched your search
@@ -1,8 +1,9 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fs::File;
|
||||
use std::future::Future;
|
||||
use std::io::{BufWriter, Seek, Write};
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -192,6 +193,85 @@ fn query_export_timeout(timeout_secs: Option<u64>) -> Option<Duration> {
|
||||
}
|
||||
}
|
||||
|
||||
struct StreamProgressClock {
|
||||
started_at: tokio::time::Instant,
|
||||
last_progress_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl StreamProgressClock {
|
||||
fn new() -> Self {
|
||||
Self { started_at: tokio::time::Instant::now(), last_progress_ms: AtomicU64::new(0) }
|
||||
}
|
||||
|
||||
fn mark(&self) {
|
||||
self.last_progress_ms.store(self.started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn elapsed_since_progress(&self) -> Duration {
|
||||
let last_progress_ms = self.last_progress_ms.load(Ordering::Relaxed);
|
||||
let elapsed_ms = self.started_at.elapsed().as_millis() as u64;
|
||||
Duration::from_millis(elapsed_ms.saturating_sub(last_progress_ms))
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_stream_with_progress_timeout<F, T>(
|
||||
stream_future: F,
|
||||
timeout: Option<Duration>,
|
||||
progress_clock: Arc<StreamProgressClock>,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
timeout_message: String,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
{
|
||||
let Some(timeout) = timeout else {
|
||||
return match cancel_token {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => Err(canceled_error()),
|
||||
result = stream_future => result,
|
||||
}
|
||||
}
|
||||
None => stream_future.await,
|
||||
};
|
||||
};
|
||||
|
||||
tokio::pin!(stream_future);
|
||||
loop {
|
||||
// The query timeout is an inactivity budget, not a cap on total export duration.
|
||||
// This keeps a stalled server bounded while allowing large local file writes to finish.
|
||||
let remaining = timeout.saturating_sub(progress_clock.elapsed_since_progress());
|
||||
if remaining.is_zero() {
|
||||
return Err(timeout_message.clone());
|
||||
}
|
||||
let sleep = tokio::time::sleep(remaining);
|
||||
tokio::pin!(sleep);
|
||||
|
||||
match cancel_token {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(canceled_error()),
|
||||
result = &mut stream_future => return result,
|
||||
_ = &mut sleep => {},
|
||||
}
|
||||
}
|
||||
None => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut stream_future => return result,
|
||||
_ = &mut sleep => {},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if progress_clock.elapsed_since_progress() >= timeout {
|
||||
return Err(timeout_message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn should_fetch_next_page(
|
||||
use_agent_result_session: bool,
|
||||
has_more: bool,
|
||||
@@ -1309,6 +1389,9 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
None
|
||||
};
|
||||
let mut xlsx = None;
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let progress_clock_for_stream = progress_clock.clone();
|
||||
let query_timeout = query_export_timeout(request.timeout_secs);
|
||||
|
||||
let mut client = match cancel_token.as_ref() {
|
||||
Some(token) => {
|
||||
@@ -1378,15 +1461,20 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Mark only after the row is fully written so local XLSX work never consumes
|
||||
// the next database inactivity window.
|
||||
progress_clock_for_stream.mark();
|
||||
Ok(())
|
||||
},
|
||||
);
|
||||
match query_export_timeout(request.timeout_secs) {
|
||||
Some(timeout) => tokio::time::timeout(timeout, stream_future)
|
||||
.await
|
||||
.map_err(|_| format!("Query timed out after {} seconds", timeout.as_secs()))??,
|
||||
None => stream_future.await?,
|
||||
};
|
||||
await_stream_with_progress_timeout(
|
||||
stream_future,
|
||||
query_timeout,
|
||||
progress_clock,
|
||||
cancel_token.as_ref(),
|
||||
format!("Query timed out after {} seconds", query_timeout.map_or(0, |timeout| timeout.as_secs())),
|
||||
)
|
||||
.await?;
|
||||
drop(client);
|
||||
|
||||
if rows_exported != last_progress_rows {
|
||||
@@ -1593,4 +1681,101 @@ mod tests {
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlserver_stream_times_out_when_database_makes_no_progress() {
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let result = await_stream_with_progress_timeout(
|
||||
std::future::pending::<Result<(), String>>(),
|
||||
Some(Duration::from_millis(20)),
|
||||
progress_clock,
|
||||
None,
|
||||
"query timeout".to_string(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result, Err("query timeout".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlserver_stream_timeout_resets_after_each_completed_row() {
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let progress_clock_for_stream = progress_clock.clone();
|
||||
let result = await_stream_with_progress_timeout(
|
||||
async move {
|
||||
for row in 1..=5 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
progress_clock_for_stream.mark();
|
||||
assert!(row <= 5);
|
||||
}
|
||||
Ok::<_, String>(5_u8)
|
||||
},
|
||||
Some(Duration::from_millis(150)),
|
||||
progress_clock,
|
||||
None,
|
||||
"query timeout".to_string(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result, Ok(5));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlserver_stream_does_not_count_synchronous_local_writes_as_database_idle_time() {
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let progress_clock_for_stream = progress_clock.clone();
|
||||
let result = await_stream_with_progress_timeout(
|
||||
async move {
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
progress_clock_for_stream.mark();
|
||||
Ok::<_, String>(())
|
||||
},
|
||||
Some(Duration::from_millis(20)),
|
||||
progress_clock,
|
||||
None,
|
||||
"query timeout".to_string(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result, Ok(()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlserver_stream_timeout_zero_disables_idle_timeout() {
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let result = await_stream_with_progress_timeout(
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
Ok::<_, String>(())
|
||||
},
|
||||
None,
|
||||
progress_clock,
|
||||
None,
|
||||
"query timeout".to_string(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result, Ok(()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlserver_stream_cancellation_wins_over_idle_timeout() {
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let cancel_token = CancellationToken::new();
|
||||
let cancel_token_for_task = cancel_token.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
await_stream_with_progress_timeout(
|
||||
async { std::future::pending::<Result<(), String>>().await },
|
||||
Some(Duration::from_secs(1)),
|
||||
progress_clock,
|
||||
Some(&cancel_token_for_task),
|
||||
"query timeout".to_string(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
cancel_token.cancel();
|
||||
|
||||
assert_eq!(task.await.unwrap(), Err(QUERY_CANCELED.to_string()));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user