fix(ssh): exclude TOFU prompt time from connect timeout

This commit is contained in:
Melody Ruch authored and GitHub committed 2026-08-19 01:51:54 +08:00
1 parent 6f5ec6c819
commit 09b2a1d552
1 file changed
+163 -30
+163 -30
View File
@@ -17,9 +17,9 @@ use russh::MethodSet;
use russh::{kex, mac, ChannelOpenFailure, Preferred}; use russh::{kex, mac, ChannelOpenFailure, Preferred};
use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener; use tokio::net::TcpListener;
use tokio::sync::Mutex; use tokio::sync::{mpsc, Mutex};
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use tokio::time::{Duration, MissedTickBehavior}; use tokio::time::{Duration, Instant, MissedTickBehavior};
use crate::db::ssh_host_key::{HostKeyState, HostKeyVerifier}; use crate::db::ssh_host_key::{HostKeyState, HostKeyVerifier};
use crate::db::ssh_prompt; use crate::db::ssh_prompt;
@@ -52,6 +52,10 @@ struct SshClient {
host_key_verifier: Arc<HostKeyVerifier>, host_key_verifier: Arc<HostKeyVerifier>,
host: String, host: String,
port: u16, port: u16,
/// Signaled once an interactive TOFU prompt actually starts waiting on
/// the user, so the caller can stop charging that think-time against
/// its short network-level connect timeout (see `connect_and_authenticate`).
prompt_started_tx: Option<mpsc::Sender<Instant>>,
} }
impl client::Handler for SshClient { impl client::Handler for SshClient {
@@ -103,6 +107,13 @@ impl SshClient {
return Ok(false); return Ok(false);
}; };
// From this point we are genuinely waiting on the user, not the
// network -- tell the caller so it can stop enforcing its short
// network-level connect timeout for the remainder of the handshake.
if let Some(tx) = &self.prompt_started_tx {
let _ = tx.try_send(Instant::now());
}
let answer = tokio::time::timeout(TOFU_PROMPT_TIMEOUT, responder_rx).await; let answer = tokio::time::timeout(TOFU_PROMPT_TIMEOUT, responder_rx).await;
match answer { match answer {
Ok(Ok(ssh_prompt::SshPromptAnswer::Accept { remember })) => { Ok(Ok(ssh_prompt::SshPromptAnswer::Accept { remember })) => {
@@ -171,6 +182,10 @@ fn ssh_client_config() -> Config {
Config { nodelay: true, keepalive_interval: Some(Duration::from_secs(30)), preferred, ..Default::default() } Config { nodelay: true, keepalive_interval: Some(Duration::from_secs(30)), preferred, ..Default::default() }
} }
fn tofu_prompt_deadline(network_deadline: Instant, prompt_started_at: Instant) -> Option<Instant> {
(prompt_started_at < network_deadline).then_some(prompt_started_at + TOFU_PROMPT_TIMEOUT)
}
/// Returns `true` only when the server explicitly advertised `password` among /// Returns `true` only when the server explicitly advertised `password` among
/// the authentication methods that may continue the dialog. /// the authentication methods that may continue the dialog.
/// ///
@@ -325,21 +340,45 @@ async fn connect_and_authenticate(
// changed keys, missing prompt gateways, timeouts, and rejection fail closed. // changed keys, missing prompt gateways, timeouts, and rejection fail closed.
let host_key_verifier = Arc::new(HostKeyVerifier::new(known_hosts_path.to_path_buf())); let host_key_verifier = Arc::new(HostKeyVerifier::new(known_hosts_path.to_path_buf()));
let mut session = tokio::time::timeout( let (started_tx, mut started_rx) = mpsc::channel::<Instant>(1);
connect_timeout, let connect_fut = client::connect(
client::connect( config,
config, (connect_host, connect_port),
(connect_host, connect_port), SshClient {
SshClient { host_key_verifier: host_key_verifier.clone(),
host_key_verifier: host_key_verifier.clone(), host: host_key_host.to_string(),
host: host_key_host.to_string(), port: host_key_port,
port: host_key_port, prompt_started_tx: Some(started_tx),
}, },
), );
) tokio::pin!(connect_fut);
.await
.map_err(|_| format!("SSH connection timed out ({connect_timeout_secs}s)"))? // `connect_timeout` is meant to bound the *network* portion of the
.map_err(|e| format!("SSH connection failed: {e}"))?; // handshake (TCP connect + KEX). But `check_server_key` can block on an
// interactive TOFU prompt while the user reads/confirms the host-key
// fingerprint, and that think-time must not be charged against the same
// short budget -- otherwise accepting the dialog after it fires still
// fails with a spurious "connection timed out". Once we're notified that
// a prompt has actually started, we hand the remaining budget off to
// TOFU_PROMPT_TIMEOUT (the real ceiling enforced inside
// `prompt_for_host_key` itself); the network-only path is unaffected.
let network_deadline = Instant::now() + connect_timeout;
let sleep = tokio::time::sleep_until(network_deadline);
tokio::pin!(sleep);
let mut extended = false;
let mut session = loop {
tokio::select! {
res = &mut connect_fut => break res.map_err(|e| format!("SSH connection failed: {e}"))?,
_ = &mut sleep => return Err(format!("SSH connection timed out ({connect_timeout_secs}s)")),
Some(prompt_started_at) = started_rx.recv(), if !extended => {
extended = true;
if let Some(prompt_deadline) = tofu_prompt_deadline(network_deadline, prompt_started_at) {
sleep.as_mut().reset(prompt_deadline);
}
}
}
};
// Probe with "none" authentication first. Some SSH proxies and jump-hosts // Probe with "none" authentication first. Some SSH proxies and jump-hosts
// accept connections without any credential, and this is also the standard // accept connections without any credential, and this is also the standard
@@ -1830,8 +1869,8 @@ mod tests {
bind_tunnel_listener, connect_and_authenticate, describe_terminal_auth_failure, effective_hop_timeout, bind_tunnel_listener, connect_and_authenticate, describe_terminal_auth_failure, effective_hop_timeout,
netcat_proxy_command, openssh_padding_len, plan_chain, read_ssh_string, netcat_proxy_command, openssh_padding_len, plan_chain, read_ssh_string,
sanitize_unencrypted_openssh_comment_bytes, server_offers_keyboard_interactive, server_offers_password, sanitize_unencrypted_openssh_comment_bytes, server_offers_keyboard_interactive, server_offers_password,
ssh_client_config, HostKeyState, HostKeyVerifier, PlannedTunnel, TunnelEntry, TunnelKind, TunnelManager, ssh_client_config, tofu_prompt_deadline, HostKeyState, HostKeyVerifier, PlannedTunnel, TunnelEntry, TunnelKind,
TunnelStatus, TunnelManager, TunnelStatus, TOFU_PROMPT_TIMEOUT,
}; };
use crate::db::ssh_prompt; use crate::db::ssh_prompt;
use crate::models::connection::{default_ssh_connect_timeout_secs, SshTunnelConfig}; use crate::models::connection::{default_ssh_connect_timeout_secs, SshTunnelConfig};
@@ -1850,6 +1889,28 @@ mod tests {
use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream}; use tokio::net::{TcpListener, TcpStream};
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tokio::time::{Duration, Instant};
#[test]
fn tofu_prompt_deadline_uses_the_actual_prompt_start() {
let now = Instant::now();
let network_deadline = now + Duration::from_secs(10);
let prompt_started_at = now + Duration::from_secs(4);
assert_eq!(
tofu_prompt_deadline(network_deadline, prompt_started_at),
Some(prompt_started_at + TOFU_PROMPT_TIMEOUT)
);
}
#[test]
fn tofu_prompt_deadline_does_not_revive_an_expired_network_budget() {
let now = Instant::now();
let network_deadline = now + Duration::from_secs(10);
assert_eq!(tofu_prompt_deadline(network_deadline, network_deadline), None);
assert_eq!(tofu_prompt_deadline(network_deadline, network_deadline + Duration::from_millis(1)), None);
}
fn push_u32(bytes: &mut Vec<u8>, value: u32) { fn push_u32(bytes: &mut Vec<u8>, value: u32) {
bytes.extend_from_slice(&value.to_be_bytes()); bytes.extend_from_slice(&value.to_be_bytes());
@@ -2201,8 +2262,12 @@ mod tests {
let key = test_server_public_key(); let key = test_server_public_key();
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: true }); install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: true });
let mut client = let mut client = SshClient {
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 }; host_key_verifier: Arc::new(verifier),
host: "db.example.com".to_string(),
port: 22,
prompt_started_tx: None,
};
let trusted = client.check_server_key(&key).await.unwrap(); let trusted = client.check_server_key(&key).await.unwrap();
assert!(trusted, "accepted host key should be trusted"); assert!(trusted, "accepted host key should be trusted");
@@ -2221,8 +2286,12 @@ mod tests {
let key = test_server_public_key(); let key = test_server_public_key();
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: false }); install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: false });
let mut client = let mut client = SshClient {
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 }; host_key_verifier: Arc::new(verifier),
host: "db.example.com".to_string(),
port: 22,
prompt_started_tx: None,
};
let trusted = client.check_server_key(&key).await.unwrap(); let trusted = client.check_server_key(&key).await.unwrap();
assert!(trusted, "accepted host key should be trusted for the session"); assert!(trusted, "accepted host key should be trusted for the session");
@@ -2243,8 +2312,12 @@ mod tests {
let key = test_server_public_key(); let key = test_server_public_key();
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Reject); install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Reject);
let mut client = let mut client = SshClient {
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 }; host_key_verifier: Arc::new(verifier),
host: "db.example.com".to_string(),
port: 22,
prompt_started_tx: None,
};
let trusted = client.check_server_key(&key).await.unwrap(); let trusted = client.check_server_key(&key).await.unwrap();
assert!(!trusted, "rejected host key must not be trusted"); assert!(!trusted, "rejected host key must not be trusted");
@@ -2266,8 +2339,12 @@ mod tests {
let verifier = HostKeyVerifier::new(path); let verifier = HostKeyVerifier::new(path);
let key = test_server_public_key(); let key = test_server_public_key();
let mut client = let mut client = SshClient {
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 }; host_key_verifier: Arc::new(verifier),
host: "db.example.com".to_string(),
port: 22,
prompt_started_tx: None,
};
let trusted = client.check_server_key(&key).await.unwrap(); let trusted = client.check_server_key(&key).await.unwrap();
// No UI to confirm -> fail-closed, host is not trusted. // No UI to confirm -> fail-closed, host is not trusted.
assert!(!trusted, "without a gateway, an unknown host must be rejected (fail-closed)"); assert!(!trusted, "without a gateway, an unknown host must be rejected (fail-closed)");
@@ -2285,8 +2362,12 @@ mod tests {
// No gateway installed, but the host is trusted so no prompt is needed. // No gateway installed, but the host is trusted so no prompt is needed.
ssh_prompt::clear_ssh_prompt_gateway(); ssh_prompt::clear_ssh_prompt_gateway();
let mut client = let mut client = SshClient {
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 }; host_key_verifier: Arc::new(verifier),
host: "db.example.com".to_string(),
port: 22,
prompt_started_tx: None,
};
let trusted = client.check_server_key(&key).await.unwrap(); let trusted = client.check_server_key(&key).await.unwrap();
assert!(trusted, "a known-trusted host must be accepted without a prompt"); assert!(trusted, "a known-trusted host must be accepted without a prompt");
} }
@@ -2881,6 +2962,53 @@ uveF/dLmnVN1IriEyEvHAAAACGRieC10ZXN0AQIDBAU=
server_task.abort(); server_task.abort();
} }
/// Accepting an unknown host key must not fail with a network-level
/// connect timeout just because the user took a couple of seconds to
/// read the fingerprint dialog and click "accept". The interactive
/// wait must not be charged against `connect_timeout_secs`.
#[tokio::test]
async fn slow_host_key_acceptance_does_not_time_out_the_connection() {
let _guard = PROMPT_TEST_LOCK.lock().await;
ssh_prompt::clear_ssh_prompt_gateway();
let (connect_port, server_task) = start_accept_none_server().await;
let dir = tempdir().unwrap();
let known_hosts_path = dir.path().join("known_hosts");
let (gateway_tx, mut gateway_rx) = mpsc::channel::<ssh_prompt::SshPromptEnvelope>(1);
tokio::spawn(async move {
let envelope = gateway_rx.recv().await.expect("host-key prompt");
// Simulate the user taking longer to read/accept the fingerprint
// dialog than the (deliberately short) network connect timeout.
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
let _ = envelope.responder.send(ssh_prompt::SshPromptAnswer::Accept { remember: true });
});
ssh_prompt::install_ssh_prompt_gateway(gateway_tx);
let session = connect_and_authenticate(
"127.0.0.1",
connect_port,
"ssh-target.invalid",
2222,
"user",
"",
"",
"",
false,
"",
"none",
1,
&known_hosts_path,
)
.await
.expect("accepting the host key after the network timeout window should still succeed");
let known_hosts = std::fs::read_to_string(&known_hosts_path).unwrap();
assert!(known_hosts.contains("[ssh-target.invalid]:2222"), "accepted key should be learned: {known_hosts}");
drop(session);
ssh_prompt::clear_ssh_prompt_gateway();
server_task.abort();
}
#[tokio::test] #[tokio::test]
async fn concurrent_tunnel_starts_share_one_handshake() { async fn concurrent_tunnel_starts_share_one_handshake() {
let _guard = PROMPT_TEST_LOCK.lock().await; let _guard = PROMPT_TEST_LOCK.lock().await;
@@ -3008,7 +3136,12 @@ uveF/dLmnVN1IriEyEvHAAAACGRieC10ZXN0AQIDBAU=
let frozen = dir.path().join("frozen"); let frozen = dir.path().join("frozen");
std::fs::write(&frozen, b"not a directory").unwrap(); std::fs::write(&frozen, b"not a directory").unwrap();
let verifier = HostKeyVerifier::new(frozen.join("known_hosts")); let verifier = HostKeyVerifier::new(frozen.join("known_hosts"));
let handler = SshClient { host_key_verifier: Arc::new(verifier), host: "127.0.0.1".to_string(), port }; let handler = SshClient {
host_key_verifier: Arc::new(verifier),
host: "127.0.0.1".to_string(),
port,
prompt_started_tx: None,
};
let client_config = Arc::new(ssh_client_config()); let client_config = Arc::new(ssh_client_config());
let connect_result = let connect_result =