fix(ssh): exclude TOFU prompt time from connect timeout
This commit is contained in:
1 parent
6f5ec6c819
commit
09b2a1d552
1 file changed
+163
-30
@@ -17,9 +17,9 @@ use russh::MethodSet;
|
|||||||
use russh::{kex, mac, ChannelOpenFailure, Preferred};
|
use russh::{kex, mac, ChannelOpenFailure, Preferred};
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::{mpsc, Mutex};
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
use tokio::time::{Duration, MissedTickBehavior};
|
use tokio::time::{Duration, Instant, MissedTickBehavior};
|
||||||
|
|
||||||
use crate::db::ssh_host_key::{HostKeyState, HostKeyVerifier};
|
use crate::db::ssh_host_key::{HostKeyState, HostKeyVerifier};
|
||||||
use crate::db::ssh_prompt;
|
use crate::db::ssh_prompt;
|
||||||
@@ -52,6 +52,10 @@ struct SshClient {
|
|||||||
host_key_verifier: Arc<HostKeyVerifier>,
|
host_key_verifier: Arc<HostKeyVerifier>,
|
||||||
host: String,
|
host: String,
|
||||||
port: u16,
|
port: u16,
|
||||||
|
/// Signaled once an interactive TOFU prompt actually starts waiting on
|
||||||
|
/// the user, so the caller can stop charging that think-time against
|
||||||
|
/// its short network-level connect timeout (see `connect_and_authenticate`).
|
||||||
|
prompt_started_tx: Option<mpsc::Sender<Instant>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl client::Handler for SshClient {
|
impl client::Handler for SshClient {
|
||||||
@@ -103,6 +107,13 @@ impl SshClient {
|
|||||||
return Ok(false);
|
return Ok(false);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// From this point we are genuinely waiting on the user, not the
|
||||||
|
// network -- tell the caller so it can stop enforcing its short
|
||||||
|
// network-level connect timeout for the remainder of the handshake.
|
||||||
|
if let Some(tx) = &self.prompt_started_tx {
|
||||||
|
let _ = tx.try_send(Instant::now());
|
||||||
|
}
|
||||||
|
|
||||||
let answer = tokio::time::timeout(TOFU_PROMPT_TIMEOUT, responder_rx).await;
|
let answer = tokio::time::timeout(TOFU_PROMPT_TIMEOUT, responder_rx).await;
|
||||||
match answer {
|
match answer {
|
||||||
Ok(Ok(ssh_prompt::SshPromptAnswer::Accept { remember })) => {
|
Ok(Ok(ssh_prompt::SshPromptAnswer::Accept { remember })) => {
|
||||||
@@ -171,6 +182,10 @@ fn ssh_client_config() -> Config {
|
|||||||
Config { nodelay: true, keepalive_interval: Some(Duration::from_secs(30)), preferred, ..Default::default() }
|
Config { nodelay: true, keepalive_interval: Some(Duration::from_secs(30)), preferred, ..Default::default() }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn tofu_prompt_deadline(network_deadline: Instant, prompt_started_at: Instant) -> Option<Instant> {
|
||||||
|
(prompt_started_at < network_deadline).then_some(prompt_started_at + TOFU_PROMPT_TIMEOUT)
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns `true` only when the server explicitly advertised `password` among
|
/// Returns `true` only when the server explicitly advertised `password` among
|
||||||
/// the authentication methods that may continue the dialog.
|
/// the authentication methods that may continue the dialog.
|
||||||
///
|
///
|
||||||
@@ -325,21 +340,45 @@ async fn connect_and_authenticate(
|
|||||||
// changed keys, missing prompt gateways, timeouts, and rejection fail closed.
|
// changed keys, missing prompt gateways, timeouts, and rejection fail closed.
|
||||||
let host_key_verifier = Arc::new(HostKeyVerifier::new(known_hosts_path.to_path_buf()));
|
let host_key_verifier = Arc::new(HostKeyVerifier::new(known_hosts_path.to_path_buf()));
|
||||||
|
|
||||||
let mut session = tokio::time::timeout(
|
let (started_tx, mut started_rx) = mpsc::channel::<Instant>(1);
|
||||||
connect_timeout,
|
let connect_fut = client::connect(
|
||||||
client::connect(
|
config,
|
||||||
config,
|
(connect_host, connect_port),
|
||||||
(connect_host, connect_port),
|
SshClient {
|
||||||
SshClient {
|
host_key_verifier: host_key_verifier.clone(),
|
||||||
host_key_verifier: host_key_verifier.clone(),
|
host: host_key_host.to_string(),
|
||||||
host: host_key_host.to_string(),
|
port: host_key_port,
|
||||||
port: host_key_port,
|
prompt_started_tx: Some(started_tx),
|
||||||
},
|
},
|
||||||
),
|
);
|
||||||
)
|
tokio::pin!(connect_fut);
|
||||||
.await
|
|
||||||
.map_err(|_| format!("SSH connection timed out ({connect_timeout_secs}s)"))?
|
// `connect_timeout` is meant to bound the *network* portion of the
|
||||||
.map_err(|e| format!("SSH connection failed: {e}"))?;
|
// handshake (TCP connect + KEX). But `check_server_key` can block on an
|
||||||
|
// interactive TOFU prompt while the user reads/confirms the host-key
|
||||||
|
// fingerprint, and that think-time must not be charged against the same
|
||||||
|
// short budget -- otherwise accepting the dialog after it fires still
|
||||||
|
// fails with a spurious "connection timed out". Once we're notified that
|
||||||
|
// a prompt has actually started, we hand the remaining budget off to
|
||||||
|
// TOFU_PROMPT_TIMEOUT (the real ceiling enforced inside
|
||||||
|
// `prompt_for_host_key` itself); the network-only path is unaffected.
|
||||||
|
let network_deadline = Instant::now() + connect_timeout;
|
||||||
|
let sleep = tokio::time::sleep_until(network_deadline);
|
||||||
|
tokio::pin!(sleep);
|
||||||
|
let mut extended = false;
|
||||||
|
|
||||||
|
let mut session = loop {
|
||||||
|
tokio::select! {
|
||||||
|
res = &mut connect_fut => break res.map_err(|e| format!("SSH connection failed: {e}"))?,
|
||||||
|
_ = &mut sleep => return Err(format!("SSH connection timed out ({connect_timeout_secs}s)")),
|
||||||
|
Some(prompt_started_at) = started_rx.recv(), if !extended => {
|
||||||
|
extended = true;
|
||||||
|
if let Some(prompt_deadline) = tofu_prompt_deadline(network_deadline, prompt_started_at) {
|
||||||
|
sleep.as_mut().reset(prompt_deadline);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Probe with "none" authentication first. Some SSH proxies and jump-hosts
|
// Probe with "none" authentication first. Some SSH proxies and jump-hosts
|
||||||
// accept connections without any credential, and this is also the standard
|
// accept connections without any credential, and this is also the standard
|
||||||
@@ -1830,8 +1869,8 @@ mod tests {
|
|||||||
bind_tunnel_listener, connect_and_authenticate, describe_terminal_auth_failure, effective_hop_timeout,
|
bind_tunnel_listener, connect_and_authenticate, describe_terminal_auth_failure, effective_hop_timeout,
|
||||||
netcat_proxy_command, openssh_padding_len, plan_chain, read_ssh_string,
|
netcat_proxy_command, openssh_padding_len, plan_chain, read_ssh_string,
|
||||||
sanitize_unencrypted_openssh_comment_bytes, server_offers_keyboard_interactive, server_offers_password,
|
sanitize_unencrypted_openssh_comment_bytes, server_offers_keyboard_interactive, server_offers_password,
|
||||||
ssh_client_config, HostKeyState, HostKeyVerifier, PlannedTunnel, TunnelEntry, TunnelKind, TunnelManager,
|
ssh_client_config, tofu_prompt_deadline, HostKeyState, HostKeyVerifier, PlannedTunnel, TunnelEntry, TunnelKind,
|
||||||
TunnelStatus,
|
TunnelManager, TunnelStatus, TOFU_PROMPT_TIMEOUT,
|
||||||
};
|
};
|
||||||
use crate::db::ssh_prompt;
|
use crate::db::ssh_prompt;
|
||||||
use crate::models::connection::{default_ssh_connect_timeout_secs, SshTunnelConfig};
|
use crate::models::connection::{default_ssh_connect_timeout_secs, SshTunnelConfig};
|
||||||
@@ -1850,6 +1889,28 @@ mod tests {
|
|||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
|
use tokio::time::{Duration, Instant};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tofu_prompt_deadline_uses_the_actual_prompt_start() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let network_deadline = now + Duration::from_secs(10);
|
||||||
|
let prompt_started_at = now + Duration::from_secs(4);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
tofu_prompt_deadline(network_deadline, prompt_started_at),
|
||||||
|
Some(prompt_started_at + TOFU_PROMPT_TIMEOUT)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tofu_prompt_deadline_does_not_revive_an_expired_network_budget() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let network_deadline = now + Duration::from_secs(10);
|
||||||
|
|
||||||
|
assert_eq!(tofu_prompt_deadline(network_deadline, network_deadline), None);
|
||||||
|
assert_eq!(tofu_prompt_deadline(network_deadline, network_deadline + Duration::from_millis(1)), None);
|
||||||
|
}
|
||||||
|
|
||||||
fn push_u32(bytes: &mut Vec<u8>, value: u32) {
|
fn push_u32(bytes: &mut Vec<u8>, value: u32) {
|
||||||
bytes.extend_from_slice(&value.to_be_bytes());
|
bytes.extend_from_slice(&value.to_be_bytes());
|
||||||
@@ -2201,8 +2262,12 @@ mod tests {
|
|||||||
let key = test_server_public_key();
|
let key = test_server_public_key();
|
||||||
|
|
||||||
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: true });
|
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: true });
|
||||||
let mut client =
|
let mut client = SshClient {
|
||||||
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 };
|
host_key_verifier: Arc::new(verifier),
|
||||||
|
host: "db.example.com".to_string(),
|
||||||
|
port: 22,
|
||||||
|
prompt_started_tx: None,
|
||||||
|
};
|
||||||
|
|
||||||
let trusted = client.check_server_key(&key).await.unwrap();
|
let trusted = client.check_server_key(&key).await.unwrap();
|
||||||
assert!(trusted, "accepted host key should be trusted");
|
assert!(trusted, "accepted host key should be trusted");
|
||||||
@@ -2221,8 +2286,12 @@ mod tests {
|
|||||||
let key = test_server_public_key();
|
let key = test_server_public_key();
|
||||||
|
|
||||||
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: false });
|
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Accept { remember: false });
|
||||||
let mut client =
|
let mut client = SshClient {
|
||||||
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 };
|
host_key_verifier: Arc::new(verifier),
|
||||||
|
host: "db.example.com".to_string(),
|
||||||
|
port: 22,
|
||||||
|
prompt_started_tx: None,
|
||||||
|
};
|
||||||
|
|
||||||
let trusted = client.check_server_key(&key).await.unwrap();
|
let trusted = client.check_server_key(&key).await.unwrap();
|
||||||
assert!(trusted, "accepted host key should be trusted for the session");
|
assert!(trusted, "accepted host key should be trusted for the session");
|
||||||
@@ -2243,8 +2312,12 @@ mod tests {
|
|||||||
let key = test_server_public_key();
|
let key = test_server_public_key();
|
||||||
|
|
||||||
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Reject);
|
install_fake_prompt_gateway(ssh_prompt::SshPromptAnswer::Reject);
|
||||||
let mut client =
|
let mut client = SshClient {
|
||||||
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 };
|
host_key_verifier: Arc::new(verifier),
|
||||||
|
host: "db.example.com".to_string(),
|
||||||
|
port: 22,
|
||||||
|
prompt_started_tx: None,
|
||||||
|
};
|
||||||
|
|
||||||
let trusted = client.check_server_key(&key).await.unwrap();
|
let trusted = client.check_server_key(&key).await.unwrap();
|
||||||
assert!(!trusted, "rejected host key must not be trusted");
|
assert!(!trusted, "rejected host key must not be trusted");
|
||||||
@@ -2266,8 +2339,12 @@ mod tests {
|
|||||||
let verifier = HostKeyVerifier::new(path);
|
let verifier = HostKeyVerifier::new(path);
|
||||||
let key = test_server_public_key();
|
let key = test_server_public_key();
|
||||||
|
|
||||||
let mut client =
|
let mut client = SshClient {
|
||||||
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 };
|
host_key_verifier: Arc::new(verifier),
|
||||||
|
host: "db.example.com".to_string(),
|
||||||
|
port: 22,
|
||||||
|
prompt_started_tx: None,
|
||||||
|
};
|
||||||
let trusted = client.check_server_key(&key).await.unwrap();
|
let trusted = client.check_server_key(&key).await.unwrap();
|
||||||
// No UI to confirm -> fail-closed, host is not trusted.
|
// No UI to confirm -> fail-closed, host is not trusted.
|
||||||
assert!(!trusted, "without a gateway, an unknown host must be rejected (fail-closed)");
|
assert!(!trusted, "without a gateway, an unknown host must be rejected (fail-closed)");
|
||||||
@@ -2285,8 +2362,12 @@ mod tests {
|
|||||||
|
|
||||||
// No gateway installed, but the host is trusted so no prompt is needed.
|
// No gateway installed, but the host is trusted so no prompt is needed.
|
||||||
ssh_prompt::clear_ssh_prompt_gateway();
|
ssh_prompt::clear_ssh_prompt_gateway();
|
||||||
let mut client =
|
let mut client = SshClient {
|
||||||
SshClient { host_key_verifier: Arc::new(verifier), host: "db.example.com".to_string(), port: 22 };
|
host_key_verifier: Arc::new(verifier),
|
||||||
|
host: "db.example.com".to_string(),
|
||||||
|
port: 22,
|
||||||
|
prompt_started_tx: None,
|
||||||
|
};
|
||||||
let trusted = client.check_server_key(&key).await.unwrap();
|
let trusted = client.check_server_key(&key).await.unwrap();
|
||||||
assert!(trusted, "a known-trusted host must be accepted without a prompt");
|
assert!(trusted, "a known-trusted host must be accepted without a prompt");
|
||||||
}
|
}
|
||||||
@@ -2881,6 +2962,53 @@ uveF/dLmnVN1IriEyEvHAAAACGRieC10ZXN0AQIDBAU=
|
|||||||
server_task.abort();
|
server_task.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Accepting an unknown host key must not fail with a network-level
|
||||||
|
/// connect timeout just because the user took a couple of seconds to
|
||||||
|
/// read the fingerprint dialog and click "accept". The interactive
|
||||||
|
/// wait must not be charged against `connect_timeout_secs`.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn slow_host_key_acceptance_does_not_time_out_the_connection() {
|
||||||
|
let _guard = PROMPT_TEST_LOCK.lock().await;
|
||||||
|
ssh_prompt::clear_ssh_prompt_gateway();
|
||||||
|
let (connect_port, server_task) = start_accept_none_server().await;
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let known_hosts_path = dir.path().join("known_hosts");
|
||||||
|
let (gateway_tx, mut gateway_rx) = mpsc::channel::<ssh_prompt::SshPromptEnvelope>(1);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let envelope = gateway_rx.recv().await.expect("host-key prompt");
|
||||||
|
// Simulate the user taking longer to read/accept the fingerprint
|
||||||
|
// dialog than the (deliberately short) network connect timeout.
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||||
|
let _ = envelope.responder.send(ssh_prompt::SshPromptAnswer::Accept { remember: true });
|
||||||
|
});
|
||||||
|
ssh_prompt::install_ssh_prompt_gateway(gateway_tx);
|
||||||
|
|
||||||
|
let session = connect_and_authenticate(
|
||||||
|
"127.0.0.1",
|
||||||
|
connect_port,
|
||||||
|
"ssh-target.invalid",
|
||||||
|
2222,
|
||||||
|
"user",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
false,
|
||||||
|
"",
|
||||||
|
"none",
|
||||||
|
1,
|
||||||
|
&known_hosts_path,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("accepting the host key after the network timeout window should still succeed");
|
||||||
|
|
||||||
|
let known_hosts = std::fs::read_to_string(&known_hosts_path).unwrap();
|
||||||
|
assert!(known_hosts.contains("[ssh-target.invalid]:2222"), "accepted key should be learned: {known_hosts}");
|
||||||
|
|
||||||
|
drop(session);
|
||||||
|
ssh_prompt::clear_ssh_prompt_gateway();
|
||||||
|
server_task.abort();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn concurrent_tunnel_starts_share_one_handshake() {
|
async fn concurrent_tunnel_starts_share_one_handshake() {
|
||||||
let _guard = PROMPT_TEST_LOCK.lock().await;
|
let _guard = PROMPT_TEST_LOCK.lock().await;
|
||||||
@@ -3008,7 +3136,12 @@ uveF/dLmnVN1IriEyEvHAAAACGRieC10ZXN0AQIDBAU=
|
|||||||
let frozen = dir.path().join("frozen");
|
let frozen = dir.path().join("frozen");
|
||||||
std::fs::write(&frozen, b"not a directory").unwrap();
|
std::fs::write(&frozen, b"not a directory").unwrap();
|
||||||
let verifier = HostKeyVerifier::new(frozen.join("known_hosts"));
|
let verifier = HostKeyVerifier::new(frozen.join("known_hosts"));
|
||||||
let handler = SshClient { host_key_verifier: Arc::new(verifier), host: "127.0.0.1".to_string(), port };
|
let handler = SshClient {
|
||||||
|
host_key_verifier: Arc::new(verifier),
|
||||||
|
host: "127.0.0.1".to_string(),
|
||||||
|
port,
|
||||||
|
prompt_started_tx: None,
|
||||||
|
};
|
||||||
let client_config = Arc::new(ssh_client_config());
|
let client_config = Arc::new(ssh_client_config());
|
||||||
|
|
||||||
let connect_result =
|
let connect_result =
|
||||||
|
|||||||
Reference in new issue
Block a user