1 file changed
+142
-91
@@ -40,38 +40,13 @@ pub struct AgentRuntimeClient {
|
||||
|
||||
impl AgentRuntimeClient {
|
||||
pub async fn spawn(launch: AgentLaunchSpec, app_version: &str) -> Result<Arc<Self>, String> {
|
||||
let mut child = spawn_agent_process(&launch)?;
|
||||
let child_stdin = child.stdin.take().ok_or("Failed to capture agent stdin")?;
|
||||
let child_stdout = child.stdout.take().ok_or("Failed to capture agent stdout")?;
|
||||
let child_stderr = child.stderr.take().ok_or("Failed to capture agent stderr")?;
|
||||
let stderr_tail = Arc::new(Mutex::new(StderrTail::default()));
|
||||
start_stderr_collector(child_stderr, stderr_tail.clone());
|
||||
|
||||
let mut stdout = BufReader::new(child_stdout);
|
||||
let stdout = tokio::time::timeout(
|
||||
Duration::from_secs(STARTUP_TIMEOUT_SECS),
|
||||
tokio::task::spawn_blocking(move || loop {
|
||||
let line = read_agent_line(&mut stdout, "startup line")?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match serde_json::from_str::<Value>(trimmed) {
|
||||
Ok(value) if value.get("ready") == Some(&Value::Bool(true)) => return Ok(stdout),
|
||||
Ok(_) => return Err(format!("Agent did not send ready signal, got: {line}")),
|
||||
Err(_) => log::warn!("[agent:stdout] ignoring non-JSON line during startup: {trimmed}"),
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| format!("Agent startup timed out ({STARTUP_TIMEOUT_SECS}s)"))?
|
||||
.map_err(|e| format!("Agent startup task failed: {e}"))??;
|
||||
let ReadyAgentProcess { child, stdin, stdout, stderr_tail } = spawn_ready_agent_process(&launch).await?;
|
||||
|
||||
let runtime = Arc::new(Self {
|
||||
child: Arc::new(Mutex::new(child)),
|
||||
child_reaper_started: Arc::new(AtomicBool::new(false)),
|
||||
child_reaped: Arc::new(AtomicBool::new(false)),
|
||||
stdin: Arc::new(Mutex::new(BufWriter::new(child_stdin))),
|
||||
stdin: Arc::new(Mutex::new(stdin)),
|
||||
pending: Arc::new(Mutex::new(HashMap::new())),
|
||||
stderr_tail,
|
||||
next_id: AtomicU64::new(0),
|
||||
@@ -873,6 +848,7 @@ const AGENT_STDOUT_NOISE_SAMPLE_CHARS: usize = 160;
|
||||
const MAX_AGENT_RESPONSE_BYTES: usize = 512 * 1024 * 1024;
|
||||
const AGENT_EXIT_DIAGNOSTIC_WAIT_MS: u64 = 1_000;
|
||||
const AGENT_EXIT_DIAGNOSTIC_POLL_MS: u64 = 10;
|
||||
const AGENT_STARTUP_RETRY_DELAY_MS: u64 = 100;
|
||||
const SHARED_RUNTIME_IDLE_GRACE_SECS: u64 = 30;
|
||||
const AGENT_JAVA_OPTS_ENV: &str = "DBX_AGENT_JAVA_OPTS";
|
||||
|
||||
@@ -1623,6 +1599,87 @@ impl StderrTail {
|
||||
}
|
||||
}
|
||||
|
||||
struct ReadyAgentProcess {
|
||||
child: Child,
|
||||
stdin: BufWriter<ChildStdin>,
|
||||
stdout: BufReader<ChildStdout>,
|
||||
stderr_tail: Arc<Mutex<StderrTail>>,
|
||||
}
|
||||
|
||||
async fn spawn_ready_agent_process(launch: &AgentLaunchSpec) -> Result<ReadyAgentProcess, String> {
|
||||
match spawn_ready_agent_process_once(launch).await {
|
||||
Ok(process) => Ok(process),
|
||||
Err(error) if is_retryable_agent_startup_error(&error) => {
|
||||
log::warn!("[agent] process exited before startup handshake; retrying once: {error}");
|
||||
tokio::time::sleep(Duration::from_millis(AGENT_STARTUP_RETRY_DELAY_MS)).await;
|
||||
spawn_ready_agent_process_once(launch).await
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_ready_agent_process_once(launch: &AgentLaunchSpec) -> Result<ReadyAgentProcess, String> {
|
||||
let mut child = spawn_agent_process(launch)?;
|
||||
let child_stdin = child.stdin.take().ok_or("Failed to capture agent stdin")?;
|
||||
let child_stdout = child.stdout.take().ok_or("Failed to capture agent stdout")?;
|
||||
let child_stderr = child.stderr.take().ok_or("Failed to capture agent stderr")?;
|
||||
let stderr_tail = Arc::new(Mutex::new(StderrTail::default()));
|
||||
start_stderr_collector(child_stderr, stderr_tail.clone());
|
||||
|
||||
let mut stdout = BufReader::new(child_stdout);
|
||||
let startup_result = tokio::time::timeout(
|
||||
Duration::from_secs(STARTUP_TIMEOUT_SECS),
|
||||
tokio::task::spawn_blocking(move || loop {
|
||||
let line = read_agent_line(&mut stdout, "startup line")?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match serde_json::from_str::<Value>(trimmed) {
|
||||
Ok(value) if value.get("ready") == Some(&Value::Bool(true)) => return Ok(stdout),
|
||||
Ok(_) => return Err(format!("Agent did not send ready signal, got: {line}")),
|
||||
Err(_) => log::warn!("[agent:stdout] ignoring non-JSON line during startup: {trimmed}"),
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let stdout = match startup_result {
|
||||
Ok(Ok(Ok(stdout))) => stdout,
|
||||
Ok(Ok(Err(error))) => return Err(clean_up_failed_agent_startup(error, &mut child, &stderr_tail)),
|
||||
Ok(Err(error)) => {
|
||||
return Err(clean_up_failed_agent_startup(
|
||||
format!("Agent startup task failed: {error}"),
|
||||
&mut child,
|
||||
&stderr_tail,
|
||||
));
|
||||
}
|
||||
Err(_) => {
|
||||
return Err(clean_up_failed_agent_startup(
|
||||
format!("Agent startup timed out ({STARTUP_TIMEOUT_SECS}s)"),
|
||||
&mut child,
|
||||
&stderr_tail,
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(ReadyAgentProcess { child, stdin: BufWriter::new(child_stdin), stdout, stderr_tail })
|
||||
}
|
||||
|
||||
fn clean_up_failed_agent_startup(error: String, child: &mut Child, stderr_tail: &Arc<Mutex<StderrTail>>) -> String {
|
||||
let error = format_agent_startup_error(&error, child, stderr_tail);
|
||||
if matches!(child.try_wait(), Ok(None)) {
|
||||
let _ = child.kill();
|
||||
}
|
||||
let _ = child.wait();
|
||||
error
|
||||
}
|
||||
|
||||
fn is_retryable_agent_startup_error(error: &str) -> bool {
|
||||
error.contains("Failed to read startup line from agent: end of stream")
|
||||
&& !error.contains(AGENT_JAVA_TOO_OLD_MESSAGE)
|
||||
}
|
||||
|
||||
impl AgentDriverClient {
|
||||
/// Spawn an agent process and wait for it to signal readiness.
|
||||
///
|
||||
@@ -1630,66 +1687,12 @@ impl AgentDriverClient {
|
||||
/// they speak the DBX stdin/stdout JSON-RPC protocol.
|
||||
/// Blocks (async) until the agent writes `{"ready":true}` to stdout.
|
||||
pub async fn spawn(launch: AgentLaunchSpec) -> Result<Self, String> {
|
||||
let mut child = spawn_agent_process(&launch)?;
|
||||
|
||||
let child_stdin = child.stdin.take().ok_or("Failed to capture agent stdin")?;
|
||||
let child_stdout = child.stdout.take().ok_or("Failed to capture agent stdout")?;
|
||||
let child_stderr = child.stderr.take().ok_or("Failed to capture agent stderr")?;
|
||||
|
||||
let stdin = BufWriter::new(child_stdin);
|
||||
let mut stdout = BufReader::new(child_stdout);
|
||||
let stderr_tail = Arc::new(Mutex::new(StderrTail::default()));
|
||||
start_stderr_collector(child_stderr, stderr_tail.clone());
|
||||
|
||||
// Wait for the agent to signal readiness with {"ready":true}.
|
||||
// Some JDBC drivers (e.g. DM8) write banners to stdout during class
|
||||
// loading. Skip non-JSON lines so driver output doesn't break the
|
||||
// JSON-RPC handshake.
|
||||
let startup_result = tokio::time::timeout(
|
||||
Duration::from_secs(STARTUP_TIMEOUT_SECS),
|
||||
tokio::task::spawn_blocking(move || loop {
|
||||
let line = read_agent_line(&mut stdout, "startup line")?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match serde_json::from_str::<Value>(trimmed) {
|
||||
Ok(v) if v.get("ready") == Some(&Value::Bool(true)) => return Ok(stdout),
|
||||
Ok(_) => return Err(format!("Agent did not send ready signal, got: {line}")),
|
||||
Err(_) => {
|
||||
log::warn!("[agent:stdout] ignoring non-JSON line during startup: {trimmed}");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let ready_stdout = match startup_result {
|
||||
Ok(Ok(Ok(stdout))) => stdout,
|
||||
Ok(Ok(Err(e))) => {
|
||||
return Err(format_agent_startup_error(&e, &mut child, &stderr_tail));
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
return Err(format_agent_startup_error(
|
||||
&format!("Agent startup task failed: {e}"),
|
||||
&mut child,
|
||||
&stderr_tail,
|
||||
));
|
||||
}
|
||||
Err(_) => {
|
||||
return Err(format_agent_startup_error(
|
||||
&format!("Agent startup timed out ({STARTUP_TIMEOUT_SECS}s)"),
|
||||
&mut child,
|
||||
&stderr_tail,
|
||||
));
|
||||
}
|
||||
};
|
||||
let ReadyAgentProcess { child, stdin, stdout, stderr_tail } = spawn_ready_agent_process(&launch).await?;
|
||||
|
||||
Ok(Self {
|
||||
child: Some(child),
|
||||
stdin: Some(stdin),
|
||||
stdout: Some(ready_stdout),
|
||||
stdout: Some(stdout),
|
||||
stderr_tail,
|
||||
handshake: None,
|
||||
next_id: 0,
|
||||
@@ -3603,14 +3606,14 @@ mod tests {
|
||||
agent_object_source_params_with_relation, agent_process_error_hint, agent_proxy_env_vars, agent_schema_params,
|
||||
agent_schema_table_params, agent_supports_capability, agent_transaction_params, append_legacy_error_context,
|
||||
decode_agent_response, format_agent_process_error, format_agent_startup_error, is_agent_rpc_response_error,
|
||||
is_unsupported_handshake_error, legacy_agent_call_error, mongo_collection_params, mongo_database_params,
|
||||
mongo_document_id_params, parse_agent_java_opts, read_agent_json_response, read_agent_line,
|
||||
read_agent_line_with_limit, start_stderr_collector, validate_dameng_java_system_properties, AgentCallError,
|
||||
AgentCapability, AgentDriverClient, AgentErrorCategory, AgentErrorContext, AgentErrorStage, AgentHandshake,
|
||||
AgentKvMethod, AgentLaunchSpec, AgentMethod, AgentOperationOutcome, AgentRuntimeClient,
|
||||
AgentSessionDisposition, AgentTableReadCloseParams, AgentTableReadPageParams, AgentTableReadStartParams,
|
||||
MongoAgentMethod, StderrTail, AGENT_JAVA_TOO_OLD_MESSAGE, AGENT_PROTOCOL_VERSION,
|
||||
AGENT_STDOUT_NOISE_SAMPLE_CHARS, MAX_CONSECUTIVE_AGENT_STDOUT_NOISE_LINES,
|
||||
is_retryable_agent_startup_error, is_unsupported_handshake_error, legacy_agent_call_error,
|
||||
mongo_collection_params, mongo_database_params, mongo_document_id_params, parse_agent_java_opts,
|
||||
read_agent_json_response, read_agent_line, read_agent_line_with_limit, start_stderr_collector,
|
||||
validate_dameng_java_system_properties, AgentCallError, AgentCapability, AgentDriverClient, AgentErrorCategory,
|
||||
AgentErrorContext, AgentErrorStage, AgentHandshake, AgentKvMethod, AgentLaunchSpec, AgentMethod,
|
||||
AgentOperationOutcome, AgentRuntimeClient, AgentSessionDisposition, AgentTableReadCloseParams,
|
||||
AgentTableReadPageParams, AgentTableReadStartParams, MongoAgentMethod, StderrTail, AGENT_JAVA_TOO_OLD_MESSAGE,
|
||||
AGENT_PROTOCOL_VERSION, AGENT_STDOUT_NOISE_SAMPLE_CHARS, MAX_CONSECUTIVE_AGENT_STDOUT_NOISE_LINES,
|
||||
};
|
||||
use crate::agent_recovery::{RecoveryDecision, RecoveryPolicy, RecoveryScope};
|
||||
use std::io::Cursor;
|
||||
@@ -4331,6 +4334,54 @@ mod tests {
|
||||
assert!(message.contains("UnsupportedClassVersionError"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_retry_only_accepts_premature_eof_without_java_version_failure() {
|
||||
assert!(is_retryable_agent_startup_error(
|
||||
"Failed to read startup line from agent: end of stream. agent process exited with exit code: 2"
|
||||
));
|
||||
assert!(!is_retryable_agent_startup_error("Agent startup timed out (15s)"));
|
||||
assert!(!is_retryable_agent_startup_error(&format!(
|
||||
"{AGENT_JAVA_TOO_OLD_MESSAGE}. details: Failed to read startup line from agent: end of stream"
|
||||
)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shared_runtime_retries_once_after_premature_startup_eof() {
|
||||
let test_id = uuid::Uuid::new_v4();
|
||||
let script_path = std::env::temp_dir().join(format!("dbx-agent-startup-retry-{test_id}.py"));
|
||||
let counter_path = std::env::temp_dir().join(format!("dbx-agent-startup-retry-{test_id}.count"));
|
||||
std::fs::write(
|
||||
&script_path,
|
||||
r#"import json, pathlib, sys
|
||||
counter_path = pathlib.Path(sys.argv[1])
|
||||
attempt = int(counter_path.read_text()) + 1 if counter_path.exists() else 1
|
||||
counter_path.write_text(str(attempt))
|
||||
if attempt == 1:
|
||||
print('transient startup failure', file=sys.stderr, flush=True)
|
||||
sys.exit(2)
|
||||
print(json.dumps({'ready': True}), flush=True)
|
||||
for line in sys.stdin:
|
||||
request = json.loads(line)
|
||||
result = {'protocolVersion': 2, 'agentProtocolVersion': 2, 'capabilities': ['multi_session']}
|
||||
print(json.dumps({'jsonrpc': '2.0', 'id': request['id'], 'result': result}), flush=True)
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let runtime = AgentRuntimeClient::spawn(
|
||||
AgentLaunchSpec::new(test_python())
|
||||
.with_args([script_path.to_string_lossy().to_string(), counter_path.to_string_lossy().to_string()]),
|
||||
"test",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read_to_string(&counter_path).unwrap(), "2");
|
||||
runtime.kill_and_wait().await;
|
||||
let _ = std::fs::remove_file(script_path);
|
||||
let _ = std::fs::remove_file(counter_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_agent_process_error_does_not_wait_for_live_child() {
|
||||
let child = test_shell_command("sleep 2", "Start-Sleep -Seconds 2")
|
||||
|
||||
Reference in new issue
Block a user