feat(ai): upgrade agent loop tool calling to true SSE streaming

Move tool calling from non-streaming (complete) API calls to real
streaming SSE, so text, reasoning, and tool call arguments all arrive
incrementally — users see the AI think and act in real time.

- Extract ~370 lines of non-streaming tool-call functions from
  agent_loop.rs (call_with_tools, call_openai_with_tools,
  call_claude_with_tools, call_gemini_with_tools) and replace with
  a single ai::stream_with_tools() dispatch call.

- Add streaming-with-tools infrastructure to ai.rs (~550 lines):
  - StreamToolEvent enum: Chunk, ToolCallStart, ToolCallDelta,
    ToolCallComplete — provider-agnostic, feeds into the
    StreamingToolCallAccumulator
  - stream_claude_with_tools(): parse SSE content_block_start/delta/
    stop events, incremental input_json_delta for tool arguments
  - stream_openai_with_tools(): parse SSE delta.tool_calls with
    incremental function.arguments
  - stream_gemini_with_tools(): parse SSE functionCall (Gemini
    sends complete objects, not deltas — emit as one chunk)
  - Cancellation support via tokio::select! in every streaming loop
  - Extended timeout for reasoning models (600s vs 120s)

- StreamingToolCallAccumulator: collect partial tool call fragments
  from streaming deltas, then deserialize into complete ToolCall
  objects in index order once the stream ends.
This commit is contained in:
runstone committed 2026-06-12 11:43:09 +08:00
1 parent 9939e3191f
commit 2d75667b9a
5 files changed
+565 -339

No files matched your search

+6 -334
View File
@@ -149,7 +149,7 @@ pub async fn run_agent_loop(
// Add tool result to conversation for the next LLM call
// Uses "tool" role per OpenAI convention; provider-specific conversion
// happens in call_*_with_tools functions when building provider API requests.
// happens in ai::stream_with_tools when building provider API requests.
conversation_messages.push(AiMessage {
role: "tool".to_string(),
content: result.content.clone(),
@@ -170,7 +170,7 @@ fn build_tool_request(
config: &AiConfig,
system_prompt: &str,
messages: &[AiMessage],
_tools: &[ToolDefinition], // Tools are injected in stream_with_tools / call_*_with_tools, not via AiCompletionRequest.
_tools: &[ToolDefinition], // Tools are injected in ai::stream_with_tools, not via AiCompletionRequest.
max_tokens: Option<u32>,
temperature: Option<f32>,
) -> AiCompletionRequest {
@@ -185,15 +185,10 @@ fn build_tool_request(
}
}
/// Collect an LLM response with tool support, parsing tool_calls.
/// Stream an LLM response with tool support, parsing tool_calls from SSE deltas.
///
/// Named "stream" for consistency with the agent loop's streaming-API contract,
/// but currently uses a non-streaming (complete) LLM call internally, then emits
/// the full text as a single delta. True streaming tool_calls is deferred to
/// Phase 2 (provider-specific delta accumulation).
///
/// Check `cancelled` before the LLM call; if the LLM request has already started
/// it will run to completion (non-streaming requests cannot be interrupted mid-flight).
/// True streaming: text, reasoning, and tool call arguments are all emitted
/// incrementally as they arrive from the provider.
async fn stream_with_tools(
config: &AiConfig,
request: &AiCompletionRequest,
@@ -208,330 +203,7 @@ async fn stream_with_tools(
return Err("Agent loop cancelled".to_string());
}
// TODO(agent-loop-phase2): Implement streaming tool_calls for each provider.
// Ref: .trellis/tasks/06-11-feat-ai-agent-loop-tool-calling-upgrade PRD Phase 2
let client = ai::build_ai_http_client(config, 120)?;
let response_text = call_with_tools(&client, config, request, &tools).await?;
// Emit the full text as a single delta for the UI
if !response_text.text.is_empty() {
on_chunk(AiStreamChunk {
session_id: session_id.to_string(),
delta: response_text.text.clone(),
reasoning_delta: None,
done: false,
});
}
on_chunk(AiStreamChunk {
session_id: session_id.to_string(),
delta: String::new(),
reasoning_delta: None,
done: true,
});
Ok(response_text.tool_calls)
}
/// Response from an LLM call that may include tool calls.
struct LlmResponse {
text: String,
tool_calls: Vec<ToolCall>,
}
/// Make a non-streaming LLM call with tools and parse the response.
async fn call_with_tools(
client: &reqwest::Client,
config: &AiConfig,
request: &AiCompletionRequest,
tools: &[ToolDefinition],
) -> Result<LlmResponse, String> {
match config.provider {
AiProvider::Claude => call_claude_with_tools(client, config, request, tools).await,
AiProvider::Gemini => call_gemini_with_tools(client, config, request, tools).await,
_ => call_openai_with_tools(client, config, request, tools).await,
}
}
/// OpenAI / compatible providers: non-streaming call with tools.
async fn call_openai_with_tools(
client: &reqwest::Client,
config: &AiConfig,
request: &AiCompletionRequest,
tools: &[ToolDefinition],
) -> Result<LlmResponse, String> {
let headers = ai::maybe_bearer_headers(config)?;
let mut messages = vec![json!({ "role": "system", "content": request.system_prompt })];
messages.extend(request.messages.iter().map(|m| {
let mut msg = json!({ "role": m.role, "content": m.content });
if m.role == "tool" {
if let Some(ref tc_id) = m.tool_call_id {
msg["tool_call_id"] = json!(tc_id);
}
} else if m.role == "assistant" && !m.tool_calls.is_empty() {
let calls: Vec<serde_json::Value> = m
.tool_calls
.iter()
.map(|tc| {
json!({
"id": tc.id,
"type": "function",
"function": {
"name": tc.name,
"arguments": tc.arguments.to_string()
}
})
})
.collect();
msg["tool_calls"] = json!(calls);
}
msg
}));
let tool_json: Vec<serde_json::Value> = tools.iter().map(|t| t.to_openai_tool()).collect();
let mut body = json!({
"model": config.model,
"messages": messages,
"max_tokens": request.max_tokens.unwrap_or(4096),
"tools": tool_json,
"tool_choice": "auto",
"stream": false,
});
ai::add_temperature_if_supported(&mut body, request);
// Responses API tool calling is deferred to a future phase.
// Use the completions endpoint regardless of api_style for now.
let endpoint = ai::resolve_endpoint(config);
let res = client
.post(&endpoint)
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| format!("AI request failed: {e}"))?;
if !res.status().is_success() {
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
return Err(ai::extract_error(&data).unwrap_or_else(|| "API error".to_string()));
}
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
let text = data["choices"].get(0).and_then(|c| c["message"]["content"].as_str()).unwrap_or("").to_string();
let mut tool_calls = Vec::new();
if let Some(calls) = data["choices"].get(0).and_then(|c| c["message"]["tool_calls"].as_array()) {
for tc in calls {
let id = tc["id"].as_str().unwrap_or_default().to_string();
let name = tc["function"]["name"].as_str().unwrap_or_default().to_string();
let args_str = tc["function"]["arguments"].as_str().unwrap_or("{}");
let args: serde_json::Value = serde_json::from_str(args_str).unwrap_or(json!({}));
tool_calls.push(ToolCall { id, name, arguments: args });
}
}
Ok(LlmResponse { text, tool_calls })
}
/// Claude: non-streaming call with tools.
async fn call_claude_with_tools(
client: &reqwest::Client,
config: &AiConfig,
request: &AiCompletionRequest,
tools: &[ToolDefinition],
) -> Result<LlmResponse, String> {
let headers = ai::claude_headers(config)?;
let mut messages: Vec<serde_json::Value> = Vec::new();
for m in &request.messages {
if m.role == "tool" {
// Convert tool results to Anthropic tool_result content blocks
messages.push(json!({
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": m.tool_call_id.as_deref().unwrap_or_default(),
"content": m.content
}]
}));
} else if m.role == "assistant" && !m.tool_calls.is_empty() {
// Reconstruct assistant message with tool_use content blocks
let mut content_blocks: Vec<serde_json::Value> = Vec::new();
if !m.content.is_empty() {
content_blocks.push(json!({
"type": "text",
"text": m.content
}));
}
for tc in &m.tool_calls {
content_blocks.push(json!({
"type": "tool_use",
"id": tc.id,
"name": tc.name,
"input": tc.arguments
}));
}
messages.push(json!({
"role": "assistant",
"content": content_blocks
}));
} else {
messages.push(json!({ "role": m.role, "content": m.content }));
}
}
let tool_json: Vec<serde_json::Value> = tools.iter().map(|t| t.to_anthropic_tool()).collect();
let body = json!({
"model": config.model,
"max_tokens": request.max_tokens.unwrap_or(4096),
"system": request.system_prompt,
"messages": messages,
"tools": tool_json,
});
let res = client
.post(ai::resolve_endpoint(config))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| format!("AI request failed: {e}"))?;
if !res.status().is_success() {
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
return Err(ai::extract_error(&data).unwrap_or_else(|| "API error".to_string()));
}
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
let mut text = String::new();
let mut tool_calls = Vec::new();
if let Some(content) = data["content"].as_array() {
for block in content {
match block["type"].as_str() {
Some("text") => {
if let Some(t) = block["text"].as_str() {
text.push_str(t);
}
}
Some("tool_use") => {
let id = block["id"].as_str().unwrap_or_default().to_string();
let name = block["name"].as_str().unwrap_or_default().to_string();
let args = block["input"].clone();
tool_calls.push(ToolCall { id, name, arguments: args });
}
_ => {}
}
}
}
Ok(LlmResponse { text, tool_calls })
}
/// Gemini: non-streaming call with tools.
async fn call_gemini_with_tools(
client: &reqwest::Client,
config: &AiConfig,
request: &AiCompletionRequest,
tools: &[ToolDefinition],
) -> Result<LlmResponse, String> {
let mut contents: Vec<serde_json::Value> = Vec::new();
for m in &request.messages {
if m.role == "tool" {
// Extract tool name from id "gemini-tc-<tool_name>-<idx>"
let tool_name = m
.tool_call_id
.as_deref()
.and_then(|s| s.strip_prefix("gemini-tc-"))
.and_then(|s| s.rsplitn(2, '-').nth(1))
.unwrap_or("unknown");
contents.push(json!({
"role": "user",
"parts": [{
"functionResponse": {
"name": tool_name,
"response": { "content": m.content }
}
}]
}));
} else if m.role == "assistant" && !m.tool_calls.is_empty() {
let mut parts: Vec<serde_json::Value> = Vec::new();
if !m.content.is_empty() {
parts.push(json!({ "text": m.content }));
}
for tc in &m.tool_calls {
parts.push(json!({
"functionCall": {
"name": tc.name,
"args": tc.arguments
}
}));
}
contents.push(json!({
"role": "model",
"parts": parts
}));
} else {
let role = if m.role == "assistant" { "model" } else { "user" };
contents.push(json!({
"role": role,
"parts": [{ "text": m.content }]
}));
}
}
let tool_declarations: Vec<serde_json::Value> = tools.iter().map(|t| t.to_gemini_tool()).collect();
let body = json!({
"contents": contents,
"systemInstruction": { "parts": [{ "text": request.system_prompt }] },
"tools": [{ "functionDeclarations": tool_declarations }],
"generationConfig": {
"maxOutputTokens": request.max_tokens.unwrap_or(4096),
}
});
let res = client
.post(ai::resolve_endpoint(config))
.query(&[("key", config.api_key.as_str())])
.header(reqwest::header::CONTENT_TYPE, "application/json")
.json(&body)
.send()
.await
.map_err(|e| format!("AI request failed: {e}"))?;
if !res.status().is_success() {
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
return Err(ai::extract_error(&data).unwrap_or_else(|| "API error".to_string()));
}
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
let mut text = String::new();
let mut tool_calls = Vec::new();
if let Some(candidates) = data["candidates"].as_array() {
if let Some(parts) = candidates[0]["content"]["parts"].as_array() {
for part in parts {
if let Some(t) = part["text"].as_str() {
text.push_str(t);
}
if let Some(fc) = part.get("functionCall") {
let name = fc["name"].as_str().unwrap_or_default().to_string();
let args = fc["args"].clone();
let id = format!("gemini-tc-{name}-{}", tool_calls.len());
tool_calls.push(ToolCall { id, name, arguments: args });
}
}
}
}
Ok(LlmResponse { text, tool_calls })
ai::stream_with_tools(config, request, session_id, &tools, cancelled, on_chunk).await
}
/// Text-only fallback for providers that don't support function calling.