feat(ai): upgrade agent loop tool calling to true SSE streaming
Move tool calling from non-streaming (complete) API calls to real
streaming SSE, so text, reasoning, and tool call arguments all arrive
incrementally — users see the AI think and act in real time.
- Extract ~370 lines of non-streaming tool-call functions from
agent_loop.rs (call_with_tools, call_openai_with_tools,
call_claude_with_tools, call_gemini_with_tools) and replace with
a single ai::stream_with_tools() dispatch call.
- Add streaming-with-tools infrastructure to ai.rs (~550 lines):
- StreamToolEvent enum: Chunk, ToolCallStart, ToolCallDelta,
ToolCallComplete — provider-agnostic, feeds into the
StreamingToolCallAccumulator
- stream_claude_with_tools(): parse SSE content_block_start/delta/
stop events, incremental input_json_delta for tool arguments
- stream_openai_with_tools(): parse SSE delta.tool_calls with
incremental function.arguments
- stream_gemini_with_tools(): parse SSE functionCall (Gemini
sends complete objects, not deltas — emit as one chunk)
- Cancellation support via tokio::select! in every streaming loop
- Extended timeout for reasoning models (600s vs 120s)
- StreamingToolCallAccumulator: collect partial tool call fragments
from streaming deltas, then deserialize into complete ToolCall
objects in index order once the stream ends.
This commit is contained in:
1 parent
9939e3191f
commit
2d75667b9a
5 files changed
+565
-339
No files matched your search
@@ -149,7 +149,7 @@ pub async fn run_agent_loop(
|
||||
|
||||
// Add tool result to conversation for the next LLM call
|
||||
// Uses "tool" role per OpenAI convention; provider-specific conversion
|
||||
// happens in call_*_with_tools functions when building provider API requests.
|
||||
// happens in ai::stream_with_tools when building provider API requests.
|
||||
conversation_messages.push(AiMessage {
|
||||
role: "tool".to_string(),
|
||||
content: result.content.clone(),
|
||||
@@ -170,7 +170,7 @@ fn build_tool_request(
|
||||
config: &AiConfig,
|
||||
system_prompt: &str,
|
||||
messages: &[AiMessage],
|
||||
_tools: &[ToolDefinition], // Tools are injected in stream_with_tools / call_*_with_tools, not via AiCompletionRequest.
|
||||
_tools: &[ToolDefinition], // Tools are injected in ai::stream_with_tools, not via AiCompletionRequest.
|
||||
max_tokens: Option<u32>,
|
||||
temperature: Option<f32>,
|
||||
) -> AiCompletionRequest {
|
||||
@@ -185,15 +185,10 @@ fn build_tool_request(
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect an LLM response with tool support, parsing tool_calls.
|
||||
/// Stream an LLM response with tool support, parsing tool_calls from SSE deltas.
|
||||
///
|
||||
/// Named "stream" for consistency with the agent loop's streaming-API contract,
|
||||
/// but currently uses a non-streaming (complete) LLM call internally, then emits
|
||||
/// the full text as a single delta. True streaming tool_calls is deferred to
|
||||
/// Phase 2 (provider-specific delta accumulation).
|
||||
///
|
||||
/// Check `cancelled` before the LLM call; if the LLM request has already started
|
||||
/// it will run to completion (non-streaming requests cannot be interrupted mid-flight).
|
||||
/// True streaming: text, reasoning, and tool call arguments are all emitted
|
||||
/// incrementally as they arrive from the provider.
|
||||
async fn stream_with_tools(
|
||||
config: &AiConfig,
|
||||
request: &AiCompletionRequest,
|
||||
@@ -208,330 +203,7 @@ async fn stream_with_tools(
|
||||
return Err("Agent loop cancelled".to_string());
|
||||
}
|
||||
|
||||
// TODO(agent-loop-phase2): Implement streaming tool_calls for each provider.
|
||||
// Ref: .trellis/tasks/06-11-feat-ai-agent-loop-tool-calling-upgrade PRD Phase 2
|
||||
let client = ai::build_ai_http_client(config, 120)?;
|
||||
let response_text = call_with_tools(&client, config, request, &tools).await?;
|
||||
|
||||
// Emit the full text as a single delta for the UI
|
||||
if !response_text.text.is_empty() {
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: response_text.text.clone(),
|
||||
reasoning_delta: None,
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
|
||||
on_chunk(AiStreamChunk {
|
||||
session_id: session_id.to_string(),
|
||||
delta: String::new(),
|
||||
reasoning_delta: None,
|
||||
done: true,
|
||||
});
|
||||
|
||||
Ok(response_text.tool_calls)
|
||||
}
|
||||
|
||||
/// Response from an LLM call that may include tool calls.
|
||||
struct LlmResponse {
|
||||
text: String,
|
||||
tool_calls: Vec<ToolCall>,
|
||||
}
|
||||
|
||||
/// Make a non-streaming LLM call with tools and parse the response.
|
||||
async fn call_with_tools(
|
||||
client: &reqwest::Client,
|
||||
config: &AiConfig,
|
||||
request: &AiCompletionRequest,
|
||||
tools: &[ToolDefinition],
|
||||
) -> Result<LlmResponse, String> {
|
||||
match config.provider {
|
||||
AiProvider::Claude => call_claude_with_tools(client, config, request, tools).await,
|
||||
AiProvider::Gemini => call_gemini_with_tools(client, config, request, tools).await,
|
||||
_ => call_openai_with_tools(client, config, request, tools).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// OpenAI / compatible providers: non-streaming call with tools.
|
||||
async fn call_openai_with_tools(
|
||||
client: &reqwest::Client,
|
||||
config: &AiConfig,
|
||||
request: &AiCompletionRequest,
|
||||
tools: &[ToolDefinition],
|
||||
) -> Result<LlmResponse, String> {
|
||||
let headers = ai::maybe_bearer_headers(config)?;
|
||||
|
||||
let mut messages = vec![json!({ "role": "system", "content": request.system_prompt })];
|
||||
messages.extend(request.messages.iter().map(|m| {
|
||||
let mut msg = json!({ "role": m.role, "content": m.content });
|
||||
if m.role == "tool" {
|
||||
if let Some(ref tc_id) = m.tool_call_id {
|
||||
msg["tool_call_id"] = json!(tc_id);
|
||||
}
|
||||
} else if m.role == "assistant" && !m.tool_calls.is_empty() {
|
||||
let calls: Vec<serde_json::Value> = m
|
||||
.tool_calls
|
||||
.iter()
|
||||
.map(|tc| {
|
||||
json!({
|
||||
"id": tc.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tc.name,
|
||||
"arguments": tc.arguments.to_string()
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
msg["tool_calls"] = json!(calls);
|
||||
}
|
||||
msg
|
||||
}));
|
||||
|
||||
let tool_json: Vec<serde_json::Value> = tools.iter().map(|t| t.to_openai_tool()).collect();
|
||||
|
||||
let mut body = json!({
|
||||
"model": config.model,
|
||||
"messages": messages,
|
||||
"max_tokens": request.max_tokens.unwrap_or(4096),
|
||||
"tools": tool_json,
|
||||
"tool_choice": "auto",
|
||||
"stream": false,
|
||||
});
|
||||
ai::add_temperature_if_supported(&mut body, request);
|
||||
|
||||
// Responses API tool calling is deferred to a future phase.
|
||||
// Use the completions endpoint regardless of api_style for now.
|
||||
let endpoint = ai::resolve_endpoint(config);
|
||||
|
||||
let res = client
|
||||
.post(&endpoint)
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("AI request failed: {e}"))?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
return Err(ai::extract_error(&data).unwrap_or_else(|| "API error".to_string()));
|
||||
}
|
||||
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
|
||||
let text = data["choices"].get(0).and_then(|c| c["message"]["content"].as_str()).unwrap_or("").to_string();
|
||||
|
||||
let mut tool_calls = Vec::new();
|
||||
if let Some(calls) = data["choices"].get(0).and_then(|c| c["message"]["tool_calls"].as_array()) {
|
||||
for tc in calls {
|
||||
let id = tc["id"].as_str().unwrap_or_default().to_string();
|
||||
let name = tc["function"]["name"].as_str().unwrap_or_default().to_string();
|
||||
let args_str = tc["function"]["arguments"].as_str().unwrap_or("{}");
|
||||
let args: serde_json::Value = serde_json::from_str(args_str).unwrap_or(json!({}));
|
||||
tool_calls.push(ToolCall { id, name, arguments: args });
|
||||
}
|
||||
}
|
||||
|
||||
Ok(LlmResponse { text, tool_calls })
|
||||
}
|
||||
|
||||
/// Claude: non-streaming call with tools.
|
||||
async fn call_claude_with_tools(
|
||||
client: &reqwest::Client,
|
||||
config: &AiConfig,
|
||||
request: &AiCompletionRequest,
|
||||
tools: &[ToolDefinition],
|
||||
) -> Result<LlmResponse, String> {
|
||||
let headers = ai::claude_headers(config)?;
|
||||
|
||||
let mut messages: Vec<serde_json::Value> = Vec::new();
|
||||
for m in &request.messages {
|
||||
if m.role == "tool" {
|
||||
// Convert tool results to Anthropic tool_result content blocks
|
||||
messages.push(json!({
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": m.tool_call_id.as_deref().unwrap_or_default(),
|
||||
"content": m.content
|
||||
}]
|
||||
}));
|
||||
} else if m.role == "assistant" && !m.tool_calls.is_empty() {
|
||||
// Reconstruct assistant message with tool_use content blocks
|
||||
let mut content_blocks: Vec<serde_json::Value> = Vec::new();
|
||||
if !m.content.is_empty() {
|
||||
content_blocks.push(json!({
|
||||
"type": "text",
|
||||
"text": m.content
|
||||
}));
|
||||
}
|
||||
for tc in &m.tool_calls {
|
||||
content_blocks.push(json!({
|
||||
"type": "tool_use",
|
||||
"id": tc.id,
|
||||
"name": tc.name,
|
||||
"input": tc.arguments
|
||||
}));
|
||||
}
|
||||
messages.push(json!({
|
||||
"role": "assistant",
|
||||
"content": content_blocks
|
||||
}));
|
||||
} else {
|
||||
messages.push(json!({ "role": m.role, "content": m.content }));
|
||||
}
|
||||
}
|
||||
|
||||
let tool_json: Vec<serde_json::Value> = tools.iter().map(|t| t.to_anthropic_tool()).collect();
|
||||
|
||||
let body = json!({
|
||||
"model": config.model,
|
||||
"max_tokens": request.max_tokens.unwrap_or(4096),
|
||||
"system": request.system_prompt,
|
||||
"messages": messages,
|
||||
"tools": tool_json,
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(ai::resolve_endpoint(config))
|
||||
.headers(headers)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("AI request failed: {e}"))?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
return Err(ai::extract_error(&data).unwrap_or_else(|| "API error".to_string()));
|
||||
}
|
||||
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut text = String::new();
|
||||
let mut tool_calls = Vec::new();
|
||||
|
||||
if let Some(content) = data["content"].as_array() {
|
||||
for block in content {
|
||||
match block["type"].as_str() {
|
||||
Some("text") => {
|
||||
if let Some(t) = block["text"].as_str() {
|
||||
text.push_str(t);
|
||||
}
|
||||
}
|
||||
Some("tool_use") => {
|
||||
let id = block["id"].as_str().unwrap_or_default().to_string();
|
||||
let name = block["name"].as_str().unwrap_or_default().to_string();
|
||||
let args = block["input"].clone();
|
||||
tool_calls.push(ToolCall { id, name, arguments: args });
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(LlmResponse { text, tool_calls })
|
||||
}
|
||||
|
||||
/// Gemini: non-streaming call with tools.
|
||||
async fn call_gemini_with_tools(
|
||||
client: &reqwest::Client,
|
||||
config: &AiConfig,
|
||||
request: &AiCompletionRequest,
|
||||
tools: &[ToolDefinition],
|
||||
) -> Result<LlmResponse, String> {
|
||||
let mut contents: Vec<serde_json::Value> = Vec::new();
|
||||
for m in &request.messages {
|
||||
if m.role == "tool" {
|
||||
// Extract tool name from id "gemini-tc-<tool_name>-<idx>"
|
||||
let tool_name = m
|
||||
.tool_call_id
|
||||
.as_deref()
|
||||
.and_then(|s| s.strip_prefix("gemini-tc-"))
|
||||
.and_then(|s| s.rsplitn(2, '-').nth(1))
|
||||
.unwrap_or("unknown");
|
||||
contents.push(json!({
|
||||
"role": "user",
|
||||
"parts": [{
|
||||
"functionResponse": {
|
||||
"name": tool_name,
|
||||
"response": { "content": m.content }
|
||||
}
|
||||
}]
|
||||
}));
|
||||
} else if m.role == "assistant" && !m.tool_calls.is_empty() {
|
||||
let mut parts: Vec<serde_json::Value> = Vec::new();
|
||||
if !m.content.is_empty() {
|
||||
parts.push(json!({ "text": m.content }));
|
||||
}
|
||||
for tc in &m.tool_calls {
|
||||
parts.push(json!({
|
||||
"functionCall": {
|
||||
"name": tc.name,
|
||||
"args": tc.arguments
|
||||
}
|
||||
}));
|
||||
}
|
||||
contents.push(json!({
|
||||
"role": "model",
|
||||
"parts": parts
|
||||
}));
|
||||
} else {
|
||||
let role = if m.role == "assistant" { "model" } else { "user" };
|
||||
contents.push(json!({
|
||||
"role": role,
|
||||
"parts": [{ "text": m.content }]
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
let tool_declarations: Vec<serde_json::Value> = tools.iter().map(|t| t.to_gemini_tool()).collect();
|
||||
|
||||
let body = json!({
|
||||
"contents": contents,
|
||||
"systemInstruction": { "parts": [{ "text": request.system_prompt }] },
|
||||
"tools": [{ "functionDeclarations": tool_declarations }],
|
||||
"generationConfig": {
|
||||
"maxOutputTokens": request.max_tokens.unwrap_or(4096),
|
||||
}
|
||||
});
|
||||
|
||||
let res = client
|
||||
.post(ai::resolve_endpoint(config))
|
||||
.query(&[("key", config.api_key.as_str())])
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("AI request failed: {e}"))?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
return Err(ai::extract_error(&data).unwrap_or_else(|| "API error".to_string()));
|
||||
}
|
||||
|
||||
let data: serde_json::Value = res.json().await.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut text = String::new();
|
||||
let mut tool_calls = Vec::new();
|
||||
|
||||
if let Some(candidates) = data["candidates"].as_array() {
|
||||
if let Some(parts) = candidates[0]["content"]["parts"].as_array() {
|
||||
for part in parts {
|
||||
if let Some(t) = part["text"].as_str() {
|
||||
text.push_str(t);
|
||||
}
|
||||
if let Some(fc) = part.get("functionCall") {
|
||||
let name = fc["name"].as_str().unwrap_or_default().to_string();
|
||||
let args = fc["args"].clone();
|
||||
let id = format!("gemini-tc-{name}-{}", tool_calls.len());
|
||||
tool_calls.push(ToolCall { id, name, arguments: args });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(LlmResponse { text, tool_calls })
|
||||
ai::stream_with_tools(config, request, session_id, &tools, cancelled, on_chunk).await
|
||||
}
|
||||
|
||||
/// Text-only fallback for providers that don't support function calling.
|
||||
|
||||
Reference in new issue
Block a user