github/codex
4
0 Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-04 00:24:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor truncation helpers into its own file (#6683)

Browse Source 
That's to centralize the truncation in one place. Next step would be to
make only two methods public: one with bytes/lines and one with tokens.
This commit is contained in:
Ahmed Ibrahim authored and GitHub committed 2025-11-15 06:44:23 +00:00
1 parent b560c5cef1
commit 94dfb211af
7 files changed
+425 -189
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

codex-rs/core/src/unified_exec/mod.rs
-31
View File
@@ -124,37 +124,6 @@ pub(crate) fn generate_chunk_id() -> String {
.collect()
}
pub(crate) fn truncate_output_to_tokens(
output: &str,
max_tokens: usize,
) -> (String, Option<usize>) {
if max_tokens == 0 {
let total_tokens = output.chars().count();
let message = format!("…{total_tokens} tokens truncated…");
return (message, Some(total_tokens));
}
let tokens: Vec<char> = output.chars().collect();
let total_tokens = tokens.len();
if total_tokens <= max_tokens {
return (output.to_string(), None);
}
let half = max_tokens / 2;
if half == 0 {
let truncated = total_tokens.saturating_sub(max_tokens);
let message = format!("…{truncated} tokens truncated…");
return (message, Some(total_tokens));
}
let truncated = total_tokens.saturating_sub(half * 2);
let mut truncated_output = String::new();
truncated_output.extend(&tokens[..half]);
truncated_output.push_str(&format!("…{truncated} tokens truncated…"));
truncated_output.extend(&tokens[total_tokens - half..]);
(truncated_output, Some(total_tokens))
}
#[cfg(test)]
#[cfg(unix)]
mod tests {