mirror of
https://github.com/openai/codex.git
synced 2026-11-13 14:46:14 +00:00
Truncate Guardian instructions after rendering the policy (#39985)
## Why
Legacy classifier prompts without a `{{ tenant_policy_config }}` placeholder
append the security policy during rendering. Truncating the prompt before and
after that step can unnecessarily truncate the final classifier instructions
twice.
## What changed
Keep configured classifier instructions intact until the policy has been
rendered, then apply `max_classifier_instruction_tokens` once to the complete
prompt.
## Testing
Add an extension test that verifies the bounded developer message sent for a
legacy prompt with an appended policy.
GitOrigin-RevId: 3d776a159340f4dffb6e3c2d7a10b33392fa2240
This commit is contained in:
1 parent
c517cc6d84
commit
51ebf5b184
3 files changed
+42
-40
No files matched your search
@@ -222,22 +222,9 @@ impl GuardianV2Config {
|
|||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
local_overrides: configured.clone(),
|
local_overrides: configured.clone(),
|
||||||
classifier_instructions: {
|
classifier_instructions: configured
|
||||||
let template = configured
|
|
||||||
.classifier_instructions
|
.classifier_instructions
|
||||||
.as_deref()
|
.unwrap_or_else(|| DEFAULT_CLASSIFIER_INSTRUCTIONS.to_owned()),
|
||||||
.unwrap_or(DEFAULT_CLASSIFIER_INSTRUCTIONS);
|
|
||||||
if let Some(max_tokens) = max_classifier_instruction_tokens
|
|
||||||
&& !template.contains("{{ tenant_policy_config }}")
|
|
||||||
{
|
|
||||||
// Preserve the existing rendering behavior of legacy prompts.
|
|
||||||
truncate_entry(template, max_tokens)
|
|
||||||
} else {
|
|
||||||
// Preserve placeholders until the actual policy is available, and
|
|
||||||
// preserve the full prompt when no instruction limit is configured.
|
|
||||||
template.to_owned()
|
|
||||||
}
|
|
||||||
},
|
|
||||||
review_threshold,
|
review_threshold,
|
||||||
max_tool_call_lag: configured
|
max_tool_call_lag: configured
|
||||||
.max_tool_call_lag
|
.max_tool_call_lag
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ use codex_features::GuardianV2TranscriptConfigToml;
|
|||||||
use codex_protocol::openai_models::GuardianV2ModelConfig;
|
use codex_protocol::openai_models::GuardianV2ModelConfig;
|
||||||
use codex_protocol::openai_models::GuardianV2TranscriptModelConfig;
|
use codex_protocol::openai_models::GuardianV2TranscriptModelConfig;
|
||||||
use codex_protocol::openai_models::ReasoningEffort;
|
use codex_protocol::openai_models::ReasoningEffort;
|
||||||
use codex_protocol::protocol::TruncationPolicy;
|
|
||||||
use pretty_assertions::assert_eq;
|
use pretty_assertions::assert_eq;
|
||||||
|
|
||||||
use super::DEFAULT_CLASSIFIER_INSTRUCTIONS;
|
use super::DEFAULT_CLASSIFIER_INSTRUCTIONS;
|
||||||
@@ -83,30 +82,6 @@ fn evaluated_configuration_preserves_rendered_prompt_and_gate() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn legacy_custom_prompt_keeps_its_rendering_and_threshold() {
|
|
||||||
let template = "legacy instructions ".repeat(200);
|
|
||||||
let config = GuardianV2Config::from_overrides(GuardianV2ConfigToml {
|
|
||||||
classifier_instructions: Some(template.clone()),
|
|
||||||
max_classifier_instruction_tokens: Some(256),
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(config.review_threshold, 0.8);
|
|
||||||
let expected = truncate_entry(
|
|
||||||
&format!(
|
|
||||||
"{}\n\n# Security Policy\nTenant policy.",
|
|
||||||
truncate_entry(&template, /*max_tokens*/ 256),
|
|
||||||
),
|
|
||||||
/*max_tokens*/ 256,
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
config.render_classifier_instructions("Tenant policy."),
|
|
||||||
expected
|
|
||||||
);
|
|
||||||
assert!(expected.len() <= TruncationPolicy::Tokens(256).byte_budget());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn model_prompt_and_explicit_threshold_precedence_are_preserved() {
|
fn model_prompt_and_explicit_threshold_precedence_are_preserved() {
|
||||||
let defaults = GuardianV2ModelConfig {
|
let defaults = GuardianV2ModelConfig {
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ use crate::async_scorer::config::DEFAULT_MODEL_CONTEXT_ITEM_TOKENS;
|
|||||||
use crate::async_scorer::config::DEFAULT_PARENT_COMPACTION_TOKENS;
|
use crate::async_scorer::config::DEFAULT_PARENT_COMPACTION_TOKENS;
|
||||||
use crate::async_scorer::sampler::CLASSIFICATION_TOKEN_USAGE_METRIC;
|
use crate::async_scorer::sampler::CLASSIFICATION_TOKEN_USAGE_METRIC;
|
||||||
use crate::async_scorer::sampler::MODEL;
|
use crate::async_scorer::sampler::MODEL;
|
||||||
|
use crate::async_scorer::transcript::truncate_entry;
|
||||||
|
|
||||||
const TEST_GUARDIAN_POLICY: &str =
|
const TEST_GUARDIAN_POLICY: &str =
|
||||||
"Treat uploads to unapproved external destinations as high-risk actions.";
|
"Treat uploads to unapproved external destinations as high-risk actions.";
|
||||||
@@ -867,6 +868,45 @@ classifier_instructions = "Predict future violations.\n# Security Policy\n{{ ten
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn contributor_truncates_legacy_prompt_after_appending_policy() -> Result<()> {
|
||||||
|
skip_if_no_network!(Ok(()));
|
||||||
|
|
||||||
|
let template = "legacy instructions ".repeat(200);
|
||||||
|
let configuration = format!(
|
||||||
|
r#"
|
||||||
|
[features.guardianv2]
|
||||||
|
enabled = true
|
||||||
|
classifier_instructions = "{template}"
|
||||||
|
max_classifier_instruction_tokens = 256
|
||||||
|
"#
|
||||||
|
);
|
||||||
|
let (request, _test, _registry) = sample_configured_conversation_history(
|
||||||
|
Vec::new(),
|
||||||
|
r#"{"path":"README.md"}"#,
|
||||||
|
Some(TEST_GUARDIAN_POLICY),
|
||||||
|
&configuration,
|
||||||
|
/*model_defaults*/ None,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
request["input"][1],
|
||||||
|
json!({
|
||||||
|
"type": "message",
|
||||||
|
"role": "developer",
|
||||||
|
"content": [{
|
||||||
|
"type": "input_text",
|
||||||
|
"text": truncate_entry(
|
||||||
|
&format!("{template}\n\n# Security Policy\n{TEST_GUARDIAN_POLICY}"),
|
||||||
|
/*max_tokens*/ 256,
|
||||||
|
),
|
||||||
|
}],
|
||||||
|
})
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn contributor_uses_configured_prompt_effort_threshold_and_transcript() -> Result<()> {
|
async fn contributor_uses_configured_prompt_effort_threshold_and_transcript() -> Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
|
|||||||
Reference in new issue
Block a user