mirror of
https://github.com/openai/codex.git
synced 2026-10-19 08:16:48 +00:00
Preserve URLs and email addresses during external agent migration (#53706)
## Why Rewriting source agent names and instruction filenames inside URLs and email addresses can change their destinations when importing external agent content. ## What changed Use `linkify` to preserve recognized URLs and email addresses while rewriting surrounding text, including Markdown link labels and titles. Continue migrating instruction filename references to `AGENTS.md`, including relative links and references with line or column suffixes. ## Testing Add rewrite coverage for URLs, email addresses, Markdown links, and surrounding Unicode text. Extend skill import and app-server configuration import tests to verify that link labels migrate while URL destinations remain unchanged. GitOrigin-RevId: c4d210a287c99b6440438c99527364d379c568bd
This commit is contained in:
jif
Generated
+1
@@ -1307,6 +1307,7 @@
|
||||
"libz-sys_1.1.25": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0.98\"},{\"kind\":\"build\",\"name\":\"cmake\",\"optional\":true,\"req\":\"^0.1.50\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.43\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.9\"},{\"kind\":\"build\",\"name\":\"vcpkg\",\"req\":\"^0.2.11\"}],\"features\":{\"asm\":[],\"default\":[\"libc\",\"stock-zlib\"],\"static\":[],\"stock-zlib\":[],\"zlib-ng\":[\"libc\",\"cmake\"],\"zlib-ng-no-cmake-experimental-community-maintained\":[\"libc\"]}}",
|
||||
"line-clipping_0.3.7": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.11\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.26\"}],\"features\":{}}",
|
||||
"link-section_0.17.2": "{\"dependencies\":[{\"features\":[\"link_section\"],\"name\":\"linktime-proc-macro\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"macrotest\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2\"}],\"features\":{\"default\":[\"proc_macro\",\"std\"],\"proc_macro\":[\"dep:linktime-proc-macro\"],\"std\":[]}}",
|
||||
"linkify_0.11.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"memchr\",\"req\":\"^2.0.1\"}],\"features\":{}}",
|
||||
"linktime-proc-macro_0.1.0": "{\"dependencies\":[],\"features\":{\"ctor\":[],\"default\":[],\"dtor\":[],\"link_section\":[]}}",
|
||||
"linux-keyutils_0.2.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"default_features\":false,\"features\":[\"std\",\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.4.11\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.132\"},{\"kind\":\"dev\",\"name\":\"zeroize\",\"req\":\"^1.5.7\"}],\"features\":{\"default\":[],\"std\":[\"bitflags/std\"]}}",
|
||||
"linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}",
|
||||
|
||||
Generated
+10
@@ -3403,6 +3403,7 @@ dependencies = [
|
||||
"codex-rollout",
|
||||
"codex-thread-store",
|
||||
"codex-utils-output-truncation",
|
||||
"linkify",
|
||||
"pretty_assertions",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -10004,6 +10005,15 @@ version = "0.17.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d1e908a416d6e9f725743b84a36feea40c4c131e805fbc26d61f9f451f36080"
|
||||
|
||||
[[package]]
|
||||
name = "linkify"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92f9ae2f3d31697697311f016320fe984127e636b45cf5eb5c175b7eb103dadb"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linktime-proc-macro"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -407,6 +407,7 @@ libc = "0.2.182"
|
||||
# Keep SQLx's bundled SQLite on a version containing the WAL-reset corruption fix:
|
||||
# https://www.sqlite.org/wal.html#the_wal_reset_bug
|
||||
libsqlite3-sys = { version = "0.37", default-features = false }
|
||||
linkify = "0.11"
|
||||
log = "0.4"
|
||||
lru = "0.18.2"
|
||||
maplit = "1.0.2"
|
||||
|
||||
@@ -401,7 +401,10 @@ async fn external_agent_config_import_source_remains_attribution_only() -> Resul
|
||||
let codex_home = TempDir::new()?;
|
||||
let source_home = external_agent_home(codex_home.path());
|
||||
std::fs::create_dir_all(&source_home)?;
|
||||
std::fs::write(source_home.join("CLAUDE.md"), "Claude guidance")?;
|
||||
std::fs::write(
|
||||
source_home.join("CLAUDE.md"),
|
||||
"Claude guidance: [Claude](https://claude.ai/CLAUDE.md)",
|
||||
)?;
|
||||
let home_dir = codex_home.path().display().to_string();
|
||||
let mut mcp = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
@@ -447,7 +450,7 @@ async fn external_agent_config_import_source_remains_attribution_only() -> Resul
|
||||
assert_eq!(completed.item_type_results[0].failures, Vec::new());
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(codex_home.path().join("AGENTS.md"))?,
|
||||
"Codex guidance"
|
||||
"Codex guidance: [Codex](https://claude.ai/CLAUDE.md)"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -27,6 +27,7 @@ codex-protocol = { workspace = true }
|
||||
codex-rollout = { workspace = true }
|
||||
codex-thread-store = { workspace = true }
|
||||
codex-utils-output-truncation = { workspace = true }
|
||||
linkify = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_yaml = { workspace = true }
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use linkify::LinkFinder;
|
||||
|
||||
/// Describes source-specific terms that should be rewritten in migrated artifacts.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct RewriteProfile {
|
||||
@@ -35,8 +37,35 @@ impl RewriteProfile {
|
||||
self.case_sensitive_term_variants
|
||||
}
|
||||
|
||||
/// Rewrites source-specific documentation names and product terms to their Codex forms.
|
||||
/// Rewrites source-specific terms outside recognized URLs and email addresses.
|
||||
pub fn rewrite(self, content: &str) -> String {
|
||||
let mut finder = LinkFinder::new();
|
||||
finder.url_must_have_scheme(/* url_must_have_scheme */ false);
|
||||
let mut rewritten = String::with_capacity(content.len());
|
||||
let mut start = 0;
|
||||
for link in finder.links(content) {
|
||||
// The .md suffix is also a TLD, but this filename must still migrate,
|
||||
// including references with a :line or :line:column suffix.
|
||||
if link.as_str().split(['?', '#']).next().is_some_and(|path| {
|
||||
let mut parts = path.splitn(3, ':');
|
||||
parts
|
||||
.next()
|
||||
.is_some_and(|name| name.eq_ignore_ascii_case(self.doc_file_name))
|
||||
&& parts.all(|part| {
|
||||
!part.is_empty() && part.as_bytes().iter().all(u8::is_ascii_digit)
|
||||
})
|
||||
}) {
|
||||
continue;
|
||||
}
|
||||
rewritten.push_str(&self.rewrite_terms(&content[start..link.start()]));
|
||||
rewritten.push_str(link.as_str());
|
||||
start = link.end();
|
||||
}
|
||||
rewritten.push_str(&self.rewrite_terms(&content[start..]));
|
||||
rewritten
|
||||
}
|
||||
|
||||
fn rewrite_terms(self, content: &str) -> String {
|
||||
let mut rewritten =
|
||||
replace_case_insensitive_with_boundaries(content, self.doc_file_name, "AGENTS.md");
|
||||
for from in self.term_variants {
|
||||
|
||||
@@ -7,7 +7,67 @@ const PROFILE: RewriteProfile = RewriteProfile::new("SOURCE.md", &["source agent
|
||||
#[test]
|
||||
fn rewrites_terms_only_at_word_boundaries() {
|
||||
assert_eq!(
|
||||
PROFILE.rewrite("SOURCE.md Source source agent source_agent"),
|
||||
"AGENTS.md Codex Codex source_agent"
|
||||
PROFILE.rewrite("SOURCE.md Source source agent source_agent Source: café ‘Source’"),
|
||||
"AGENTS.md Codex Codex source_agent Codex: café ‘Codex’"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_urls_and_emails_while_rewriting_surrounding_text() {
|
||||
assert_eq!(
|
||||
PROFILE.rewrite(
|
||||
r#"Source: https://Source.example/SOURCE.md?product=Source#Source
|
||||
[Source](HTTPS://Source.example/a_(Source)/source.md "Source")Source
|
||||
<http://Source.example/SOURCE.md>，Source
|
||||
“https://Source.example/é/SOURCE.md” Source
|
||||
`file:///Source/SOURCE.md` Source://Source/SOURCE.md
|
||||
[Source](mailto:Source@example.com) [SOURCE.md](./SOURCE.md)
|
||||
WWW.Source.example/SOURCE.md Source
|
||||
https://Source.example/it's/SOURCE.md Source
|
||||
<Source@example.com> Source@example.com support@Source.example Source
|
||||
[Source](/Source/reference) [SOURCE.md](./Source/SOURCE.md)
|
||||
SOURCE.md?section=details
|
||||
[Source](#Source)
|
||||
<a id="Source"></a>
|
||||
|
||||
[![logo][SOURCE.md logo]](https://Source.example/Source)
|
||||
|
||||
[SOURCE.md logo]: ./logo.svg
|
||||
|
||||
| URL | Agent | Instructions |
|
||||
| --- | --- | --- |
|
||||
|https://example.com|Source|[docs](SOURCE.md)|
|
||||
|
||||
[Source][Source] [Source][] [Source]
|
||||
|
||||
[Source]: https://example.com/Source/SOURCE.md "Source"
|
||||
"#,
|
||||
),
|
||||
r#"Codex: https://Source.example/SOURCE.md?product=Source#Source
|
||||
[Codex](HTTPS://Source.example/a_(Source)/source.md "Codex")Codex
|
||||
<http://Source.example/SOURCE.md>，Codex
|
||||
“https://Source.example/é/SOURCE.md” Codex
|
||||
`file:///Source/SOURCE.md` Source://Source/SOURCE.md
|
||||
[Codex](mailto:Source@example.com) [AGENTS.md](./AGENTS.md)
|
||||
WWW.Source.example/SOURCE.md Codex
|
||||
https://Source.example/it's/SOURCE.md Codex
|
||||
<Source@example.com> Source@example.com support@Source.example Codex
|
||||
[Codex](/Codex/reference) [AGENTS.md](./Codex/AGENTS.md)
|
||||
AGENTS.md?section=details
|
||||
[Codex](#Codex)
|
||||
<a id="Codex"></a>
|
||||
|
||||
[![logo][AGENTS.md logo]](https://Source.example/Source)
|
||||
|
||||
[AGENTS.md logo]: ./logo.svg
|
||||
|
||||
| URL | Agent | Instructions |
|
||||
| --- | --- | --- |
|
||||
|https://example.com|Codex|[docs](AGENTS.md)|
|
||||
|
||||
[Codex][Codex] [Codex][] [Codex]
|
||||
|
||||
[Codex]: https://example.com/Source/SOURCE.md "Codex"
|
||||
"#
|
||||
);
|
||||
}
|
||||
|
||||
@@ -139,7 +139,10 @@ async fn import_home_migrates_supported_config_fields_skills_and_agents_md() {
|
||||
.join("skill-a")
|
||||
.join("SKILL.md"),
|
||||
format!(
|
||||
"Use {SOURCE_EXTERNAL_AGENT_PRODUCT_NAME} and {SOURCE_EXTERNAL_AGENT_UPPER_NAME} utilities."
|
||||
"Use {SOURCE_EXTERNAL_AGENT_PRODUCT_NAME} and {SOURCE_EXTERNAL_AGENT_UPPER_NAME} utilities.\n\
|
||||
Open [{SOURCE_EXTERNAL_AGENT_DISPLAY_NAME} settings](https://{SOURCE_EXTERNAL_AGENT_NAME}.ai/settings)\n\
|
||||
or <HTTPS://{SOURCE_EXTERNAL_AGENT_UPPER_NAME}.ai/{EXTERNAL_AGENT_CONFIG_MD}>.\n\
|
||||
Read [{EXTERNAL_AGENT_CONFIG_MD}](./{EXTERNAL_AGENT_CONFIG_MD})."
|
||||
),
|
||||
)
|
||||
.expect("write skill");
|
||||
@@ -199,7 +202,12 @@ MY_TEAM = "codex"
|
||||
assert_eq!(
|
||||
fs::read_to_string(agents_skills.join("skill-a").join("SKILL.md"))
|
||||
.expect("read copied skill"),
|
||||
"Use Codex and Codex utilities."
|
||||
format!(
|
||||
"Use Codex and Codex utilities.\n\
|
||||
Open [Codex settings](https://{SOURCE_EXTERNAL_AGENT_NAME}.ai/settings)\n\
|
||||
or <HTTPS://{SOURCE_EXTERNAL_AGENT_UPPER_NAME}.ai/{EXTERNAL_AGENT_CONFIG_MD}>.\n\
|
||||
Read [AGENTS.md](./AGENTS.md)."
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user