mirror of
https://github.com/openai/codex.git
synced 2026-10-19 16:17:49 +00:00
Stream daemon executable hashing off the async runtime (#47591)
## Why Debug executables can be hundreds of megabytes. Computing their identity previously loaded the entire file into memory and hashed it on an async runtime thread. ## What changed Stream executable contents into the BLAKE3 hasher using `tokio::task::spawn_blocking`, preserving the existing digest format while avoiding whole-file allocation and moving hashing off the async runtime. ## Testing Expand the executable identity test to verify compatibility with whole-file hashing for a file spanning multiple reads with a partial final buffer and for an empty file, and to confirm that changing a byte changes the identity. GitOrigin-RevId: 528103397395313d170ec720fefc8c5ab3676956
This commit is contained in:
@@ -187,16 +187,26 @@ pub(crate) struct ExecutableIdentity {
|
||||
}
|
||||
|
||||
pub(crate) async fn executable_identity(executable: &Path) -> Result<ExecutableIdentity> {
|
||||
let bytes = fs::read(executable)
|
||||
.await
|
||||
.with_context(|| format!("failed to read executable {}", executable.display()))?;
|
||||
Ok(executable_identity_from_bytes(&bytes))
|
||||
let executable = executable.to_path_buf();
|
||||
// Debug executables can be hundreds of MB. Stream the digest off the async
|
||||
// runtime instead of allocating the whole file and blocking a runtime thread.
|
||||
tokio::task::spawn_blocking(move || {
|
||||
std::fs::File::open(&executable)
|
||||
.and_then(executable_identity_from_reader)
|
||||
.with_context(|| format!("failed to read executable {}", executable.display()))
|
||||
})
|
||||
.await
|
||||
.context("executable identity task failed")?
|
||||
}
|
||||
|
||||
pub(crate) fn executable_identity_from_bytes(bytes: &[u8]) -> ExecutableIdentity {
|
||||
ExecutableIdentity {
|
||||
digest: *blake3::hash(bytes).as_bytes(),
|
||||
}
|
||||
pub(crate) fn executable_identity_from_reader(
|
||||
reader: impl std::io::Read,
|
||||
) -> std::io::Result<ExecutableIdentity> {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update_reader(reader)?;
|
||||
Ok(ExecutableIdentity {
|
||||
digest: *hasher.finalize().as_bytes(),
|
||||
})
|
||||
}
|
||||
|
||||
fn managed_codex_file_name() -> &'static str {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use super::executable_identity_from_bytes;
|
||||
use super::ExecutableIdentity;
|
||||
use super::executable_identity;
|
||||
use super::parse_codex_version;
|
||||
|
||||
#[test]
|
||||
@@ -16,12 +17,30 @@ fn rejects_malformed_codex_cli_version_output() {
|
||||
assert!(parse_codex_version("codex\n").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn executable_identity_uses_binary_contents() {
|
||||
let old = executable_identity_from_bytes(b"old");
|
||||
let same = executable_identity_from_bytes(b"old");
|
||||
let new = executable_identity_from_bytes(b"new");
|
||||
|
||||
assert_eq!(old, same);
|
||||
assert_ne!(old, new);
|
||||
#[tokio::test]
|
||||
async fn executable_identity_uses_binary_contents() {
|
||||
let directory = tempfile::tempdir().expect("temporary directory");
|
||||
let executable = directory.path().join("codex");
|
||||
// Span multiple reads, including a partial final buffer, and preserve the
|
||||
// digest stored by older clients that hashed the complete file in memory.
|
||||
let mut bytes: Vec<u8> = (0..200_003).map(|index| (index % 251) as u8).collect();
|
||||
for contents in [&bytes[..], &[][..]] {
|
||||
std::fs::write(&executable, contents).expect("write executable");
|
||||
assert_eq!(
|
||||
executable_identity(&executable).await.expect("identity"),
|
||||
ExecutableIdentity {
|
||||
digest: *blake3::hash(contents).as_bytes(),
|
||||
}
|
||||
);
|
||||
}
|
||||
std::fs::write(&executable, &bytes).expect("write executable");
|
||||
let old = executable_identity(&executable).await.expect("identity");
|
||||
bytes[100_000] ^= 1;
|
||||
std::fs::write(&executable, bytes).expect("replace executable");
|
||||
assert_ne!(
|
||||
executable_identity(&executable)
|
||||
.await
|
||||
.expect("new identity"),
|
||||
old
|
||||
);
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::UpdateStatus;
|
||||
#[cfg(unix)]
|
||||
use crate::managed_install::executable_identity;
|
||||
#[cfg(unix)]
|
||||
use crate::managed_install::executable_identity_from_bytes;
|
||||
use crate::managed_install::executable_identity_from_reader;
|
||||
|
||||
#[tokio::test]
|
||||
async fn installer_fetch_uses_exact_url_and_preserves_bytes() {
|
||||
@@ -772,7 +772,7 @@ async fn check_manual_update_restart(package_directory: &str) {
|
||||
super::run_with_http(
|
||||
&http,
|
||||
&updater_daemon,
|
||||
&executable_identity_from_bytes(b"updater"),
|
||||
&executable_identity_from_reader(&b"updater"[..]).expect("updater identity"),
|
||||
restore_release,
|
||||
)
|
||||
.await
|
||||
@@ -849,7 +849,7 @@ async fn check_manual_update_restart(package_directory: &str) {
|
||||
let output = manual_update_once(
|
||||
&no_op,
|
||||
&daemon,
|
||||
&executable_identity_from_bytes(b"updater"),
|
||||
&executable_identity_from_reader(&b"updater"[..]).expect("updater identity"),
|
||||
&mut test_terminate(),
|
||||
super::UpdateTrigger::Manual,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user