Prune unused standalone releases after successful installation (#55149)

## What changed

Clean up older standalone releases after verifying the installed command, while holding the install lock. Retain the current and previously selected releases, along with releases used by running Codex processes or bundled helpers.

Record selection history for all standalone installs and preserve it on no-op installs. Only remove validated, installer-owned release directories; skip cleanup when filesystem or process inspection cannot establish that removal is safe. Recheck process usage after moving a release aside and restore it if it is in use.

## Testing

Add installer regression coverage for running helpers, helpers started during cleanup, repeated installs, failed installs, downgrades, and preservation of unowned directories and symlinks.

GitOrigin-RevId: 8fb363750ace14b4f804fdcfcdb210540f49c928
This commit is contained in:
Eric Traut
2026-10-19 00:13:28 +00:00
committed by copyberry
parent 70b3bb6d88
commit 89461ea683
2 changed files with 395 additions and 6 deletions
+160 -3
View File
@@ -214,8 +214,9 @@ parse_release_metadata() {
# so the record boundaries inserted by fold do not change the document.
LC_ALL=C fold -b -w 4096 | LC_ALL=C awk '
function finish_string(value) {
if (object_depth == 1 && key == "tag_name") {
print "tag_name\t" value
if (object_depth == 1 &&
(key == "tag_name" || key == "version" || key == "target" || key == "entrypoint")) {
print key "\t" value
} else if (object_depth == asset_object_depth) {
if (key == "name") {
asset_name = value
@@ -758,6 +759,157 @@ cleanup_stale_install_artifacts() {
fi
}
record_standalone_releases_in_use() {
# /proc also catches old clients with no session records. On macOS lsof
# checks every open file in the release, including running helper programs.
if [ "$os" = "linux" ]; then
[ -r /proc/self/mountinfo ] && [ -L /proc/self/exe ] || return 1
# hidepid can omit other users' processes entirely; do not assume an
# incomplete process listing proves an older release is unused.
if grep -E '(^|[ ,])hidepid=[^0, ]' /proc/self/mountinfo >/dev/null; then
return 1
fi
: > "$tmp_dir/in-use" || return 1
for process in /proc/[0-9]*; do
if executable="$(readlink "$process/exe" 2>/dev/null)"; then
printf 'n%s\n' "$executable" >> "$tmp_dir/in-use" || return 1
elif [ -d "$process" ]; then
# Even same-user ssh-agent/systemd processes may deny exe inspection.
# Only an unreadable Codex or bundled helper prevents safe pruning.
process_name="$(cat "$process/comm" 2>/dev/null)" || {
[ ! -d "$process" ] && continue
return 1
}
# Linux truncates comm to 15 bytes, including the code-mode host.
case "$process_name" in
codex* | rg | bwrap)
# A zombie has no files left open, even if its parent never reaps it.
grep -Eq '^State:[[:space:]]*Z' "$process/status" 2>/dev/null && continue
[ ! -d "$process" ] && continue
return 1
;;
esac
fi
done
else
# macOS lsof silently omits other users' processes, even with no errors.
# ps can still report their executable names. As on Linux, decline to
# prune when an uninspectable Codex or helper may be using a release.
current_uid="$(id -u)" || return 1
ps -axo uid=,comm= > "$tmp_dir/processes" || return 1
[ -s "$tmp_dir/processes" ] || return 1
while read -r process_uid process_name; do
[ "$process_uid" = "$current_uid" ] && continue
case "${process_name##*/}" in
codex* | rg | bwrap) return 1 ;;
esac
done < "$tmp_dir/processes"
if /usr/sbin/lsof -nP -F n +D "$releases" > "$tmp_dir/in-use" 2> "$tmp_dir/lsof-error"; then
:
else
# lsof returns 1 for an empty result as well as for errors.
[ ! -s "$tmp_dir/in-use" ] && [ ! -s "$tmp_dir/lsof-error" ] || return 1
fi
[ ! -s "$tmp_dir/lsof-error" ] || return 1
fi
}
# Run under the install lock, after the visible command has been verified.
# Only retire ordinary, manifest-owned packages; never retry a retired tree.
prune_standalone_releases() (
[ ! -L "$CODEX_HOME_DIR/packages" ] &&
[ ! -L "$STANDALONE_ROOT" ] && [ ! -L "$RELEASES_DIR" ] || return 1
releases="$(cd -P "$RELEASES_DIR" && pwd)" || return 1
# Local process inspection cannot establish whether a network home is in
# use on another host. df -l excludes remotely mounted filesystems.
df -lP "$releases" > /dev/null 2> "$tmp_dir/df-error" || return 1
[ ! -s "$tmp_dir/df-error" ] || return 1
selected="$(cd -P "$CURRENT_LINK" && pwd)" || return 1
history="$(cat "$STANDALONE_ROOT/selection-history")" || return 1
previous="$(printf '%s\n' "$history" | sed -n '2p')"
case "$previous" in
*/* | . | ..) return 1 ;;
esac
identity="$(LC_ALL=C stat -c '%d %i %.9Z' "$CURRENT_LINK" 2>/dev/null ||
LC_ALL=C stat -f '%d %i %Fc' "$CURRENT_LINK")" || return 1
[ "$selected" = "$releases/${selected##*/}" ] &&
[ "$history" = "$(printf '%s\n%s\n%s' "${selected##*/}" "$previous" "$identity")" ] ||
return 1
record_standalone_releases_in_use || return 1
restore_candidate=""
# A stopped install must not strand a late-starting release in retirement.
trap 'if [ -n "$restore_candidate" ] && [ -d "$retired/release" ] &&
[ ! -e "$candidate" ] && [ ! -L "$candidate" ]; then
mv "$retired/release" "$candidate" && rmdir "$retired"
fi' EXIT
trap 'exit 1' HUP INT TERM
for candidate in "$releases"/*; do
[ -d "$candidate" ] && [ ! -L "$candidate" ] &&
[ "$candidate" != "$selected" ] && [ "${candidate##*/}" != "$previous" ] || continue
grep -F "n$candidate/" "$tmp_dir/in-use" >/dev/null && continue
[ -f "$candidate/codex-package.json" ] && [ ! -L "$candidate/codex-package.json" ] || continue
metadata="$(parse_release_metadata < "$candidate/codex-package.json")" || continue
version="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "version" {print $2}')"
target="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "target" {print $2}')"
entrypoint="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "entrypoint" {print $2}')"
[ "${candidate##*/}" = "$version-$target" ] && [ "$version" != "latest" ] &&
[ "$entrypoint" = "bin/codex" ] && [ -f "$candidate/bin/codex" ] || continue
validate_version "$version" >/dev/null 2>&1 || continue
case "$target" in
aarch64-apple-darwin | x86_64-apple-darwin | aarch64-unknown-linux-musl | x86_64-unknown-linux-musl) ;;
*) continue ;;
esac
# Reject symlinks (except the installer's own alias), special files and
# mounts before rm -r. find -xdev alone would not protect nested mounts.
[ "$(readlink "$candidate/codex" 2>/dev/null)" = "bin/codex" ] || continue
if ! find "$candidate" -xdev ! -type f ! -type d ! -path "$candidate/codex" -print > "$tmp_dir/unowned"; then
continue
fi
[ ! -s "$tmp_dir/unowned" ] || continue
if [ "$os" = "linux" ]; then
# mountinfo escapes spaces, tabs, newlines, and backslashes in paths.
if ! CODEX_PRUNE_RELEASE="$candidate" LC_ALL=C awk '
BEGIN { release = ENVIRON["CODEX_PRUNE_RELEASE"] }
{ path = $5; gsub(/\\040/, " ", path); gsub(/\\011/, "\t", path)
gsub(/\\012/, "\n", path); gsub(/\\134/, "\\", path)
if (path == release || index(path, release "/") == 1) found = 1 }
END { exit found ? 1 : 0 }
' /proc/self/mountinfo; then
continue
fi
else
device="$(stat -f '%d' "$releases")" || return 1
if ! find "$candidate" -xdev -exec stat -f '%d' {} + > "$tmp_dir/devices"; then
continue
fi
grep -vxF "$device" "$tmp_dir/devices" >/dev/null && continue
fi
retired="$(mktemp -d "$releases/.retired.XXXXXXXX")" || return 1
restore_candidate=1
if mv "$candidate" "$retired/release"; then
# Block new launches at the original path before the final use check.
# Some systems report the old executable path, so check both names.
if ! record_standalone_releases_in_use ||
grep -F -e "n$candidate/" -e "n$retired/release/" "$tmp_dir/in-use" >/dev/null; then
[ ! -e "$candidate" ] && [ ! -L "$candidate" ] || return 1
mv "$retired/release" "$candidate" && rmdir "$retired" || return 1
restore_candidate=""
continue
fi
# Once proved unused, never restore a possibly partially deleted tree.
restore_candidate=""
find "$retired" -xdev -depth -delete || return 1
else
restore_candidate=""
rmdir "$retired"
return 1
fi
done
)
replace_path_with_symlink() {
link_path="$1"
link_target="$2"
@@ -1267,6 +1419,8 @@ if [ "$DAEMON_ONLY" = "1" ] && [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ];
echo "The production release does not support daemon-owned packages; the current selection was left unchanged." >&2
exit 1
fi
fi
if [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ]; then
# A no-op must not replace the selection link: history is bound to it.
selected_dir="$(cd -P "$release_dir" && pwd)"
previous_dir="$(cd -P "$CURRENT_LINK" 2>/dev/null && pwd)" || previous_dir=""
@@ -1280,7 +1434,7 @@ if [ "$DAEMON_ONLY" = "1" ] && [ "${CODEX_INSTALL_DEFER_SELECTION:-0}" != "1" ];
printf '%s\n%s\n%s\n' "$release_name" "${previous_dir##*/}" "$selection_id" > "$history.tmp.$$"
mv -f "$history.tmp.$$" "$history"
); then
warn "Could not record daemon selection; pruning will be skipped."
warn "Could not record installation selection; pruning will be skipped."
fi
fi
else
@@ -1300,6 +1454,9 @@ fi
update_visible_command "$release_dir"
add_to_path
verify_visible_command
if ! prune_standalone_releases; then
warn "Could not safely inspect old Codex releases; skipped remaining cleanup."
fi
release_install_lock
handle_conflicting_install
+235 -3
View File
@@ -1,9 +1,13 @@
#!/usr/bin/env python3
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import select
import shlex
import shutil
import subprocess
import tarfile
import tempfile
@@ -317,6 +321,224 @@ class InstallShTest(unittest.TestCase):
self.assertEqual(guarded.returncode, 0, guarded.stderr)
self.assertTrue(managed.exists())
def test_standalone_retains_previous_and_running_releases_across_installs(
self,
) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
# Exercise real process inspection, including paths containing spaces.
root = Path(temp_dir).resolve() / "codex install"
root.mkdir()
system = os.uname()
if system.sysname == "Linux":
# The race requires pruning to proceed. A live but unreadable
# Codex/helper (e.g. the agent's sandbox) must prevent that.
for process in Path("/proc").glob("[0-9]*"):
try:
(process / "exe").readlink()
except OSError:
try:
name = (process / "comm").read_text().strip()
status = (process / "status").read_text()
except OSError:
continue # The process may have exited.
if (
name.startswith("codex") or name in ("rg", "bwrap")
) and not any(
line.startswith("State:") and "Z" in line
for line in status.splitlines()
):
self.skipTest(
f"Cannot inspect live {name} process {process.name}"
)
arch = "aarch64" if system.machine in ("arm64", "aarch64") else "x86_64"
target = f"{arch}-{'apple-darwin' if system.sysname == 'Darwin' else 'unknown-linux-musl'}"
packages = root / "codex-home/packages/standalone"
releases = packages / "releases"
versions = ("0.140.0", "0.141.0", VERSION)
install_options = {}
for version in versions:
source = root / version
source.mkdir()
archive, checksum, metadata = create_package_release(
source,
metadata_version=version,
binary_version=version,
target=target,
)
install_options[version] = dict(
metadata_json=metadata, archive_path=archive, checksum_path=checksum
)
first, _ = run_installer_in(
root, versions[0], **install_options[versions[0]]
)
self.assertEqual(first.returncode, 0, first.stderr)
oldest = releases / f"{versions[0]}-{target}"
helper = oldest / "bin/codex-code-mode-host"
shutil.copy("/bin/sleep", helper)
if system.sysname == "Darwin":
# A copied macOS system binary cannot run with its platform signature.
subprocess.run(
["/usr/bin/codesign", "--force", "--sign", "-", str(helper)],
check=True,
capture_output=True,
)
# A helper can outlive its parent CLI; retain its whole release.
with subprocess.Popen([str(helper), "300"]) as running:
try:
second, _ = run_installer_in(
root, versions[1], **install_options[versions[1]]
)
self.assertEqual(second.returncode, 0, second.stderr)
unowned = releases / f"0.139.0-{target}"
unowned.mkdir()
(unowned / "codex-package.json").write_text("{}")
retired = releases / ".retired.old-attempt"
retired.mkdir()
(retired / "leave-alone").write_text("not retried")
external = root / "external"
external.mkdir()
(external / "user-file").write_text("untouched")
(releases / f"0.138.0-{target}").symlink_to(external)
# Failed installs must not advance history or prune anything.
history = packages / "selection-history"
before_failure = history.read_bytes()
bad_source = root / "wrong-version"
bad_source.mkdir()
archive, checksum, metadata = create_package_release(
bad_source, metadata_version="0.199.0", target=target
)
failed, _ = run_installer_in(
root,
"0.199.0",
metadata_json=metadata,
archive_path=archive,
checksum_path=checksum,
)
self.assertNotEqual(failed.returncode, 0, failed.stderr)
self.assertEqual(history.read_bytes(), before_failure)
self.assertTrue(oldest.is_dir())
latest, _ = run_installer_in(
root, "latest", **install_options[VERSION]
)
self.assertEqual(latest.returncode, 0, latest.stderr)
self.assertIsNone(running.poll())
self.assertTrue(oldest.is_dir(), latest.stderr)
previous = releases / f"{versions[1]}-{target}"
current = releases / f"{VERSION}-{target}"
self.assertEqual((packages / "current").resolve(), current)
self.assertTrue(previous.is_dir())
self.assertEqual(
history.read_text().splitlines()[:2],
[current.name, previous.name],
)
recorded = history.read_bytes()
finally:
running.terminate()
running.wait()
if system.sysname == "Darwin":
# lsof sees no running helper now. ps can still see names for
# a different user, whose open files lsof silently omits.
fake_ps = root / "bin/ps"
foreign_uid = 0 if os.geteuid() != 0 else 501
try:
for executable in (oldest / "bin/codex", helper):
with self.subTest(foreign_executable=executable.name):
fake_ps.write_text(
"#!/bin/sh\n"
'case "$*" in\n'
" '-axo uid=,comm=')\n"
f" printf '%s\\n' {shlex.quote(f'{foreign_uid} {executable}')}\n"
" ;;\n"
' *) exec /bin/ps "$@" ;;\n'
"esac\n"
)
fake_ps.chmod(0o755)
retained, _ = run_installer_in(
root, "latest", **install_options[VERSION]
)
self.assertEqual(retained.returncode, 0, retained.stderr)
self.assertTrue(oldest.is_dir(), retained.stderr)
self.assertEqual(history.read_bytes(), recorded)
finally:
fake_ps.unlink()
# Start the helper after the first scan but immediately before the
# retirement rename. The second scan must restore its release.
fake_mv = root / "bin/mv"
ready = root / "retirement-ready"
resume = root / "retirement-resume"
os.mkfifo(ready)
os.mkfifo(resume)
real_mv = shutil.which("mv")
fake_mv.write_text(
"#!/bin/sh\n"
'case "$2" in\n'
" */.retired.*/release)\n"
f' printf "ready\\n" > {shlex.quote(str(ready))}\n'
f" read -r _ < {shlex.quote(str(resume))}\n"
" ;;\n"
"esac\n"
f'exec {shlex.quote(real_mv)} "$@"\n'
)
fake_mv.chmod(0o755)
ready_fd = os.open(ready, os.O_RDWR | os.O_NONBLOCK)
resume_fd = os.open(resume, os.O_RDWR | os.O_NONBLOCK)
try:
with ThreadPoolExecutor(max_workers=1) as executor:
installing = executor.submit(
run_installer_in, root, "latest", **install_options[VERSION]
)
try:
readable, _, _ = select.select([ready_fd], [], [], 30)
self.assertTrue(
readable, "installer did not attempt retirement"
)
self.assertEqual(os.read(ready_fd, 128), b"ready\n")
with subprocess.Popen([str(helper), "300"]) as late_helper:
try:
os.write(resume_fd, b"continue\n")
raced, _ = installing.result(timeout=30)
self.assertEqual(raced.returncode, 0, raced.stderr)
self.assertTrue(oldest.is_dir(), raced.stderr)
self.assertEqual(history.read_bytes(), recorded)
finally:
late_helper.terminate()
late_helper.wait()
finally:
# Release the intercepted mv even if an assertion fails.
os.write(resume_fd, b"continue\n")
finally:
os.close(ready_fd)
os.close(resume_fd)
fake_mv.unlink()
# A no-op latest install (also the codex update path) cleans up the
# now-unused release without rotating previous to current.
repeated, _ = run_installer_in(root, "latest", **install_options[VERSION])
self.assertEqual(repeated.returncode, 0, repeated.stderr)
self.assertFalse(oldest.exists(), repeated.stderr)
self.assertTrue(previous.is_dir())
self.assertEqual(history.read_bytes(), recorded)
self.assertTrue(unowned.is_dir())
self.assertEqual((retired / "leave-alone").read_text(), "not retried")
self.assertEqual((external / "user-file").read_text(), "untouched")
# Downgrades keep the actual previously selected version.
pinned, _ = run_installer_in(
root, versions[1], **install_options[versions[1]]
)
self.assertEqual(pinned.returncode, 0, pinned.stderr)
self.assertEqual((packages / "current").resolve(), previous)
self.assertEqual(
history.read_text().splitlines()[:2], [previous.name, current.name]
)
self.assertTrue(current.is_dir())
def test_uninspectable_legacy_updater_does_not_clear_pin(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
@@ -937,22 +1159,32 @@ def create_package_release(
root: Path,
*,
metadata_version: str = VERSION,
binary_version: str = VERSION,
target: str = "aarch64-apple-darwin",
) -> tuple[Path, Path, str]:
package_dir = root / "package"
(package_dir / "bin").mkdir(parents=True)
(package_dir / "codex-path").mkdir()
(package_dir / "codex-package.json").write_text("{}\n", encoding="utf-8")
(package_dir / "codex-package.json").write_text(
json.dumps(
{"version": binary_version, "target": target, "entrypoint": "bin/codex"}
),
encoding="utf-8",
)
write_executable(
package_dir / "bin" / "codex",
f"#!/bin/sh\nprintf 'codex-cli {VERSION}\\n'\n",
f"#!/bin/sh\nprintf 'codex-cli {binary_version}\\n'\n",
)
write_executable(
package_dir / "bin" / "codex-code-mode-host",
"#!/bin/sh\nexit 0\n",
)
write_executable(package_dir / "codex-path" / "rg", "#!/bin/sh\nexit 0\n")
if "linux" in target:
(package_dir / "codex-resources").mkdir()
write_executable(package_dir / "codex-resources/bwrap", "#!/bin/sh\nexit 0\n")
asset = "codex-package-aarch64-apple-darwin.tar.gz"
asset = f"codex-package-{target}.tar.gz"
archive_path = root / asset
with tarfile.open(archive_path, "w:gz") as archive:
for path in package_dir.iterdir():