Format Python scripts across the repository (#42109)

## Why

The root Python formatting pass only covered `scripts/`, leaving Python utilities elsewhere in the repository outside `just fmt` and `just fmt-check`.

## What changed

- Run the scripts Ruff formatter from the repository root.
- Exclude `sdk/python`, which has its own formatter pass, and `codex-rs/vendor`.
- Apply Ruff formatting to the newly covered Python files and update the formatter coverage test.

GitOrigin-RevId: 73ac82b8bb9c713d589724f6c4f926399febb513
This commit is contained in:
Charlie Marsh
2026-09-01 17:47:29 +00:00
committed by copyberry
parent 6a479e1813
commit 84aa75204a
31 changed files with 1020 additions and 342 deletions
+83 -23
View File
@@ -60,7 +60,9 @@ def parse_args():
) )
parser.add_argument("--pr", default="auto", help="auto, PR number, or PR URL") parser.add_argument("--pr", default="auto", help="auto, PR number, or PR URL")
parser.add_argument("--repo", help="Optional OWNER/REPO override") parser.add_argument("--repo", help="Optional OWNER/REPO override")
parser.add_argument("--poll-seconds", type=int, default=30, help="Watch poll interval") parser.add_argument(
"--poll-seconds", type=int, default=30, help="Watch poll interval"
)
parser.add_argument( parser.add_argument(
"--max-flaky-retries", "--max-flaky-retries",
type=int, type=int,
@@ -68,8 +70,12 @@ def parse_args():
help="Max rerun cycles per head SHA before stop recommendation", help="Max rerun cycles per head SHA before stop recommendation",
) )
parser.add_argument("--state-file", help="Path to state JSON file") parser.add_argument("--state-file", help="Path to state JSON file")
parser.add_argument("--once", action="store_true", help="Emit one snapshot and exit") parser.add_argument(
parser.add_argument("--watch", action="store_true", help="Continuously emit JSONL snapshots") "--once", action="store_true", help="Emit one snapshot and exit"
)
parser.add_argument(
"--watch", action="store_true", help="Continuously emit JSONL snapshots"
)
parser.add_argument( parser.add_argument(
"--retry-failed-now", "--retry-failed-now",
action="store_true", action="store_true",
@@ -128,7 +134,9 @@ def gh_json(args, repo=None):
try: try:
return json.loads(raw) return json.loads(raw)
except json.JSONDecodeError as err: except json.JSONDecodeError as err:
raise GhCommandError(f"Failed to parse JSON from gh output for {' '.join(args)}") from err raise GhCommandError(
f"Failed to parse JSON from gh output for {' '.join(args)}"
) from err
def parse_pr_spec(pr_spec): def parse_pr_spec(pr_spec):
@@ -210,6 +218,8 @@ def extract_repo_from_pr_view(data):
if owner and name: if owner and name:
return f"{owner}/{name}" return f"{owner}/{name}"
return None return None
def extract_repo_from_pr_url(pr_url): def extract_repo_from_pr_url(pr_url):
parsed = urlparse(pr_url) parsed = urlparse(pr_url)
parts = [p for p in parsed.path.split("/") if p] parts = [p for p in parsed.path.split("/") if p]
@@ -242,7 +252,9 @@ def load_state(path):
def save_state(path, state): def save_state(path, state):
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
payload = json.dumps(state, indent=2, sort_keys=True) + "\n" payload = json.dumps(state, indent=2, sort_keys=True) + "\n"
fd, tmp_name = tempfile.mkstemp(prefix=f"{path.name}.", suffix=".tmp", dir=path.parent) fd, tmp_name = tempfile.mkstemp(
prefix=f"{path.name}.", suffix=".tmp", dir=path.parent
)
tmp_path = Path(tmp_name) tmp_path = Path(tmp_name)
try: try:
with os.fdopen(fd, "w", encoding="utf-8") as tmp_file: with os.fdopen(fd, "w", encoding="utf-8") as tmp_file:
@@ -304,7 +316,16 @@ def summarize_checks(checks):
def get_workflow_runs_for_sha(repo, head_sha): def get_workflow_runs_for_sha(repo, head_sha):
endpoint = f"repos/{repo}/actions/runs" endpoint = f"repos/{repo}/actions/runs"
data = gh_json( data = gh_json(
["api", endpoint, "-X", "GET", "-f", f"head_sha={head_sha}", "-f", "per_page=100"], [
"api",
endpoint,
"-X",
"GET",
"-f",
f"head_sha={head_sha}",
"-f",
"per_page=100",
],
repo=repo, repo=repo,
) )
if not isinstance(data, dict): if not isinstance(data, dict):
@@ -334,7 +355,12 @@ def failed_runs_from_workflow_runs(runs, head_sha):
"html_url": str(run.get("html_url") or ""), "html_url": str(run.get("html_url") or ""),
} }
) )
failed_runs.sort(key=lambda item: (str(item.get("workflow_name") or ""), str(item.get("run_id") or ""))) failed_runs.sort(
key=lambda item: (
str(item.get("workflow_name") or ""),
str(item.get("run_id") or ""),
)
)
return failed_runs return failed_runs
@@ -361,7 +387,10 @@ def failed_jobs_from_workflow_runs(repo, runs, head_sha):
continue continue
run_status = str(run.get("status") or "") run_status = str(run.get("status") or "")
run_conclusion = str(run.get("conclusion") or "") run_conclusion = str(run.get("conclusion") or "")
if run_status.lower() == "completed" and run_conclusion not in FAILED_RUN_CONCLUSIONS: if (
run_status.lower() == "completed"
and run_conclusion not in FAILED_RUN_CONCLUSIONS
):
continue continue
jobs = get_jobs_for_run(repo, run_id) jobs = get_jobs_for_run(repo, run_id)
for job in jobs: for job in jobs:
@@ -401,7 +430,9 @@ def failed_jobs_from_workflow_runs(repo, runs, head_sha):
def get_authenticated_login(): def get_authenticated_login():
data = gh_json(["api", "user"]) data = gh_json(["api", "user"])
if not isinstance(data, dict) or not data.get("login"): if not isinstance(data, dict) or not data.get("login"):
raise GhCommandError("Unable to determine authenticated GitHub login from `gh api user`") raise GhCommandError(
"Unable to determine authenticated GitHub login from `gh api user`"
)
return str(data["login"]) return str(data["login"])
@@ -492,7 +523,9 @@ def normalize_reviews(items):
"id": str(item.get("id") or ""), "id": str(item.get("id") or ""),
"author": extract_login(item.get("user")), "author": extract_login(item.get("user")),
"author_association": str(item.get("author_association") or ""), "author_association": str(item.get("author_association") or ""),
"created_at": str(item.get("submitted_at") or item.get("created_at") or ""), "created_at": str(
item.get("submitted_at") or item.get("created_at") or ""
),
"body": str(item.get("body") or ""), "body": str(item.get("body") or ""),
"path": None, "path": None,
"line": None, "line": None,
@@ -535,7 +568,9 @@ def fetch_new_review_items(pr, state, fresh_state, authenticated_login=None):
endpoints = comment_endpoints(repo, pr_number) endpoints = comment_endpoints(repo, pr_number)
issue_payload = gh_api_list_paginated(endpoints["issue_comment"], repo=repo) issue_payload = gh_api_list_paginated(endpoints["issue_comment"], repo=repo)
review_comment_payload = gh_api_list_paginated(endpoints["review_comment"], repo=repo) review_comment_payload = gh_api_list_paginated(
endpoints["review_comment"], repo=repo
)
review_payload = gh_api_list_paginated(endpoints["review"], repo=repo) review_payload = gh_api_list_paginated(endpoints["review"], repo=repo)
issue_items = normalize_issue_comments(issue_payload) issue_items = normalize_issue_comments(issue_payload)
@@ -545,7 +580,9 @@ def fetch_new_review_items(pr, state, fresh_state, authenticated_login=None):
if isinstance(item, dict) and item.get("id") not in (None, "") if isinstance(item, dict) and item.get("id") not in (None, "")
} }
pending_review_ids = { pending_review_ids = {
review_id for review_id, review_state in review_states.items() if review_state == "PENDING" review_id
for review_id, review_state in review_states.items()
if review_state == "PENDING"
} }
pending_review_comment_ids = { pending_review_comment_ids = {
str(item.get("id")) str(item.get("id"))
@@ -554,7 +591,9 @@ def fetch_new_review_items(pr, state, fresh_state, authenticated_login=None):
and item.get("id") not in (None, "") and item.get("id") not in (None, "")
and str(item.get("pull_request_review_id") or "") in pending_review_ids and str(item.get("pull_request_review_id") or "") in pending_review_ids
} }
review_comment_items = normalize_review_comments(review_comment_payload, review_states) review_comment_items = normalize_review_comments(
review_comment_payload, review_states
)
review_items = normalize_reviews(review_payload) review_items = normalize_reviews(review_payload)
all_items = issue_items + review_comment_items + review_items all_items = issue_items + review_comment_items + review_items
@@ -598,7 +637,13 @@ def fetch_new_review_items(pr, state, fresh_state, authenticated_login=None):
elif kind == "review": elif kind == "review":
seen_review.add(item_id) seen_review.add(item_id)
new_items.sort(key=lambda item: (item.get("created_at") or "", item.get("kind") or "", item.get("id") or "")) new_items.sort(
key=lambda item: (
item.get("created_at") or "",
item.get("kind") or "",
item.get("id") or "",
)
)
state["seen_issue_comment_ids"] = sorted(seen_issue) state["seen_issue_comment_ids"] = sorted(seen_issue)
state["seen_review_comment_ids"] = sorted(seen_review_comment) state["seen_review_comment_ids"] = sorted(seen_review_comment)
state["seen_review_ids"] = sorted(seen_review) state["seen_review_ids"] = sorted(seen_review)
@@ -650,7 +695,15 @@ def is_pr_ready_to_merge(pr, checks_summary, new_review_items):
return True return True
def recommend_actions(pr, checks_summary, failed_runs, failed_jobs, new_review_items, retries_used, max_retries): def recommend_actions(
pr,
checks_summary,
failed_runs,
failed_jobs,
new_review_items,
retries_used,
max_retries,
):
actions = [] actions = []
if pr["closed"] or pr["merged"]: if pr["closed"] or pr["merged"]:
if new_review_items: if new_review_items:
@@ -671,7 +724,11 @@ def recommend_actions(pr, checks_summary, failed_runs, failed_jobs, new_review_i
actions.append("stop_exhausted_retries") actions.append("stop_exhausted_retries")
else: else:
actions.append("diagnose_ci_failure") actions.append("diagnose_ci_failure")
if checks_summary["all_terminal"] and failed_runs and retries_used < max_retries: if (
checks_summary["all_terminal"]
and failed_runs
and retries_used < max_retries
):
actions.append("retry_failed_checks") actions.append("retry_failed_checks")
if not actions: if not actions:
@@ -681,7 +738,9 @@ def recommend_actions(pr, checks_summary, failed_runs, failed_jobs, new_review_i
def collect_snapshot(args): def collect_snapshot(args):
pr = resolve_pr(args.pr, repo_override=args.repo) pr = resolve_pr(args.pr, repo_override=args.repo)
state_path = Path(args.state_file) if args.state_file else default_state_file_for(pr) state_path = (
Path(args.state_file) if args.state_file else default_state_file_for(pr)
)
state, fresh_state = load_state(state_path) state, fresh_state = load_state(state_path)
if not state.get("started_at"): if not state.get("started_at"):
@@ -703,7 +762,9 @@ def collect_snapshot(args):
checks_summary = summarize_checks(checks) checks_summary = summarize_checks(checks)
workflow_runs = get_workflow_runs_for_sha(pr["repo"], pr["head_sha"]) workflow_runs = get_workflow_runs_for_sha(pr["repo"], pr["head_sha"])
failed_runs = failed_runs_from_workflow_runs(workflow_runs, pr["head_sha"]) failed_runs = failed_runs_from_workflow_runs(workflow_runs, pr["head_sha"])
failed_jobs = failed_jobs_from_workflow_runs(pr["repo"], workflow_runs, pr["head_sha"]) failed_jobs = failed_jobs_from_workflow_runs(
pr["repo"], workflow_runs, pr["head_sha"]
)
retries_used = current_retry_count(state, pr["head_sha"]) retries_used = current_retry_count(state, pr["head_sha"])
actions = recommend_actions( actions = recommend_actions(
@@ -845,11 +906,10 @@ def run_watch(args):
}, },
) )
actions = set(snapshot.get("actions") or []) actions = set(snapshot.get("actions") or [])
if ( if "stop_pr_closed" in actions or "stop_exhausted_retries" in actions:
"stop_pr_closed" in actions print_event(
or "stop_exhausted_retries" in actions "stop", {"actions": snapshot.get("actions"), "pr": snapshot.get("pr")}
): )
print_event("stop", {"actions": snapshot.get("actions"), "pr": snapshot.get("pr")})
return 0 return 0
current_change_key = snapshot_change_key(snapshot) current_change_key = snapshot_change_key(snapshot)
@@ -49,10 +49,14 @@ class NotarizationConfiguration:
"vault_name": "AZURE_KEYVAULT_NAME", "vault_name": "AZURE_KEYVAULT_NAME",
"vault_key_name": "APPLE_NOTARIZATION_AKV_KEY_NAME", "vault_key_name": "APPLE_NOTARIZATION_AKV_KEY_NAME",
} }
values = {field: os.environ.get(name, "").strip() for field, name in required.items()} values = {
field: os.environ.get(name, "").strip() for field, name in required.items()
}
missing = [name for field, name in required.items() if not values[field]] missing = [name for field, name in required.items() if not values[field]]
if missing: if missing:
raise NotarizationError("Missing notarization configuration: " + ", ".join(missing)) raise NotarizationError(
"Missing notarization configuration: " + ", ".join(missing)
)
if not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]{2,23}", values["vault_name"]): if not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]{2,23}", values["vault_name"]):
raise NotarizationError("Invalid signing vault name") raise NotarizationError("Invalid signing vault name")
if not re.fullmatch(r"[A-Za-z0-9-]+", values["vault_key_name"]): if not re.fullmatch(r"[A-Za-z0-9-]+", values["vault_key_name"]):
@@ -90,7 +94,9 @@ class NotarizationConfiguration:
try: try:
result = subprocess.run(command, check=True, capture_output=True, text=True) result = subprocess.run(command, check=True, capture_output=True, text=True)
except (OSError, subprocess.CalledProcessError) as error: except (OSError, subprocess.CalledProcessError) as error:
raise NotarizationError("Notarization signing key could not be read") from error raise NotarizationError(
"Notarization signing key could not be read"
) from error
try: try:
metadata = json.loads(result.stdout) metadata = json.loads(result.stdout)
returned_key_id = metadata["id"] returned_key_id = metadata["id"]
@@ -99,10 +105,10 @@ class NotarizationConfiguration:
except (json.JSONDecodeError, KeyError, TypeError) as error: except (json.JSONDecodeError, KeyError, TypeError) as error:
raise NotarizationError("Notarization key metadata is invalid") from error raise NotarizationError("Notarization key metadata is invalid") from error
key_id_prefix = ( key_id_prefix = f"https://{values['vault_name']}.vault.azure.net/keys/{values['vault_key_name']}/"
f"https://{values['vault_name']}.vault.azure.net/keys/{values['vault_key_name']}/" if not isinstance(returned_key_id, str) or not returned_key_id.startswith(
) key_id_prefix
if not isinstance(returned_key_id, str) or not returned_key_id.startswith(key_id_prefix): ):
raise NotarizationError("Unexpected notarization signing key") raise NotarizationError("Unexpected notarization signing key")
returned_key_version = returned_key_id[len(key_id_prefix) :] returned_key_version = returned_key_id[len(key_id_prefix) :]
if not re.fullmatch(r"[0-9a-fA-F]{32}", returned_key_version): if not re.fullmatch(r"[0-9a-fA-F]{32}", returned_key_version):
@@ -120,7 +126,9 @@ class NotarizationConfiguration:
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}",
apple_issuer_id.strip(), apple_issuer_id.strip(),
): ):
raise NotarizationError("Notarization apple-issuer-id tag must contain a valid UUID") raise NotarizationError(
"Notarization apple-issuer-id tag must contain a valid UUID"
)
values["issuer_id"] = apple_issuer_id.strip() values["issuer_id"] = apple_issuer_id.strip()
values["apple_key_id"] = apple_key_id.strip() values["apple_key_id"] = apple_key_id.strip()
@@ -147,12 +155,18 @@ def base64url_decode(value: str) -> bytes:
"""Decode a base64url value and reject malformed input.""" """Decode a base64url value and reject malformed input."""
try: try:
return base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True) return base64.b64decode(
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
)
except (ValueError, UnicodeEncodeError) as error: except (ValueError, UnicodeEncodeError) as error:
raise NotarizationError("Signing service returned an invalid signature") from error raise NotarizationError(
"Signing service returned an invalid signature"
) from error
def create_apple_jwt(configuration: NotarizationConfiguration, *, issued_at: int) -> str: def create_apple_jwt(
configuration: NotarizationConfiguration, *, issued_at: int
) -> str:
"""Create an authentication token for notarization requests.""" """Create an authentication token for notarization requests."""
header = {"alg": "ES256", "kid": configuration.apple_key_id, "typ": "JWT"} header = {"alg": "ES256", "kid": configuration.apple_key_id, "typ": "JWT"}
@@ -164,8 +178,12 @@ def create_apple_jwt(configuration: NotarizationConfiguration, *, issued_at: int
"aud": "appstoreconnect-v1", "aud": "appstoreconnect-v1",
"scope": ["/notary/v2"], "scope": ["/notary/v2"],
} }
encoded_header = base64url_encode(json.dumps(header, separators=(",", ":")).encode("utf-8")) encoded_header = base64url_encode(
encoded_claims = base64url_encode(json.dumps(claims, separators=(",", ":")).encode("utf-8")) json.dumps(header, separators=(",", ":")).encode("utf-8")
)
encoded_claims = base64url_encode(
json.dumps(claims, separators=(",", ":")).encode("utf-8")
)
signing_input = f"{encoded_header}.{encoded_claims}" signing_input = f"{encoded_header}.{encoded_claims}"
digest = hashlib.sha256(signing_input.encode("ascii")).digest() digest = hashlib.sha256(signing_input.encode("ascii")).digest()
key_url = configuration.versioned_key_id key_url = configuration.versioned_key_id
@@ -206,7 +224,9 @@ def create_apple_jwt(configuration: NotarizationConfiguration, *, issued_at: int
returned_key_id = response["kid"] returned_key_id = response["kid"]
encoded_signature = response["value"] encoded_signature = response["value"]
except (json.JSONDecodeError, KeyError, TypeError) as error: except (json.JSONDecodeError, KeyError, TypeError) as error:
raise NotarizationError("Signing service returned an invalid response") from error raise NotarizationError(
"Signing service returned an invalid response"
) from error
# Reject signatures generated with a different key version. # Reject signatures generated with a different key version.
if returned_key_id != key_url: if returned_key_id != key_url:
@@ -220,7 +240,9 @@ def create_apple_jwt(configuration: NotarizationConfiguration, *, issued_at: int
return f"{signing_input}.{base64url_encode(signature)}" return f"{signing_input}.{base64url_encode(signature)}"
def json_request(url: str, token: str, *, body: dict[str, Any] | None = None) -> dict[str, Any]: def json_request(
url: str, token: str, *, body: dict[str, Any] | None = None
) -> dict[str, Any]:
"""Send an authenticated JSON request to Apple's notarization service.""" """Send an authenticated JSON request to Apple's notarization service."""
data = None if body is None else json.dumps(body).encode("utf-8") data = None if body is None else json.dumps(body).encode("utf-8")
@@ -239,7 +261,9 @@ def json_request(url: str, token: str, *, body: dict[str, Any] | None = None) ->
result = json.load(response) result = json.load(response)
except urllib.error.HTTPError as error: except urllib.error.HTTPError as error:
detail = error.read(1024).decode("utf-8", errors="replace") detail = error.read(1024).decode("utf-8", errors="replace")
raise NotarizationError(f"Apple Notary API returned HTTP {error.code}: {detail}") from error raise NotarizationError(
f"Apple Notary API returned HTTP {error.code}: {detail}"
) from error
except (OSError, json.JSONDecodeError) as error: except (OSError, json.JSONDecodeError) as error:
raise NotarizationError("Apple Notary API request failed") from error raise NotarizationError("Apple Notary API request failed") from error
@@ -281,9 +305,12 @@ def upload_to_apple_s3(
"object", "object",
) )
if any( if any(
not isinstance(credentials.get(name), str) or not credentials[name] for name in required not isinstance(credentials.get(name), str) or not credentials[name]
for name in required
): ):
raise NotarizationError("Apple returned incomplete temporary upload credentials") raise NotarizationError(
"Apple returned incomplete temporary upload credentials"
)
size = path.stat().st_size size = path.stat().st_size
if not 0 < size <= MAX_SINGLE_UPLOAD_BYTES: if not 0 < size <= MAX_SINGLE_UPLOAD_BYTES:
@@ -293,7 +320,9 @@ def upload_to_apple_s3(
amz_date = now.strftime("%Y%m%dT%H%M%SZ") amz_date = now.strftime("%Y%m%dT%H%M%SZ")
date_stamp = now.strftime("%Y%m%d") date_stamp = now.strftime("%Y%m%d")
host = f"{credentials['bucket']}.s3.{AWS_REGION}.amazonaws.com" host = f"{credentials['bucket']}.s3.{AWS_REGION}.amazonaws.com"
object_path = "/" + urllib.parse.quote(credentials["object"].lstrip("/"), safe="/-_.~") object_path = "/" + urllib.parse.quote(
credentials["object"].lstrip("/"), safe="/-_.~"
)
headers = { headers = {
"content-type": "application/octet-stream", "content-type": "application/octet-stream",
"host": host, "host": host,
@@ -319,7 +348,9 @@ def upload_to_apple_s3(
signing_key = ("AWS4" + credentials["awsSecretAccessKey"]).encode("utf-8") signing_key = ("AWS4" + credentials["awsSecretAccessKey"]).encode("utf-8")
for component in (date_stamp, AWS_REGION, "s3", "aws4_request"): for component in (date_stamp, AWS_REGION, "s3", "aws4_request"):
signing_key = hmac_sha256(signing_key, component) signing_key = hmac_sha256(signing_key, component)
signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() signature = hmac.new(
signing_key, string_to_sign.encode("utf-8"), hashlib.sha256
).hexdigest()
authorization = ( authorization = (
"AWS4-HMAC-SHA256 " "AWS4-HMAC-SHA256 "
f"Credential={credentials['awsAccessKeyId']}/{credential_scope}, " f"Credential={credentials['awsAccessKeyId']}/{credential_scope}, "
@@ -346,10 +377,14 @@ def upload_to_apple_s3(
f"Apple notarization payload upload failed with HTTP {error.code}" f"Apple notarization payload upload failed with HTTP {error.code}"
) from error ) from error
except OSError as error: except OSError as error:
raise NotarizationError("Apple notarization payload upload failed") from error raise NotarizationError(
"Apple notarization payload upload failed"
) from error
def write_developer_log(submission_id: str, token: str, destination: Path | None) -> None: def write_developer_log(
submission_id: str, token: str, destination: Path | None
) -> None:
"""Write the submission's optional diagnostic report.""" """Write the submission's optional diagnostic report."""
if destination is None: if destination is None:
@@ -358,9 +393,13 @@ def write_developer_log(submission_id: str, token: str, destination: Path | None
try: try:
log_url = response["data"]["attributes"]["developerLogUrl"] log_url = response["data"]["attributes"]["developerLogUrl"]
except (KeyError, TypeError) as error: except (KeyError, TypeError) as error:
raise NotarizationError("Apple did not provide a notarization developer log") from error raise NotarizationError(
"Apple did not provide a notarization developer log"
) from error
if not isinstance(log_url, str) or not log_url.startswith("https://"): if not isinstance(log_url, str) or not log_url.startswith("https://"):
raise NotarizationError("Apple returned an invalid notarization developer log URL") raise NotarizationError(
"Apple returned an invalid notarization developer log URL"
)
try: try:
# Do not forward unrelated authorization headers to the download URL. # Do not forward unrelated authorization headers to the download URL.
with urllib.request.urlopen(log_url, timeout=60) as response: with urllib.request.urlopen(log_url, timeout=60) as response:
@@ -370,7 +409,9 @@ def write_developer_log(submission_id: str, token: str, destination: Path | None
f"Apple notarization developer log download failed with HTTP {error.code}" f"Apple notarization developer log download failed with HTTP {error.code}"
) from error ) from error
except OSError as error: except OSError as error:
raise NotarizationError("Apple notarization developer log download failed") from error raise NotarizationError(
"Apple notarization developer log download failed"
) from error
def notarize( def notarize(
@@ -395,11 +436,15 @@ def notarize(
submission_id = response["data"]["id"] submission_id = response["data"]["id"]
upload_credentials = response["data"]["attributes"] upload_credentials = response["data"]["attributes"]
except (KeyError, TypeError) as error: except (KeyError, TypeError) as error:
raise NotarizationError("Apple returned an invalid notarization submission") from error raise NotarizationError(
"Apple returned an invalid notarization submission"
) from error
if not isinstance(submission_id, str) or not submission_id: if not isinstance(submission_id, str) or not submission_id:
raise NotarizationError("Apple did not return a notarization submission ID") raise NotarizationError("Apple did not return a notarization submission ID")
if not isinstance(upload_credentials, dict): if not isinstance(upload_credentials, dict):
raise NotarizationError("Apple returned invalid notarization upload credentials") raise NotarizationError(
"Apple returned invalid notarization upload credentials"
)
print(f"Uploading notarization submission {submission_id} for {path.name}") print(f"Uploading notarization submission {submission_id} for {path.name}")
upload_to_apple_s3(path, file_digest, upload_credentials) upload_to_apple_s3(path, file_digest, upload_credentials)
@@ -416,7 +461,9 @@ def notarize(
try: try:
status = result["data"]["attributes"]["status"] status = result["data"]["attributes"]["status"]
except (KeyError, TypeError) as error: except (KeyError, TypeError) as error:
raise NotarizationError("Apple returned an invalid notarization status") from error raise NotarizationError(
"Apple returned an invalid notarization status"
) from error
if status in {"Accepted", "Invalid", "Rejected"}: if status in {"Accepted", "Invalid", "Rejected"}:
# Save diagnostics for both accepted and rejected submissions. # Save diagnostics for both accepted and rejected submissions.
@@ -449,7 +496,9 @@ def main() -> int:
try: try:
if not arguments.file.is_file(): if not arguments.file.is_file():
raise NotarizationError(f"Notarization payload does not exist: {arguments.file}") raise NotarizationError(
f"Notarization payload does not exist: {arguments.file}"
)
if arguments.max_wait_seconds < 0: if arguments.max_wait_seconds < 0:
raise NotarizationError("--max-wait-seconds must be non-negative") raise NotarizationError("--max-wait-seconds must be non-negative")
if arguments.report_log is not None: if arguments.report_log is not None:
@@ -35,7 +35,9 @@ CREDENTIALS = {
def response(payload): def response(payload):
return io.BytesIO(json.dumps(payload).encode() if isinstance(payload, dict) else payload) return io.BytesIO(
json.dumps(payload).encode() if isinstance(payload, dict) else payload
)
def azure_response(payload): def azure_response(payload):
@@ -66,10 +68,16 @@ class NotarizationTest(unittest.TestCase):
notary.NotarizationConfiguration.from_environment() notary.NotarizationConfiguration.from_environment()
for issuer_id in (None, "", " "): for issuer_id in (None, "", " "):
with self.subTest(issuer_id=issuer_id): with self.subTest(issuer_id=issuer_id):
show.return_value = azure_response({**metadata, "apple_issuer_id": issuer_id}) show.return_value = azure_response(
with self.assertRaisesRegex(notary.NotarizationError, "apple-issuer-id tag"): {**metadata, "apple_issuer_id": issuer_id}
)
with self.assertRaisesRegex(
notary.NotarizationError, "apple-issuer-id tag"
):
notary.NotarizationConfiguration.from_environment() notary.NotarizationConfiguration.from_environment()
show.return_value = azure_response({**metadata, "apple_issuer_id": "invalid-issuer"}) show.return_value = azure_response(
{**metadata, "apple_issuer_id": "invalid-issuer"}
)
with self.assertRaisesRegex(notary.NotarizationError, "valid UUID"): with self.assertRaisesRegex(notary.NotarizationError, "valid UUID"):
notary.NotarizationConfiguration.from_environment() notary.NotarizationConfiguration.from_environment()
self.assertIn( self.assertIn(
@@ -78,7 +86,10 @@ class NotarizationTest(unittest.TestCase):
show.call_args.args[0], show.call_args.args[0],
) )
invalid_environment = {**ENVIRONMENT, "APPLE_NOTARIZATION_AKV_KEY_NAME": "../bad"} invalid_environment = {
**ENVIRONMENT,
"APPLE_NOTARIZATION_AKV_KEY_NAME": "../bad",
}
with ( with (
patch.dict(os.environ, invalid_environment, clear=True), patch.dict(os.environ, invalid_environment, clear=True),
self.assertRaisesRegex(notary.NotarizationError, "key name"), self.assertRaisesRegex(notary.NotarizationError, "key name"),
@@ -91,10 +102,14 @@ class NotarizationTest(unittest.TestCase):
"kid": CONFIGURATION.versioned_key_id, "kid": CONFIGURATION.versioned_key_id,
"value": notary.base64url_encode(signature), "value": notary.base64url_encode(signature),
} }
with patch.object(notary.subprocess, "run", return_value=azure_response(payload)) as sign: with patch.object(
notary.subprocess, "run", return_value=azure_response(payload)
) as sign:
token = notary.create_apple_jwt(CONFIGURATION, issued_at=1_780_000_000) token = notary.create_apple_jwt(CONFIGURATION, issued_at=1_780_000_000)
header, claims, encoded_signature = token.split(".") header, claims, encoded_signature = token.split(".")
self.assertEqual(json.loads(notary.base64url_decode(header))["kid"], "APPLEKEY01") self.assertEqual(
json.loads(notary.base64url_decode(header))["kid"], "APPLEKEY01"
)
self.assertEqual( self.assertEqual(
json.loads(notary.base64url_decode(claims))["iss"], CONFIGURATION.issuer_id json.loads(notary.base64url_decode(claims))["iss"], CONFIGURATION.issuer_id
) )
@@ -113,7 +128,11 @@ class NotarizationTest(unittest.TestCase):
def test_rejects_wrong_key_versions_and_invalid_signatures(self) -> None: def test_rejects_wrong_key_versions_and_invalid_signatures(self) -> None:
cases = ( cases = (
(CONFIGURATION.versioned_key_id + "wrong", bytes(64), "unexpected key version"), (
CONFIGURATION.versioned_key_id + "wrong",
bytes(64),
"unexpected key version",
),
(CONFIGURATION.versioned_key_id, b"short", "64 JOSE"), (CONFIGURATION.versioned_key_id, b"short", "64 JOSE"),
) )
for key_id, signature, message in cases: for key_id, signature, message in cases:
@@ -141,7 +160,15 @@ class NotarizationTest(unittest.TestCase):
response(b""), response(b""),
response({"data": {"attributes": {"status": "In Progress"}}}), response({"data": {"attributes": {"status": "In Progress"}}}),
response({"data": {"attributes": {"status": "Accepted"}}}), response({"data": {"attributes": {"status": "Accepted"}}}),
response({"data": {"attributes": {"developerLogUrl": "https://logs.example.com/log"}}}), response(
{
"data": {
"attributes": {
"developerLogUrl": "https://logs.example.com/log"
}
}
}
),
response(b'{"status":"Accepted"}'), response(b'{"status":"Accepted"}'),
] ]
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
@@ -153,7 +180,9 @@ class NotarizationTest(unittest.TestCase):
output = io.StringIO() output = io.StringIO()
with ( with (
patch.object(notary.subprocess, "run", return_value=signature), patch.object(notary.subprocess, "run", return_value=signature),
patch.object(notary.urllib.request, "urlopen", side_effect=responses) as requests, patch.object(
notary.urllib.request, "urlopen", side_effect=responses
) as requests,
patch.object(notary.time, "sleep") as sleep, patch.object(notary.time, "sleep") as sleep,
contextlib.redirect_stdout(output), contextlib.redirect_stdout(output),
): ):
@@ -185,9 +214,15 @@ class NotarizationWrapperTest(unittest.TestCase):
"CALL_LOG": str(call_log), "CALL_LOG": str(call_log),
} }
for name in ("az", "python3", "rcodesign"): for name in ("az", "python3", "rcodesign"):
body = "exit 0" if name == "az" else f'printf "{name} %s\\n" "$*" >> "$CALL_LOG"' body = (
"exit 0"
if name == "az"
else f'printf "{name} %s\\n" "$*" >> "$CALL_LOG"'
)
executable = tools / name executable = tools / name
executable.write_text(f"#!/usr/bin/env bash\nset -euo pipefail\n{body}\n") executable.write_text(
f"#!/usr/bin/env bash\nset -euo pipefail\n{body}\n"
)
executable.chmod(0o755) executable.chmod(0o755)
for kind in ("binary", "dmg"): for kind in ("binary", "dmg"):
with self.subTest(kind=kind): with self.subTest(kind=kind):
@@ -195,7 +230,9 @@ class NotarizationWrapperTest(unittest.TestCase):
artifact.write_bytes(b"signed release artifact") artifact.write_bytes(b"signed release artifact")
subprocess.run( subprocess.run(
[ [
str(SIGNING_DIRECTORY / f"notarize_macos_{kind}_with_akv.sh"), str(
SIGNING_DIRECTORY / f"notarize_macos_{kind}_with_akv.sh"
),
f"--{kind}", f"--{kind}",
str(artifact), str(artifact),
"--report-dir", "--report-dir",
+1 -6
View File
@@ -236,12 +236,7 @@ class RustyV8BazelTest(unittest.TestCase):
TemporaryDirectory() as target_dir, TemporaryDirectory() as target_dir,
TemporaryDirectory() as output_dir, TemporaryDirectory() as output_dir,
): ):
gn_out = ( gn_out = Path(target_dir) / "x86_64-pc-windows-msvc" / "release" / "gn_out"
Path(target_dir)
/ "x86_64-pc-windows-msvc"
/ "release"
/ "gn_out"
)
(gn_out / "obj").mkdir(parents=True) (gn_out / "obj").mkdir(parents=True)
(gn_out / "obj" / "rusty_v8.lib").write_bytes(b"archive") (gn_out / "obj" / "rusty_v8.lib").write_bytes(b"archive")
(gn_out / "src_binding.rs").write_text("binding") (gn_out / "src_binding.rs").write_text("binding")
+1 -3
View File
@@ -168,9 +168,7 @@ def main() -> None:
canary = canary_required(files, base_version, head_version) canary = canary_required(files, base_version, head_version)
windows_source = windows_source_required(files, base_version, head_version) windows_source = windows_source_required(files, base_version, head_version)
if base_version != head_version: if base_version != head_version:
canary_reason = ( canary_reason = f"v8 version changed from {base_version} to {head_version}"
f"v8 version changed from {base_version} to {head_version}"
)
windows_source_reason = canary_reason windows_source_reason = canary_reason
else: else:
canary_reason = ( canary_reason = (
@@ -167,7 +167,9 @@ def manifest_errors(
"create crate features" "create crate features"
) )
if not is_internal_dependency(path, dependency_name, dependency, internal_package_names): if not is_internal_dependency(
path, dependency_name, dependency, internal_package_names
):
continue continue
dependency_features = dependency.get("features") dependency_features = dependency.get("features")
@@ -354,7 +356,9 @@ def add_unused_exception_errors(
) )
def add_failure(failures_by_path: dict[str, list[str]], path_key: str, error: str) -> None: def add_failure(
failures_by_path: dict[str, list[str]], path_key: str, error: str
) -> None:
failures_by_path.setdefault(path_key, []).append(error) failures_by_path.setdefault(path_key, []).append(error)
+4 -2
View File
@@ -66,7 +66,7 @@ def dependency_sections(manifest: dict) -> list[tuple[str, dict]]:
for section_name in ("dependencies", "dev-dependencies", "build-dependencies"): for section_name in ("dependencies", "dev-dependencies", "build-dependencies"):
dependencies = target.get(section_name) dependencies = target.get(section_name)
if isinstance(dependencies, dict): if isinstance(dependencies, dict):
sections.append((f'target.{target_name}.{section_name}', dependencies)) sections.append((f"target.{target_name}.{section_name}", dependencies))
return sections return sections
@@ -77,7 +77,9 @@ def source_failures() -> list[str]:
text = path.read_text() text = path.read_text()
for line_number, line in enumerate(text.splitlines(), start=1): for line_number, line in enumerate(text.splitlines(), start=1):
if any(pattern.search(line) for pattern in FORBIDDEN_SOURCE_PATTERNS): if any(pattern.search(line) for pattern in FORBIDDEN_SOURCE_PATTERNS):
failures.append(f"{relative_path(path)}:{line_number} imports `codex_core`") failures.append(
f"{relative_path(path)}:{line_number} imports `codex_core`"
)
return failures return failures
+15 -7
View File
@@ -88,8 +88,11 @@ PACKAGE_TARGET_FILTERS: dict[str, str] = {
PACKAGE_CHOICES = tuple(PACKAGE_NATIVE_COMPONENTS) PACKAGE_CHOICES = tuple(PACKAGE_NATIVE_COMPONENTS)
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build or stage the Codex CLI npm package.") parser = argparse.ArgumentParser(
description="Build or stage the Codex CLI npm package."
)
parser.add_argument( parser.add_argument(
"--package", "--package",
choices=PACKAGE_CHOICES, choices=PACKAGE_CHOICES,
@@ -102,9 +105,7 @@ def parse_args() -> argparse.Namespace:
) )
parser.add_argument( parser.add_argument(
"--release-version", "--release-version",
help=( help=("Version to stage for npm release."),
"Version to stage for npm release."
),
) )
parser.add_argument( parser.add_argument(
"--staging-dir", "--staging-dir",
@@ -141,7 +142,9 @@ def main() -> int:
release_version = args.release_version release_version = args.release_version
if release_version: if release_version:
if version and version != release_version: if version and version != release_version:
raise RuntimeError("--version and --release-version must match when both are provided.") raise RuntimeError(
"--version and --release-version must match when both are provided."
)
version = release_version version = release_version
if not version: if not version:
@@ -272,7 +275,9 @@ def stage_sources(staging_dir: Path, version: str, package: str) -> None:
elif package == "codex-responses-api-proxy": elif package == "codex-responses-api-proxy":
bin_dir = staging_dir / "bin" bin_dir = staging_dir / "bin"
bin_dir.mkdir(parents=True, exist_ok=True) bin_dir.mkdir(parents=True, exist_ok=True)
launcher_src = RESPONSES_API_PROXY_NPM_ROOT / "bin" / "codex-responses-api-proxy.js" launcher_src = (
RESPONSES_API_PROXY_NPM_ROOT / "bin" / "codex-responses-api-proxy.js"
)
shutil.copy2(launcher_src, bin_dir / "codex-responses-api-proxy.js") shutil.copy2(launcher_src, bin_dir / "codex-responses-api-proxy.js")
readme_src = RESPONSES_API_PROXY_NPM_ROOT / "README.md" readme_src = RESPONSES_API_PROXY_NPM_ROOT / "README.md"
@@ -405,7 +410,10 @@ def copy_native_binaries(
missing_targets = sorted(target_filter - copied_targets) missing_targets = sorted(target_filter - copied_targets)
if missing_targets: if missing_targets:
missing_list = ", ".join(missing_targets) missing_list = ", ".join(missing_targets)
raise RuntimeError(f"Missing target directories in vendor source: {missing_list}") raise RuntimeError(
f"Missing target directories in vendor source: {missing_list}"
)
def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: def run_npm_pack(staging_dir: Path, output_path: Path) -> Path:
output_path = output_path.resolve() output_path = output_path.resolve()
@@ -33,9 +33,7 @@ def main() -> None:
env = os.environ.copy() env = os.environ.copy()
env["CODEX_APP_SERVER_SCHEMA_ROOT"] = str(schema_root) env["CODEX_APP_SERVER_SCHEMA_ROOT"] = str(schema_root)
env["CODEX_APP_SERVER_SCHEMA_EXPERIMENTAL"] = ( env["CODEX_APP_SERVER_SCHEMA_EXPERIMENTAL"] = "1" if args.experimental else "0"
"1" if args.experimental else "0"
)
if args.prettier: if args.prettier:
env["CODEX_APP_SERVER_SCHEMA_PRETTIER"] = str(args.prettier) env["CODEX_APP_SERVER_SCHEMA_PRETTIER"] = str(args.prettier)
@@ -132,7 +132,9 @@ def _validate_gpt_image_2_size(size: str) -> None:
total_pixels = width * height total_pixels = width * height
if max_edge > GPT_IMAGE_2_MAX_EDGE: if max_edge > GPT_IMAGE_2_MAX_EDGE:
_die("gpt-image-2 size maximum edge length must be less than or equal to 3840px.") _die(
"gpt-image-2 size maximum edge length must be less than or equal to 3840px."
)
if width % 16 != 0 or height % 16 != 0: if width % 16 != 0 or height % 16 != 0:
_die("gpt-image-2 size width and height must be multiples of 16px.") _die("gpt-image-2 size width and height must be multiples of 16px.")
if max_edge / min_edge > GPT_IMAGE_2_MAX_RATIO: if max_edge / min_edge > GPT_IMAGE_2_MAX_RATIO:
@@ -257,7 +259,9 @@ def _augment_prompt(args: argparse.Namespace, prompt: str) -> str:
return _augment_prompt_fields(args.augment, prompt, fields) return _augment_prompt_fields(args.augment, prompt, fields)
def _augment_prompt_fields(augment: bool, prompt: str, fields: Dict[str, Optional[str]]) -> str: def _augment_prompt_fields(
augment: bool, prompt: str, fields: Dict[str, Optional[str]]
) -> str:
if not augment: if not augment:
return prompt return prompt
@@ -280,7 +284,7 @@ def _augment_prompt_fields(augment: bool, prompt: str, fields: Dict[str, Optiona
if fields.get("materials"): if fields.get("materials"):
sections.append(f"Materials/textures: {fields['materials']}") sections.append(f"Materials/textures: {fields['materials']}")
if fields.get("text"): if fields.get("text"):
sections.append(f"Text (verbatim): \"{fields['text']}\"") sections.append(f'Text (verbatim): "{fields["text"]}"')
if fields.get("constraints"): if fields.get("constraints"):
sections.append(f"Constraints: {fields['constraints']}") sections.append(f"Constraints: {fields['constraints']}")
if fields.get("negative"): if fields.get("negative"):
@@ -327,7 +331,9 @@ def _derive_downscale_path(path: Path, suffix: str) -> Path:
return path.with_name(f"{path.stem}{suffix}{path.suffix}") return path.with_name(f"{path.stem}{suffix}{path.suffix}")
def _downscale_image_bytes(image_bytes: bytes, *, max_dim: int, output_format: str) -> bytes: def _downscale_image_bytes(
image_bytes: bytes, *, max_dim: int, output_format: str
) -> bytes:
try: try:
from PIL import Image from PIL import Image
except Exception: except Exception:
@@ -342,16 +348,22 @@ def _downscale_image_bytes(image_bytes: bytes, *, max_dim: int, output_format: s
scale = min(1.0, float(max_dim) / float(max(w, h))) scale = min(1.0, float(max_dim) / float(max(w, h)))
target = (max(1, int(round(w * scale))), max(1, int(round(h * scale)))) target = (max(1, int(round(w * scale))), max(1, int(round(h * scale))))
resized = img if target == (w, h) else img.resize(target, Image.Resampling.LANCZOS) resized = (
img if target == (w, h) else img.resize(target, Image.Resampling.LANCZOS)
)
fmt = output_format.lower() fmt = output_format.lower()
if fmt == "jpg": if fmt == "jpg":
fmt = "jpeg" fmt = "jpeg"
if fmt == "jpeg": if fmt == "jpeg":
if resized.mode in ("RGBA", "LA") or ("transparency" in getattr(resized, "info", {})): if resized.mode in ("RGBA", "LA") or (
"transparency" in getattr(resized, "info", {})
):
bg = Image.new("RGB", resized.size, (255, 255, 255)) bg = Image.new("RGB", resized.size, (255, 255, 255))
bg.paste(resized.convert("RGBA"), mask=resized.convert("RGBA").split()[-1]) bg.paste(
resized.convert("RGBA"), mask=resized.convert("RGBA").split()[-1]
)
resized = bg resized = bg
else: else:
resized = resized.convert("RGB") resized = resized.convert("RGB")
@@ -389,7 +401,9 @@ def _decode_write_and_downscale(
if derived.exists() and not force: if derived.exists() and not force:
_die(f"Output already exists: {derived} (use --force to overwrite)") _die(f"Output already exists: {derived} (use --force to overwrite)")
derived.parent.mkdir(parents=True, exist_ok=True) derived.parent.mkdir(parents=True, exist_ok=True)
resized = _downscale_image_bytes(raw, max_dim=downscale_max_dim, output_format=output_format) resized = _downscale_image_bytes(
raw, max_dim=downscale_max_dim, output_format=output_format
)
derived.write_bytes(resized) derived.write_bytes(resized)
print(f"Wrote {derived}") print(f"Wrote {derived}")
@@ -398,7 +412,9 @@ def _create_client():
try: try:
from openai import OpenAI from openai import OpenAI
except ImportError: except ImportError:
_die(f"openai SDK not installed in the active environment. {_dependency_hint('openai')}") _die(
f"openai SDK not installed in the active environment. {_dependency_hint('openai')}"
)
return OpenAI() return OpenAI()
@@ -500,10 +516,7 @@ def _job_output_paths(
if n == 1: if n == 1:
return [base] return [base]
return [ return [base.with_name(f"{base.stem}-{i}{base.suffix}") for i in range(1, n + 1)]
base.with_name(f"{base.stem}-{i}{base.suffix}")
for i in range(1, n + 1)
]
def _extract_retry_after_seconds(exc: Exception) -> Optional[float]: def _extract_retry_after_seconds(exc: Exception) -> Optional[float]:
@@ -589,17 +602,25 @@ async def _run_generate_batch(args: argparse.Namespace) -> int:
prompt = str(job["prompt"]).strip() prompt = str(job["prompt"]).strip()
fields = _merge_non_null(base_fields, job.get("fields", {})) fields = _merge_non_null(base_fields, job.get("fields", {}))
# Allow flat job keys as well (use_case, scene, etc.) # Allow flat job keys as well (use_case, scene, etc.)
fields = _merge_non_null(fields, {k: job.get(k) for k in base_fields.keys()}) fields = _merge_non_null(
fields, {k: job.get(k) for k in base_fields.keys()}
)
augmented = _augment_prompt_fields(args.augment, prompt, fields) augmented = _augment_prompt_fields(args.augment, prompt, fields)
job_payload = dict(base_payload) job_payload = dict(base_payload)
job_payload["prompt"] = augmented job_payload["prompt"] = augmented
job_payload = _merge_non_null(job_payload, {k: job.get(k) for k in base_payload.keys()}) job_payload = _merge_non_null(
job_payload, {k: job.get(k) for k in base_payload.keys()}
)
job_payload = {k: v for k, v in job_payload.items() if v is not None} job_payload = {k: v for k, v in job_payload.items() if v is not None}
_validate_generate_payload(job_payload) _validate_generate_payload(job_payload)
effective_output_format = _normalize_output_format(job_payload.get("output_format")) effective_output_format = _normalize_output_format(
_validate_transparency(job_payload.get("background"), effective_output_format) job_payload.get("output_format")
)
_validate_transparency(
job_payload.get("background"), effective_output_format
)
job_payload["output_format"] = effective_output_format job_payload["output_format"] = effective_output_format
n = int(job_payload.get("n", 1)) n = int(job_payload.get("n", 1))
@@ -614,7 +635,8 @@ async def _run_generate_batch(args: argparse.Namespace) -> int:
downscaled = None downscaled = None
if args.downscale_max_dim is not None: if args.downscale_max_dim is not None:
downscaled = [ downscaled = [
str(_derive_downscale_path(p, args.downscale_suffix)) for p in outputs str(_derive_downscale_path(p, args.downscale_suffix))
for p in outputs
] ]
_print_request( _print_request(
{ {
@@ -688,7 +710,9 @@ async def _run_generate_batch(args: argparse.Namespace) -> int:
raise raise
return i, str(exc) return i, str(exc)
tasks = [asyncio.create_task(run_job(i, job)) for i, job in enumerate(jobs, start=1)] tasks = [
asyncio.create_task(run_job(i, job)) for i, job in enumerate(jobs, start=1)
]
try: try:
await asyncio.gather(*tasks) await asyncio.gather(*tasks)
@@ -730,7 +754,9 @@ def _generate(args: argparse.Namespace) -> None:
output_paths = _build_output_paths(args.out, output_format, args.n, args.out_dir) output_paths = _build_output_paths(args.out, output_format, args.n, args.out_dir)
downscaled = None downscaled = None
if args.downscale_max_dim is not None: if args.downscale_max_dim is not None:
downscaled = [str(_derive_downscale_path(p, args.downscale_suffix)) for p in output_paths] downscaled = [
str(_derive_downscale_path(p, args.downscale_suffix)) for p in output_paths
]
if args.dry_run: if args.dry_run:
_print_request( _print_request(
@@ -799,7 +825,9 @@ def _edit(args: argparse.Namespace) -> None:
output_paths = _build_output_paths(args.out, output_format, args.n, args.out_dir) output_paths = _build_output_paths(args.out, output_format, args.n, args.out_dir)
downscaled = None downscaled = None
if args.downscale_max_dim is not None: if args.downscale_max_dim is not None:
downscaled = [str(_derive_downscale_path(p, args.downscale_suffix)) for p in output_paths] downscaled = [
str(_derive_downscale_path(p, args.downscale_suffix)) for p in output_paths
]
if args.dry_run: if args.dry_run:
payload_preview = dict(payload) payload_preview = dict(payload)
@@ -949,7 +977,9 @@ def main() -> int:
help="Generate multiple prompts concurrently (JSONL input)", help="Generate multiple prompts concurrently (JSONL input)",
) )
_add_shared_args(batch_parser) _add_shared_args(batch_parser)
batch_parser.add_argument("--input", required=True, help="Path to JSONL file (one job per line)") batch_parser.add_argument(
"--input", required=True, help="Path to JSONL file (one job per line)"
)
batch_parser.add_argument("--concurrency", type=int, default=DEFAULT_CONCURRENCY) batch_parser.add_argument("--concurrency", type=int, default=DEFAULT_CONCURRENCY)
batch_parser.add_argument("--max-attempts", type=int, default=3) batch_parser.add_argument("--max-attempts", type=int, default=3)
batch_parser.add_argument("--fail-fast", action="store_true") batch_parser.add_argument("--fail-fast", action="store_true")
@@ -969,11 +999,16 @@ def main() -> int:
_die("--concurrency must be between 1 and 25") _die("--concurrency must be between 1 and 25")
if getattr(args, "max_attempts", 3) < 1 or getattr(args, "max_attempts", 3) > 10: if getattr(args, "max_attempts", 3) < 1 or getattr(args, "max_attempts", 3) > 10:
_die("--max-attempts must be between 1 and 10") _die("--max-attempts must be between 1 and 10")
if args.output_compression is not None and not (0 <= args.output_compression <= 100): if args.output_compression is not None and not (
0 <= args.output_compression <= 100
):
_die("--output-compression must be between 0 and 100") _die("--output-compression must be between 0 and 100")
if args.command == "generate-batch" and not args.out_dir: if args.command == "generate-batch" and not args.out_dir:
_die("generate-batch requires --out-dir") _die("generate-batch requires --out-dir")
if getattr(args, "downscale_max_dim", None) is not None and args.downscale_max_dim < 1: if (
getattr(args, "downscale_max_dim", None) is not None
and args.downscale_max_dim < 1
):
_die("--downscale-max-dim must be >= 1") _die("--downscale-max-dim must be >= 1")
_validate_model(args.model) _validate_model(args.model)
@@ -95,7 +95,9 @@ def _smoothstep(value: float) -> float:
return value * value * (3.0 - 2.0 * value) return value * value * (3.0 - 2.0 * value)
def _soft_alpha(distance: int, transparent_threshold: float, opaque_threshold: float) -> int: def _soft_alpha(
distance: int, transparent_threshold: float, opaque_threshold: float
) -> int:
if distance <= transparent_threshold: if distance <= transparent_threshold:
return 0 return 0
if distance >= opaque_threshold: if distance >= opaque_threshold:
@@ -132,7 +134,9 @@ def _spill_channels(key: Color) -> list[int]:
key_max = max(key) key_max = max(key)
if key_max < 128: if key_max < 128:
return [] return []
return [idx for idx, value in enumerate(key) if value >= key_max - 16 and value >= 128] return [
idx for idx, value in enumerate(key) if value >= key_max - 16 and value >= 128
]
def _key_channel_dominance(rgb: Color, key: Color) -> float: def _key_channel_dominance(rgb: Color, key: Color) -> float:
@@ -358,7 +362,10 @@ def _remove_chroma_key(args: argparse.Namespace) -> None:
print(f"Transparent pixels: {transparent_after}/{total}") print(f"Transparent pixels: {transparent_after}/{total}")
print(f"Partially transparent pixels: {partial_after}/{total}") print(f"Partially transparent pixels: {partial_after}/{total}")
if transparent == 0: if transparent == 0:
print("Warning: no pixels matched the key color before feathering.", file=sys.stderr) print(
"Warning: no pixels matched the key color before feathering.",
file=sys.stderr,
)
def _build_parser() -> argparse.ArgumentParser: def _build_parser() -> argparse.ArgumentParser:
@@ -425,7 +432,9 @@ def _build_parser() -> argparse.ArgumentParser:
action="store_true", action="store_true",
help="Alias for --spill-cleanup; decontaminate key-color edge spill.", help="Alias for --spill-cleanup; decontaminate key-color edge spill.",
) )
parser.add_argument("--force", action="store_true", help="Overwrite an existing output file.") parser.add_argument(
"--force", action="store_true", help="Overwrite an existing output file."
)
return parser return parser
@@ -49,7 +49,9 @@ def display_name_from_plugin_name(plugin_name: str) -> str:
return " ".join(part.capitalize() for part in re.split(r"[-_]+", plugin_name)) return " ".join(part.capitalize() for part in re.split(r"[-_]+", plugin_name))
def build_plugin_json(plugin_name: str, *, with_mcp: bool, with_apps: bool) -> dict[str, Any]: def build_plugin_json(
plugin_name: str, *, with_mcp: bool, with_apps: bool
) -> dict[str, Any]:
display_name = display_name_from_plugin_name(plugin_name) display_name = display_name_from_plugin_name(plugin_name)
payload: dict[str, Any] = { payload: dict[str, Any] = {
"name": plugin_name, "name": plugin_name,
@@ -126,7 +128,9 @@ def load_validated_marketplace(
if marketplace_path.exists(): if marketplace_path.exists():
payload = load_json(marketplace_path) payload = load_json(marketplace_path)
else: else:
payload = build_default_marketplace(marketplace_name or DEFAULT_MARKETPLACE_NAME) payload = build_default_marketplace(
marketplace_name or DEFAULT_MARKETPLACE_NAME
)
if not isinstance(payload, dict): if not isinstance(payload, dict):
raise ValueError(f"{marketplace_path} must contain a JSON object.") raise ValueError(f"{marketplace_path} must contain a JSON object.")
@@ -134,7 +138,10 @@ def load_validated_marketplace(
validate_marketplace_interface(payload) validate_marketplace_interface(payload)
existing_marketplace_name = payload.get("name") existing_marketplace_name = payload.get("name")
if not isinstance(existing_marketplace_name, str) or not existing_marketplace_name.strip(): if (
not isinstance(existing_marketplace_name, str)
or not existing_marketplace_name.strip()
):
raise ValueError(f"{marketplace_path} must contain a non-empty string 'name'.") raise ValueError(f"{marketplace_path} must contain a non-empty string 'name'.")
validate_marketplace_name(existing_marketplace_name) validate_marketplace_name(existing_marketplace_name)
@@ -150,7 +157,8 @@ def load_validated_marketplace(
if not isinstance(plugins, list): if not isinstance(plugins, list):
raise ValueError(f"{marketplace_path} field 'plugins' must be an array.") raise ValueError(f"{marketplace_path} field 'plugins' must be an array.")
if not force and any( if not force and any(
isinstance(entry, dict) and entry.get("name") == plugin_name for entry in plugins isinstance(entry, dict) and entry.get("name") == plugin_name
for entry in plugins
): ):
raise FileExistsError( raise FileExistsError(
f"Marketplace entry '{plugin_name}' already exists in {marketplace_path}. " f"Marketplace entry '{plugin_name}' already exists in {marketplace_path}. "
@@ -169,10 +177,14 @@ def update_marketplace_json(
category: str, category: str,
force: bool, force: bool,
) -> None: ) -> None:
payload = load_validated_marketplace(marketplace_path, marketplace_name, plugin_name, force) payload = load_validated_marketplace(
marketplace_path, marketplace_name, plugin_name, force
)
plugins = payload["plugins"] plugins = payload["plugins"]
new_entry = build_marketplace_entry(plugin_name, install_policy, auth_policy, category) new_entry = build_marketplace_entry(
plugin_name, install_policy, auth_policy, category
)
for index, entry in enumerate(plugins): for index, entry in enumerate(plugins):
if isinstance(entry, dict) and entry.get("name") == plugin_name: if isinstance(entry, dict) and entry.get("name") == plugin_name:
@@ -215,12 +227,24 @@ def parse_args() -> argparse.Namespace:
"Pass an explicit repo path only when a repo/team plugin is intended." "Pass an explicit repo path only when a repo/team plugin is intended."
), ),
) )
parser.add_argument("--with-skills", action="store_true", help="Create skills/ directory") parser.add_argument(
parser.add_argument("--with-hooks", action="store_true", help="Create hooks/ directory") "--with-skills", action="store_true", help="Create skills/ directory"
parser.add_argument("--with-scripts", action="store_true", help="Create scripts/ directory") )
parser.add_argument("--with-assets", action="store_true", help="Create assets/ directory") parser.add_argument(
parser.add_argument("--with-mcp", action="store_true", help="Create .mcp.json placeholder") "--with-hooks", action="store_true", help="Create hooks/ directory"
parser.add_argument("--with-apps", action="store_true", help="Create .app.json placeholder") )
parser.add_argument(
"--with-scripts", action="store_true", help="Create scripts/ directory"
)
parser.add_argument(
"--with-assets", action="store_true", help="Create assets/ directory"
)
parser.add_argument(
"--with-mcp", action="store_true", help="Create .mcp.json placeholder"
)
parser.add_argument(
"--with-apps", action="store_true", help="Create .app.json placeholder"
)
parser.add_argument( parser.add_argument(
"--with-marketplace", "--with-marketplace",
action="store_true", action="store_true",
@@ -271,7 +295,9 @@ def main() -> None:
raw_plugin_name = args.plugin_name raw_plugin_name = args.plugin_name
plugin_name = normalize_plugin_name(raw_plugin_name) plugin_name = normalize_plugin_name(raw_plugin_name)
if plugin_name != raw_plugin_name: if plugin_name != raw_plugin_name:
print(f"Note: Normalized plugin name from '{raw_plugin_name}' to '{plugin_name}'.") print(
f"Note: Normalized plugin name from '{raw_plugin_name}' to '{plugin_name}'."
)
validate_plugin_name(plugin_name) validate_plugin_name(plugin_name)
marketplace_name = None marketplace_name = None
if args.marketplace_name is not None: if args.marketplace_name is not None:
@@ -280,15 +306,19 @@ def main() -> None:
if args.with_marketplace: if args.with_marketplace:
marketplace_path = Path(args.marketplace_path).expanduser().resolve() marketplace_path = Path(args.marketplace_path).expanduser().resolve()
load_validated_marketplace(marketplace_path, marketplace_name, plugin_name, args.force) load_validated_marketplace(
marketplace_path, marketplace_name, plugin_name, args.force
)
plugin_root = (Path(args.path).expanduser().resolve() / plugin_name) plugin_root = Path(args.path).expanduser().resolve() / plugin_name
plugin_root.mkdir(parents=True, exist_ok=True) plugin_root.mkdir(parents=True, exist_ok=True)
plugin_json_path = plugin_root / ".codex-plugin" / "plugin.json" plugin_json_path = plugin_root / ".codex-plugin" / "plugin.json"
write_json( write_json(
plugin_json_path, plugin_json_path,
build_plugin_json(plugin_name, with_mcp=args.with_mcp, with_apps=args.with_apps), build_plugin_json(
plugin_name, with_mcp=args.with_mcp, with_apps=args.with_apps
),
args.force, args.force,
) )
@@ -62,6 +62,8 @@ def load_manifest(manifest_path: Path) -> dict[str, object]:
if not isinstance(payload, dict): if not isinstance(payload, dict):
raise ValueError(f"{manifest_path} must contain a JSON object.") raise ValueError(f"{manifest_path} must contain a JSON object.")
return payload return payload
def sanitize_cachebuster(value: str) -> str: def sanitize_cachebuster(value: str) -> str:
sanitized = re.sub(r"[^a-z0-9-]+", "-", value.strip().lower()) sanitized = re.sub(r"[^a-z0-9-]+", "-", value.strip().lower())
sanitized = re.sub(r"-{2,}", "-", sanitized).strip("-") sanitized = re.sub(r"-{2,}", "-", sanitized).strip("-")
@@ -187,7 +187,9 @@ def validate_manifest_shape(
if not isinstance(capabilities, list) or not all( if not isinstance(capabilities, list) or not all(
isinstance(value, str) and value.strip() for value in capabilities isinstance(value, str) and value.strip() for value in capabilities
): ):
errors.append("plugin.json field `interface.capabilities` must be an array of strings") errors.append(
"plugin.json field `interface.capabilities` must be an array of strings"
)
for field in ("websiteURL", "privacyPolicyURL", "termsOfServiceURL"): for field in ("websiteURL", "privacyPolicyURL", "termsOfServiceURL"):
validate_optional_https_url(interface, field, errors, prefix="interface") validate_optional_https_url(interface, field, errors, prefix="interface")
brand_color = interface.get("brandColor") brand_color = interface.get("brandColor")
@@ -260,7 +262,9 @@ def reject_unknown_fields(
errors: list[str], errors: list[str],
) -> None: ) -> None:
for key in sorted(set(payload) - allowed_keys): for key in sorted(set(payload) - allowed_keys):
errors.append(f"plugin.json field `{prefix}.{key}` is not accepted by plugin validation") errors.append(
f"plugin.json field `{prefix}.{key}` is not accepted by plugin validation"
)
def validate_optional_https_url( def validate_optional_https_url(
@@ -275,7 +279,9 @@ def validate_optional_https_url(
return return
parsed = urlparse(value) if isinstance(value, str) else None parsed = urlparse(value) if isinstance(value, str) else None
if parsed is None or parsed.scheme != "https" or not parsed.netloc: if parsed is None or parsed.scheme != "https" or not parsed.netloc:
errors.append(f"plugin.json field `{prefix}.{key}` must be an absolute `https://` URL") errors.append(
f"plugin.json field `{prefix}.{key}` must be an absolute `https://` URL"
)
def validate_optional_contract_path( def validate_optional_contract_path(
@@ -344,9 +350,13 @@ def validate_app_manifest(path: Path, errors: list[str]) -> None:
) )
app_id = value.get("id") app_id = value.get("id")
if not isinstance(app_id, str) or not app_id.strip(): if not isinstance(app_id, str) or not app_id.strip():
errors.append(f"`.app.json` app `{key}` field `id` must be a non-empty string") errors.append(
f"`.app.json` app `{key}` field `id` must be a non-empty string"
)
category = value.get("category") category = value.get("category")
if category is not None and (not isinstance(category, str) or not category.strip()): if category is not None and (
not isinstance(category, str) or not category.strip()
):
errors.append( errors.append(
f"`.app.json` app `{key}` field `category` must be a non-empty string" f"`.app.json` app `{key}` field `category` must be a non-empty string"
) )
@@ -448,7 +458,9 @@ def validate_skill_manifest(skill_root: Path, errors: list[str]) -> None:
return return
skill_name = frontmatter.get("name") skill_name = frontmatter.get("name")
if not isinstance(skill_name, str) or not skill_name.strip(): if not isinstance(skill_name, str) or not skill_name.strip():
errors.append(f"skill `{skill_root.name}` frontmatter field `name` must be non-empty") errors.append(
f"skill `{skill_root.name}` frontmatter field `name` must be non-empty"
)
description = frontmatter.get("description") description = frontmatter.get("description")
if not isinstance(description, str) or not description.strip(): if not isinstance(description, str) or not description.strip():
errors.append( errors.append(
@@ -498,7 +510,9 @@ def validate_skill_agent_manifest(
) )
interface = payload.get("interface") interface = payload.get("interface")
if not isinstance(interface, dict): if not isinstance(interface, dict):
errors.append(f"skill `{skill_root.name}` agent field `interface` must be an object") errors.append(
f"skill `{skill_root.name}` agent field `interface` must be an object"
)
return return
reject_skill_agent_unknown_fields( reject_skill_agent_unknown_fields(
interface, interface,
@@ -547,7 +561,9 @@ def validate_skill_agent_manifest(
policy = payload.get("policy") policy = payload.get("policy")
if policy is not None: if policy is not None:
if not isinstance(policy, dict): if not isinstance(policy, dict):
errors.append(f"skill `{skill_root.name}` agent field `policy` must be an object") errors.append(
f"skill `{skill_root.name}` agent field `policy` must be an object"
)
else: else:
reject_skill_agent_unknown_fields( reject_skill_agent_unknown_fields(
policy, policy,
@@ -624,7 +640,9 @@ def validate_asset_path(
errors.append(f"{label} must be a non-empty relative path") errors.append(f"{label} must be a non-empty relative path")
return return
candidate = PurePosixPath(raw_path.replace("\\", "/")) candidate = PurePosixPath(raw_path.replace("\\", "/"))
if candidate.is_absolute() or any(part in {"", ".", ".."} for part in candidate.parts): if candidate.is_absolute() or any(
part in {"", ".", ".."} for part in candidate.parts
):
errors.append(f"{label} must stay inside the plugin archive") errors.append(f"{label} must stay inside the plugin archive")
return return
resolved_path = (base_dir / candidate.as_posix()).resolve() resolved_path = (base_dir / candidate.as_posix()).resolve()
@@ -148,7 +148,10 @@ def parse_interface_overrides(raw_overrides):
print(f"[ERROR] Unknown interface field '{key}'. Allowed: {allowed}") print(f"[ERROR] Unknown interface field '{key}'. Allowed: {allowed}")
return None, None return None, None
overrides[key] = value overrides[key] = value
if key not in ("display_name", "short_description") and key not in optional_order: if (
key not in ("display_name", "short_description")
and key not in optional_order
):
optional_order.append(key) optional_order.append(key)
return overrides, optional_order return overrides, optional_order
@@ -159,7 +162,9 @@ def write_openai_yaml(skill_dir, skill_name, raw_overrides):
return None return None
display_name = overrides.get("display_name") or format_display_name(skill_name) display_name = overrides.get("display_name") or format_display_name(skill_name)
short_description = overrides.get("short_description") or generate_short_description(display_name) short_description = overrides.get(
"short_description"
) or generate_short_description(display_name)
if not (25 <= len(short_description) <= 64): if not (25 <= len(short_description) <= 64):
print( print(
@@ -123,7 +123,9 @@ def parse_resources(raw_resources):
return deduped return deduped
def create_resource_dirs(skill_dir, skill_name, skill_title, resources, include_examples): def create_resource_dirs(
skill_dir, skill_name, skill_title, resources, include_examples
):
for resource in resources: for resource in resources:
resource_dir = skill_dir / resource resource_dir = skill_dir / resource
resource_dir.mkdir(exist_ok=True) resource_dir.mkdir(exist_ok=True)
@@ -138,7 +140,9 @@ def create_resource_dirs(skill_dir, skill_name, skill_title, resources, include_
elif resource == "references": elif resource == "references":
if include_examples: if include_examples:
example_reference = resource_dir / "api_reference.md" example_reference = resource_dir / "api_reference.md"
example_reference.write_text(EXAMPLE_REFERENCE.format(skill_title=skill_title)) example_reference.write_text(
EXAMPLE_REFERENCE.format(skill_title=skill_title)
)
print("[OK] Created references/api_reference.md") print("[OK] Created references/api_reference.md")
else: else:
print("[OK] Created references/") print("[OK] Created references/")
@@ -182,7 +186,9 @@ def init_skill(skill_name, path, resources, include_examples, interface_override
# Create SKILL.md from template # Create SKILL.md from template
skill_title = title_case_skill_name(skill_name) skill_title = title_case_skill_name(skill_name)
skill_content = SKILL_TEMPLATE.format(skill_name=skill_name, skill_title=skill_title) skill_content = SKILL_TEMPLATE.format(
skill_name=skill_name, skill_title=skill_title
)
skill_md_path = skill_dir / "SKILL.md" skill_md_path = skill_dir / "SKILL.md"
try: try:
@@ -204,7 +210,9 @@ def init_skill(skill_name, path, resources, include_examples, interface_override
# Create resource directories if requested # Create resource directories if requested
if resources: if resources:
try: try:
create_resource_dirs(skill_dir, skill_name, skill_title, resources, include_examples) create_resource_dirs(
skill_dir, skill_name, skill_title, resources, include_examples
)
except Exception as e: except Exception as e:
print(f"[ERROR] Error creating resource directories: {e}") print(f"[ERROR] Error creating resource directories: {e}")
return None return None
@@ -215,14 +223,20 @@ def init_skill(skill_name, path, resources, include_examples, interface_override
print("1. Edit SKILL.md to complete the TODO items and update the description") print("1. Edit SKILL.md to complete the TODO items and update the description")
if resources: if resources:
if include_examples: if include_examples:
print("2. Customize or delete the example files in scripts/, references/, and assets/") print(
"2. Customize or delete the example files in scripts/, references/, and assets/"
)
else: else:
print("2. Add resources to scripts/, references/, and assets/ as needed") print("2. Add resources to scripts/, references/, and assets/ as needed")
else: else:
print("2. Create resource directories only if needed (scripts/, references/, assets/)") print(
"2. Create resource directories only if needed (scripts/, references/, assets/)"
)
print("3. Update agents/openai.yaml if the UI metadata should differ") print("3. Update agents/openai.yaml if the UI metadata should differ")
print("4. Run the validator when ready to check the skill structure") print("4. Run the validator when ready to check the skill structure")
print("5. Consider independent forward-testing only when complexity or risk warrants it") print(
"5. Consider independent forward-testing only when complexity or risk warrants it"
)
return skill_dir return skill_dir
@@ -109,7 +109,9 @@ def validate_skill(skill_path):
fence_length = 0 fence_length = 0
continue continue
if fence_marker is None and re.fullmatch(r"[ ]{0,3}\[TODO:[^\n]*\][ \t]*", line): if fence_marker is None and re.fullmatch(
r"[ ]{0,3}\[TODO:[^\n]*\][ \t]*", line
):
return False, "Skill instructions contain an unfinished TODO placeholder" return False, "Skill instructions contain an unfinished TODO placeholder"
return True, "Skill is valid!" return True, "Skill is valid!"
@@ -15,6 +15,7 @@ import urllib.parse
import zipfile import zipfile
from github_utils import github_request from github_utils import github_request
DEFAULT_REF = "main" DEFAULT_REF = "main"
@@ -97,7 +98,9 @@ def _download_repo_zip(owner: str, repo: str, ref: str, dest_dir: str) -> str:
def _run_git(args: list[str]) -> None: def _run_git(args: list[str]) -> None:
result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) result = subprocess.run(
args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)
if result.returncode != 0: if result.returncode != 0:
raise InstallError(result.stderr.strip() or "Git command failed.") raise InstallError(result.stderr.strip() or "Git command failed.")
@@ -125,7 +128,9 @@ def _validate_skill_name(name: str) -> None:
raise InstallError("Invalid skill name.") raise InstallError("Invalid skill name.")
def _git_sparse_checkout(repo_url: str, ref: str, paths: list[str], dest_dir: str) -> str: def _git_sparse_checkout(
repo_url: str, ref: str, paths: list[str], dest_dir: str
) -> str:
repo_dir = os.path.join(dest_dir, "repo") repo_dir = os.path.join(dest_dir, "repo")
clone_cmd = [ clone_cmd = [
"git", "git",
@@ -165,7 +170,10 @@ def _validate_skill(path: str, repo_root: str) -> None:
resolved_repo_root = os.path.realpath(repo_root) resolved_repo_root = os.path.realpath(repo_root)
resolved_path = os.path.realpath(path) resolved_path = os.path.realpath(path)
try: try:
inside_repo = os.path.commonpath([resolved_repo_root, resolved_path]) == resolved_repo_root inside_repo = (
os.path.commonpath([resolved_repo_root, resolved_path])
== resolved_repo_root
)
except ValueError: except ValueError:
inside_repo = False inside_repo = False
if not inside_repo: if not inside_repo:
@@ -194,12 +202,15 @@ def _validate_skill(path: str, repo_root: str) -> None:
resolved_entry = os.path.realpath(entry_path) resolved_entry = os.path.realpath(entry_path)
try: try:
inside_skill = ( inside_skill = (
os.path.commonpath([resolved_path, resolved_entry]) == resolved_path os.path.commonpath([resolved_path, resolved_entry])
== resolved_path
) )
except ValueError: except ValueError:
inside_skill = False inside_skill = False
if not inside_skill or not os.path.isfile(resolved_entry): if not inside_skill or not os.path.isfile(resolved_entry):
raise InstallError(f"Unsupported symbolic link in skill: {relative_entry}") raise InstallError(
f"Unsupported symbolic link in skill: {relative_entry}"
)
continue continue
if not os.path.isdir(entry_path) and not os.path.isfile(entry_path): if not os.path.isdir(entry_path) and not os.path.isfile(entry_path):
raise InstallError(f"Unsupported file type in skill: {relative_entry}") raise InstallError(f"Unsupported file type in skill: {relative_entry}")
@@ -54,8 +54,7 @@ def _list_skills(repo: str, path: str, ref: str) -> list[str]:
except urllib.error.HTTPError as exc: except urllib.error.HTTPError as exc:
if exc.code == 404: if exc.code == 404:
raise ListError( raise ListError(
"Skills path not found: " f"Skills path not found: https://github.com/{repo}/tree/{ref}/{path}"
f"https://github.com/{repo}/tree/{ref}/{path}"
) from exc ) from exc
raise ListError(f"Failed to fetch skills: HTTP {exc.code}") from exc raise ListError(f"Failed to fetch skills: HTTP {exc.code}") from exc
data = json.loads(payload.decode("utf-8")) data = json.loads(payload.decode("utf-8"))
+24 -6
View File
@@ -33,7 +33,9 @@ class PluginCreatorSecurityTests(unittest.TestCase):
self.marketplace_path = self.root / "marketplace.json" self.marketplace_path = self.root / "marketplace.json"
self.plugin_root = self.root / "plugins" / "demo" self.plugin_root = self.root / "plugins" / "demo"
def run_script(self, script: str, *arguments: str) -> subprocess.CompletedProcess[str]: def run_script(
self, script: str, *arguments: str
) -> subprocess.CompletedProcess[str]:
return subprocess.run( return subprocess.run(
[sys.executable, "-B", str(SCRIPTS / script), *arguments], [sys.executable, "-B", str(SCRIPTS / script), *arguments],
capture_output=True, capture_output=True,
@@ -65,7 +67,9 @@ class PluginCreatorSecurityTests(unittest.TestCase):
scaffold = runpy.run_path(str(SCRIPTS / "create_basic_plugin.py")) scaffold = runpy.run_path(str(SCRIPTS / "create_basic_plugin.py"))
validator = runpy.run_path(str(SCRIPTS / "validate_plugin.py")) validator = runpy.run_path(str(SCRIPTS / "validate_plugin.py"))
manifest = scaffold["build_plugin_json"]("demo", with_mcp=False, with_apps=False) manifest = scaffold["build_plugin_json"](
"demo", with_mcp=False, with_apps=False
)
manifest["name"] = name manifest["name"] = name
errors: list[str] = [] errors: list[str] = []
validator["validate_manifest_shape"](self.plugin_root, manifest, errors) validator["validate_manifest_shape"](self.plugin_root, manifest, errors)
@@ -163,7 +167,9 @@ class PluginCreatorSecurityTests(unittest.TestCase):
with self.subTest(name=name): with self.subTest(name=name):
manifest_path = self.write_plugin(name) manifest_path = self.write_plugin(name)
original = manifest_path.read_bytes() original = manifest_path.read_bytes()
result = self.run_script("update_plugin_cachebuster.py", str(self.plugin_root)) result = self.run_script(
"update_plugin_cachebuster.py", str(self.plugin_root)
)
self.assertNotEqual(result.returncode, 0) self.assertNotEqual(result.returncode, 0)
self.assertEqual(result.stdout, "") self.assertEqual(result.stdout, "")
self.assertEqual(manifest_path.read_bytes(), original) self.assertEqual(manifest_path.read_bytes(), original)
@@ -206,7 +212,12 @@ class PluginCreatorSecurityTests(unittest.TestCase):
app_manifest.write_text('{"apps":{"existing":{}}}', encoding="utf-8") app_manifest.write_text('{"apps":{"existing":{}}}', encoding="utf-8")
originals = { originals = {
path: path.read_bytes() path: path.read_bytes()
for path in (self.marketplace_path, plugin_manifest, mcp_manifest, app_manifest) for path in (
self.marketplace_path,
plugin_manifest,
mcp_manifest,
app_manifest,
)
} }
result = self.run_script( result = self.run_script(
@@ -241,7 +252,12 @@ class PluginCreatorSecurityTests(unittest.TestCase):
app_manifest.write_text('{"apps":{"existing":{}}}', encoding="utf-8") app_manifest.write_text('{"apps":{"existing":{}}}', encoding="utf-8")
originals = { originals = {
path: path.read_bytes() path: path.read_bytes()
for path in (self.marketplace_path, plugin_manifest, mcp_manifest, app_manifest) for path in (
self.marketplace_path,
plugin_manifest,
mcp_manifest,
app_manifest,
)
} }
result = self.run_script( result = self.run_script(
@@ -264,7 +280,9 @@ class PluginCreatorSecurityTests(unittest.TestCase):
with self.subTest(path=path): with self.subTest(path=path):
self.assertEqual(path.read_bytes(), original) self.assertEqual(path.read_bytes(), original)
def test_scaffold_rejects_duplicate_marketplace_entry_before_creating_files(self) -> None: def test_scaffold_rejects_duplicate_marketplace_entry_before_creating_files(
self,
) -> None:
self.marketplace_path.write_text( self.marketplace_path.write_text(
json.dumps({"name": "team-local", "plugins": [{"name": "demo"}]}), json.dumps({"name": "team-local", "plugins": [{"name": "demo"}]}),
encoding="utf-8", encoding="utf-8",
+412 -104
View File
@@ -14,6 +14,7 @@ from pathlib import Path
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
from urllib.parse import urlsplit from urllib.parse import urlsplit
def _resolve_codex_cmd() -> List[str]: def _resolve_codex_cmd() -> List[str]:
"""Resolve the Codex CLI to invoke `codex sandbox windows`. """Resolve the Codex CLI to invoke `codex sandbox windows`.
@@ -30,10 +31,12 @@ def _resolve_codex_cmd() -> List[str]:
] ]
if cargo_target: if cargo_target:
cargo_base = Path(cargo_target) cargo_base = Path(cargo_target)
candidates.extend([ candidates.extend(
cargo_base / "debug" / "codex.exe", [
cargo_base / "release" / "codex.exe", cargo_base / "debug" / "codex.exe",
]) cargo_base / "release" / "codex.exe",
]
)
for candidate in candidates: for candidate in candidates:
if candidate.exists(): if candidate.exists():
@@ -49,20 +52,27 @@ def _resolve_codex_cmd() -> List[str]:
" cargo build -p codex-cli\n" " cargo build -p codex-cli\n"
) )
CODEX_CMD = _resolve_codex_cmd() CODEX_CMD = _resolve_codex_cmd()
print(CODEX_CMD) print(CODEX_CMD)
TIMEOUT_SEC = 20 TIMEOUT_SEC = 20
WS_ROOT = Path(os.environ["USERPROFILE"]) / "sbx_ws_tests" WS_ROOT = Path(os.environ["USERPROFILE"]) / "sbx_ws_tests"
OUTSIDE = Path(os.environ["USERPROFILE"]) / "sbx_ws_outside" # outside CWD for deny checks OUTSIDE = (
EXTRA_ROOT = Path(os.environ["USERPROFILE"]) / "WorkspaceRoot" # additional writable root Path(os.environ["USERPROFILE"]) / "sbx_ws_outside"
) # outside CWD for deny checks
EXTRA_ROOT = (
Path(os.environ["USERPROFILE"]) / "WorkspaceRoot"
) # additional writable root
ENV_BASE = {} # extend if needed ENV_BASE = {} # extend if needed
class CaseResult: class CaseResult:
def __init__(self, name: str, ok: bool, detail: str = ""): def __init__(self, name: str, ok: bool, detail: str = ""):
self.name, self.ok, self.detail = name, ok, detail self.name, self.ok, self.detail = name, ok, detail
def run_sbx( def run_sbx(
policy: str, policy: str,
cmd_argv: List[str], cmd_argv: List[str],
@@ -90,38 +100,61 @@ def run_sbx(
f'sandbox_workspace_write.writable_roots=["{additional_root.as_posix()}"]', f'sandbox_workspace_write.writable_roots=["{additional_root.as_posix()}"]',
] ]
argv = [*CODEX_CMD, "sandbox", "windows", *policy_flags, *overrides, "--", *cmd_argv] argv = [
*CODEX_CMD,
"sandbox",
"windows",
*policy_flags,
*overrides,
"--",
*cmd_argv,
]
print(cmd_argv) print(cmd_argv)
cp = subprocess.run(argv, cwd=str(cwd), env=env, cp = subprocess.run(
stdout=subprocess.PIPE, stderr=subprocess.PIPE, argv,
timeout=TIMEOUT_SEC, text=True) cwd=str(cwd),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=TIMEOUT_SEC,
text=True,
)
return cp.returncode, cp.stdout, cp.stderr return cp.returncode, cp.stdout, cp.stderr
def have(cmd: str) -> bool: def have(cmd: str) -> bool:
return shutil.which(cmd) is not None return shutil.which(cmd) is not None
def make_dir_clean(p: Path) -> None: def make_dir_clean(p: Path) -> None:
if p.exists(): if p.exists():
shutil.rmtree(p, ignore_errors=True) shutil.rmtree(p, ignore_errors=True)
p.mkdir(parents=True, exist_ok=True) p.mkdir(parents=True, exist_ok=True)
def write_file(p: Path, content: str = "x") -> None: def write_file(p: Path, content: str = "x") -> None:
p.parent.mkdir(parents=True, exist_ok=True) p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(content, encoding="utf-8") p.write_text(content, encoding="utf-8")
def remove_if_exists(p: Path) -> None: def remove_if_exists(p: Path) -> None:
try: try:
if p.is_dir(): shutil.rmtree(p, ignore_errors=True) if p.is_dir():
elif p.exists(): p.unlink(missing_ok=True) shutil.rmtree(p, ignore_errors=True)
elif p.exists():
p.unlink(missing_ok=True)
except Exception: except Exception:
pass pass
def assert_exists(p: Path) -> bool: def assert_exists(p: Path) -> bool:
return p.exists() return p.exists()
def assert_not_exists(p: Path) -> bool: def assert_not_exists(p: Path) -> bool:
return not p.exists() return not p.exists()
def make_junction(link: Path, target: Path) -> bool: def make_junction(link: Path, target: Path) -> bool:
"""Create a directory junction; return True if it exists afterward.""" """Create a directory junction; return True if it exists afterward."""
remove_if_exists(link) remove_if_exists(link)
@@ -130,6 +163,7 @@ def make_junction(link: Path, target: Path) -> bool:
cp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) cp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return cp.returncode == 0 and link.exists() return cp.returncode == 0 and link.exists()
def make_symlink(link: Path, target: Path) -> bool: def make_symlink(link: Path, target: Path) -> bool:
"""Create a directory symlink; return True if it exists afterward.""" """Create a directory symlink; return True if it exists afterward."""
remove_if_exists(link) remove_if_exists(link)
@@ -142,10 +176,12 @@ def make_symlink(link: Path, target: Path) -> bool:
cp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) cp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return cp.returncode == 0 and link.exists() return cp.returncode == 0 and link.exists()
class _QuietHandler(http.server.BaseHTTPRequestHandler): class _QuietHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, format, *args): def log_message(self, format, *args):
pass pass
class _TargetHandler(_QuietHandler): class _TargetHandler(_QuietHandler):
def do_GET(self): def do_GET(self):
body = b"proxy-ok" body = b"proxy-ok"
@@ -155,6 +191,7 @@ class _TargetHandler(_QuietHandler):
self.end_headers() self.end_headers()
self.wfile.write(body) self.wfile.write(body)
class _ProxyHandler(_QuietHandler): class _ProxyHandler(_QuietHandler):
def do_GET(self): def do_GET(self):
parsed = urlsplit(self.path) parsed = urlsplit(self.path)
@@ -169,7 +206,9 @@ class _ProxyHandler(_QuietHandler):
path = f"{path}?{parsed.query}" path = f"{path}?{parsed.query}"
conn = None conn = None
try: try:
conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=2) conn = http.client.HTTPConnection(
parsed.hostname, parsed.port or 80, timeout=2
)
conn.request("GET", path) conn.request("GET", path)
upstream = conn.getresponse() upstream = conn.getresponse()
body = upstream.read() body = upstream.read()
@@ -186,6 +225,7 @@ class _ProxyHandler(_QuietHandler):
self.end_headers() self.end_headers()
self.wfile.write(body) self.wfile.write(body)
@contextlib.contextmanager @contextlib.contextmanager
def start_loopback_proxy_fixture(): def start_loopback_proxy_fixture():
target = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _TargetHandler) target = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _TargetHandler)
@@ -204,16 +244,21 @@ def start_loopback_proxy_fixture():
proxy.server_close() proxy.server_close()
target.server_close() target.server_close()
def summarize(results: List[CaseResult]) -> int: def summarize(results: List[CaseResult]) -> int:
ok = sum(1 for r in results if r.ok) ok = sum(1 for r in results if r.ok)
total = len(results) total = len(results)
print("\n" + "=" * 72) print("\n" + "=" * 72)
print(f"Sandbox smoke tests: {ok}/{total} passed") print(f"Sandbox smoke tests: {ok}/{total} passed")
for r in results: for r in results:
print(f"[{'PASS' if r.ok else 'FAIL'}] {r.name}" + (f" :: {r.detail.strip()}" if r.detail and not r.ok else "")) print(
f"[{'PASS' if r.ok else 'FAIL'}] {r.name}"
+ (f" :: {r.detail.strip()}" if r.detail and not r.ok else "")
)
print("=" * 72) print("=" * 72)
return 0 if ok == total else 1 return 0 if ok == total else 1
def main() -> int: def main() -> int:
results: List[CaseResult] = [] results: List[CaseResult] = []
make_dir_clean(WS_ROOT) make_dir_clean(WS_ROOT)
@@ -229,26 +274,44 @@ def main() -> int:
ro_temp_denied = probe_rc != 0 ro_temp_denied = probe_rc != 0
def add(name: str, ok: bool, detail: str = ""): def add(name: str, ok: bool, detail: str = ""):
print('running', name) print("running", name)
results.append(CaseResult(name, ok, detail)) results.append(CaseResult(name, ok, detail))
# 1. RO: deny write in CWD # 1. RO: deny write in CWD
target = WS_ROOT / "ro_should_fail.txt" target = WS_ROOT / "ro_should_fail.txt"
remove_if_exists(target) remove_if_exists(target)
rc, out, err = run_sbx("read-only", ["cmd", "/c", "echo nope > ro_should_fail.txt"], WS_ROOT) rc, out, err = run_sbx(
add("RO: write in CWD denied", rc != 0 and assert_not_exists(target), f"rc={rc}, err={err}") "read-only", ["cmd", "/c", "echo nope > ro_should_fail.txt"], WS_ROOT
)
add(
"RO: write in CWD denied",
rc != 0 and assert_not_exists(target),
f"rc={rc}, err={err}",
)
# 2. WS: allow write in CWD # 2. WS: allow write in CWD
target = WS_ROOT / "ws_ok.txt" target = WS_ROOT / "ws_ok.txt"
remove_if_exists(target) remove_if_exists(target)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo ok > ws_ok.txt"], WS_ROOT) rc, out, err = run_sbx(
add("WS: write in CWD allowed", rc == 0 and assert_exists(target), f"rc={rc}, err={err}") "workspace-write", ["cmd", "/c", "echo ok > ws_ok.txt"], WS_ROOT
)
add(
"WS: write in CWD allowed",
rc == 0 and assert_exists(target),
f"rc={rc}, err={err}",
)
# 3. WS: deny write outside workspace # 3. WS: deny write outside workspace
outside_file = OUTSIDE / "blocked.txt" outside_file = OUTSIDE / "blocked.txt"
remove_if_exists(outside_file) remove_if_exists(outside_file)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", f"echo nope > {outside_file}"], WS_ROOT) rc, out, err = run_sbx(
add("WS: write outside workspace denied", rc != 0 and assert_not_exists(outside_file), f"rc={rc}") "workspace-write", ["cmd", "/c", f"echo nope > {outside_file}"], WS_ROOT
)
add(
"WS: write outside workspace denied",
rc != 0 and assert_not_exists(outside_file),
f"rc={rc}",
)
# 3b. WS: allow write in additional workspace root # 3b. WS: allow write in additional workspace root
extra_target = EXTRA_ROOT / "extra_ok.txt" extra_target = EXTRA_ROOT / "extra_ok.txt"
@@ -259,7 +322,11 @@ def main() -> int:
WS_ROOT, WS_ROOT,
additional_root=EXTRA_ROOT, additional_root=EXTRA_ROOT,
) )
add("WS: write in additional root allowed", rc == 0 and assert_exists(extra_target), f"rc={rc}, err={err}") add(
"WS: write in additional root allowed",
rc == 0 and assert_exists(extra_target),
f"rc={rc}, err={err}",
)
# 3c. RO: deny write in additional workspace root # 3c. RO: deny write in additional workspace root
ro_extra_target = EXTRA_ROOT / "extra_ro.txt" ro_extra_target = EXTRA_ROOT / "extra_ro.txt"
@@ -277,11 +344,17 @@ def main() -> int:
) )
# 4. WS: allow TEMP write # 4. WS: allow TEMP write
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo tempok > %TEMP%\\ws_temp_ok.txt"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write",
["cmd", "/c", "echo tempok > %TEMP%\\ws_temp_ok.txt"],
WS_ROOT,
)
add("WS: TEMP write allowed", rc == 0, f"rc={rc}") add("WS: TEMP write allowed", rc == 0, f"rc={rc}")
# 5. RO: deny TEMP write # 5. RO: deny TEMP write
rc, out, err = run_sbx("read-only", ["cmd", "/c", "echo tempno > %TEMP%\\ro_temp_fail.txt"], WS_ROOT) rc, out, err = run_sbx(
"read-only", ["cmd", "/c", "echo tempno > %TEMP%\\ro_temp_fail.txt"], WS_ROOT
)
if ro_temp_denied: if ro_temp_denied:
add("RO: TEMP write denied", rc != 0, f"rc={rc}") add("RO: TEMP write denied", rc != 0, f"rc={rc}")
else: else:
@@ -289,62 +362,145 @@ def main() -> int:
# 6. WS: append OK in CWD # 6. WS: append OK in CWD
target = WS_ROOT / "append.txt" target = WS_ROOT / "append.txt"
remove_if_exists(target); write_file(target, "line1\n") remove_if_exists(target)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo line2 >> append.txt"], WS_ROOT) write_file(target, "line1\n")
add("WS: append allowed", rc == 0 and target.read_text().strip().endswith("line2"), f"rc={rc}") rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "echo line2 >> append.txt"], WS_ROOT
)
add(
"WS: append allowed",
rc == 0 and target.read_text().strip().endswith("line2"),
f"rc={rc}",
)
# 7. RO: append denied # 7. RO: append denied
target = WS_ROOT / "ro_append.txt" target = WS_ROOT / "ro_append.txt"
write_file(target, "line1\n") write_file(target, "line1\n")
rc, out, err = run_sbx("read-only", ["cmd", "/c", "echo line2 >> ro_append.txt"], WS_ROOT) rc, out, err = run_sbx(
"read-only", ["cmd", "/c", "echo line2 >> ro_append.txt"], WS_ROOT
)
add("RO: append denied", rc != 0 and target.read_text() == "line1\n", f"rc={rc}") add("RO: append denied", rc != 0 and target.read_text() == "line1\n", f"rc={rc}")
# 8. WS: PowerShell Set-Content in CWD (OK) # 8. WS: PowerShell Set-Content in CWD (OK)
target = WS_ROOT / "ps_ok.txt" target = WS_ROOT / "ps_ok.txt"
remove_if_exists(target) remove_if_exists(target)
rc, out, err = run_sbx("workspace-write", rc, out, err = run_sbx(
["powershell", "-NoLogo", "-NoProfile", "-Command", "workspace-write",
"Set-Content -LiteralPath ps_ok.txt -Value 'hello' -Encoding ASCII"], WS_ROOT) [
add("WS: PowerShell Set-Content allowed", rc == 0 and assert_exists(target), f"rc={rc}, err={err}") "powershell",
"-NoLogo",
"-NoProfile",
"-Command",
"Set-Content -LiteralPath ps_ok.txt -Value 'hello' -Encoding ASCII",
],
WS_ROOT,
)
add(
"WS: PowerShell Set-Content allowed",
rc == 0 and assert_exists(target),
f"rc={rc}, err={err}",
)
# 9. RO: PowerShell Set-Content denied # 9. RO: PowerShell Set-Content denied
target = WS_ROOT / "ps_ro_fail.txt" target = WS_ROOT / "ps_ro_fail.txt"
remove_if_exists(target) remove_if_exists(target)
rc, out, err = run_sbx("read-only", rc, out, err = run_sbx(
["powershell", "-NoLogo", "-NoProfile", "-Command", "read-only",
"Set-Content -LiteralPath ps_ro_fail.txt -Value 'x'"], WS_ROOT) [
add("RO: PowerShell Set-Content denied", rc != 0 and assert_not_exists(target), f"rc={rc}") "powershell",
"-NoLogo",
"-NoProfile",
"-Command",
"Set-Content -LiteralPath ps_ro_fail.txt -Value 'x'",
],
WS_ROOT,
)
add(
"RO: PowerShell Set-Content denied",
rc != 0 and assert_not_exists(target),
f"rc={rc}",
)
# 10. WS: mkdir and write (OK) # 10. WS: mkdir and write (OK)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "mkdir sub && echo hi > sub\\in_sub.txt"], WS_ROOT) rc, out, err = run_sbx(
add("WS: mkdir+write allowed", rc == 0 and (WS_ROOT / "sub/in_sub.txt").exists(), f"rc={rc}") "workspace-write",
["cmd", "/c", "mkdir sub && echo hi > sub\\in_sub.txt"],
WS_ROOT,
)
add(
"WS: mkdir+write allowed",
rc == 0 and (WS_ROOT / "sub/in_sub.txt").exists(),
f"rc={rc}",
)
# 11. WS: rename (EXPECTED SUCCESS on this host) # 11. WS: rename (EXPECTED SUCCESS on this host)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo x > r.txt & ren r.txt r2.txt"], WS_ROOT) rc, out, err = run_sbx(
add("WS: rename succeeds (expected on this host)", rc == 0 and (WS_ROOT / "r2.txt").exists(), f"rc={rc}, err={err}") "workspace-write", ["cmd", "/c", "echo x > r.txt & ren r.txt r2.txt"], WS_ROOT
)
add(
"WS: rename succeeds (expected on this host)",
rc == 0 and (WS_ROOT / "r2.txt").exists(),
f"rc={rc}, err={err}",
)
# 12. WS: delete (EXPECTED SUCCESS on this host) # 12. WS: delete (EXPECTED SUCCESS on this host)
target = WS_ROOT / "delme.txt"; write_file(target, "x") target = WS_ROOT / "delme.txt"
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "del /q delme.txt"], WS_ROOT) write_file(target, "x")
add("WS: delete succeeds (expected on this host)", rc == 0 and not target.exists(), f"rc={rc}, err={err}") rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "del /q delme.txt"], WS_ROOT
)
add(
"WS: delete succeeds (expected on this host)",
rc == 0 and not target.exists(),
f"rc={rc}, err={err}",
)
# 13. RO: python tries to write (denied) # 13. RO: python tries to write (denied)
pyfile = WS_ROOT / "py_should_fail.txt"; remove_if_exists(pyfile) pyfile = WS_ROOT / "py_should_fail.txt"
rc, out, err = run_sbx("read-only", ["python", "-c", "open('py_should_fail.txt','w').write('x')"], WS_ROOT) remove_if_exists(pyfile)
add("RO: python file write denied", rc != 0 and assert_not_exists(pyfile), f"rc={rc}") rc, out, err = run_sbx(
"read-only",
["python", "-c", "open('py_should_fail.txt','w').write('x')"],
WS_ROOT,
)
add(
"RO: python file write denied",
rc != 0 and assert_not_exists(pyfile),
f"rc={rc}",
)
# 14. WS: python writes file (OK) # 14. WS: python writes file (OK)
pyfile = WS_ROOT / "py_ok.txt"; remove_if_exists(pyfile) pyfile = WS_ROOT / "py_ok.txt"
rc, out, err = run_sbx("workspace-write", ["python", "-c", "open('py_ok.txt','w').write('x')"], WS_ROOT) remove_if_exists(pyfile)
add("WS: python file write allowed", rc == 0 and assert_exists(pyfile), f"rc={rc}, err={err}") rc, out, err = run_sbx(
"workspace-write", ["python", "-c", "open('py_ok.txt','w').write('x')"], WS_ROOT
)
add(
"WS: python file write allowed",
rc == 0 and assert_exists(pyfile),
f"rc={rc}, err={err}",
)
# 15. WS: curl network blocked (short timeout) # 15. WS: curl network blocked (short timeout)
rc, out, err = run_sbx("workspace-write", ["curl", "--connect-timeout", "1", "--max-time", "2", "https://example.com"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write",
["curl", "--connect-timeout", "1", "--max-time", "2", "https://example.com"],
WS_ROOT,
)
add("WS: curl network blocked", rc != 0, f"rc={rc}") add("WS: curl network blocked", rc != 0, f"rc={rc}")
# 16. WS: iwr network blocked (HTTP) # 16. WS: iwr network blocked (HTTP)
rc, out, err = run_sbx("workspace-write", ["powershell", "-NoLogo", "-NoProfile", "-Command", rc, out, err = run_sbx(
"try { iwr http://neverssl.com -TimeoutSec 2 } catch { exit 1 }"], WS_ROOT) "workspace-write",
[
"powershell",
"-NoLogo",
"-NoProfile",
"-Command",
"try { iwr http://neverssl.com -TimeoutSec 2 } catch { exit 1 }",
],
WS_ROOT,
)
add("WS: iwr network blocked", rc != 0, f"rc={rc}") add("WS: iwr network blocked", rc != 0, f"rc={rc}")
# 17. WS: direct loopback blocked, proxy loopback allowed via env proxy # 17. WS: direct loopback blocked, proxy loopback allowed via env proxy
@@ -401,14 +557,28 @@ def main() -> int:
WS_ROOT, WS_ROOT,
env_extra={"CODEX_HOME": str(proxy_home)}, env_extra={"CODEX_HOME": str(proxy_home)},
) )
add("WS: direct loopback blocked", rc_direct != 0, f"rc={rc_direct}, err={err_direct}") add(
"WS: direct loopback blocked",
rc_direct != 0,
f"rc={rc_direct}, err={err_direct}",
)
else: else:
add("WS: direct/proxy loopback tests (curl missing)", True, "curl not installed") add(
"WS: direct/proxy loopback tests (curl missing)", True, "curl not installed"
)
# 18. RO: deny TEMP writes via PowerShell # 18. RO: deny TEMP writes via PowerShell
rc, out, err = run_sbx("read-only", rc, out, err = run_sbx(
["powershell", "-NoLogo", "-NoProfile", "-Command", "read-only",
"Set-Content -LiteralPath $env:TEMP\\ro_tmpfail.txt -Value 'x'"], WS_ROOT) [
"powershell",
"-NoLogo",
"-NoProfile",
"-Command",
"Set-Content -LiteralPath $env:TEMP\\ro_tmpfail.txt -Value 'x'",
],
WS_ROOT,
)
if ro_temp_denied: if ro_temp_denied:
add("RO: TEMP write denied (PS)", rc != 0, f"rc={rc}") add("RO: TEMP write denied (PS)", rc != 0, f"rc={rc}")
else: else:
@@ -416,14 +586,18 @@ def main() -> int:
# 19. WS: curl version check — don't rely on stub, just succeed # 19. WS: curl version check — don't rely on stub, just succeed
if have("curl"): if have("curl"):
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "curl --version"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "curl --version"], WS_ROOT
)
add("WS: curl present (version prints)", rc == 0, f"rc={rc}, err={err}") add("WS: curl present (version prints)", rc == 0, f"rc={rc}, err={err}")
else: else:
add("WS: curl present (optional, skipped)", True) add("WS: curl present (optional, skipped)", True)
# 20. Optional: ripgrep version # 20. Optional: ripgrep version
if have("rg"): if have("rg"):
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "rg --version"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "rg --version"], WS_ROOT
)
add("WS: rg --version (optional)", rc == 0, f"rc={rc}, err={err}") add("WS: rg --version (optional)", rc == 0, f"rc={rc}, err={err}")
else: else:
add("WS: rg --version (optional, skipped)", True) add("WS: rg --version (optional, skipped)", True)
@@ -436,30 +610,71 @@ def main() -> int:
add("WS: git --version (optional, skipped)", True) add("WS: git --version (optional, skipped)", True)
# 24. WS: PS bytes write (OK) # 24. WS: PS bytes write (OK)
rc, out, err = run_sbx("workspace-write", rc, out, err = run_sbx(
["powershell", "-NoLogo", "-NoProfile", "-Command", "workspace-write",
"[IO.File]::WriteAllBytes('bytes_ok.bin',[byte[]](0..255))"], WS_ROOT) [
add("WS: PS bytes write allowed", rc == 0 and (WS_ROOT / "bytes_ok.bin").exists(), f"rc={rc}") "powershell",
"-NoLogo",
"-NoProfile",
"-Command",
"[IO.File]::WriteAllBytes('bytes_ok.bin',[byte[]](0..255))",
],
WS_ROOT,
)
add(
"WS: PS bytes write allowed",
rc == 0 and (WS_ROOT / "bytes_ok.bin").exists(),
f"rc={rc}",
)
# 25. RO: PS bytes write denied # 25. RO: PS bytes write denied
rc, out, err = run_sbx("read-only", rc, out, err = run_sbx(
["powershell", "-NoLogo", "-NoProfile", "-Command", "read-only",
"[IO.File]::WriteAllBytes('bytes_fail.bin',[byte[]](0..10))"], WS_ROOT) [
add("RO: PS bytes write denied", rc != 0 and not (WS_ROOT / "bytes_fail.bin").exists(), f"rc={rc}") "powershell",
"-NoLogo",
"-NoProfile",
"-Command",
"[IO.File]::WriteAllBytes('bytes_fail.bin',[byte[]](0..10))",
],
WS_ROOT,
)
add(
"RO: PS bytes write denied",
rc != 0 and not (WS_ROOT / "bytes_fail.bin").exists(),
f"rc={rc}",
)
# 26. WS: deep mkdir and write (OK) # 26. WS: deep mkdir and write (OK)
rc, out, err = run_sbx("workspace-write", rc, out, err = run_sbx(
["cmd", "/c", "mkdir deep\\nest && echo ok > deep\\nest\\f.txt"], WS_ROOT) "workspace-write",
add("WS: deep mkdir+write allowed", rc == 0 and (WS_ROOT / "deep/nest/f.txt").exists(), f"rc={rc}") ["cmd", "/c", "mkdir deep\\nest && echo ok > deep\\nest\\f.txt"],
WS_ROOT,
)
add(
"WS: deep mkdir+write allowed",
rc == 0 and (WS_ROOT / "deep/nest/f.txt").exists(),
f"rc={rc}",
)
# 27. WS: move (EXPECTED SUCCESS on this host) # 27. WS: move (EXPECTED SUCCESS on this host)
rc, out, err = run_sbx("workspace-write", rc, out, err = run_sbx(
["cmd", "/c", "echo x > m1.txt & move /y m1.txt m2.txt"], WS_ROOT) "workspace-write",
add("WS: move succeeds (expected on this host)", rc == 0 and (WS_ROOT / "m2.txt").exists(), f"rc={rc}, err={err}") ["cmd", "/c", "echo x > m1.txt & move /y m1.txt m2.txt"],
WS_ROOT,
)
add(
"WS: move succeeds (expected on this host)",
rc == 0 and (WS_ROOT / "m2.txt").exists(),
f"rc={rc}, err={err}",
)
# 28. RO: cmd redirection denied # 28. RO: cmd redirection denied
target = WS_ROOT / "cmd_ro.txt"; remove_if_exists(target) target = WS_ROOT / "cmd_ro.txt"
rc, out, err = run_sbx("read-only", ["cmd", "/c", "echo nope > cmd_ro.txt"], WS_ROOT) remove_if_exists(target)
rc, out, err = run_sbx(
"read-only", ["cmd", "/c", "echo nope > cmd_ro.txt"], WS_ROOT
)
add("RO: cmd redirection denied", rc != 0 and not target.exists(), f"rc={rc}") add("RO: cmd redirection denied", rc != 0 and not target.exists(), f"rc={rc}")
# 29. WS: CWD junction poisoning denied (allowlist should not follow to OUTSIDE) # 29. WS: CWD junction poisoning denied (allowlist should not follow to OUTSIDE)
@@ -467,10 +682,20 @@ def main() -> int:
if make_junction(poison_cwd, OUTSIDE): if make_junction(poison_cwd, OUTSIDE):
target = OUTSIDE / "poisoned.txt" target = OUTSIDE / "poisoned.txt"
remove_if_exists(target) remove_if_exists(target)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo poison > poisoned.txt"], poison_cwd) rc, out, err = run_sbx(
add("WS: junction poisoning via CWD denied", rc != 0 and assert_not_exists(target), f"rc={rc}, err={err}") "workspace-write", ["cmd", "/c", "echo poison > poisoned.txt"], poison_cwd
)
add(
"WS: junction poisoning via CWD denied",
rc != 0 and assert_not_exists(target),
f"rc={rc}, err={err}",
)
else: else:
add("WS: junction poisoning via CWD denied (setup skipped)", True, "junction creation failed") add(
"WS: junction poisoning via CWD denied (setup skipped)",
True,
"junction creation failed",
)
# 30. WS: junction into Windows denied # 30. WS: junction into Windows denied
sys_link = WS_ROOT / "sys_link" sys_link = WS_ROOT / "sys_link"
@@ -479,44 +704,76 @@ def main() -> int:
if sys_file.exists(): if sys_file.exists():
remove_if_exists(sys_file) remove_if_exists(sys_file)
if make_junction(sys_link, sys_target): if make_junction(sys_link, sys_target):
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo bad > sys_link\\system32\\sbx_junc.txt"], WS_ROOT) rc, out, err = run_sbx(
add("WS: junction into Windows denied", rc != 0 and not sys_file.exists(), f"rc={rc}, err={err}") "workspace-write",
["cmd", "/c", "echo bad > sys_link\\system32\\sbx_junc.txt"],
WS_ROOT,
)
add(
"WS: junction into Windows denied",
rc != 0 and not sys_file.exists(),
f"rc={rc}, err={err}",
)
else: else:
add("WS: junction into Windows denied (setup skipped)", True, "junction creation failed") add(
"WS: junction into Windows denied (setup skipped)",
True,
"junction creation failed",
)
# 31. WS: device/pipe access blocked # 31. WS: device/pipe access blocked
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "type \\\\.\\PhysicalDrive0"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "type \\\\.\\PhysicalDrive0"], WS_ROOT
)
add("WS: raw device access denied", rc != 0, f"rc={rc}") add("WS: raw device access denied", rc != 0, f"rc={rc}")
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo hi > \\\\.\\pipe\\codex_testpipe"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write",
["cmd", "/c", "echo hi > \\\\.\\pipe\\codex_testpipe"],
WS_ROOT,
)
add("WS: named pipe creation denied", rc != 0, f"rc={rc}") add("WS: named pipe creation denied", rc != 0, f"rc={rc}")
# 32. WS: ADS/long-path escape denied # 32. WS: ADS/long-path escape denied
ads_base = WS_ROOT / "ads_base.txt" ads_base = WS_ROOT / "ads_base.txt"
remove_if_exists(ads_base) remove_if_exists(ads_base)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo secret > ads_base.txt:stream"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "echo secret > ads_base.txt:stream"], WS_ROOT
)
add("WS: ADS write denied", rc != 0 and assert_not_exists(ads_base), f"rc={rc}") add("WS: ADS write denied", rc != 0 and assert_not_exists(ads_base), f"rc={rc}")
lp_target = Path(r"\\?\C:\sbx_longpath_test.txt") lp_target = Path(r"\\?\C:\sbx_longpath_test.txt")
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo long > \\\\?\\C:\\sbx_longpath_test.txt"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write",
["cmd", "/c", "echo long > \\\\?\\C:\\sbx_longpath_test.txt"],
WS_ROOT,
)
add("WS: long-path escape denied", rc != 0 and not lp_target.exists(), f"rc={rc}") add("WS: long-path escape denied", rc != 0 and not lp_target.exists(), f"rc={rc}")
# 33. WS: case-insensitive protected path bypass denied (.GiT) # 33. WS: case-insensitive protected path bypass denied (.GiT)
git_variation = WS_ROOT / ".GiT" / "config" git_variation = WS_ROOT / ".GiT" / "config"
remove_if_exists(git_variation.parent) remove_if_exists(git_variation.parent)
git_variation.parent.mkdir(exist_ok=True) git_variation.parent.mkdir(exist_ok=True)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo hack > .GiT\\config"], WS_ROOT) rc, out, err = run_sbx(
add("WS: protected path case-variation denied", rc != 0 and assert_not_exists(git_variation), f"rc={rc}") "workspace-write", ["cmd", "/c", "echo hack > .GiT\\config"], WS_ROOT
)
add(
"WS: protected path case-variation denied",
rc != 0 and assert_not_exists(git_variation),
f"rc={rc}",
)
# 34. WS: policy tamper (.codex artifacts) denied # 34. WS: policy tamper (.codex artifacts) denied
codex_home = Path(os.environ["USERPROFILE"]) / ".codex" codex_home = Path(os.environ["USERPROFILE"]) / ".codex"
cap_sid_target = codex_home / "cap_sid" cap_sid_target = codex_home / "cap_sid"
rc, out, err = run_sbx( rc, out, err = run_sbx(
"workspace-write", "workspace-write",
["cmd", "/c", f"echo tamper > \"{cap_sid_target}\""], ["cmd", "/c", f'echo tamper > "{cap_sid_target}"'],
WS_ROOT, WS_ROOT,
) )
rc2, out2, err2 = run_sbx("workspace-write", ["cmd", "/c", "echo tamper > .codex\\policy.json"], WS_ROOT) rc2, out2, err2 = run_sbx(
"workspace-write", ["cmd", "/c", "echo tamper > .codex\\policy.json"], WS_ROOT
)
add("WS: .codex cap_sid tamper denied", rc != 0, f"rc={rc}, err={err}") add("WS: .codex cap_sid tamper denied", rc != 0, f"rc={rc}, err={err}")
add("WS: .codex policy tamper denied", rc2 != 0, f"rc={rc2}, err={err2}") add("WS: .codex policy tamper denied", rc2 != 0, f"rc={rc2}, err={err2}")
@@ -531,7 +788,9 @@ def main() -> int:
shim = tools_dir / "ssh.bat" shim = tools_dir / "ssh.bat"
shim.write_text("@echo off\r\necho stubbed\r\n", encoding="utf-8") shim.write_text("@echo off\r\necho stubbed\r\n", encoding="utf-8")
env = {"PATH": f"{tools_dir};%PATH%"} env = {"PATH": f"{tools_dir};%PATH%"}
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "ssh"], WS_ROOT, env_extra=env) rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "ssh"], WS_ROOT, env_extra=env
)
add("WS: PATH stub bypass denied", "stubbed" in out, f"rc={rc}, out={out}") add("WS: PATH stub bypass denied", "stubbed" in out, f"rc={rc}, out={out}")
else: else:
add("WS: PATH stub bypass denied (ssh missing)", True, "ssh not installed") add("WS: PATH stub bypass denied (ssh missing)", True, "ssh not installed")
@@ -553,44 +812,84 @@ def main() -> int:
"/c", "/c",
f'for /L %i in (1,1,400) do (rmdir flip & mklink /D flip "{inside_abs}" >NUL & rmdir flip & mklink /D flip "{outside_abs}" >NUL)', f'for /L %i in (1,1,400) do (rmdir flip & mklink /D flip "{inside_abs}" >NUL & rmdir flip & mklink /D flip "{outside_abs}" >NUL)',
] ]
subprocess.Popen(toggle, cwd=str(race_root), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) subprocess.Popen(
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo race > flip\\race.txt"], race_root) toggle, cwd=str(race_root), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
add("WS: symlink race write denied (best-effort)", rc != 0 and not (outside / "race.txt").exists(), f"rc={rc}") )
rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "echo race > flip\\race.txt"], race_root
)
add(
"WS: symlink race write denied (best-effort)",
rc != 0 and not (outside / "race.txt").exists(),
f"rc={rc}",
)
# 37. WS: audit blind spots – deep junction/world-writable denied # 37. WS: audit blind spots – deep junction/world-writable denied
deep = WS_ROOT / "deep" / "redir" deep = WS_ROOT / "deep" / "redir"
unsafe_dir = WS_ROOT / "deep" / "unsafe" unsafe_dir = WS_ROOT / "deep" / "unsafe"
make_junction(deep, Path("C:/Windows")) make_junction(deep, Path("C:/Windows"))
unsafe_dir.mkdir(parents=True, exist_ok=True) unsafe_dir.mkdir(parents=True, exist_ok=True)
subprocess.run(["icacls", str(unsafe_dir), "/grant", "Everyone:(F)"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) subprocess.run(
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo probe > deep\\redir\\system32\\audit_gap.txt"], WS_ROOT) ["icacls", str(unsafe_dir), "/grant", "Everyone:(F)"],
add("WS: deep junction/world-writable escape denied", rc != 0, f"rc={rc}, err={err}") stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
rc, out, err = run_sbx(
"workspace-write",
["cmd", "/c", "echo probe > deep\\redir\\system32\\audit_gap.txt"],
WS_ROOT,
)
add(
"WS: deep junction/world-writable escape denied", rc != 0, f"rc={rc}, err={err}"
)
# 38. WS: policy poisoning via workspace symlink root denied # 38. WS: policy poisoning via workspace symlink root denied
# Simulate workspace replaced by symlink to C:\; expect writes to be denied. # Simulate workspace replaced by symlink to C:\; expect writes to be denied.
fake_root = WS_ROOT / "fake_root" fake_root = WS_ROOT / "fake_root"
if make_symlink(fake_root, Path("C:/")): if make_symlink(fake_root, Path("C:/")):
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo owned > codex_escape.txt"], fake_root) rc, out, err = run_sbx(
"workspace-write", ["cmd", "/c", "echo owned > codex_escape.txt"], fake_root
)
add("WS: workspace-root symlink poisoning denied", rc != 0, f"rc={rc}") add("WS: workspace-root symlink poisoning denied", rc != 0, f"rc={rc}")
else: else:
add("WS: workspace-root symlink poisoning denied (setup skipped)", True, "symlink creation failed") add(
"WS: workspace-root symlink poisoning denied (setup skipped)",
True,
"symlink creation failed",
)
# 39. WS: UNC/other-drive canonicalization denied # 39. WS: UNC/other-drive canonicalization denied
unc_link = WS_ROOT / "unc_link" unc_link = WS_ROOT / "unc_link"
other_to = Path(r"\\\\localhost\\C$") other_to = Path(r"\\\\localhost\\C$")
if make_symlink(unc_link, other_to): if make_symlink(unc_link, other_to):
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo unc > unc_link\\unc_test.txt"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write",
["cmd", "/c", "echo unc > unc_link\\unc_test.txt"],
WS_ROOT,
)
add("WS: UNC link escape denied", rc != 0, f"rc={rc}") add("WS: UNC link escape denied", rc != 0, f"rc={rc}")
else: else:
add("WS: UNC link escape denied (setup skipped)", True, "symlink creation failed") add(
"WS: UNC link escape denied (setup skipped)",
True,
"symlink creation failed",
)
other_drive = WS_ROOT / "other_drive" other_drive = WS_ROOT / "other_drive"
other_target = Path("D:/") # best-effort; may not exist other_target = Path("D:/") # best-effort; may not exist
if make_symlink(other_drive, other_target): if make_symlink(other_drive, other_target):
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo drive > other_drive\\drive.txt"], WS_ROOT) rc, out, err = run_sbx(
"workspace-write",
["cmd", "/c", "echo drive > other_drive\\drive.txt"],
WS_ROOT,
)
add("WS: other-drive link escape denied", rc != 0, f"rc={rc}") add("WS: other-drive link escape denied", rc != 0, f"rc={rc}")
else: else:
add("WS: other-drive link escape denied (setup skipped)", True, "symlink creation failed") add(
"WS: other-drive link escape denied (setup skipped)",
True,
"symlink creation failed",
)
# 40. WS: timeout cleanup still denies outside write # 40. WS: timeout cleanup still denies outside write
slow_ps = WS_ROOT / "sleep.ps1" slow_ps = WS_ROOT / "sleep.ps1"
@@ -601,8 +900,16 @@ def main() -> int:
pass pass
outside_after_timeout = OUTSIDE / "timeout_leak.txt" outside_after_timeout = OUTSIDE / "timeout_leak.txt"
remove_if_exists(outside_after_timeout) remove_if_exists(outside_after_timeout)
rc, out, err = run_sbx("workspace-write", ["cmd", "/c", f"echo leak > {outside_after_timeout}"], WS_ROOT) rc, out, err = run_sbx(
add("WS: post-timeout outside write still denied", rc != 0 and assert_not_exists(outside_after_timeout), f"rc={rc}") "workspace-write",
["cmd", "/c", f"echo leak > {outside_after_timeout}"],
WS_ROOT,
)
add(
"WS: post-timeout outside write still denied",
rc != 0 and assert_not_exists(outside_after_timeout),
f"rc={rc}",
)
# 41. RO: Start-Process https blocked (KNOWN FAIL until GUI escape fixed) # 41. RO: Start-Process https blocked (KNOWN FAIL until GUI escape fixed)
rc, out, err = run_sbx( rc, out, err = run_sbx(
@@ -624,5 +931,6 @@ def main() -> int:
return summarize(results) return summarize(results)
if __name__ == "__main__": if __name__ == "__main__":
sys.exit(main()) sys.exit(main())
+1
View File
@@ -0,0 +1 @@
extend-exclude = ["sdk/python", "codex-rs/vendor"]
+2 -12
View File
@@ -116,20 +116,10 @@ def python_sdk_formatter_group(*, check: bool) -> FormatterGroup:
def python_scripts_formatter_group(*, check: bool) -> FormatterGroup: def python_scripts_formatter_group(*, check: bool) -> FormatterGroup:
# The SDK and internal scripts intentionally use separate project roots so args = ["uv", "run", "--frozen", "--project", "scripts", "ruff", "format"]
# uv and Ruff retain each project's configuration context.
args = [
"uv",
"run",
"--frozen",
"--project",
"scripts",
"ruff",
"format",
]
if check: if check:
args.append("--check") args.append("--check")
args.append("scripts") args.append(".")
return FormatterGroup("Python scripts", (Command(tuple(args)),)) return FormatterGroup("Python scripts", (Command(tuple(args)),))
@@ -272,11 +272,11 @@ def test_root_format_driver_covers_all_formatter_groups(
) )
assert [group.commands[-1].args[-3:] for group in formatters[3:]] == [ assert [group.commands[-1].args[-3:] for group in formatters[3:]] == [
("ruff", "format", "sdk/python"), ("ruff", "format", "sdk/python"),
("ruff", "format", "scripts"), ("ruff", "format", "."),
] ]
assert [group.commands[-1].args[-4:] for group in checks[3:]] == [ assert [group.commands[-1].args[-4:] for group in checks[3:]] == [
("ruff", "format", "--check", "sdk/python"), ("ruff", "format", "--check", "sdk/python"),
("ruff", "format", "--check", "scripts"), ("ruff", "format", "--check", "."),
] ]
+26 -8
View File
@@ -11,17 +11,26 @@ import shutil
def assemble(package: Path, helper: Path, voice_target: str, commit: str, output: Path): def assemble(package: Path, helper: Path, voice_target: str, commit: str, output: Path):
package, helper = package.resolve(strict=True), helper.resolve(strict=True) package, helper = package.resolve(strict=True), helper.resolve(strict=True)
output = output.absolute() output = output.absolute()
if output.exists() or output.is_symlink() or output.resolve().is_relative_to(package): if (
output.exists()
or output.is_symlink()
or output.resolve().is_relative_to(package)
):
raise ValueError("output must be fresh and outside the input package") raise ValueError("output must be fresh and outside the input package")
if not re.fullmatch(r"[0-9a-f]{40}", commit): if not re.fullmatch(r"[0-9a-f]{40}", commit):
raise ValueError("a full build commit is required; dev builds are not distributable") raise ValueError(
"a full build commit is required; dev builds are not distributable"
)
metadata = json.loads((package / "codex-package.json").read_text()) metadata = json.loads((package / "codex-package.json").read_text())
app_target = metadata["target"] app_target = metadata["target"]
targets = { targets = {
f"{arch}-{suffix}": f"{arch}-{suffix.replace('musl', 'gnu')}" f"{arch}-{suffix}": f"{arch}-{suffix.replace('musl', 'gnu')}"
for arch in ("aarch64", "x86_64") for arch in ("aarch64", "x86_64")
for suffix in ( for suffix in (
"apple-darwin", "unknown-linux-gnu", "unknown-linux-musl", "pc-windows-msvc" "apple-darwin",
"unknown-linux-gnu",
"unknown-linux-musl",
"pc-windows-msvc",
) )
} }
if targets.get(app_target) != voice_target: if targets.get(app_target) != voice_target:
@@ -29,8 +38,11 @@ def assemble(package: Path, helper: Path, voice_target: str, commit: str, output
suffix = ".exe" if app_target.endswith("windows-msvc") else "" suffix = ".exe" if app_target.endswith("windows-msvc") else ""
entrypoint = f"bin/codex{suffix}" entrypoint = f"bin/codex{suffix}"
expected = { expected = {
"layoutVersion": 1, "variant": "codex", "entrypoint": entrypoint, "layoutVersion": 1,
"resourcesDir": "codex-resources", "pathDir": "codex-path", "variant": "codex",
"entrypoint": entrypoint,
"resourcesDir": "codex-resources",
"pathDir": "codex-path",
} }
if any(metadata.get(key) != value for key, value in expected.items()): if any(metadata.get(key) != value for key, value in expected.items()):
raise ValueError("input is not a canonical Codex package") raise ValueError("input is not a canonical Codex package")
@@ -57,8 +69,12 @@ def assemble(package: Path, helper: Path, voice_target: str, commit: str, output
with (output / relative).open("rb") as source: with (output / relative).open("rb") as source:
digests[relative] = hashlib.file_digest(source, "sha256").hexdigest() digests[relative] = hashlib.file_digest(source, "sha256").hexdigest()
manifest = { manifest = {
"schemaVersion": 1, "buildCommit": commit, "appTarget": app_target, "schemaVersion": 1,
"voiceTarget": voice_target, "appVersion": metadata["version"], "sha256": digests, "buildCommit": commit,
"appTarget": app_target,
"voiceTarget": voice_target,
"appVersion": metadata["version"],
"sha256": digests,
} }
(destination.parent.parent / "manifest.json").write_text( (destination.parent.parent / "manifest.json").write_text(
json.dumps(manifest, indent=2) + "\n", encoding="utf-8" json.dumps(manifest, indent=2) + "\n", encoding="utf-8"
@@ -76,4 +92,6 @@ if __name__ == "__main__":
parser.add_argument("--build-commit", required=True) parser.add_argument("--build-commit", required=True)
parser.add_argument("--output", type=Path, required=True) parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args() args = parser.parse_args()
assemble(args.package, args.helper, args.voice_target, args.build_commit, args.output) assemble(
args.package, args.helper, args.voice_target, args.build_commit, args.output
)
+62 -19
View File
@@ -21,9 +21,13 @@ class AssembleTests(unittest.TestCase):
(self.package / "codex-path").mkdir() (self.package / "codex-path").mkdir()
self.commit = "a" * 40 self.commit = "a" * 40
self.metadata = { self.metadata = {
"layoutVersion": 1, "version": f"0.0.0+{self.commit}", "layoutVersion": 1,
"target": "aarch64-unknown-linux-musl", "variant": "codex", "version": f"0.0.0+{self.commit}",
"entrypoint": "bin/codex", "resourcesDir": "codex-resources", "pathDir": "codex-path", "target": "aarch64-unknown-linux-musl",
"variant": "codex",
"entrypoint": "bin/codex",
"resourcesDir": "codex-resources",
"pathDir": "codex-path",
} }
(self.package / "codex-package.json").write_text(json.dumps(self.metadata)) (self.package / "codex-package.json").write_text(json.dumps(self.metadata))
(self.package / "bin/codex").write_bytes(b"unchanged app") (self.package / "bin/codex").write_bytes(b"unchanged app")
@@ -33,7 +37,13 @@ class AssembleTests(unittest.TestCase):
self.output = self.root / "installed copy" self.output = self.root / "installed copy"
def test_copies_app_unchanged_and_records_distinct_linux_targets(self): def test_copies_app_unchanged_and_records_distinct_linux_targets(self):
assemble(self.package, self.helper, "aarch64-unknown-linux-gnu", self.commit, self.output) assemble(
self.package,
self.helper,
"aarch64-unknown-linux-gnu",
self.commit,
self.output,
)
self.assertEqual((self.output / "bin/codex").read_bytes(), b"unchanged app") self.assertEqual((self.output / "bin/codex").read_bytes(), b"unchanged app")
self.assertEqual((self.package / "bin/codex").read_bytes(), b"unchanged app") self.assertEqual((self.package / "bin/codex").read_bytes(), b"unchanged app")
self.assertFalse((self.package / "codex-resources/voice").exists()) self.assertFalse((self.package / "codex-resources/voice").exists())
@@ -42,13 +52,20 @@ class AssembleTests(unittest.TestCase):
(self.package / "codex-package.json").read_bytes(), (self.package / "codex-package.json").read_bytes(),
) )
self.assertEqual( self.assertEqual(
json.loads((self.output / "codex-resources/voice/manifest.json").read_text()), json.loads(
(self.output / "codex-resources/voice/manifest.json").read_text()
),
{ {
"schemaVersion": 1, "buildCommit": self.commit, "schemaVersion": 1,
"appTarget": self.metadata["target"], "voiceTarget": "aarch64-unknown-linux-gnu", "buildCommit": self.commit,
"appVersion": self.metadata["version"], "sha256": { "appTarget": self.metadata["target"],
"voiceTarget": "aarch64-unknown-linux-gnu",
"appVersion": self.metadata["version"],
"sha256": {
"bin/codex": hashlib.sha256(b"unchanged app").hexdigest(), "bin/codex": hashlib.sha256(b"unchanged app").hexdigest(),
"codex-resources/voice/bin/codex-voice-host": hashlib.sha256(b"private helper").hexdigest(), "codex-resources/voice/bin/codex-voice-host": hashlib.sha256(
b"private helper"
).hexdigest(),
}, },
}, },
) )
@@ -60,7 +77,10 @@ class AssembleTests(unittest.TestCase):
("aarch64-unknown-linux-gnu", "dev"), ("aarch64-unknown-linux-gnu", "dev"),
("aarch64-unknown-linux-gnu", "b" * 40), ("aarch64-unknown-linux-gnu", "b" * 40),
]: ]:
with self.subTest(target=target, commit=commit), self.assertRaises(ValueError): with (
self.subTest(target=target, commit=commit),
self.assertRaises(ValueError),
):
assemble(self.package, self.helper, target, commit, self.output) assemble(self.package, self.helper, target, commit, self.output)
self.assertFalse(self.output.exists()) self.assertFalse(self.output.exists())
@@ -69,38 +89,61 @@ class AssembleTests(unittest.TestCase):
target = f"{architecture}-unknown-linux-gnu" target = f"{architecture}-unknown-linux-gnu"
with self.subTest(target=target): with self.subTest(target=target):
self.metadata["target"] = target self.metadata["target"] = target
(self.package / "codex-package.json").write_text(json.dumps(self.metadata)) (self.package / "codex-package.json").write_text(
json.dumps(self.metadata)
)
output = self.root / target output = self.root / target
assemble(self.package, self.helper, target, self.commit, output) assemble(self.package, self.helper, target, self.commit, output)
manifest = json.loads((output / "codex-resources/voice/manifest.json").read_text()) manifest = json.loads(
(output / "codex-resources/voice/manifest.json").read_text()
)
self.assertEqual( self.assertEqual(
manifest, manifest,
{ {
"schemaVersion": 1, "buildCommit": self.commit, "schemaVersion": 1,
"appTarget": target, "voiceTarget": target, "buildCommit": self.commit,
"appVersion": self.metadata["version"], "sha256": { "appTarget": target,
"voiceTarget": target,
"appVersion": self.metadata["version"],
"sha256": {
"bin/codex": hashlib.sha256(b"unchanged app").hexdigest(), "bin/codex": hashlib.sha256(b"unchanged app").hexdigest(),
"codex-resources/voice/bin/codex-voice-host": hashlib.sha256(b"private helper").hexdigest(), "codex-resources/voice/bin/codex-voice-host": hashlib.sha256(
b"private helper"
).hexdigest(),
}, },
}, },
) )
self.assertEqual((output / "bin/codex").read_bytes(), b"unchanged app") self.assertEqual((output / "bin/codex").read_bytes(), b"unchanged app")
self.assertEqual( self.assertEqual(
(output / "codex-resources/voice/bin/codex-voice-host").read_bytes(), (
output / "codex-resources/voice/bin/codex-voice-host"
).read_bytes(),
self.helper.read_bytes(), self.helper.read_bytes(),
) )
def test_never_replaces_existing_or_nested_outputs(self): def test_never_replaces_existing_or_nested_outputs(self):
for output in (self.package, self.package / "nested", self.helper): for output in (self.package, self.package / "nested", self.helper):
with self.subTest(output=output), self.assertRaises(ValueError): with self.subTest(output=output), self.assertRaises(ValueError):
assemble(self.package, self.helper, "aarch64-unknown-linux-gnu", self.commit, output) assemble(
self.package,
self.helper,
"aarch64-unknown-linux-gnu",
self.commit,
output,
)
self.assertEqual(self.helper.read_bytes(), b"private helper") self.assertEqual(self.helper.read_bytes(), b"private helper")
self.assertFalse((self.package / "nested").exists()) self.assertFalse((self.package / "nested").exists())
def test_cleans_only_its_new_copy_on_failure(self): def test_cleans_only_its_new_copy_on_failure(self):
with patch("assemble_package.shutil.copy2", side_effect=OSError("copy failed")): with patch("assemble_package.shutil.copy2", side_effect=OSError("copy failed")):
with self.assertRaises(OSError): with self.assertRaises(OSError):
assemble(self.package, self.helper, "aarch64-unknown-linux-gnu", self.commit, self.output) assemble(
self.package,
self.helper,
"aarch64-unknown-linux-gnu",
self.commit,
self.output,
)
self.assertFalse(self.output.exists()) self.assertFalse(self.output.exists())
self.assertEqual((self.package / "bin/codex").read_bytes(), b"unchanged app") self.assertEqual((self.package / "bin/codex").read_bytes(), b"unchanged app")
+21 -17
View File
@@ -207,9 +207,10 @@ class NativeBuildTests(unittest.TestCase):
with self.subTest(architecture=architecture): with self.subTest(architecture=architecture):
self.args.target = f"{architecture}-pc-windows-msvc" self.args.target = f"{architecture}-pc-windows-msvc"
self.args.output = self.root / architecture self.args.output = self.root / architecture
with patch( with (
"build_native.platform.system", return_value="Windows" patch("build_native.platform.system", return_value="Windows"),
), patch("build_native.platform.machine", return_value=architecture): patch("build_native.platform.machine", return_value=architecture),
):
build = NativeBuild(self.args, self.environment) build = NativeBuild(self.args, self.environment)
self.assertEqual( self.assertEqual(
build.environment["USERPROFILE"], self.environment["USERPROFILE"] build.environment["USERPROFILE"], self.environment["USERPROFILE"]
@@ -222,20 +223,23 @@ class NativeBuildTests(unittest.TestCase):
"-ID:/private/include -LD:/private/lib -lffi\n" "-ID:/private/include -LD:/private/lib -lffi\n"
) )
with patch( with (
"build_native.prepare_sources", patch(
side_effect=lambda *args: (build.output / "build").mkdir(), "build_native.prepare_sources",
), patch.object(build, "cmake") as cmake, patch.object( side_effect=lambda *args: (build.output / "build").mkdir(),
build, "meson" ),
), patch.object( patch.object(build, "cmake") as cmake,
build, "run", side_effect=record patch.object(build, "meson"),
), patch.object( patch.object(build, "run", side_effect=record),
build, patch.object(
"posix_path", build,
side_effect=lambda path: "/cygdrive/d/" + path.name, "posix_path",
), patch( side_effect=lambda path: "/cygdrive/d/" + path.name,
"build_native.subprocess.check_output", ),
return_value="/usr/share/automake-1.18\n", patch(
"build_native.subprocess.check_output",
return_value="/usr/share/automake-1.18\n",
),
): ):
build.build() build.build()
opus_options = next( opus_options = next(
+2 -3
View File
@@ -80,6 +80,7 @@ class PreparationTests(unittest.TestCase):
}, },
}, },
) )
def test_rejects_changed_archive_before_extracting(self): def test_rejects_changed_archive_before_extracting(self):
source = self.source([("fixture-1/file", tarfile.REGTYPE, b"original")]) source = self.source([("fixture-1/file", tarfile.REGTYPE, b"original")])
(self.root / source["archive"]).write_bytes(b"changed") (self.root / source["archive"]).write_bytes(b"changed")
@@ -106,9 +107,7 @@ class PreparationTests(unittest.TestCase):
] ]
) )
with patch.object(preparation, "MAX_SOURCE_BYTES", 10): with patch.object(preparation, "MAX_SOURCE_BYTES", 10):
with self.assertRaisesRegex( with self.assertRaisesRegex(ValueError, "Expanded source exceeds limits"):
ValueError, "Expanded source exceeds limits"
):
self.prepare(source) self.prepare(source)
self.assertFalse(self.output.exists()) self.assertFalse(self.output.exists())
+6 -1
View File
@@ -27,7 +27,12 @@ def main() -> "Never":
ensure_source_prerequisites(env) ensure_source_prerequisites(env)
set_default_lint_env(env) set_default_lint_env(env)
command = ["cargo", "dylint", "--path", str(root / "tools" / "argument-comment-lint")] command = [
"cargo",
"dylint",
"--path",
str(root / "tools" / "argument-comment-lint"),
]
if not parsed.has_library_selection: if not parsed.has_library_selection:
command.append("--all") command.append("--all")
command.extend(final_args) command.extend(final_args)
@@ -14,7 +14,9 @@ import wrapper_common
class WrapperCommonTest(unittest.TestCase): class WrapperCommonTest(unittest.TestCase):
def test_defaults_to_workspace_and_all_targets(self) -> None: def test_defaults_to_workspace_and_all_targets(self) -> None:
parsed = wrapper_common.parse_wrapper_args([]) parsed = wrapper_common.parse_wrapper_args([])
final_args = wrapper_common.build_final_args(parsed, Path("/repo/codex-rs/Cargo.toml")) final_args = wrapper_common.build_final_args(
parsed, Path("/repo/codex-rs/Cargo.toml")
)
self.assertEqual( self.assertEqual(
final_args, final_args,
@@ -29,8 +31,12 @@ class WrapperCommonTest(unittest.TestCase):
) )
def test_forwarded_cargo_args_keep_single_separator(self) -> None: def test_forwarded_cargo_args_keep_single_separator(self) -> None:
parsed = wrapper_common.parse_wrapper_args(["-p", "codex-core", "--", "--tests"]) parsed = wrapper_common.parse_wrapper_args(
final_args = wrapper_common.build_final_args(parsed, Path("/repo/codex-rs/Cargo.toml")) ["-p", "codex-core", "--", "--tests"]
)
final_args = wrapper_common.build_final_args(
parsed, Path("/repo/codex-rs/Cargo.toml")
)
self.assertEqual( self.assertEqual(
final_args, final_args,
@@ -47,7 +53,9 @@ class WrapperCommonTest(unittest.TestCase):
def test_fix_does_not_add_all_targets(self) -> None: def test_fix_does_not_add_all_targets(self) -> None:
parsed = wrapper_common.parse_wrapper_args(["--fix", "-p", "codex-core"]) parsed = wrapper_common.parse_wrapper_args(["--fix", "-p", "codex-core"])
final_args = wrapper_common.build_final_args(parsed, Path("/repo/codex-rs/Cargo.toml")) final_args = wrapper_common.build_final_args(
parsed, Path("/repo/codex-rs/Cargo.toml")
)
self.assertEqual( self.assertEqual(
final_args, final_args,
@@ -72,7 +80,9 @@ class WrapperCommonTest(unittest.TestCase):
"--bins", "--bins",
] ]
) )
final_args = wrapper_common.build_final_args(parsed, Path("/repo/codex-rs/Cargo.toml")) final_args = wrapper_common.build_final_args(
parsed, Path("/repo/codex-rs/Cargo.toml")
)
self.assertEqual( self.assertEqual(
final_args, final_args,
@@ -93,7 +103,9 @@ class WrapperCommonTest(unittest.TestCase):
"/tmp/custom/Cargo.toml", "/tmp/custom/Cargo.toml",
] ]
) )
final_args = wrapper_common.build_final_args(parsed, Path("/repo/codex-rs/Cargo.toml")) final_args = wrapper_common.build_final_args(
parsed, Path("/repo/codex-rs/Cargo.toml")
)
self.assertEqual( self.assertEqual(
final_args, final_args,
+10 -6
View File
@@ -31,9 +31,7 @@ _TARGET_SELECTION_ARGS = {
} }
_TARGET_SELECTION_PREFIXES = ("--bin=", "--test=", "--example=", "--bench=") _TARGET_SELECTION_PREFIXES = ("--bin=", "--test=", "--example=", "--bench=")
_TARGET_SELECTION_WITH_VALUE = {"--bin", "--test", "--example", "--bench"} _TARGET_SELECTION_WITH_VALUE = {"--bin", "--test", "--example", "--bench"}
_NIGHTLY_LIBRARY_PATTERN = re.compile( _NIGHTLY_LIBRARY_PATTERN = re.compile(r"^(.+@nightly-[0-9]{4}-[0-9]{2}-[0-9]{2})-.+$")
r"^(.+@nightly-[0-9]{4}-[0-9]{2}-[0-9]{2})-.+$"
)
@dataclass @dataclass
@@ -153,7 +151,9 @@ def require_command(name: str, install_message: str | None = None) -> str:
return executable return executable
def run_capture(args: Sequence[str], env: MutableMapping[str, str] | None = None) -> str: def run_capture(
args: Sequence[str], env: MutableMapping[str, str] | None = None
) -> str:
try: try:
completed = subprocess.run( completed = subprocess.run(
list(args), list(args),
@@ -230,14 +230,18 @@ def prefer_rustup_shims(env: MutableMapping[str, str]) -> None:
env["RUSTUP_HOME"] = rustup_home env["RUSTUP_HOME"] = rustup_home
def fetch_packaged_entrypoint(dotslash_manifest: Path, env: MutableMapping[str, str]) -> Path: def fetch_packaged_entrypoint(
dotslash_manifest: Path, env: MutableMapping[str, str]
) -> Path:
require_command( require_command(
"dotslash", "dotslash",
"argument-comment-lint prebuilt wrapper requires dotslash.\n" "argument-comment-lint prebuilt wrapper requires dotslash.\n"
"Install dotslash, or use:\n" "Install dotslash, or use:\n"
" ./tools/argument-comment-lint/run.py ...", " ./tools/argument-comment-lint/run.py ...",
) )
entrypoint = run_capture(["dotslash", "--", "fetch", str(dotslash_manifest)], env=env) entrypoint = run_capture(
["dotslash", "--", "fetch", str(dotslash_manifest)], env=env
)
return Path(entrypoint).resolve() return Path(entrypoint).resolve()