diff --git a/MODULE.bazel b/MODULE.bazel index ab81530b0b..1906a03473 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -182,6 +182,23 @@ voice_python.defaults(python_version = "3.11") voice_python.toolchain(python_version = "3.12") use_repo(voice_python, "python_3_12") +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + download_only = True, + hub_name = "schema_python", + python_version = "3.12", + target_platforms = [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_aarch64", + "windows_x86_64", + ], + uv_lock = "//sdk/python:uv.lock", +) +use_repo(pip, "schema_python") + # Use the existing pinned Make build for foreign_cc consumers, including the # native voice pkg-config bootstrap, instead of a PATH-provided Make. register_toolchains("@rules_foreign_cc//toolchains:built_make_toolchain") diff --git a/bazel/rules/schema_bundle.bzl b/bazel/rules/schema_bundle.bzl index 5f8e2eae01..4a9e559042 100644 --- a/bazel/rules/schema_bundle.bzl +++ b/bazel/rules/schema_bundle.bzl @@ -1,4 +1,4 @@ -"""Cache schema generation from the filtered public Rust test executable.""" +"""Generate API artifacts with the existing Rust and Python exporters.""" def _schema_bundle_impl(ctx): outputs = [] @@ -27,7 +27,10 @@ def _schema_bundle_impl(ctx): # can normalize the JSON without a separate `bazel run` invocation. zstd = ctx.actions.declare_file(ctx.label.name + ".zstd") ctx.actions.symlink(output = zstd, target_file = ctx.executable.zstd, is_executable = True) - return [DefaultInfo(files = depset(outputs + [zstd]))] + return [ + DefaultInfo(files = depset(outputs + [zstd])), + OutputGroupInfo(stable = depset([outputs[0]]), experimental = depset([outputs[1]])), + ] schema_bundle = rule( implementation = _schema_bundle_impl, @@ -37,3 +40,71 @@ schema_bundle = rule( }, doc = "Generates both schema modes as cacheable actions with explicit tool dependencies.", ) + +def _schema_generator_sources_impl(ctx): + sources = [] + crate_root = None + prefix = ctx.label.package + "/" + for source in ctx.files.srcs: + if not source.short_path.startswith(prefix): + fail("Schema generator sources must belong to " + ctx.label.package) + relative = source.short_path[len(prefix):] + output = ctx.actions.declare_file(ctx.label.name + "/" + relative) + ctx.actions.symlink(output = output, target_file = source) + sources.append(output) + if relative == "src/lib.rs": + crate_root = output + if crate_root == None: + fail("Schema generator sources must include src/lib.rs") + + # The ignored generation function derives schemas from Rust. These embedded + # bundles are unused on that path, so do not read the committed artifacts. + precomputed = [ + ctx.actions.declare_file(ctx.label.name + "/schema/precomputed/app-server-exports-" + mode + ".json.zst") + for mode in ["stable", "experimental"] + ] + for output in precomputed: + ctx.actions.write(output, "") + return [ + DefaultInfo(files = depset(sources)), + OutputGroupInfo(crate_root = depset([crate_root]), precomputed = depset(precomputed)), + ] + +schema_generator_sources = rule( + implementation = _schema_generator_sources_impl, + attrs = {"srcs": attr.label_list(allow_files = [".rs"])}, +) + +def _python_sdk_exports_impl(ctx): + output = ctx.actions.declare_directory(ctx.label.name) + manifest = ctx.actions.declare_file(ctx.label.name + ".json") + prefix = ctx.label.package + "/" + files = [] + for source in ctx.files.srcs: + if not source.short_path.startswith(prefix): + fail("Python SDK generator inputs must belong to " + ctx.label.package) + files.append([source.path, source.short_path[len(prefix):]]) + ctx.actions.write(manifest, json.encode(files)) + ctx.actions.run( + executable = ctx.executable._generator, + arguments = [manifest.path, ctx.file.schema.path + "/json", output.path], + inputs = ctx.files.srcs + [ctx.file.schema, manifest], + tools = [ctx.attr._generator[DefaultInfo].files_to_run], + outputs = [output], + env = {"PYTHONDONTWRITEBYTECODE": "1", "PYTHONUTF8": "1"}, + mnemonic = "PythonSdkExports", + ) + return [DefaultInfo(files = depset([output]))] + +python_sdk_exports = rule( + implementation = _python_sdk_exports_impl, + attrs = { + "srcs": attr.label_list(allow_files = True), + "schema": attr.label(mandatory = True, allow_single_file = True), + "_generator": attr.label( + default = "//bazel/schema/tools:python-codegen", + executable = True, + cfg = "exec", + ), + }, +) diff --git a/bazel/schema/BUILD.bazel b/bazel/schema/BUILD.bazel index 401d30b00f..c7dbbc93bf 100644 --- a/bazel/schema/BUILD.bazel +++ b/bazel/schema/BUILD.bazel @@ -6,3 +6,18 @@ schema_bundle( generator = "//codex-rs/app-server-protocol:schema-generator", tags = ["manual"], ) + +schema_bundle( + name = "app-server", + generator = "//codex-rs/app-server-protocol:schema-generator-on-demand", + tags = ["manual"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "stable", + srcs = [":app-server"], + output_group = "stable", + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/bazel/schema/tools/BUILD.bazel b/bazel/schema/tools/BUILD.bazel new file mode 100644 index 0000000000..b2ffe5478d --- /dev/null +++ b/bazel/schema/tools/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_python//python:defs.bzl", "py_binary") +load("@schema_python//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "python-codegen", + srcs = ["python_codegen.py"], + main = "python_codegen.py", + python_version = "3.12", + tags = ["manual"], + deps = [ + requirement("datamodel-code-generator"), + requirement("packaging"), + ], +) diff --git a/bazel/schema/tools/python_codegen.py b/bazel/schema/tools/python_codegen.py new file mode 100644 index 0000000000..ff3ad00fc3 --- /dev/null +++ b/bazel/schema/tools/python_codegen.py @@ -0,0 +1,51 @@ +"""Generate SDK build outputs in scratch space without formatting.""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +def main() -> None: + files = json.loads(Path(sys.argv[1]).read_text()) + schema = Path(sys.argv[2]).resolve() + output = Path(sys.argv[3]).resolve() + with tempfile.TemporaryDirectory() as temporary: + sdk = Path(temporary) / "sdk/python" + for source, relative in files: + destination = sdk / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + environment = { + **os.environ, + "PYTHONPATH": os.pathsep.join(sys.path), + } + subprocess.run( + [ + sys.executable, + str(sdk / "scripts/update_sdk_artifacts.py"), + "generate-types", + "--no-format", + "--schema-dir", + str(schema), + ], + cwd=sdk, + env=environment, + check=True, + ) + for relative in ( + "src/openai_codex/api.py", + "src/openai_codex/generated/v2_all.py", + "src/openai_codex/generated/notification_registry.py", + ): + destination = output / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(sdk / relative, destination) + + +if __name__ == "__main__": + main() diff --git a/codex-rs/app-server-protocol/BUILD.bazel b/codex-rs/app-server-protocol/BUILD.bazel index 4ecf754479..32ae594c77 100644 --- a/codex-rs/app-server-protocol/BUILD.bazel +++ b/codex-rs/app-server-protocol/BUILD.bazel @@ -1,4 +1,7 @@ -load("//:defs.bzl", "codex_rust_crate") +load("@crates//:defs.bzl", "all_crate_deps") +load("@rules_rust//rust:defs.bzl", "rust_binary") +load("//:defs.bzl", "WINDOWS_RUSTC_LINK_FLAGS", "codex_rust_crate") +load("//bazel/rules:schema_bundle.bzl", "schema_generator_sources") codex_rust_crate( name = "app-server-protocol", @@ -17,3 +20,45 @@ alias( tags = ["manual"], visibility = ["//bazel/schema:__pkg__"], ) + +schema_generator_sources( + name = "schema-generator-sources", + srcs = glob(["src/**/*.rs"]), + tags = ["manual"], +) + +filegroup( + name = "schema-generator-root", + srcs = [":schema-generator-sources"], + output_group = "crate_root", + tags = ["manual"], +) + +filegroup( + name = "schema-generator-placeholders", + srcs = [":schema-generator-sources"], + output_group = "precomputed", + tags = ["manual"], +) + +# Compile the crate's test harness, including its unit tests. +# Each schema action runs only write_schema_fixtures_from_env via --exact --ignored. +rust_binary( + name = "schema-generator-on-demand", + srcs = [":schema-generator-sources"], + compile_data = [":schema-generator-placeholders"], + crate_name = "codex_app_server_protocol", + crate_root = ":schema-generator-root", + edition = "2024", + rustc_env = { + "BAZEL_PACKAGE": "codex-rs/app-server-protocol", + "CARGO_PKG_NAME": "codex-app-server-protocol", + }, + rustc_flags = ["--test"] + WINDOWS_RUSTC_LINK_FLAGS, + tags = ["manual"], + visibility = ["//bazel/schema:__pkg__"], + deps = all_crate_deps( + normal = True, + normal_dev = True, + ), +) diff --git a/sdk/python/BUILD.bazel b/sdk/python/BUILD.bazel new file mode 100644 index 0000000000..002e8094db --- /dev/null +++ b/sdk/python/BUILD.bazel @@ -0,0 +1,21 @@ +load("//bazel/rules:schema_bundle.bzl", "python_sdk_exports") + +exports_files(["uv.lock"]) + +python_sdk_exports( + name = "generated", + srcs = [ + "pyproject.toml", + "release_version.py", + "scripts/update_sdk_artifacts.py", + ] + glob( + ["src/**/*.py"], + exclude = [ + "src/openai_codex/generated/notification_registry.py", + "src/openai_codex/generated/v2_all.py", + ], + ), + schema = "//bazel/schema:stable", + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index 9451aebc20..d6663c65ab 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -14,6 +14,7 @@ import tempfile import types import typing from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import Any, Callable, Sequence, get_args, get_origin @@ -545,7 +546,7 @@ def _normalized_schema_bundle_text(schema_dir: Path) -> str: return json.dumps(schema, indent=2, sort_keys=True) + "\n" -def generate_v2_all(schema_dir: Path) -> None: +def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None: """Regenerate the Pydantic v2 protocol model module from app-server schemas.""" out_path = sdk_root() / "src" / "openai_codex" / "generated" / "v2_all.py" out_dir = out_path.parent @@ -556,6 +557,9 @@ def generate_v2_all(schema_dir: Path) -> None: with tempfile.TemporaryDirectory() as td: normalized_bundle = Path(td) / schema_bundle_path(schema_dir).name normalized_bundle.write_text(_normalized_schema_bundle_text(schema_dir)) + if not format_output: + # The pinned generator accepts an empty formatter list through TOML. + (Path(td) / "pyproject.toml").write_text("[tool.datamodel-codegen]\nformatters = []\n") run_python_module( "datamodel_code_generator", [ @@ -582,18 +586,13 @@ def generate_v2_all(schema_dir: Path) -> None: "--use-annotated", "--use-union-operator", "--disable-timestamp", - # Keep the generated file formatted deterministically so the - # checked-in artifact only changes when the schema does. - "--formatters", - "ruff-format", - ], - cwd=sdk_root(), + ] + + (["--formatters", "ruff-format"] if format_output else []), + cwd=sdk_root() if format_output else Path(td), ) _preserve_inline_image_class_names(out_path) _require_nullable_field(out_path, "ChatgptAccount", r"email: str \| None") - _require_nullable_field( - out_path, "McpResourceReadTarget", r"link_id: Annotated\[\n(?: .*\n)+ \]" - ) + _require_nullable_field(out_path, "McpResourceReadTarget", r"link_id: Annotated\[[\s\S]*?\]") _preserve_open_enum(out_path, "CodexErrorInfoValue", allow_empty=True) _preserve_reasoning_effort_enum(out_path) _preserve_thread_source_enum(out_path) @@ -1351,7 +1350,7 @@ def _render_thread_block(turn_fields: list[PublicFieldSpec], *, is_async: bool = return "\n".join(lines) -def generate_public_api_flat_methods() -> None: +def generate_public_api_flat_methods(*, format_output: bool = True) -> None: """Regenerate the public convenience methods from generated protocol models.""" src_dir = sdk_root() / "src" public_api_path = src_dir / "openai_codex" / "api.py" @@ -1400,15 +1399,16 @@ def generate_public_api_flat_methods() -> None: _render_thread_block(turn_start_fields, is_async=True), ) public_api_path.write_text(source) - run_python_module("ruff", ["format", str(public_api_path)], cwd=sdk_root()) + if format_output: + run_python_module("ruff", ["format", str(public_api_path)], cwd=sdk_root()) -def generate_types_from_schema_dir(schema_dir: Path) -> None: +def generate_types_from_schema_dir(schema_dir: Path, *, format_output: bool = True) -> None: """Regenerate every SDK artifact derived from an existing schema directory.""" # v2_all is the authoritative generated surface. - generate_v2_all(schema_dir) + generate_v2_all(schema_dir, format_output=format_output) generate_notification_registry(schema_dir) - generate_public_api_flat_methods() + generate_public_api_flat_methods(format_output=format_output) def build_parser() -> argparse.ArgumentParser: @@ -1423,6 +1423,12 @@ def build_parser() -> argparse.ArgumentParser: type=Path, help="App-server JSON schema directory (defaults to tool.codex.codegen.schema-dir)", ) + generate_types_parser.add_argument( + "--no-format", + dest="format_output", + action="store_false", + help="Generate build outputs without running formatters", + ) stage_sdk_parser = subparsers.add_parser( "stage-sdk", @@ -1483,9 +1489,13 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return build_parser().parse_args(list(argv) if argv is not None else None) -def default_cli_ops() -> CliOps: +def default_cli_ops(*, format_output: bool = True) -> CliOps: return CliOps( - generate_types=generate_types_from_schema_dir, + generate_types=( + generate_types_from_schema_dir + if format_output + else partial(generate_types_from_schema_dir, format_output=False) + ), stage_python_sdk_package=stage_python_sdk_package, stage_python_runtime_package=stage_python_runtime_package, ) @@ -1520,7 +1530,7 @@ def run_command(args: argparse.Namespace, ops: CliOps) -> None: def main(argv: Sequence[str] | None = None, ops: CliOps | None = None) -> None: args = parse_args(argv) - run_command(args, ops or default_cli_ops()) + run_command(args, ops or default_cli_ops(format_output=getattr(args, "format_output", True))) print("Done.") diff --git a/sdk/python/tests/test_contract_generation.py b/sdk/python/tests/test_contract_generation.py index 2b610c46bd..11acc14205 100644 --- a/sdk/python/tests/test_contract_generation.py +++ b/sdk/python/tests/test_contract_generation.py @@ -1,7 +1,9 @@ from __future__ import annotations +import ast import os import runpy +import shutil import subprocess import sys from pathlib import Path @@ -36,23 +38,46 @@ def _snapshot_targets(root: Path) -> dict[str, dict[str, bytes] | bytes | None]: return {str(rel_path): _snapshot_target(root, rel_path) for rel_path in GENERATED_TARGETS} -def test_generated_files_are_up_to_date(): - """Regenerating from repository schemas should leave reviewed artifacts unchanged.""" +@pytest.mark.parametrize("format_output", [True, False]) +def test_generated_files_are_up_to_date(tmp_path, format_output): + """Both generation modes preserve the reviewed API, with optional formatting.""" before = _snapshot_targets(ROOT) + sdk = tmp_path / "sdk" / "python" + sdk.mkdir(parents=True) + for directory in ("src", "scripts"): + shutil.copytree( + ROOT / directory, sdk / directory, ignore=shutil.ignore_patterns("__pycache__") + ) + for name in ("pyproject.toml", "release_version.py"): + shutil.copyfile(ROOT / name, sdk / name) env = os.environ.copy() python_bin = str(Path(sys.executable).parent) env["PATH"] = f"{python_bin}{os.pathsep}{env.get('PATH', '')}" + command = [ + sys.executable, + "scripts/update_sdk_artifacts.py", + "generate-types", + "--schema-dir", + str(ROOT.parents[1] / "codex-rs/app-server-protocol/schema/json"), + ] + if not format_output: + command.append("--no-format") subprocess.run( - [sys.executable, "scripts/update_sdk_artifacts.py", "generate-types"], - cwd=ROOT, + command, + cwd=sdk, check=True, env=env, ) - after = _snapshot_targets(ROOT) - assert before == after, "Generated files drifted after regeneration" + after = _snapshot_targets(sdk) + if format_output: + assert before == after, "Generated files drifted after regeneration" + else: + assert [ast.dump(ast.parse((ROOT / path).read_bytes())) for path in GENERATED_TARGETS] == [ + ast.dump(ast.parse((sdk / path).read_bytes())) for path in GENERATED_TARGETS + ] @pytest.mark.parametrize("mode", ["repository", "scratch", "experimental"])