Add a Bazel target for native voice dependencies (#43111)

## What changed

Add `//third_party/voice:native_prefix` to run the existing native build recipe with Bazel-declared archives and toolchain inputs on matching macOS and Linux GNU 2.28 targets for x64 and ARM64. The manual target exports `prefix.tar` and a `built.json` receipt through the `receipt` output group.

Keep macOS actions local to access Apple tools, use Apple's linker for libffi partial links, and preserve compiler runtime flags through libtool on Linux. Surface upstream build failure logs in the Bazel action output.

Document the build's host utility requirements and the distinction between the raw prefix and a relocatable SDK or runtime package.

GitOrigin-RevId: b4917943f621dec07bb994392619e8a42e496b8b

.bazelrc

@@ -83,6 +83,8 @@ common --jobs=30
 # configurations include this group; CI configs override TestRunner below
 # when tests must remain local on their runner.
 common:remote --strategy=remote
+# Native Mac voice actions need the local Apple tools; Linux remains remote.
+common:macos --strategy=VoiceNativePrefix=remote,sandboxed,local
 common:remote --extra_execution_platforms=//:rbe
 common:remote --jobs=800
 # TODO(team): Evaluate if this actually helps, zbarsky is not sure, everything seems bottlenecked on `core` either way.

MODULE.bazel

@@ -110,6 +110,10 @@ osx.frameworks(names = [
     "AVFAudio",
     "AVRouting",
     "CFNetwork",
+    "Cocoa",
+    "CoreData",
+    "PrintCore",
+    "Symbols",
     "FontServices",
     "AudioUnit",
     "CoreAudio",
@@ -157,6 +161,13 @@ single_version_override(
     version = "0.15.1",
 )
 
+bazel_dep(name = "rules_python", version = "1.7.0")
+
+voice_python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+voice_python.defaults(python_version = "3.11")
+voice_python.toolchain(python_version = "3.12")
+use_repo(voice_python, "python_3_12")
+
 # Use the existing pinned Make build for foreign_cc consumers, including the
 # native voice pkg-config bootstrap, instead of a PATH-provided Make.
 register_toolchains("@rules_foreign_cc//toolchains:built_make_toolchain")
@@ -491,6 +502,17 @@ include("//bazel/modules:wine.MODULE.bazel")
 voice_sources = use_extension("//third_party/voice:extensions.bzl", "voice_sources")
 use_repo(
     voice_sources,
+    "voice_archive_glib",
+    "voice_archive_gst_plugins_base",
+    "voice_archive_gst_plugins_good",
+    "voice_archive_gstreamer",
+    "voice_archive_libffi",
+    "voice_archive_meson",
+    "voice_archive_ninja",
+    "voice_archive_opus",
+    "voice_archive_pcre2",
+    "voice_archive_proxy_libintl",
+    "voice_archive_zlib",
     "voice_glib",
     "voice_gst_plugins_base",
     "voice_gst_plugins_good",

third_party/voice/BUILD.bazel

@@ -1,4 +1,5 @@
 load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain")
+load(":native.bzl", "native_prefix")
 load(":pkg_config.bzl", "pkg_config")
 
 # All native build consumers must receive the same pinned source manifest.
@@ -49,6 +50,14 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
+filegroup(
+    name = "native_recipe",
+    srcs = [
+        "build_native.py",
+        ":source_inputs",
+    ],
+)
+
 alias(
     name = "pkg_config_linker",
     actual = select({
@@ -103,3 +112,68 @@ native_tool_toolchain(
     tags = ["manual"],
     target = ":pkg_config",
 )
+
+filegroup(
+    name = "archives",
+    srcs = [
+        "@voice_archive_glib//file",
+        "@voice_archive_gst_plugins_base//file",
+        "@voice_archive_gst_plugins_good//file",
+        "@voice_archive_gstreamer//file",
+        "@voice_archive_libffi//file",
+        "@voice_archive_meson//file",
+        "@voice_archive_ninja//file",
+        "@voice_archive_opus//file",
+        "@voice_archive_pcre2//file",
+        "@voice_archive_proxy_libintl//file",
+        "@voice_archive_zlib//file",
+    ],
+    tags = ["manual"],
+)
+
+# Configure probes execute target binaries: constrain each action's host too.
+_NATIVE_PLATFORMS = [
+    ("macos", "aarch64", "apple-darwin"),
+    ("macos", "x86_64", "apple-darwin"),
+    ("linux", "aarch64", "unknown-linux-gnu"),
+    ("linux", "x86_64", "unknown-linux-gnu"),
+]
+
+[
+    config_setting(
+        name = os + "_" + cpu,
+        constraint_values = [
+            "@platforms//os:" + os,
+            "@platforms//cpu:" + cpu,
+        ] + (["@llvm//constraints/libc:gnu.2.28"] if os == "linux" else []),
+    )
+    for os, cpu, _ in _NATIVE_PLATFORMS
+]
+
+[
+    native_prefix(
+        name = "native_prefix_" + os + "_" + cpu,
+        archives = [":archives"],
+        exec_compatible_with = [
+            "@platforms//os:" + os,
+            "@platforms//cpu:" + cpu,
+        ],
+        tags = ["manual"],
+        target = cpu + "-" + suffix,
+        target_compatible_with = select({
+            ":" + os + "_" + cpu: [],
+            "//conditions:default": ["@platforms//:incompatible"],
+        }),
+    )
+    for os, cpu, suffix in _NATIVE_PLATFORMS
+]
+
+alias(
+    name = "native_prefix",
+    actual = select({
+        ":" + os + "_" + cpu: ":native_prefix_" + os + "_" + cpu
+        for os, cpu, _ in _NATIVE_PLATFORMS
+    }),
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
+)

third_party/voice/README.md

@@ -65,8 +65,8 @@ and use `pkg-config --define-prefix` for libffi/PCRE2/zlib metadata too. Only th
 required native metadata is exported; capture Opus uses `opusic-sys`. Native
 library loader paths are not changed by SDK export. These build inputs do not replace the
 separate runtime projection and are never copied into users' Codex packages.
-Cargo/Bazel native providers, final helper linkage and moved-package execution
-remain separate integration work; exporting an SDK does not enable voice.
+Final helper linkage and moved-package execution remain separate integration
+work; exporting an SDK does not enable voice.
 
 The Rust GStreamer bindings also link GLib's GIO library. Its pkg-config metadata
 is included in the SDK, and runtime preparation treats its loader identity as an
@@ -76,6 +76,25 @@ transitive imports must satisfy the same private-library and system-import check
 
 ## Native build recipe
 
+`bazel build //third_party/voice:native_prefix` runs this same recipe with
+Bazel-declared source archives, C/C++ compiler and SDK flags, LLVM ranlib,
+CMake, Make, pkg-config and Python. It supports matching native macOS and Linux
+GNU 2.28 toolchain targets on x64/ARM64; Windows continues to use the standalone
+recipe below. The macOS minimum comes from the selected CcToolchain. This manual
+target is not part of ordinary wildcard CLI builds.
+Mac libffi partial links require Apple's `/usr/bin/ld` from Command Line Tools;
+the LLVM compiler and normal final-link flags remain in use.
+
+The root registers the existing pinned built-Make toolchain for all foreign_cc
+Make consumers, including pkg-config's bootstrap, instead of selecting host Make.
+The recipe still needs the host's standard Unix utilities and `/bin/bash`;
+it is not a fully hermetic build.
+
+The target exports raw `prefix.tar` and a `built.json` receipt (the `receipt`
+output group). These are build inputs, not a relocatable SDK or runtime package.
+No Rust GStreamer build-script annotations or helper loader-path changes are
+provided by this target. Upstream build failure logs appear in the action output.
+
 `build_native.py` runs the unmodified upstream build systems in a new output
 directory, using the same archives. Unix builds accept repeated `--c-flag=...`,
 `--cxx-flag=...`, `--link-flag=...` and optional `--ar`/`--ranlib` inputs.

third_party/voice/bazel_native.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Run the existing native recipe with Bazel-declared inputs; export a raw prefix."""
+
+import json
+import os
+from pathlib import Path
+import shutil
+import sys
+import tarfile
+import tempfile
+from types import SimpleNamespace
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from build_native import NativeBuild
+
+
+def main():
+    config = json.loads(Path(sys.argv[1]).read_text())
+    root = str(Path.cwd())
+    # Toolchain flags may name execroot-relative SDK files even though upstream
+    # build systems change directory. Substitute a literal marker, never a shell.
+    config = json.loads(json.dumps(config).replace("@VOICE_EXECROOT@", root))
+    with tempfile.TemporaryDirectory(prefix="voice-native-") as temporary:
+        temporary = Path(temporary)
+        archives = temporary / "archives"
+        archives.mkdir()
+        for archive in config.pop("archives"):
+            source = Path(archive)
+            shutil.copyfile(source, archives / source.name)
+        prefix = Path(config.pop("prefix")).absolute()
+        receipt = Path(config.pop("receipt")).absolute()
+        ld = Path(config.pop("ld")).absolute()
+        for name in (
+            "cc",
+            "cxx",
+            "ar",
+            "ranlib",
+            "cmake",
+            "make",
+            "pkg_config",
+            "shell",
+        ):
+            config[name] = Path(config[name]).absolute()
+        args = SimpleNamespace(
+            **config,
+            archives=archives,
+            output=temporary / "build",
+            bootstrap_make=None,
+        )
+        builder = NativeBuild(args, os.environ)
+        # Libtool probes the raw linker separately from compiler link flags.
+        builder.environment["LD"] = str(ld)
+        try:
+            builder.build()
+        except Exception:
+            # Preserve the failing upstream diagnostic in the Bazel action log.
+            if builder.record["steps"]:
+                log = builder.output / (builder.record["steps"][-1]["name"] + ".log")
+                if log.is_file():
+                    print(log.read_text(errors="replace"), file=sys.stderr)
+            raise
+        with tarfile.open(prefix, "w") as archive:
+            archive.add(builder.prefix, arcname=".")
+        shutil.copyfile(builder.output / "built.json", receipt)
+
+
+if __name__ == "__main__":
+    main()

third_party/voice/build_native.py

@@ -379,6 +379,9 @@ class NativeBuild:
                 }
             )
         else:
+            if self.args.target.endswith("apple-darwin"):
+                # Libtool's partial links need -r, which ld64.lld does not support.
+                environment["CC"] += " -fuse-ld=/usr/bin/ld"
             for name in ("ar", "ranlib"):
                 if name in self.toolchain:
                     path = str(self.toolchain[name])
@@ -408,6 +411,13 @@ class NativeBuild:
                         + "\n"
                     )
                     environment[name] = f"@{response}"
+            if self.args.target.endswith("unknown-linux-gnu"):
+                # Libtool otherwise drops Clang's declared CRT/runtime options.
+                environment["AM_LTLDFLAGS"] = shlex.join(
+                    argument
+                    for flag in self.flags["LDFLAGS"]
+                    for argument in ("-Xcompiler", flag)
+                )
         self.run(
             "libffi-configure",
             [

third_party/voice/extensions.bzl

@@ -1,6 +1,6 @@
 """Use standard Bazel archives for the standalone builder's pinned sources."""
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
 
 _BUILD_FILE = """
 filegroup(
@@ -13,6 +13,12 @@ filegroup(
 def _voice_sources_impl(module_ctx):
     manifest = json.decode(module_ctx.read(Label("//third_party/voice:sources.json")))
     for source in manifest["sources"]:
+        http_file(
+            name = "voice_archive_" + source["name"].replace("-", "_"),
+            urls = [source["url"]],
+            sha256 = source["sha256"],
+            downloaded_file_path = source["archive"],
+        )
         http_archive(
             name = "voice_" + source["name"].replace("-", "_"),
             urls = [source["url"]],

third_party/voice/native.bzl

@@ -0,0 +1,109 @@
+"""Build-only native voice prefix using the same recipe as release packaging."""
+
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+load("@rules_cc//cc/common:cc_common.bzl", "cc_common")
+load("@rules_foreign_cc//foreign_cc/private:cc_toolchain_util.bzl", "absolutize_path_in_str", "get_env_vars", "get_flags_info", "get_tools_info")
+load("@rules_python//python:py_runtime_info.bzl", "PyRuntimeInfo")
+
+def _native_prefix_impl(ctx):
+    cc = find_cpp_toolchain(ctx)
+    features = cc_common.configure_features(
+        ctx = ctx,
+        cc_toolchain = cc,
+        requested_features = ctx.features,
+        unsupported_features = ctx.disabled_features,
+    )
+    runtime = cc.static_runtime_lib(feature_configuration = features)
+    tools = get_tools_info(ctx)
+    flags = get_flags_info(ctx)
+    minimums = [flag.removeprefix("-mmacosx-version-min=") for flag in flags.cc if flag.startswith("-mmacosx-version-min=")]
+    if ctx.attr.target.endswith("apple-darwin") and len(minimums) != 1:
+        fail("The native prefix requires one macOS minimum from the CcToolchain")
+    python = ctx.attr._python[PyRuntimeInfo]
+    if not python.interpreter:
+        fail("The native prefix requires a declared Python interpreter")
+    prefix = ctx.actions.declare_file(ctx.label.name + "/prefix.tar")
+    receipt = ctx.actions.declare_file(ctx.label.name + "/built.json")
+    config = ctx.actions.declare_file(ctx.label.name + ".json")
+    values = {
+        "archives": [file.path for file in ctx.files.archives],
+        "prefix": prefix.path,
+        "receipt": receipt.path,
+        "target": ctx.attr.target,
+        "deployment_target": minimums[0] if minimums else None,
+        "jobs": 8,
+        "cc": tools.cc,
+        "cxx": tools.cxx,
+        "ar": tools.cxx_linker_static,
+        "ranlib": ctx.executable._ranlib.path,
+        "ld": ctx.executable._ld.path,
+        "shell": "/bin/bash",
+        "pkg_config": ctx.file._pkg_config.path,
+    }
+    inputs = [cc.all_files, python.files, runtime, ctx.attr._pkg_config.files]
+    for name in ("cmake", "make"):
+        tool = ctx.toolchains["@rules_foreign_cc//toolchains:" + name + "_toolchain"].data
+        if not tool.target:
+            fail("The native prefix requires a declared " + name + " tool")
+        values[name] = tool.path
+
+        # Prebuilt tools expose a package-relative path; built tools already
+        # include their tree artifact. Match the upstream tool-access convention.
+        for file in tool.target.files.to_list():
+            if file.path.endswith("/" + tool.path):
+                values[name] = file.path
+                break
+        inputs.append(tool.target.files)
+    for name, arguments in {
+        "c_flag": flags.cc,
+        "cxx_flag": flags.cxx,
+        # Upstream selects executable vs shared output. Retain their common
+        # driver/SDK flags without forcing one output kind on configure probes.
+        "link_flag": [flag for flag in flags.cxx_linker_shared if flag in flags.cxx_linker_executable] + [file.path for file in runtime.to_list()],
+    }.items():
+        values[name] = [absolutize_path_in_str(ctx.workspace_name, "@VOICE_EXECROOT@/", flag) for flag in arguments]
+    ctx.actions.write(config, json.encode(values))
+    ctx.actions.run(
+        executable = python.interpreter,
+        arguments = [ctx.file._driver.path, config.path],
+        inputs = depset(
+            [config, ctx.file._driver, python.interpreter] + ctx.files.archives + ctx.files._recipe,
+            transitive = inputs,
+        ),
+        tools = [
+            ctx.attr._ranlib[DefaultInfo].files_to_run,
+            ctx.attr._ld[DefaultInfo].files_to_run,
+        ],
+        outputs = [prefix, receipt],
+        env = get_env_vars(ctx),
+        execution_requirements = {
+            "no-remote-exec": "1",
+            "no-remote-cache": "1",
+        } if ctx.attr.target.endswith("apple-darwin") else {},
+        mnemonic = "VoiceNativePrefix",
+        progress_message = "Building native voice prefix for " + ctx.attr.target,
+    )
+    return [
+        DefaultInfo(files = depset([prefix])),
+        OutputGroupInfo(receipt = depset([receipt])),
+    ]
+
+native_prefix = rule(
+    implementation = _native_prefix_impl,
+    attrs = {
+        "archives": attr.label_list(allow_files = True, mandatory = True),
+        "target": attr.string(mandatory = True),
+        "_driver": attr.label(default = "//third_party/voice:bazel_native.py", allow_single_file = True),
+        "_recipe": attr.label(default = "//third_party/voice:native_recipe"),
+        "_python": attr.label(default = "@python_3_12//:py3_runtime", cfg = "exec"),
+        "_ranlib": attr.label(default = "@llvm//tools:llvm-ranlib", executable = True, allow_files = True, cfg = "exec"),
+        "_ld": attr.label(default = "//third_party/voice:pkg_config_linker", executable = True, allow_files = True, cfg = "exec"),
+        "_pkg_config": attr.label(default = "//third_party/voice:pkg_config", allow_single_file = True, cfg = "exec"),
+    },
+    fragments = ["cpp"],
+    toolchains = [
+        "@bazel_tools//tools/cpp:toolchain_type",
+        "@rules_foreign_cc//toolchains:cmake_toolchain",
+        "@rules_foreign_cc//toolchains:make_toolchain",
+    ],
+)