diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 1a6c678b8b..85feb73574 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -39,6 +39,7 @@ filegroup( "macos_runtime.py", "package_runtime.py", "runtime.py", + "sdk.py", "windows_runtime.py", ":source_inputs", ], diff --git a/third_party/voice/README.md b/third_party/voice/README.md index 50df282e25..61b3281c6f 100644 --- a/third_party/voice/README.md +++ b/third_party/voice/README.md @@ -50,6 +50,24 @@ a shared Opus build with Rust consumers or a reduced dependency count. Rust `opus` 0.4.0 is available through Socket. Adding Rust transport dependencies and establishing a shared Opus build remain separate integration work. +## Native development inputs + +The private native CI job also emits `sdk.tar.gz` from the same inspected prefix. +It contains headers (including the target's GLib configuration), development +library names, import/static libraries and pkg-config metadata. `sdk.json` +records the target, source commit, pinned manifest and every exported file hash. +Shared-library bytes must match the native inspection receipt; other development +files are hashed during export. This is provenance, not an authenticity check. + +Meson generates relocatable pkg-config metadata using its standard option. +Consumers must restrict `PKG_CONFIG_LIBDIR` to the SDK, clear `PKG_CONFIG_PATH`, +and use `pkg-config --define-prefix` for libffi/PCRE2/zlib metadata too. Only the +required native metadata is exported; capture Opus uses `opusic-sys`. Native +library loader paths are not changed by SDK export. These build inputs do not replace the +separate runtime projection and are never copied into users' Codex packages. +Cargo/Bazel native providers, final helper linkage and moved-package execution +remain separate integration work; exporting an SDK does not enable voice. + ## Native build recipe `build_native.py` runs the unmodified upstream build systems in a new output diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.py index adb75b6e4a..3ae5669557 100644 --- a/third_party/voice/build_native.py +++ b/third_party/voice/build_native.py @@ -248,6 +248,7 @@ class NativeBuild: "--wrap-mode=nofallback", "-Dauto_features=disabled", "-Ddefault_library=shared", + "-Dpkgconfig.relocatable=true", *options, ], ) diff --git a/third_party/voice/sdk.py b/third_party/voice/sdk.py new file mode 100644 index 0000000000..bc7f05037b --- /dev/null +++ b/third_party/voice/sdk.py @@ -0,0 +1,140 @@ +"""Export development inputs from the same inspected build as the private runtime. + +SDKs are build inputs, never installed user payloads. Their receipt records byte +identity and provenance, not authenticity or approval. Native loader relocation +and final helper linking remain the runtime preparer and consumer's jobs. +""" + +import json +from pathlib import Path +import re +import shutil + +from runtime import digest + +MODULES = ( + "glib-2.0", + "gobject-2.0", + "gmodule-no-export-2.0", + "gstreamer-1.0", + "gstreamer-base-1.0", + "gstreamer-app-1.0", + "gstreamer-audio-1.0", + "gstreamer-tag-1.0", +) + + +def export_sdk(prefix: Path, receipts: Path, target: str, output: Path): + prefix, receipts = prefix.resolve(strict=True), receipts.resolve(strict=True) + output = output.absolute() + if ( + output.exists() + or output.is_symlink() + or any(output.resolve().is_relative_to(root) for root in (prefix, receipts)) + ): + raise ValueError("SDK output must be fresh and outside the inputs") + if not re.fullmatch( + r"(?:aarch64|x86_64)-(?:apple-darwin|unknown-linux-gnu|pc-windows-msvc)", + target, + ): + raise ValueError("unsupported SDK target") + ci = json.loads((receipts / "ci.json").read_text()) + source_hash = digest(Path(__file__).with_name("sources.json")) + if ( + ci.get("target") != target + or ci.get("build_complete") is not True + or ci.get("inspection_complete") is not True + or ci.get("manifest_sha256") != source_hash + or not re.fullmatch(r"[0-9a-f]{40}", ci.get("commit", "")) + ): + raise ValueError("SDK build receipt does not match the sources and target") + inventory = json.loads((receipts / "inspection/binaries.json").read_text()) + if not 1 <= len(inventory) <= 128: + raise ValueError("unexpected native inventory size") + binaries = {} + for record in inventory: + name = record["path"].replace("\\", "/") + if not re.fullmatch(r"(?:lib(?:/gstreamer-1\.0)?|bin)/[A-Za-z0-9_+.-]+", name): + raise ValueError("invalid native inventory path") + path = prefix / name + if ( + name in binaries + or path.is_symlink() + or not path.resolve(strict=True).is_relative_to(prefix) + or record["target"] != target + or digest(path) != record["sha256"] + ): + raise ValueError("native SDK library does not match its receipt") + binaries[name] = record["sha256"] + selected = [] + names = set() + size = 0 + for path in sorted(prefix.rglob("*")): + name = path.relative_to(prefix).as_posix() + if path.is_dir(): + if path.is_symlink(): + raise ValueError("SDK input directories must not be links") + continue + if not ( + name.startswith("include/") + or name == "lib/glib-2.0/include/glibconfig.h" + or name + in { + f"lib/pkgconfig/{module}.pc" + for module in (*MODULES, "libffi", "libpcre2-8", "zlib") + } + or re.fullmatch( + r"lib/[A-Za-z0-9_+.-]+\.(?:a|lib|dylib|so(?:\.[0-9]+)*)", name + ) + ): + continue + source = path.resolve(strict=True) + if not source.is_relative_to(prefix) or not source.is_file(): + raise ValueError("SDK inputs must be files inside the prefix") + if name.casefold() in names: + raise ValueError("SDK paths collide across platforms") + names.add(name.casefold()) + size += source.stat().st_size + if len(selected) >= 4096 or size > 512 * 1024 * 1024: + raise ValueError("SDK inputs exceed the size limit") + source_name = source.relative_to(prefix).as_posix() + expected = digest(source) + if re.search(r"\.(?:dylib|so(?:\.[0-9]+)*)$", name): + if binaries.get(source_name) != expected: + raise ValueError("SDK shared library is missing from the receipt") + selected.append((name, source, expected)) + for module in MODULES: + path = prefix / f"lib/pkgconfig/{module}.pc" + if path.relative_to(prefix).as_posix().casefold() not in names or not re.search( + r"^prefix=\$\{pcfiledir\}/\.\./\.\.$", path.read_text(), re.MULTILINE + ): + raise ValueError("rebuild the SDK with relocatable Meson pkg-config files") + if "lib/glib-2.0/include/glibconfig.h" not in names: + raise ValueError("SDK is missing the target's GLib configuration header") + output.mkdir() + try: + files = [] + for name, source, expected in selected: + destination = output / name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + if digest(destination) != expected: + raise ValueError("SDK input changed while copying") + files.append({"path": name, "sha256": expected}) + (output / "sdk.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "target": target, + "sourceCommit": ci["commit"], + "sourceManifestSha256": source_hash, + "files": files, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + except BaseException: + shutil.rmtree(output) + raise diff --git a/third_party/voice/test_sdk.py b/third_party/voice/test_sdk.py new file mode 100644 index 0000000000..e31d543d04 --- /dev/null +++ b/third_party/voice/test_sdk.py @@ -0,0 +1,153 @@ +"""Exercise exported bytes, development aliases and moved pkg-config inputs.""" + +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from runtime import digest +from sdk import MODULES, export_sdk + + +class SdkTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="voice SDK ") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.prefix = self.root / "prefix" + self.receipts = self.root / "receipts" + self.output = self.root / "sdk" + self.target = "aarch64-apple-darwin" + for name in ("include/glib-2.0", "lib/glib-2.0/include", "lib/pkgconfig"): + (self.prefix / name).mkdir(parents=True) + (self.prefix / "include/glib-2.0/glib.h").write_text("/* SDK fixture */") + (self.prefix / "lib/glib-2.0/include/glibconfig.h").write_text("/* target */") + self.library = self.prefix / "lib/libfixture.0.dylib" + self.library.write_bytes(b"receipt-verified fixture bytes") + for module in (*MODULES, "zlib"): + (self.prefix / f"lib/pkgconfig/{module}.pc").write_text( + "prefix=${pcfiledir}/../..\n" + f"Name: {module}\nDescription: SDK fixture\nVersion: 1.0\n" + "Cflags: -I${prefix}/include\nLibs: -L${prefix}/lib -lfixture\n" + ) + audio = self.prefix / "lib/pkgconfig/gstreamer-audio-1.0.pc" + audio.write_text( + audio.read_text() + "Requires.private: gstreamer-tag-1.0, zlib\n" + ) + (self.receipts / "inspection").mkdir(parents=True) + self.ci = { + "target": self.target, + "build_complete": True, + "inspection_complete": True, + "manifest_sha256": digest(Path(__file__).with_name("sources.json")), + "commit": "a" * 40, + } + (self.receipts / "ci.json").write_text(json.dumps(self.ci)) + (self.receipts / "inspection/binaries.json").write_text( + json.dumps( + [ + { + "path": "lib/libfixture.0.dylib", + "target": self.target, + "sha256": digest(self.library), + } + ] + ) + ) + + def test_exported_bytes_and_receipt_survive_move_without_original(self): + export_sdk(self.prefix, self.receipts, self.target, self.output) + moved = self.root / "moved SDK" + self.output.rename(moved) + shutil.rmtree(self.prefix) + manifest = json.loads((moved / "sdk.json").read_text()) + actual = { + path.relative_to(moved).as_posix(): digest(path) + for path in moved.rglob("*") + if path.is_file() and path.name != "sdk.json" + } + self.assertEqual( + manifest, + { + "schemaVersion": 1, + "target": self.target, + "sourceCommit": "a" * 40, + "sourceManifestSha256": self.ci["manifest_sha256"], + "files": [ + {"path": name, "sha256": actual[name]} for name in sorted(actual) + ], + }, + ) + pkg_config = os.environ.get("VOICE_PKG_CONFIG") or shutil.which("pkg-config") + if not pkg_config: + self.skipTest("pkg-config required for the moved development-input probe") + result = subprocess.check_output( + [ + pkg_config, + "--define-prefix", + "--cflags", + "--libs", + "gstreamer-audio-1.0", + ], + env={ + **os.environ, + "PKG_CONFIG_PATH": "", + "PKG_CONFIG_LIBDIR": str(moved / "lib/pkgconfig"), + }, + text=True, + ).strip() + self.assertEqual( + shlex.split(result), + [f"-I{moved.as_posix()}/include", f"-L{moved.as_posix()}/lib", "-lfixture"], + ) + + def test_native_mutation_and_wrong_target_are_rejected(self): + with self.assertRaisesRegex(ValueError, "sources and target"): + export_sdk(self.prefix, self.receipts, "x86_64-apple-darwin", self.output) + self.library.write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "does not match"): + export_sdk(self.prefix, self.receipts, self.target, self.output) + self.assertFalse(self.output.exists()) + + def test_development_alias_is_materialized_and_escape_is_rejected(self): + alias = self.prefix / "lib/libfixture.dylib" + try: + alias.symlink_to(self.library.name) + except OSError: + self.skipTest("host cannot create the SDK's development symlink") + export_sdk(self.prefix, self.receipts, self.target, self.output) + self.assertFalse((self.output / "lib/libfixture.dylib").is_symlink()) + self.assertEqual( + (self.output / "lib/libfixture.dylib").read_bytes(), + self.library.read_bytes(), + ) + shutil.rmtree(self.output) + alias.unlink() + alias.symlink_to(self.receipts / "ci.json") + with self.assertRaisesRegex(ValueError, "inside the prefix"): + export_sdk(self.prefix, self.receipts, self.target, self.output) + + def test_nonrelocatable_metadata_and_failed_copy_leave_no_output(self): + metadata = self.prefix / "lib/pkgconfig/gstreamer-1.0.pc" + original = metadata.read_text() + metadata.write_text(original.replace("${pcfiledir}/../..", "/old/build")) + with self.assertRaisesRegex(ValueError, "rebuild the SDK"): + export_sdk(self.prefix, self.receipts, self.target, self.output) + metadata.write_text(original) + with patch("sdk.shutil.copy2", side_effect=OSError("copy failed")): + with self.assertRaises(OSError): + export_sdk(self.prefix, self.receipts, self.target, self.output) + self.assertFalse(self.output.exists()) + + def test_output_cannot_overwrite_or_nest_inside_inputs(self): + for output in (self.prefix, self.prefix / "sdk", self.receipts / "sdk"): + with ( + self.subTest(output=output), + self.assertRaisesRegex(ValueError, "fresh"), + ): + export_sdk(self.prefix, self.receipts, self.target, output)