diff --git a/.github/actions/setup-rusty-v8/action.yml b/.github/actions/setup-rusty-v8/action.yml index 449ab653fd..7b78d8cbdb 100644 --- a/.github/actions/setup-rusty-v8/action.yml +++ b/.github/actions/setup-rusty-v8/action.yml @@ -33,11 +33,25 @@ runs: archive_path="${binding_dir}/${archive_name}" binding_path="${binding_dir}/${binding_name}" checksums_path="${binding_dir}/${checksums_name}" + trusted_checksums="${GITHUB_WORKSPACE}/third_party/v8/rusty_v8_${version//./_}_release_manifests.sha256" + + if command -v sha256sum >/dev/null 2>&1; then + checksum_command=(sha256sum --check -) + else + checksum_command=(shasum -a 256 --check -) + fi mkdir -p "${binding_dir}" + curl -fsSL "${base_url}/${checksums_name}" -o "${checksums_path}" + # Check the original manifest bytes even when either checksum file uses CRLF. + expected_manifest_checksum="$(grep -F " ${checksums_name}" "${trusted_checksums}" | cut -d ' ' -f 1)" + actual_manifest_checksum="$(python3 -c 'import hashlib, pathlib, sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "${checksums_path}")" + if [[ "${actual_manifest_checksum}" != "${expected_manifest_checksum}" ]]; then + echo "Checksum mismatch for ${checksums_name}: expected ${expected_manifest_checksum}, got ${actual_manifest_checksum}" >&2 + exit 1 + fi curl -fsSL "${base_url}/${archive_name}" -o "${archive_path}" curl -fsSL "${base_url}/${binding_name}" -o "${binding_path}" - curl -fsSL "${base_url}/${checksums_name}" -o "${checksums_path}" if [[ "$(wc -l < "${checksums_path}")" -ne 2 ]]; then echo "Expected exactly two checksums for ${TARGET} in ${checksums_path}" >&2 @@ -45,10 +59,6 @@ runs: fi # Existing Windows-built release manifests use CRLF line endings. - if command -v sha256sum >/dev/null 2>&1; then - (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | sha256sum -c -) - else - (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | shasum -a 256 -c -) - fi + (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | "${checksum_command[@]}") echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}" echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.yml index 37c8c65370..32ec3af836 100644 --- a/.github/workflows/rusty-v8-release.yml +++ b/.github/workflows/rusty-v8-release.yml @@ -354,22 +354,18 @@ jobs: restore-keys: | rusty-v8-source-${{ matrix.target }}-sandbox- - - name: Install and start sccache + - name: Install sccache + uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 + with: + tool: sccache@0.8.2 + + - name: Start sccache shell: pwsh env: SCCACHE_CACHE_SIZE: 256M SCCACHE_DIR: ${{ github.workspace }}/upstream-rusty-v8/target/sccache SCCACHE_IDLE_TIMEOUT: 0 - run: | - $version = "v0.8.2" - $platform = "x86_64-pc-windows-msvc" - $basename = "sccache-$version-$platform" - $url = "https://github.com/mozilla/sccache/releases/download/$version/$basename.tar.gz" - cd ~ - curl -LO $url - tar -xzvf "$basename.tar.gz" - . $basename/sccache --start-server - echo "$(pwd)/$basename" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + run: sccache --start-server - name: Install Chromium clang for ARM64 MSVC cross build if: matrix.target == 'aarch64-pc-windows-msvc' @@ -437,7 +433,6 @@ jobs: steps: - name: Check whether release already exists - id: release env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }} @@ -445,10 +440,14 @@ jobs: run: | set -euo pipefail - if gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" > /dev/null 2>&1; then - echo "exists=true" >> "${GITHUB_OUTPUT}" - else - echo "exists=false" >> "${GITHUB_OUTPUT}" + if is_draft="$(gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq '.isDraft' 2>/dev/null)"; then + if [[ "${is_draft}" != "true" ]]; then + echo "Refusing to replace existing published release ${RELEASE_TAG}." >&2 + exit 1 + fi + + echo "Removing unfinished draft release for ${RELEASE_TAG}." + gh release delete "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --yes fi - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -456,22 +455,25 @@ jobs: path: dist - name: Create GitHub Release - if: ${{ steps.release.outputs.exists != 'true' }} - uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 - with: - tag_name: ${{ needs.metadata.outputs.release_tag }} - name: ${{ needs.metadata.outputs.release_tag }} - files: dist/** - # Keep V8 artifact releases out of Codex's normal "latest release" channel. - prerelease: true + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }} + shell: bash + run: | + set -euo pipefail + assets=() + while IFS= read -r -d '' asset; do + assets+=("${asset}") + done < <(find dist -type f -print0) + if [[ "${#assets[@]}" -eq 0 ]]; then + echo "No V8 release artifacts were found." >&2 + exit 1 + fi - - name: Amend existing GitHub Release - if: ${{ steps.release.outputs.exists == 'true' }} - uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 - with: - tag_name: ${{ needs.metadata.outputs.release_tag }} - name: ${{ needs.metadata.outputs.release_tag }} - files: dist/** - overwrite_files: true - # Keep V8 artifact releases out of Codex's normal "latest release" channel. - prerelease: true + # GitHub CLI keeps asset-bearing releases in draft until all uploads finish. + gh release create "${RELEASE_TAG}" \ + --repo "${GITHUB_REPOSITORY}" \ + --title "${RELEASE_TAG}" \ + --verify-tag \ + --prerelease \ + "${assets[@]}" diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index 29cb72e7c5..94f1aa0199 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -372,22 +372,18 @@ jobs: restore-keys: | rusty-v8-source-${{ matrix.target }}-${{ matrix.runner }}-sandbox- - - name: Install and start sccache + - name: Install sccache + uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 + with: + tool: sccache@0.8.2 + + - name: Start sccache shell: pwsh env: SCCACHE_CACHE_SIZE: 256M SCCACHE_DIR: ${{ steps.setup_ci.outputs.cargo-target-dir }}/sccache SCCACHE_IDLE_TIMEOUT: 0 - run: | - $version = "v0.8.2" - $platform = "x86_64-pc-windows-msvc" - $basename = "sccache-$version-$platform" - $url = "https://github.com/mozilla/sccache/releases/download/$version/$basename.tar.gz" - cd ~ - curl -LO $url - tar -xzvf "$basename.tar.gz" - . $basename/sccache --start-server - echo "$(pwd)/$basename" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + run: sccache --start-server - name: Install Chromium clang for ARM64 MSVC cross build if: matrix.target == 'aarch64-pc-windows-msvc' diff --git a/scripts/codex_package/test_v8.py b/scripts/codex_package/test_v8.py new file mode 100644 index 0000000000..a47f15a348 --- /dev/null +++ b/scripts/codex_package/test_v8.py @@ -0,0 +1,152 @@ +import hashlib +import sys +import tempfile +import unittest +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import MagicMock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from codex_package import v8 +from codex_package.targets import TARGET_SPECS, TargetSpec + + +class FetchCodexV8ArtifactsTest(unittest.TestCase): + version = "150.4.0" + + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + @contextmanager + def release( + self, + target: str, + *, + line_ending: bytes = b"\n", + trusted_digest: str | None = None, + trusted_name: str | None = None, + create_pins: bool = True, + ) -> Iterator[tuple[TargetSpec, MagicMock, str]]: + spec = TARGET_SPECS[target] + profile = v8.V8_ARTIFACT_PROFILE + archive_name = ( + f"rusty_v8_{profile}_{target}.lib.gz" + if spec.is_windows + else f"librusty_v8_{profile}_{target}.a.gz" + ) + binding_name = f"src_binding_{profile}_{target}.rs" + manifest_name = f"rusty_v8_{profile}_{target}.sha256" + archive = b"trusted V8 archive" + binding = b"trusted V8 binding" + manifest = ( + line_ending.join( + ( + f"{hashlib.sha256(archive).hexdigest()} {archive_name}".encode(), + f"{hashlib.sha256(binding).hexdigest()} {binding_name}".encode(), + ) + ) + + line_ending + ) + payloads = { + manifest_name: manifest, + archive_name: archive, + binding_name: binding, + } + + if create_pins: + pins = ( + self.root / "third_party/v8/rusty_v8_150_4_0_release_manifests.sha256" + ) + pins.parent.mkdir(parents=True) + digest = trusted_digest or hashlib.sha256(manifest).hexdigest() + name = trusted_name or manifest_name + pins.write_bytes(f"{digest} {name}".encode() + line_ending) + + def download(_url: str, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(payloads[destination.name]) + + with ( + patch.object(v8, "REPO_ROOT", self.root), + patch.object(v8, "download_file", side_effect=download) as download_file, + ): + yield spec, download_file, manifest_name + + def test_fetches_artifacts_after_authenticating_manifest(self) -> None: + with self.release("x86_64-unknown-linux-gnu") as ( + spec, + download, + manifest_name, + ): + artifacts = v8.fetch_codex_v8_artifacts( + spec, version=self.version, cache_root=self.root / "cache" + ) + + self.assertEqual(artifacts.archive.read_bytes(), b"trusted V8 archive") + self.assertEqual(artifacts.binding.read_bytes(), b"trusted V8 binding") + self.assertEqual(download.call_args_list[0].args[1].name, manifest_name) + self.assertEqual(download.call_count, 3) + + def test_authenticates_windows_manifest_with_crlf(self) -> None: + with self.release("x86_64-pc-windows-msvc", line_ending=b"\r\n") as ( + spec, + download, + _manifest_name, + ): + artifacts = v8.fetch_codex_v8_artifacts( + spec, version=self.version, cache_root=self.root / "cache" + ) + + self.assertEqual(artifacts.archive.read_bytes(), b"trusted V8 archive") + self.assertEqual(artifacts.binding.read_bytes(), b"trusted V8 binding") + self.assertEqual(download.call_count, 3) + + def test_rejects_tampered_manifest_before_downloading_artifacts(self) -> None: + with self.release("x86_64-unknown-linux-gnu", trusted_digest="0" * 64) as ( + spec, + download, + manifest_name, + ): + with self.assertRaisesRegex( + RuntimeError, "does not match its trusted SHA-256" + ): + v8.fetch_codex_v8_artifacts( + spec, version=self.version, cache_root=self.root / "cache" + ) + + download.assert_called_once() + self.assertEqual(download.call_args.args[1].name, manifest_name) + + def test_rejects_missing_manifest_pin_before_downloading_artifacts(self) -> None: + with self.release( + "x86_64-unknown-linux-gnu", trusted_name="another-target.sha256" + ) as (spec, download, manifest_name): + with self.assertRaisesRegex(RuntimeError, "has no trusted SHA-256"): + v8.fetch_codex_v8_artifacts( + spec, version=self.version, cache_root=self.root / "cache" + ) + + download.assert_called_once() + self.assertEqual(download.call_args.args[1].name, manifest_name) + + def test_rejects_missing_pin_file_before_downloading_artifacts(self) -> None: + with self.release("x86_64-unknown-linux-gnu", create_pins=False) as ( + spec, + download, + manifest_name, + ): + with self.assertRaises(FileNotFoundError): + v8.fetch_codex_v8_artifacts( + spec, version=self.version, cache_root=self.root / "cache" + ) + + download.assert_called_once() + self.assertEqual(download.call_args.args[1].name, manifest_name) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/codex_package/v8.py b/scripts/codex_package/v8.py index 66ecb6351e..f2ac15dd09 100644 --- a/scripts/codex_package/v8.py +++ b/scripts/codex_package/v8.py @@ -74,6 +74,7 @@ def fetch_codex_v8_artifacts( checksums = cache_dir / checksums_name download_file(f"{release_url}/{checksums.name}", checksums) + verify_release_checksum_manifest(checksums, version=version) expected_checksums = load_checksums(checksums, {archive.name, binding.name}) for artifact in [archive, binding]: ensure_valid_artifact( @@ -107,6 +108,32 @@ def default_cache_root() -> Path: return Path(tempfile.gettempdir()) / "codex-package" +def verify_release_checksum_manifest(checksums_path: Path, *, version: str) -> None: + version_suffix = version.replace(".", "_") + trusted_checksums = ( + REPO_ROOT + / "third_party" + / "v8" + / f"rusty_v8_{version_suffix}_release_manifests.sha256" + ) + + for line in trusted_checksums.read_text(encoding="utf-8").splitlines(): + digest, artifact_name = line.split(maxsplit=1) + if artifact_name != checksums_path.name: + continue + if has_checksum(checksums_path, digest): + return + + checksums_path.unlink(missing_ok=True) + raise RuntimeError( + f"V8 checksum manifest {checksums_path} does not match its trusted SHA-256." + ) + + raise RuntimeError( + f"V8 checksum manifest {checksums_path.name} has no trusted SHA-256 for {version}." + ) + + def load_checksums(checksums_path: Path, artifact_names: set[str]) -> dict[str, str]: checksums: dict[str, str] = {} lines = checksums_path.read_text(encoding="utf-8").splitlines() diff --git a/third_party/v8/README.md b/third_party/v8/README.md index 21adf3899d..fc9879b80d 100644 --- a/third_party/v8/README.md +++ b/third_party/v8/README.md @@ -30,7 +30,10 @@ Use this as the maintainer flow for a version bump: matching checksum manifest and generated checksums as described below. 3. Publish a release-candidate PR and validate that `v8-canary` passes. 4. If the canary is green, publish the release tag and release build. -5. Once the release build completes, rerun the build on the candidate branch +5. Independently verify the published Codex-built checksum manifests and record + their SHA-256 digests in + `third_party/v8/rusty_v8__release_manifests.sha256`. +6. Once the release build completes, rerun the build on the candidate branch and verify that the final artifact builds and tests pass. When changing the remaining prebuilt `rusty_v8` `http_file` inputs, keep the diff --git a/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256 b/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256 new file mode 100644 index 0000000000..ec1ee4a92f --- /dev/null +++ b/third_party/v8/rusty_v8_150_4_0_release_manifests.sha256 @@ -0,0 +1,8 @@ +4079b4b84a8b4fcf34a2a5ca7f080dffd0e1b53404b0032087b821e247febb43 rusty_v8_ptrcomp_sandbox_release_aarch64-apple-darwin.sha256 +9d153e6534d50961329132a64dd7f7cd18ba96501a4a43c1a6d8bfaeec454b2b rusty_v8_ptrcomp_sandbox_release_aarch64-pc-windows-msvc.sha256 +4ee879a8bc7b0f482cac891415e22300dff4429a28897fddac88bb296ce07920 rusty_v8_ptrcomp_sandbox_release_aarch64-unknown-linux-gnu.sha256 +9c40a51e4d5fcedaec527757b8660115b2a10ca3e2ddacadc3075924ad005b66 rusty_v8_ptrcomp_sandbox_release_aarch64-unknown-linux-musl.sha256 +d85c7ae0cf437a4415376c4b5b7daba50b0dcbe30f52612d21df8ce52eb8ada0 rusty_v8_ptrcomp_sandbox_release_x86_64-apple-darwin.sha256 +a4d6221dddb4b5724b23411eaac47caf6095489fbf9d126f65b33cef96a0a8ef rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.sha256 +6774b42c9424c098c72a805c08d4e94be17c591cf02b1dc2633060255a8a61be rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-gnu.sha256 +9bd5beb3a7bfa4f95bc887476ec3e4d564254c1815efe63296740e09bcc8665b rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.sha256