diff --git a/.github/scripts/configure-docker-mirror.py b/.github/scripts/configure-docker-mirror.py new file mode 100644 index 0000000000..883d2f02fb --- /dev/null +++ b/.github/scripts/configure-docker-mirror.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Add Google's public Docker Hub cache without replacing daemon settings.""" + +import argparse +import json +import os +import shutil +import stat +import subprocess +import tempfile +from pathlib import Path + +MIRROR = "https://mirror.gcr.io" + + +def configure(path, validate=None): + path = Path(path) + config = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + if not isinstance(config, dict): + raise ValueError("Docker daemon config must be a JSON object") + mirrors = config.get("registry-mirrors", []) + if not isinstance(mirrors, list) or any(not isinstance(item, str) for item in mirrors): + raise ValueError("registry-mirrors must be an array of strings") + if any(item.rstrip("/") == MIRROR for item in mirrors): + return + config["registry-mirrors"] = [*mirrors, MIRROR] + original_stat = path.stat() if path.exists() else None + candidate = None + try: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, + prefix="daemon-mirror-", suffix=".json", delete=False) as output: + candidate = Path(output.name) + output.write(json.dumps(config, indent=2) + "\n") + candidate.chmod(stat.S_IMODE(original_stat.st_mode) if original_stat else 0o644) + if original_stat and hasattr(os, "chown"): + os.chown(candidate, original_stat.st_uid, original_stat.st_gid) + if validate: + validate(candidate) + if original_stat: + backup = path.with_name(path.name + ".before-dbx-mirror") + if not backup.exists(): + descriptor = os.open(backup, os.O_WRONLY | os.O_CREAT | os.O_EXCL, + stat.S_IMODE(original_stat.st_mode)) + with os.fdopen(descriptor, "wb") as output: + output.write(path.read_bytes()) + shutil.copystat(path, backup) + if hasattr(os, "chown"): + os.chown(backup, original_stat.st_uid, original_stat.st_gid) + os.replace(candidate, path) + finally: + if candidate and candidate.exists(): + candidate.unlink() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config", type=Path) + parser.add_argument("--validate", action="store_true", help="Validate with dockerd before replacing config") + args = parser.parse_args() + + def validate(candidate): + subprocess.run(["dockerd", "--validate", "--config-file", str(candidate)], check=True, timeout=15) + + configure(args.config, validate if args.validate else None) diff --git a/.github/scripts/configure-docker-mirror.test.py b/.github/scripts/configure-docker-mirror.test.py new file mode 100644 index 0000000000..f295a37553 --- /dev/null +++ b/.github/scripts/configure-docker-mirror.test.py @@ -0,0 +1,115 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location( + "docker_mirror", Path(__file__).with_name("configure-docker-mirror.py") +) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class DockerMirrorTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.path = Path(self.directory.name) / "daemon.json" + + def test_missing_config(self): + module.configure(self.path) + self.assertEqual(json.loads(self.path.read_text()), {"registry-mirrors": [module.MIRROR]}) + self.assertEqual(list(self.path.parent.iterdir()), [self.path]) + + def test_preserves_settings_and_existing_mirror_order(self): + original = {"log-driver": "json-file", "features": {"containerd-snapshotter": True}, + "registry-mirrors": ["https://existing.example", "https://second.example/"]} + self.path.write_text(json.dumps(original)) + module.configure(self.path) + expected = {**original, "registry-mirrors": [*original["registry-mirrors"], module.MIRROR]} + self.assertEqual(json.loads(self.path.read_text()), expected) + + def test_second_run_does_not_write(self): + module.configure(self.path) + before = self.path.read_bytes() + modified = self.path.stat().st_mtime_ns + module.configure(self.path) + self.assertEqual(self.path.read_bytes(), before) + self.assertEqual(self.path.stat().st_mtime_ns, modified) + + def test_existing_mirror_with_trailing_slash_does_not_write(self): + before = '{"registry-mirrors": ["https://mirror.gcr.io/"], "debug": false}\n' + self.path.write_text(before) + module.configure(self.path) + self.assertEqual(self.path.read_text(), before) + + def test_invalid_json_fails_without_writing(self): + before = b'{"debug":' + self.path.write_bytes(before) + with self.assertRaises(json.JSONDecodeError): + module.configure(self.path) + self.assertEqual(self.path.read_bytes(), before) + + def test_invalid_structure_fails_without_writing(self): + for value in [[], {"registry-mirrors": "https://existing.example"}, + {"registry-mirrors": [None]}]: + with self.subTest(value=value): + before = json.dumps(value).encode() + self.path.write_bytes(before) + with self.assertRaises(ValueError): + module.configure(self.path) + self.assertEqual(self.path.read_bytes(), before) + + def test_existing_file_permissions_preserved(self): + self.path.write_text('{"debug": false}') + self.path.chmod(0o600) + mode = self.path.stat().st_mode + module.configure(self.path) + self.assertEqual(self.path.stat().st_mode, mode) + + def test_backup_preserves_original_bytes_and_mode(self): + before = b'{ "debug": false, "log-driver": "json-file" }\n' + self.path.write_bytes(before) + self.path.chmod(0o600) + mode = self.path.stat().st_mode + module.configure(self.path) + backup = self.path.with_name(self.path.name + ".before-dbx-mirror") + self.assertEqual(backup.read_bytes(), before) + self.assertEqual(backup.stat().st_mode, mode) + module.configure(self.path) + self.assertEqual(backup.read_bytes(), before) + + def test_validator_failure_preserves_original_and_removes_candidate(self): + before = b'{"debug": false}\n' + self.path.write_bytes(before) + + def reject(candidate): + self.assertEqual(json.loads(candidate.read_text())["registry-mirrors"], [module.MIRROR]) + self.assertEqual(self.path.read_bytes(), before) + raise ValueError("daemon validation failed") + + with self.assertRaisesRegex(ValueError, "daemon validation failed"): + module.configure(self.path, reject) + self.assertEqual(self.path.read_bytes(), before) + self.assertEqual(list(self.path.parent.iterdir()), [self.path]) + + def test_validation_precedes_atomic_replace(self): + before = b'{"debug": true}\n' + self.path.write_bytes(before) + candidates = [] + + def validate(candidate): + self.assertEqual(candidate.parent, self.path.parent) + self.assertEqual(self.path.read_bytes(), before) + self.assertTrue(json.loads(candidate.read_text())["debug"]) + candidates.append(candidate) + + module.configure(self.path, validate) + self.assertEqual(len(candidates), 1) + self.assertFalse(candidates[0].exists()) + self.assertIn(module.MIRROR, json.loads(self.path.read_text())["registry-mirrors"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a12874d411..9ebfa66146 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,9 @@ jobs: - name: GitHub script tests run: node --test .github/scripts/*.test.mjs + - name: Docker mirror config tests + run: python3 .github/scripts/configure-docker-mirror.test.py + - name: PowerShell MCP installer tests shell: pwsh run: ./.github/scripts/install-mcp-powershell.test.ps1 @@ -872,6 +875,16 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Configure public Docker Hub cache + if: runner.os == 'Linux' + run: | + source /etc/os-release + [[ "$ID" == ubuntu ]] || exit 0 + sudo python3 .github/scripts/configure-docker-mirror.py /etc/docker/daemon.json --validate + timeout 30s sudo systemctl restart docker + timeout 15s docker info --format '{{json .RegistryConfig.Mirrors}}' \ + | python3 -c 'import json, sys; assert any(x.rstrip("/") == "https://mirror.gcr.io" for x in json.load(sys.stdin))' + - name: Install system dependencies run: | sudo timeout --kill-after=30s 10m apt-get \ @@ -1509,6 +1522,16 @@ jobs: working-directory: agents steps: - uses: actions/checkout@v7 + - name: Configure public Docker Hub cache + if: runner.os == 'Linux' + working-directory: ${{ github.workspace }} + run: | + source /etc/os-release + [[ "$ID" == ubuntu ]] || exit 0 + sudo python3 .github/scripts/configure-docker-mirror.py /etc/docker/daemon.json --validate + timeout 30s sudo systemctl restart docker + timeout 15s docker info --format '{{json .RegistryConfig.Mirrors}}' \ + | python3 -c 'import json, sys; assert any(x.rstrip("/") == "https://mirror.gcr.io" for x in json.load(sys.stdin))' - name: Setup RocketMQ server Java if: matrix.scenario == 'rocketmq' uses: actions/setup-java@v6