diff --git a/.github/workflows/agents-release.yml b/.github/workflows/agents-release.yml index dfe2736d3c..03caf57e05 100644 --- a/.github/workflows/agents-release.yml +++ b/.github/workflows/agents-release.yml @@ -666,6 +666,7 @@ jobs: CARGO_INCREMENTAL: "0" CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }} steps: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@1.97.1 @@ -685,6 +686,36 @@ jobs: - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 with: version: "v0.10.0" + - name: Configure S3 sccache + if: env.SCCACHE_GHA_ENABLED != 'true' + shell: bash + env: + CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }} + CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }} + CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }} + CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }} + CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }} + CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }} + run: | + { + echo "SCCACHE_BUCKET=${CACHE_BUCKET}" + echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}" + echo "SCCACHE_REGION=${CACHE_REGION}" + echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}" + echo "SCCACHE_S3_USE_SSL=true" + echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}" + echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}" + echo "SCCACHE_IDLE_TIMEOUT=0" + # libduckdb C++ compilation dominates this build. The manylinux + # legs build inside a container and enable CC/CXX wrapping there; + # MSVC stays unwrapped. + case "${RUNNER_OS}" in + macOS) + echo "CC=${SCCACHE_PATH} clang" + echo "CXX=${SCCACHE_PATH} clang++" + ;; + esac + } >> "$GITHUB_ENV" - name: Build DuckDB native driver shell: bash run: | @@ -697,6 +728,15 @@ jobs: -e HOME=/tmp/dbx-rust-home \ -e CARGO_INCREMENTAL=0 \ -e CARGO_TARGET_DIR=/workspace/target/duckdb-driver \ + -e RUSTC_WRAPPER \ + -e SCCACHE_BUCKET \ + -e SCCACHE_ENDPOINT \ + -e SCCACHE_REGION \ + -e SCCACHE_S3_KEY_PREFIX \ + -e SCCACHE_S3_USE_SSL \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -e SCCACHE_IDLE_TIMEOUT \ -v "${{ github.workspace }}:/workspace" \ -w /workspace/agents/drivers/duckdb \ "${{ matrix.manylinux_image }}" \ @@ -704,12 +744,27 @@ jobs: mkdir -p "$HOME" curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1 source "$HOME/.cargo/env" + if [ -n "${SCCACHE_BUCKET:-}" ]; then + case "$(uname -m)" in + x86_64) SCCACHE_ARCH=x86_64-unknown-linux-musl ;; + aarch64) SCCACHE_ARCH=aarch64-unknown-linux-musl ;; + esac + curl -sSfL "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${SCCACHE_ARCH}.tar.gz" \ + | tar -xz --strip-components=1 -C "$HOME/.cargo/bin" + export CC="sccache cc" CXX="sccache c++" + sccache --version + else + unset RUSTC_WRAPPER + fi cargo build \ --manifest-path Cargo.toml \ --locked \ --release \ --bin dbx-duckdb-driver \ --target "${{ matrix.target }}" + if [ -n "${SCCACHE_BUCKET:-}" ]; then + sccache --show-stats || true + fi ' else BUILD_STD=() @@ -741,6 +796,11 @@ jobs: "${{ matrix.manylinux_image }}" \ "/driver/dbx-agent-duckdb-${{ matrix.platform }}" < /dev/null fi + - name: Show sccache stats + if: always() + continue-on-error: true + shell: bash + run: ${SCCACHE_PATH} --show-stats - uses: actions/upload-artifact@v4 with: name: duckdb-native-${{ matrix.platform }} @@ -839,6 +899,7 @@ jobs: CARGO_INCREMENTAL: "0" CARGO_TARGET_DIR: ${{ github.workspace }}/target/tdengine-driver RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }} steps: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@1.97.1 @@ -854,6 +915,36 @@ jobs: - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 with: version: "v0.10.0" + - name: Configure S3 sccache + if: env.SCCACHE_GHA_ENABLED != 'true' + shell: bash + env: + CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }} + CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }} + CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }} + CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }} + CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }} + CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }} + run: | + { + echo "SCCACHE_BUCKET=${CACHE_BUCKET}" + echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}" + echo "SCCACHE_REGION=${CACHE_REGION}" + echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}" + echo "SCCACHE_S3_USE_SSL=true" + echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}" + echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}" + echo "SCCACHE_IDLE_TIMEOUT=0" + # taos client C/C++ compilation dominates this build. The manylinux + # legs build inside a container and enable CC/CXX wrapping there; + # MSVC stays unwrapped. + case "${RUNNER_OS}" in + macOS) + echo "CC=${SCCACHE_PATH} clang" + echo "CXX=${SCCACHE_PATH} clang++" + ;; + esac + } >> "$GITHUB_ENV" - name: Build TDengine native driver shell: bash run: | @@ -863,6 +954,15 @@ jobs: -e HOME=/tmp/dbx-rust-home \ -e CARGO_INCREMENTAL=0 \ -e CARGO_TARGET_DIR=/workspace/target/tdengine-driver \ + -e RUSTC_WRAPPER \ + -e SCCACHE_BUCKET \ + -e SCCACHE_ENDPOINT \ + -e SCCACHE_REGION \ + -e SCCACHE_S3_KEY_PREFIX \ + -e SCCACHE_S3_USE_SSL \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -e SCCACHE_IDLE_TIMEOUT \ -v "${{ github.workspace }}:/workspace" \ -w /workspace \ "${{ matrix.manylinux_image }}" \ @@ -870,12 +970,27 @@ jobs: mkdir -p "$HOME" curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1 source "$HOME/.cargo/env" + if [ -n "${SCCACHE_BUCKET:-}" ]; then + case "$(uname -m)" in + x86_64) SCCACHE_ARCH=x86_64-unknown-linux-musl ;; + aarch64) SCCACHE_ARCH=aarch64-unknown-linux-musl ;; + esac + curl -sSfL "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${SCCACHE_ARCH}.tar.gz" \ + | tar -xz --strip-components=1 -C "$HOME/.cargo/bin" + export CC="sccache cc" CXX="sccache c++" + sccache --version + else + unset RUSTC_WRAPPER + fi cargo build \ --manifest-path agents/drivers/tdengine/Cargo.toml \ --locked \ --release \ --bin dbx-tdengine-driver \ --target "${{ matrix.target }}" + if [ -n "${SCCACHE_BUCKET:-}" ]; then + sccache --show-stats || true + fi ' else BUILD_STD=() @@ -903,6 +1018,11 @@ jobs: elif [ "${{ matrix.smoke }}" = "true" ]; then "release-native/dbx-agent-tdengine-${{ matrix.platform }}${{ matrix.extension }}" < /dev/null fi + - name: Show sccache stats + if: always() + continue-on-error: true + shell: bash + run: ${SCCACHE_PATH} --show-stats - uses: actions/upload-artifact@v4 with: name: tdengine-native-${{ matrix.platform }}