EasyTier/.github/workflows/bpf-diagnostic.yml

name: Validate BPF veth initialization

on:
  push:
    branches: ["diagnose/bpf-2710"]

permissions:
  contents: read
  actions: read

jobs:
  diagnose:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v5
      - name: Wait for the fixed test artifact
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          for attempt in $(seq 1 180); do
            if gh api repos/EasyTier/EasyTier/actions/runs/38869099758/artifacts --jq '.artifacts[].name' | grep -qx tests; then
              exit 0
            fi
            sleep 5
          done
          exit 1
      - uses: actions/download-artifact@v4
        with:
          name: tests
          run-id: 38869099758
          github-token: ${{ github.token }}
      - name: Prepare fixed CI binary
        run: |
          mkdir -p diagnostics
          tar --zstd -xf tests.tar.zst
          python3 - <<'PY'
          import pathlib, re
          binaries = [p for p in pathlib.Path('target/debug/deps').iterdir() if re.fullmatch(r'easytier-[0-9a-f]+', p.name)]
          assert len(binaries) == 1, binaries
          pathlib.Path('diagnostics/binary-path').write_text(str(binaries[0].resolve()))
          PY
          sudo apt-get update -qq
          sudo apt-get install -y -qq strace tcpdump
          uname -a > diagnostics/environment.txt
          systemctl is-active systemd-udevd systemd-networkd NetworkManager >> diagnostics/environment.txt || true
          sudo ip -ts -s -d monitor link > diagnostics/links.log &
          sudo tcpdump -i any -U -w diagnostics/frames.pcap 'tcp port 23456' > diagnostics/tcpdump.log 2>&1 &
          sudo bash -c 'echo "p:bpf_noop noop_enqueue" >> /sys/kernel/tracing/kprobe_events; echo 1 > /sys/kernel/tracing/events/kprobes/bpf_noop/enable' || true
          sudo cat /sys/kernel/tracing/trace_pipe > diagnostics/kernel-trace.log &
      - name: Build send observation hook
        run: |
          cat > diagnostics/send-observer.c <<'EOF'
          #define _GNU_SOURCE
          #include <dlfcn.h>
          #include <stdio.h>
          #include <sys/socket.h>
          #include <netpacket/packet.h>
          #include <net/if.h>
          #include <errno.h>
          #include <time.h>
          static ssize_t (*real_sendto)(int,const void*,size_t,int,const struct sockaddr*,socklen_t);
          __attribute__((constructor)) static void initialize(void) {
              real_sendto = dlsym(RTLD_NEXT, "sendto");
          }
          static void read_value(const char *name, const char *key, char *out) {
              char path[256];
              snprintf(path, sizeof(path), "/sys/class/net/%s/%s", name, key);
              FILE *f = fopen(path, "r");
              if (f) { if (!fgets(out, 64, f)) out[0] = 0; fclose(f); }
          }
          ssize_t sendto(int fd, const void *buf, size_t len, int flags,
                         const struct sockaddr *addr, socklen_t addrlen) {
              ssize_t ret = real_sendto(fd, buf, len, flags, addr, addrlen);
              int saved_errno = errno;
              if (addr && addr->sa_family == AF_PACKET) {
                  const struct sockaddr_ll *ll = (const struct sockaddr_ll *)addr;
                  char name[IF_NAMESIZE] = {0}, drops[64] = {0}, state[64] = {0}, carrier[64] = {0};
                  char packets[64] = {0};
                  struct timespec timestamp;
                  clock_gettime(CLOCK_REALTIME, &timestamp);
                  if_indextoname(ll->sll_ifindex, name);
                  read_value(name, "statistics/tx_dropped", drops);
                  read_value(name, "operstate", state);
                  read_value(name, "carrier", carrier);
                  read_value(name, "statistics/tx_packets", packets);
                  fprintf(stderr, "SEND_OBSERVE time=%ld.%09ld dev=%s len=%zu ret=%zd errno=%d tx_dropped=%s operstate=%s carrier=%s tx_packets=%s\n", timestamp.tv_sec, timestamp.tv_nsec, name, len, ret, saved_errno, drops, state, carrier, packets);
              }
              errno = saved_errno;
              return ret;
          }
          EOF
          gcc -shared -fPIC -O2 diagnostics/send-observer.c -ldl -o diagnostics/send-observer.so
      - name: Repeat fixed test with post-send observation
        run: |
          sudo -E python3 -u - <<'PY'
          import os, pathlib, subprocess, time
          root = pathlib.Path.cwd()
          env = os.environ.copy()
          env['EASYTIER_LINUX_BPF_INTEGRATION'] = 'required'
          env['LD_PRELOAD'] = str(root / 'diagnostics/send-observer.so')
          binary = (root / 'diagnostics/binary-path').read_text()
          case = 'socket::fake_tcp::netfilter::linux_bpf::tests::linux_bpf_tun_sends_ipv4_frame_to_ethernet_interface'
          start = time.monotonic()
          failures = 0
          timeouts = 0
          for i in range(1, 21):
              result = subprocess.run([binary, case, '--exact', '--nocapture'], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
              if i <= 3 or result.returncode:
                  (root / f'diagnostics/attempt-{i}.log').write_text(result.stdout)
              if result.returncode:
                  failures += 1
                  print('REPRODUCED', i, 'elapsed', time.monotonic() - start, flush=True)
                  print(result.stdout, flush=True)
                  if 'timed out waiting for Ethernet frame' in result.stdout:
                      timeouts += 1
                  elif 'assertion `left == right` failed' not in result.stdout:
                      break
                  if timeouts == 3:
                      break
              if i % 100 == 0:
                  print('PASS', i, 'elapsed', time.monotonic() - start, flush=True)
          else:
              print('COMPLETED 20 ATTEMPTS', flush=True)
          print('TOTAL FAILURES', failures, 'TIMEOUTS', timeouts, flush=True)
          assert failures == 0
          PY
      - name: Repeat fixed binary without observation hooks
        run: |
          sudo -E python3 -u - <<'PY'
          import os, pathlib, subprocess, time
          root = pathlib.Path.cwd()
          env = os.environ.copy()
          env['EASYTIER_LINUX_BPF_INTEGRATION'] = 'required'
          binary = (root / 'diagnostics/binary-path').read_text()
          case = 'socket::fake_tcp::netfilter::linux_bpf::tests::linux_bpf_tun_sends_ipv4_frame_to_ethernet_interface'
          start = time.monotonic()
          failures = 0
          for i in range(1, 3001):
              result = subprocess.run([binary, case, '--exact', '--nocapture'], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
              if result.returncode:
                  failures += 1
                  (root / f'diagnostics/baseline-{i}.log').write_text(result.stdout)
                  if 'timed out waiting for Ethernet frame' in result.stdout:
                      print('ORIGINAL TIMEOUT', i, 'elapsed', time.monotonic() - start, flush=True)
                      print(result.stdout, flush=True)
                      break
                  if 'assertion `left == right` failed' not in result.stdout:
                      print('UNEXPECTED', i, result.stdout, flush=True)
                      break
              if i % 500 == 0:
                  print('BASELINE', i, 'failures', failures, 'elapsed', time.monotonic() - start, flush=True)
          else:
              print('COMPLETED 3000 FIXED ATTEMPTS', flush=True)
          assert failures == 0
          PY
      - name: Validate fixed fixture under worker scheduling pressure
        run: |
          cat > diagnostics/cpu-pressure.c <<'EOF'
          #define _GNU_SOURCE
          #include <sched.h>
          #include <stdlib.h>
          #include <time.h>
          #include <stdio.h>
          int main(int argc, char **argv) {
              cpu_set_t mask;
              CPU_ZERO(&mask);
              CPU_SET(atoi(argv[1]), &mask);
              if (sched_setaffinity(0, sizeof(mask), &mask)) { perror("affinity"); return 1; }
              struct sched_param param = {.sched_priority = 10};
              if (sched_setscheduler(0, SCHED_FIFO, &param)) { perror("scheduler"); return 1; }
              struct timespec start, now;
              clock_gettime(CLOCK_MONOTONIC, &start);
              do { clock_gettime(CLOCK_MONOTONIC, &now); } while (now.tv_sec - start.tv_sec < 300);
              return 0;
          }
          EOF
          gcc -O2 diagnostics/cpu-pressure.c -o diagnostics/cpu-pressure
          sudo -E python3 -u - <<'PY'
          import os, pathlib, subprocess, time
          root = pathlib.Path.cwd()
          env = os.environ.copy()
          env['EASYTIER_LINUX_BPF_INTEGRATION'] = 'required'
          env['LD_PRELOAD'] = str(root / 'diagnostics/send-observer.so')
          binary = (root / 'diagnostics/binary-path').read_text()
          case = 'socket::fake_tcp::netfilter::linux_bpf::tests::linux_bpf_tun_sends_ipv4_frame_to_ethernet_interface'
          hogs = []
          try:
              os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(50))
              for cpu in sorted(os.sched_getaffinity(0)):
                  hogs.append(subprocess.Popen([str(root / 'diagnostics/cpu-pressure'), str(cpu)]))
              failures = 0
              timeouts = 0
              for i in range(1, 51):
                  assert all(proc.poll() is None for proc in hogs), 'CPU load ended early'
                  result = subprocess.run(['chrt', '-f', '20', binary, case, '--exact', '--nocapture'], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=10)
                  (root / f'diagnostics/pressure-fixed-{i}.log').write_text(result.stdout)
                  failures += bool(result.returncode)
                  timeouts += 'timed out waiting for Ethernet frame' in result.stdout
                  print('PRESSURE fixed', i, result.returncode, result.stdout, flush=True)
                  assert 'tx_dropped=0\n' in result.stdout
              assert all(proc.poll() is None for proc in hogs), 'CPU load ended early'
              print('PRESSURE TOTAL fixed', 'failures', failures, 'timeouts', timeouts, flush=True)
              assert failures == 0
          finally:
              for proc in hogs:
                  proc.kill()
              for proc in hogs:
                  proc.wait()
              os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
          PY
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bpf-diagnostics
          path: diagnostics