bench.yml

name: Bench
on:
  workflow_call:
  workflow_dispatch:
  schedule:
    # Run at 1 AM each day, so there is a `main`-branch baseline in the cache.
    - cron: '0 1 * * *'
env:
  CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
  CARGO_PROFILE_RELEASE_DEBUG: true
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  TOOLCHAIN: nightly
  RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
  PERF_OPT: record -F997 --call-graph fp -g

jobs:
  bench:
    name: Benchmark
    runs-on: self-hosted
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout neqo
        uses: actions/checkout@v4

      - name: Checkout msquic
        uses: actions/checkout@v4
        with:
          repository: microsoft/msquic
          ref: main
          path: msquic
          submodules: true

      - name: Set PATH
        run: echo "/home/bench/.cargo/bin" >> "${GITHUB_PATH}"

      - name: Install Rust
        uses: ./.github/actions/rust
        with:
          version: $TOOLCHAIN
          components: rustfmt

      - name: Fetch and build NSS and NSPR
        uses: ./.github/actions/nss

      - name: Build neqo
        run: |
          cargo "+$TOOLCHAIN" bench --features bench --no-run
          cargo "+$TOOLCHAIN" build --release

      - name: Build msquic
        run: |
          mkdir -p msquic/build
          cd msquic/build
          cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
          cmake --build .

      - name: Download cached main-branch results
        id: criterion-cache
        uses: actions/cache/restore@v4
        with:
          path: ./target/criterion
          key: criterion-${{ runner.name }}-${{ github.sha }}
          restore-keys: criterion-${{ runner.name }}-

      # Disable turboboost, hyperthreading and use performance governor.
      - name: Prepare machine
        run: sudo /root/bin/prep.sh

      - name: Run cargo bench
        run: |
          # Pin all but neqo-bin benchmarks to CPU 0. neqo-bin benchmarks run
          # both a client and a server, thus benefiting from multiple CPU cores.
          #
          # Run all benchmarks at elevated priority.
          taskset -c 0 nice -n -20 cargo "+$TOOLCHAIN" bench --workspace --exclude neqo-bin --features bench -- --noplot | tee results.txt
          nice -n -20 cargo "+$TOOLCHAIN" bench --package neqo-bin --features bench -- --noplot | tee -a results.txt


      # Compare various configurations of neqo against msquic, and gather perf data
      # during the hyperfine runs.
      - name: Compare neqo and msquic
        env:
          HOST: 127.0.0.1
          PORT: 4433
          SIZE: 134217728 # 128 MB
        run: |
          TMP=$(mktemp -d)
          # Make a cert and key for msquic.
          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
          # Make a test file for msquic to serve.
          truncate -s "$SIZE" "$TMP/$SIZE"
          # Define the commands to run for each client and server.
          declare -A client_cmd=(
            ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
          )
          declare -A server_cmd=(
            ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT 2> /dev/null"
            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit > /dev/null || true"
          )

          # Replace various placeholders in the commands with the actual values.
          # Also generate an extension to append to the file name.
          function transmogrify {
            CMD=$1
            local cc=$2
            local pacing=$3
            if [ "$cc" != "" ]; then
              CMD=${CMD//_cc/--cc $cc}
              EXT="-$cc"
            fi
            if [ "$pacing" == "on" ]; then
              CMD=${CMD//_pacing/}
              EXT="$EXT-pacing"
            else
              CMD=${CMD//_pacing/--no-pacing}
              EXT="$EXT-nopacing"
            fi
          }

          for server in msquic neqo; do
            for client in msquic neqo; do
              # msquic doesn't let us configure the congestion control or pacing.
              if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
                cc_opt=("")
                pacing_opt=("")
              else
                cc_opt=("reno" "cubic")
                pacing_opt=("on" "")
              fi
              for cc in "${cc_opt[@]}"; do
                for pacing in "${pacing_opt[@]}"; do
                  # Make a tag string for this test, for the results.
                  TAG="$client,$server,$cc,$pacing"
                  echo "Running benchmarks for $TAG" | tee -a comparison.txt
                  transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                  # shellcheck disable=SC2086
                  taskset -c 0 nice -n -20 \
                    perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                  PID=$!
                  transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
                  # shellcheck disable=SC2086
                  taskset -c 1 nice -n -20 \
                    perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
                      hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                      tee -a comparison.txt
                  echo >> comparison.txt
                  kill $PID
                  cat step.md >> steps.md
                  # Sanity check the size of the last retrieved file.
                  [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
                done
              done
            done
          done
          # Merge the results tables generated by hyperfine into a single table.
          echo "Transfer of $SIZE bytes over loopback." > comparison.md
          awk '(!/^\| Command/ || !c++) && (!/^\|:/ || !d++)' < steps.md |\
            sed -E 's/`//g; s/^\|:/\|:---\|:---\|:---\|:/g; s/,/ \| /g; s/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
          rm -r "$TMP"

      # Re-enable turboboost, hyperthreading and use powersave governor.
      - name: Restore machine
        run: sudo /root/bin/unprep.sh
        if: success() || failure() || cancelled()

      - name: Post-process perf data
        run: |
          for f in *.perf; do
            # Convert for profiler.firefox.com
            perf script -i "$f" -F +pid > "$f.fx" &
            # Generate perf reports
            perf report -i "$f" --no-children --stdio > "$f.txt" &
            # Generate flamegraphs
            flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}" &
          done
          wait
          rm neqo.svg

      - name: Format results as Markdown
        id: results
        run: |
          {
            echo "### Benchmark results"
            echo
          } > results.md
          SHA=$(cat target/criterion/baseline-sha.txt || true)
          if [ -n "$SHA" ]; then
            {
              echo "Performance differences relative to $SHA."
              echo
            } >> results.md
          fi
          grep -Ev 'ignored|running \d+ tests|%\)' results.txt |\
            sed -E  -e 's/(Performance has regressed.)/:broken_heart: **\1**/gi' \
                    -e 's/(Performance has improved.)/:green_heart: **\1**/gi' \
                    -e 's/^ +/   /gi' \
                    -e 's/^([a-z0-9].*)$/* **\1**/gi' \
                    -e 's/(change:[^%]*% )([^%]*%)(.*)/\1**\2**\3/gi' \
              >> results.md
          {
            echo "### Client/server transfer results"
            cat comparison.md
          } >> results.md
          cat results.md > "$GITHUB_STEP_SUMMARY"

      - name: Remember main-branch push URL
        if: github.ref == 'refs/heads/main'
        run: echo "${{ github.sha }}" > target/criterion/baseline-sha.txt

      - name: Store history
        if: github.ref == 'refs/heads/main'
        run: |
          mkdir -p target/criterion-history
          cp -r target/criterion "target/criterion-history/$(date +%s)-${{ github.sha }}"

      - name: Cache main-branch results
        if: github.ref == 'refs/heads/main'
        uses: actions/cache/save@v4
        with:
          path: ./target/criterion
          key: criterion-${{ runner.name }}-${{ github.sha }}

      - name: Export perf data
        id: export
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.event.repository.name }}-${{ github.sha }}
          path: |
            *.svg
            *.perf
            *.perf.fx
            *.txt
            results.*
            target/criterion*
          compression-level: 9

      - name: Export PR comment data
        uses: ./.github/actions/pr-comment-data-export
        with:
          name: bench
          contents: results.md
          log-url: ${{ steps.export.outputs.artifact-url }}