weekly-benchmarks.yml

name: Weekly benchmarks

# Runs the whole suite once a week on GitHub-hosted runners and publishes the results to the
# `results` branch of this repository:
#   latest/benchmarks.json         every target of the newest run (median across rounds), with runner specs
#   latest/raw/bench-<group>/      the per-round suite reports and runner info, as produced
#   history/<YYYY-MM-DD>/...       the same files, kept per run
#   index.json                     list of history entries
# The FeatherPanel website reads latest/benchmarks.json at build time, so no cross-repo token
# is needed here.

on:
  schedule:
    - cron: "17 3 * * 1" # Mondays 03:17 UTC
  workflow_dispatch:
    inputs:
      panel_rounds:
        description: "How many times to run the whole panel group (median is published)"
        required: false
        default: "3"
      daemon_rounds:
        description: "How many times to run the whole daemon group (median is published)"
        required: false
        default: "3"

concurrency:
  group: weekly-benchmarks
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  bench:
    # Fairness: every panel runs on the same machine, one after another, and the same goes for every
    # daemon. Panels are only compared with panels and daemons with daemons, so the two groups may use
    # different runners. Rounds repeat the whole group in alternating order (A, B, C, then C, B, A,
    # then A, B, C), so no target always runs first on a fresh runner or last on a warm one, and a
    # noisy minute does not land on one target only. The published figure is the median across rounds.
    # Every panel runs with its API rate limiting switched off through config (see README).
    name: ${{ matrix.group }}
    runs-on: ubuntu-latest
    timeout-minutes: 350
    strategy:
      fail-fast: false
      matrix:
        include:
          - group: panels
            target: panel:all
            rounds: ${{ github.event.inputs.panel_rounds || '3' }}
            # Stop starting new rounds after this many minutes so the job ends inside the 6h limit.
            budget: 240
          - group: daemons
            target: daemon:all
            rounds: ${{ github.event.inputs.daemon_rounds || '3' }}
            budget: 230
    steps:
      - name: Free disk space
        run: |
          df -h /
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup \
            /opt/hostedtoolcache/CodeQL /usr/local/share/boost /usr/share/swift || true
          docker system prune -af --volumes >/dev/null || true
          df -h /

      - name: Checkout sources
        uses: actions/checkout@v6

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Setup pnpm
        uses: pnpm/action-setup@v5
        with:
          version: 11
          run_install: false

      - name: Install oha
        run: |
          sudo curl -fsSL -o /usr/local/bin/oha \
            https://github.com/hatoo/oha/releases/latest/download/oha-linux-amd64
          sudo chmod +x /usr/local/bin/oha
          oha --version
          docker compose version

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Record runner info
        run: |
          mkdir -p results
          ci/runner-info.sh "${{ matrix.group }}" results/runner.json
          cat results/runner.json

      - name: Run benchmark rounds
        env:
          TARGET: ${{ matrix.target }}
          ROUNDS: ${{ matrix.rounds }}
          BUDGET_MIN: ${{ matrix.budget }}
        run: |
          start=$(date +%s)
          status=0
          for round in $(seq 1 "$ROUNDS"); do
            if [ "$round" -gt 1 ] && [ $(( ($(date +%s) - start) / 60 )) -ge "$BUDGET_MIN" ]; then
              echo "::warning::time budget reached, stopping after $((round - 1)) round(s)"
              break
            fi
            echo "::group::round $round of $ROUNDS"
            # Daemons prune fixtures as each target finishes, so only one ~3.7 GB set is on disk.
            order=""
            if [ $(( round % 2 )) -eq 0 ]; then order="--reverse"; fi
            pnpm run bench "$TARGET" $order --output-dir "results/round-$round" --prune || status=$?
            echo "::endgroup::"
          done
          # Image digests are only known once the images have been pulled.
          ci/runner-info.sh "${{ matrix.group }}" results/runner.json
          exit $status

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: bench-${{ matrix.group }}
          path: results/
          retention-days: 90

  publish:
    name: Publish results
    needs: bench
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout sources
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Download results
        uses: actions/download-artifact@v8
        with:
          pattern: bench-*
          path: artifacts

      - name: Combine
        env:
          EXPECTED_TARGETS: >-
            panel:featherpanel panel:pterodactyl panel:pelican panel:hydrodactyl
            daemon:featherwings daemon:pterodactyl daemon:pelican
        run: |
          node ci/combine.mjs artifacts benchmarks.json
          # Rate limiting is off for every panel; a 429 here means a limit was missed, so say so loudly.
          jq -r '.targets | to_entries[] | select(.value.report != null)
            | "\(.key) \([.value.report.variants[].results[] | .ratelimited // 0] | add)"' benchmarks.json \
            | while read -r target n; do
                if [ "$n" != "0" ]; then echo "::warning::$target got $n rate-limited (429) responses"; fi
              done

      - name: Push to the results branch
        run: |
          set -euo pipefail
          day="$(date -u +%F)"
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          if git ls-remote --exit-code --heads origin results >/dev/null; then
            git fetch origin results
            git worktree add out origin/results
            cd out && git checkout -B results
          else
            git worktree add --detach out
            cd out && git checkout --orphan results && git rm -rfq .
          fi
          rm -rf latest "history/$day"
          mkdir -p latest "history/$day"
          cp ../benchmarks.json latest/benchmarks.json
          mkdir -p latest/raw
          cp -r ../artifacts/. latest/raw/
          cp -r latest/. "history/$day/"
          ls history | sort -r | jq -R . | jq -s '{history: .}' > index.json
          cat > README.md <<'MD'
          # Benchmark results

          Published by the `Weekly benchmarks` workflow on `main`. Do not edit by hand.

          - `latest/benchmarks.json`: newest run, every target plus the runner it ran on
          - `history/<date>/`: previous runs
          MD
          git add -A
          git commit -qm "results: $day (run ${GITHUB_RUN_ID})" || { echo "nothing to commit"; exit 0; }
          git push origin results