ete_test_gpu.yaml

name: ete_test_gpu
run-name: ete_test_gpu-${{ github.event.inputs.run_case || 'all' }}
permissions:
  contents: read

on:
  workflow_dispatch:
    inputs:
      repo_org:
        required: false
        description: 'Tested repository organization name. Default is InternLM'
        type: string
        default: 'InternLM/xtuner'
      repo_ref:
        required: false
        description: 'Set branch or tag or commit id. Default is "main"'
        type: string
        default: 'main'
      run_case:
        required: false
        description: >-
          Case name(s) from autotest/config.yaml. Use "all" for every case,
          one name (e.g. "qwen3-rl-lmdeploy"), or comma-separated names
          (e.g. "qwen3-5-rl-vl-lmdeploy-grpo,s2-preview-rl-vl-lmdeploy-grpo").
        type: string
        default: 'all'
      parallel:
        required: false
        description: >-
          pytest-xdist worker count (-n). Cases from config.yaml run in parallel.
          Use a number (e.g. 4), or "all" for one worker per selected case.
          Resume phases within one case stay sequential.
        type: string
        default: '4'
      CI_E2E_DEBUG:
        required: false
        description: >-
          CI_E2E_DEBUG:if true, start "sleep inf &" after install and before training so the
          cluster job stays alive for interactive debug (including after OOM).
          Default false.
        type: boolean
        default: false
      train_image:
        required: false
        description: >-
          Override train image tag only (e.g. ailab-llmrazor/xtuner:pt29_latest).
          Empty uses config.yaml default.
        type: string
        default: ''
      clusterx_partition:
        required: false
        description: >-
          clusterx partition (yidian group), e.g. llmrazor_gpu.
          Empty uses llmrazor_gpu.
        type: string
        default: 'llmrazor_gpu'
      clusterx_project_name:
        required: false
        description: >-
          clusterx project namespace, e.g. ailab-llmrazor.
          Empty uses ailab-llmrazor.
        type: string
        default: 'ailab-llmrazor'
      skip_xtuner_install:
        required: false
        description: >-
          If true, skip ``pip install -e .`` in train jobs (use when the image
          already contains a preinstalled xtuner).
        type: boolean
        default: false
      runner_label:
        required: false
        description: >-
          GitHub Actions runner label for the ETE job.
          Default is h_cluster_ete.
        type: string
        default: 'h_cluster_ete'

env:
  CI_GPU_IMAGE_REGISTRY: ${{ secrets.CI_GPU_IMAGE_REGISTRY }}
  CI_ETE_TRAIN_IMAGE: ${{ github.event.inputs.train_image }}
  CI_ETE_CLUSTERX_PARTITION: ${{ github.event.inputs.clusterx_partition || 'llmrazor_gpu' }}
  CI_ETE_CLUSTERX_PROJECT_NAME: ${{ github.event.inputs.clusterx_project_name || 'ailab-llmrazor' }}
  CI_ETE_SKIP_XTUNER_INSTALL: ${{ github.event.inputs.skip_xtuner_install }}
  # ETE PNG reports: llmcitester/xtuner @ branch reports, under ete/{run_id}/
  CI_REPORTS_REPOSITORY: ${{ vars.CI_REPORTS_REPOSITORY || 'llmcitester/xtuner' }}
  CI_REPORTS_BRANCH: ${{ vars.CI_REPORTS_BRANCH || 'reports' }}
  CI_REPORTS_RAW_URL_BASE: ${{ vars.CI_REPORTS_RAW_URL_BASE || 'https://raw.githubusercontent.com/llmcitester/xtuner/reports/ete' }}
  CI_REPORTS_BROWSE_URL_BASE: ${{ vars.CI_REPORTS_BROWSE_URL_BASE || 'https://github.com/llmcitester/xtuner/tree/reports/ete' }}
  CI_REPORTS_KEEP_RUNS: ${{ vars.CI_REPORTS_KEEP_RUNS || '30' }}

jobs:
  ete_test:
    if: ${{!cancelled() }}
    runs-on: ${{ github.event.inputs.runner_label || 'h_cluster_ete' }}
    # Self-hosted job limit. 1440 (1 day) was cancelling long ETE runs.
    timeout-minutes: 10080
    steps:
      - name: Clean workdir
        run: sudo git clean -ffdx
      - name: Clone repository
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.inputs.repo_org || 'InternLM/xtuner' }}
          ref: ${{github.event.inputs.repo_ref || 'main'}}
      - name: Init job summary
        run: |
          {
            echo "## ETE test report"
            echo ""
            echo "- Workflow run: [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})"
            echo "- Reports: [ete/${{ github.run_id }}](${{ env.CI_REPORTS_BROWSE_URL_BASE }}/${{ github.run_id }})"
            echo "- Train image: \`${{ env.CI_ETE_TRAIN_IMAGE || 'ailab-llmrazor/xtuner:pt29_latest (config default)' }}\`"
            echo "- clusterx partition: \`${{ env.CI_ETE_CLUSTERX_PARTITION }}\`"
            echo "- clusterx project: \`${{ env.CI_ETE_CLUSTERX_PROJECT_NAME }}\`"
            echo "- runner label: \`${{ github.event.inputs.runner_label || 'h_cluster_ete' }}\`"
            echo "- skip xtuner install: \`${{ env.CI_ETE_SKIP_XTUNER_INSTALL }}\`"
            echo ""
          } >> "$GITHUB_STEP_SUMMARY"
      - name: run-test
        env:
          GITHUB_RUN_ID: ${{ github.run_id }}
          CI_REPORTS_RAW_URL_BASE: ${{ env.CI_REPORTS_RAW_URL_BASE }}
          CI_E2E_DEBUG: ${{ github.event.inputs.CI_E2E_DEBUG || 'false' }}
          CI_ETE_TRAIN_IMAGE: ${{ env.CI_ETE_TRAIN_IMAGE }}
          CI_ETE_CLUSTERX_PARTITION: ${{ env.CI_ETE_CLUSTERX_PARTITION }}
          CI_ETE_CLUSTERX_PROJECT_NAME: ${{ env.CI_ETE_CLUSTERX_PROJECT_NAME }}
          CI_ETE_SKIP_XTUNER_INSTALL: ${{ env.CI_ETE_SKIP_XTUNER_INSTALL }}
        run: |
          source /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/miniconda3/bin/activate
          conda activate clusterx 
          conda env list
          unset HTTP_PROXY;unset HTTPS_PROXY;unset http_proxy;unset https_proxy;
          echo "Train image: ${CI_ETE_TRAIN_IMAGE:-ailab-llmrazor/xtuner:pt29_latest (config default)}"
          echo "clusterx partition: ${CI_ETE_CLUSTERX_PARTITION}"
          echo "clusterx project: ${CI_ETE_CLUSTERX_PROJECT_NAME}"
          echo "runner label: ${{ github.event.inputs.runner_label || 'h_cluster_ete' }}"
          echo "skip xtuner install: ${CI_ETE_SKIP_XTUNER_INSTALL:-false}"
          CASE_INPUT="${{ github.event.inputs.run_case || 'all' }}"
          PARALLEL_INPUT="${{ github.event.inputs.parallel || '4' }}"
          PYTEST_NODES=()
          if [ "$CASE_INPUT" = "all" ]; then
            mapfile -t CASE_NAMES < <(python -c 'import yaml; cfg=yaml.safe_load(open("autotest/config.yaml",encoding="utf-8")); print("\n".join(cfg["case"]))')
            for case_name in "${CASE_NAMES[@]}"; do
              [ -z "$case_name" ] && continue
              PYTEST_NODES+=("autotest/test_all.py::test_all[${case_name}]")
            done
          else
            # Support one or more cases: "case_a" or "case_a,case_b, case_c"
            IFS=',' read -ra RAW_CASES <<< "$CASE_INPUT"
            for raw in "${RAW_CASES[@]}"; do
              case_name="$(echo "$raw" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
              [ -z "$case_name" ] && continue
              PYTEST_NODES+=("autotest/test_all.py::test_all[${case_name}]")
            done
          fi
          if [ "${#PYTEST_NODES[@]}" -eq 0 ]; then
            echo "No valid case names parsed from run_case='${CASE_INPUT}'"
            exit 1
          fi
          CASE_COUNT="${#PYTEST_NODES[@]}"
          if [ "$PARALLEL_INPUT" = "all" ]; then
            PARALLEL="$CASE_COUNT"
          elif [[ "$PARALLEL_INPUT" =~ ^[1-9][0-9]*$ ]]; then
            PARALLEL="$PARALLEL_INPUT"
          else
            echo "Invalid parallel='${PARALLEL_INPUT}', fallback to 4"
            PARALLEL=4
          fi
          if [ "$PARALLEL" -gt "$CASE_COUNT" ]; then
            PARALLEL="$CASE_COUNT"
          fi
          echo "Running ${CASE_COUNT} case(s) with pytest-xdist -n ${PARALLEL}"
          echo "Cases: ${PYTEST_NODES[*]}"
          pytest "${PYTEST_NODES[@]}" -m all -n "${PARALLEL}" -s -vv --run_id ${{ github.run_id }}

      - name: Check report files
        if: always()
        id: reports
        run: |
          REPORT_DIR="${{ github.workspace }}/${{ github.run_id }}"
          if compgen -G "${REPORT_DIR}/*.png" > /dev/null; then
            echo "has_reports=true" >> "$GITHUB_OUTPUT"
            echo "Found $(ls -1 "${REPORT_DIR}"/*.png | wc -l) comparison plot(s)."
          else
            echo "has_reports=false" >> "$GITHUB_OUTPUT"
            echo "No comparison plots found under ${REPORT_DIR}, skip publish steps."
          fi

      - name: Upload comparison plots
        if: always() && steps.reports.outputs.has_reports == 'true'
        uses: actions/upload-artifact@v4
        with:
          path: ${{ github.workspace }}/${{ github.run_id }}/*.png
          if-no-files-found: error
          retention-days: 7
          name: xtuner-ete-${{ github.run_id }}

      - name: Publish and prune ete reports
        if: always() && steps.reports.outputs.has_reports == 'true'
        env:
          GH_TOKEN: ${{ secrets.CI_REPORTS_DEPLOY_TOKEN }}
        run: |
          set -euo pipefail
          REPO="${{ env.CI_REPORTS_REPOSITORY }}"
          BRANCH="${{ env.CI_REPORTS_BRANCH }}"
          KEEP="${{ env.CI_REPORTS_KEEP_RUNS }}"
          SRC="${{ github.workspace }}/${{ github.run_id }}"
          DEST="ete/${{ github.run_id }}"
          git clone --depth=1 -b "${BRANCH}" "https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" pages
          cd pages
          rm -rf "${DEST}"
          mkdir -p "$(dirname "${DEST}")"
          cp -r "${SRC}" "${DEST}"
          if [ -d ete ]; then
            cd ete
            mapfile -t RUN_DIRS < <(find . -maxdepth 1 -type d -regextype posix-egrep -regex './[0-9]+' -printf '%f\n' | sort -rn)
            if [ "${#RUN_DIRS[@]}" -gt "${KEEP}" ]; then
              for old_run in "${RUN_DIRS[@]:KEEP}"; do
                rm -rf "${old_run}"
                echo "Removed old report: ${old_run}"
              done
            fi
            cd ..
          fi
          git add -A
          if git diff --staged --quiet; then
            echo "No report changes to commit."
          else
            git -c user.name="github-actions[bot]" -c user.email="github-actions[bot]@users.noreply.github.com" \
              commit -m "chore: publish ete report ${{ github.run_id }}, keep latest ${KEEP} runs"
            git push
          fi