Add a modified-only dispatch input to the PR review evaluation

.github/workflows/pr-review-evaluation.yml

@@ -25,6 +25,11 @@ on:
         required: false
         default: true
         type: boolean
+      modified-only:
+        description: "Only evaluate added or modified entries (never publishes results)"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -43,7 +48,7 @@ on:
         type: string
 
 concurrency:
-  group: pr-review-evaluation-${{ inputs.test-run && 'test' || 'full' }}
+  group: pr-review-evaluation-${{ inputs.modified-only && 'modified' || inputs.test-run && 'test' || 'full' }}
   cancel-in-progress: false
 
 env:
@@ -56,12 +61,14 @@ jobs:
       contents: write
     with:
       agent: pr-review
-      test-run: ${{ inputs.test-run }}
+      # A modified-only run never requeues, so it must not leave an ephemeral tag behind.
+      test-run: ${{ inputs.test-run || inputs.modified-only }}
       repeat: ${{ inputs.repeat }}
 
   get-entries:
     uses: $/.github/workflows/get-entries.yml
     with:
+      modified-only: ${{ inputs.modified-only }}
       test-run: ${{ inputs.test-run }}
       category: code-review
 
@@ -133,7 +140,7 @@ jobs:
         with:
           name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }}
           path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
-          retention-days: ${{ inputs.test-run && 1 || 30 }}
+          retention-days: ${{ (inputs.test-run || inputs.modified-only) && 1 || 30 }}
 
   summarize-results:
     needs: evaluate-with-pr-review
@@ -145,14 +152,16 @@ jobs:
       results-dir: ${{ needs.evaluate-with-pr-review.outputs.results-dir }}
       model: ${{ inputs.model }}
       agent: "BC PR Review"
-      mock: ${{ inputs.test-run }}
+      # Scores only the modified entries, so publishing it to Braintrust/Kusto or the
+      # leaderboard would record a partial corpus as a benchmark result.
+      mock: ${{ inputs.test-run || inputs.modified-only }}
       category: code-review
       git-ref: ${{ inputs.git-ref || github.ref_name }}
     secrets: inherit
 
   requeue:
     needs: [summarize-results, pin-commit]
-    if: ${{ !cancelled() && !failure() && !inputs.test-run }}
+    if: ${{ !cancelled() && !failure() && !inputs.test-run && !inputs.modified-only }}
     uses: $/.github/workflows/requeue-evaluation.yml
     permissions:
       contents: write

tests/test_review_workflows.py

@@ -49,10 +49,32 @@ def test_pr_review_workflow_is_fixed_to_code_review() -> None:
     assert "mai-code-1-flash-picker" not in workflow
     assert '"gemini-3.7-flash"' in workflow
     assert "gemini-3.6-flash" not in workflow
-    for input_name in ("model:", "test-run:", "repeat:", "git-ref:"):
+    for input_name in ("model:", "test-run:", "repeat:", "git-ref:", "modified-only:"):
         assert input_name in workflow
 
 
+def test_pr_review_workflow_propagates_modified_only() -> None:
+    workflow = _workflow("pr-review-evaluation.yml")
+
+    # Entry selection is delegated to get-entries.yml, which already implements --modified-only.
+    assert "modified-only: ${{ inputs.modified-only }}" in workflow
+
+
+def test_pr_review_workflow_treats_modified_only_as_a_partial_run() -> None:
+    """A modified-only run scores a subset, so it must not be recorded as a benchmark result."""
+    workflow = _workflow("pr-review-evaluation.yml")
+
+    # Publishing to Braintrust/Kusto and the leaderboard is gated on `mock`.
+    assert "mock: ${{ inputs.test-run || inputs.modified-only }}" in workflow
+    assert "retention-days: ${{ (inputs.test-run || inputs.modified-only) && 1 || 30 }}" in workflow
+    # One run verifies the new entries; repeats are for the full corpus after merge.
+    assert "!inputs.test-run && !inputs.modified-only" in workflow
+    # With requeue disabled, pin-commit must not create a tag nothing would clean up.
+    assert "test-run: ${{ inputs.test-run || inputs.modified-only }}" in workflow
+    # A requeued run is always a full-corpus run, so the payload must not carry the flag.
+    assert "modified-only" not in workflow.split("workflow-inputs:")[1]
+
+
 def test_agent_harness_action_pins_published_copilot_version() -> None:
     action = (ACTIONS / "install-agent-harnesses" / "action.yml").read_text(encoding="utf-8")