From 62133e7b15992c044ef2e04fdf42fdfd894bd3ee Mon Sep 17 00:00:00 2001
From: wenjiefan <wenjiefan@microsoft.com>
Date: Fri, 4 Sep 2026 13:51:04 +0200
Subject: [PATCH 1/4] Add a modified-only dispatch input to the PR review
 evaluation

Wires the existing get-entries --modified-only selection through to pr-review-evaluation.yml so a run can score only the dataset entries a pull request adds or changes. Opt-in and default-false, so existing dispatches are unaffected. Also propagates the flag through requeue, which would otherwise re-run the full corpus on repeat>1, and gives modified-only runs their own concurrency bucket so they do not queue behind a full-corpus run.
---
 .github/workflows/pr-review-evaluation.yml | 10 ++++++++--
 tests/test_review_workflows.py             | 11 ++++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml
index 05cb8764f..65caa8cd3 100644
--- a/.github/workflows/pr-review-evaluation.yml
+++ b/.github/workflows/pr-review-evaluation.yml
@@ -25,6 +25,11 @@ on:
         required: false
         default: true
         type: boolean
+      modified-only:
+        description: "Only evaluate entries added or modified against origin/main (takes precedence over test-run)"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -43,7 +48,7 @@ on:
         type: string
 
 concurrency:
-  group: pr-review-evaluation-${{ inputs.test-run && 'test' || 'full' }}
+  group: pr-review-evaluation-${{ inputs.modified-only && 'modified' || inputs.test-run && 'test' || 'full' }}
   cancel-in-progress: false
 
 env:
@@ -62,6 +67,7 @@ jobs:
   get-entries:
     uses: $/.github/workflows/get-entries.yml
     with:
+      modified-only: ${{ inputs.modified-only }}
       test-run: ${{ inputs.test-run }}
       category: code-review
 
@@ -162,4 +168,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "modified-only": "${{ inputs.modified-only }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py
index 7548ccf02..825297906 100644
--- a/tests/test_review_workflows.py
+++ b/tests/test_review_workflows.py
@@ -49,10 +49,19 @@ def test_pr_review_workflow_is_fixed_to_code_review() -> None:
     assert "mai-code-1-flash-picker" not in workflow
     assert '"gemini-3.7-flash"' in workflow
     assert "gemini-3.6-flash" not in workflow
-    for input_name in ("model:", "test-run:", "repeat:", "git-ref:"):
+    for input_name in ("model:", "test-run:", "repeat:", "git-ref:", "modified-only:"):
         assert input_name in workflow
 
 
+def test_pr_review_workflow_propagates_modified_only() -> None:
+    workflow = _workflow("pr-review-evaluation.yml")
+
+    # Entry selection is delegated to get-entries.yml, which already implements --modified-only.
+    assert "modified-only: ${{ inputs.modified-only }}" in workflow
+    # A requeue that dropped the flag would silently re-run the full corpus.
+    assert '"modified-only": "${{ inputs.modified-only }}"' in workflow
+
+
 def test_agent_harness_action_pins_published_copilot_version() -> None:
     action = (ACTIONS / "install-agent-harnesses" / "action.yml").read_text(encoding="utf-8")
 

From 120d082a1a8df79e429273af5f9967d355ed1cd9 Mon Sep 17 00:00:00 2001
From: wenjiefan <wenjiefan@microsoft.com>
Date: Fri, 4 Sep 2026 15:52:21 +0200
Subject: [PATCH 2/4] Treat a modified-only run as a partial run that never
 publishes

A modified-only run scores a subset of the corpus. Publishing it to Braintrust, Kusto, or the leaderboard would record that subset as a benchmark result, so it now sets the same mock flag a test run does. Requeue is disabled for the same reason, and pin-commit is told not to create an ephemeral tag it would never clean up.
---
 .github/workflows/pr-review-evaluation.yml | 13 ++++++++-----
 tests/test_review_workflows.py             | 12 ++++++++++++
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml
index 65caa8cd3..250225124 100644
--- a/.github/workflows/pr-review-evaluation.yml
+++ b/.github/workflows/pr-review-evaluation.yml
@@ -26,7 +26,7 @@ on:
         default: true
         type: boolean
       modified-only:
-        description: "Only evaluate entries added or modified against origin/main (takes precedence over test-run)"
+        description: "Only evaluate entries added or modified against origin/main (takes precedence over test-run). Scores a partial corpus, so the run never publishes results or requeues."
         required: false
         default: false
         type: boolean
@@ -61,7 +61,8 @@ jobs:
       contents: write
     with:
       agent: pr-review
-      test-run: ${{ inputs.test-run }}
+      # A modified-only run never requeues, so it must not leave an ephemeral tag behind.
+      test-run: ${{ inputs.test-run || inputs.modified-only }}
       repeat: ${{ inputs.repeat }}
 
   get-entries:
@@ -139,7 +140,7 @@ jobs:
         with:
           name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }}
           path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
-          retention-days: ${{ inputs.test-run && 1 || 30 }}
+          retention-days: ${{ (inputs.test-run || inputs.modified-only) && 1 || 30 }}
 
   summarize-results:
     needs: evaluate-with-pr-review
@@ -151,14 +152,16 @@ jobs:
       results-dir: ${{ needs.evaluate-with-pr-review.outputs.results-dir }}
       model: ${{ inputs.model }}
       agent: "BC PR Review"
-      mock: ${{ inputs.test-run }}
+      # Scores only the modified entries, so publishing it to Braintrust/Kusto or the
+      # leaderboard would record a partial corpus as a benchmark result.
+      mock: ${{ inputs.test-run || inputs.modified-only }}
       category: code-review
       git-ref: ${{ inputs.git-ref || github.ref_name }}
     secrets: inherit
 
   requeue:
     needs: [summarize-results, pin-commit]
-    if: ${{ !cancelled() && !failure() && !inputs.test-run }}
+    if: ${{ !cancelled() && !failure() && !inputs.test-run && !inputs.modified-only }}
     uses: $/.github/workflows/requeue-evaluation.yml
     permissions:
       contents: write
diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py
index 825297906..b3792a2e3 100644
--- a/tests/test_review_workflows.py
+++ b/tests/test_review_workflows.py
@@ -62,6 +62,18 @@ def test_pr_review_workflow_propagates_modified_only() -> None:
     assert '"modified-only": "${{ inputs.modified-only }}"' in workflow
 
 
+def test_pr_review_workflow_treats_modified_only_as_a_partial_run() -> None:
+    """A modified-only run scores a subset, so it must not be recorded as a benchmark result."""
+    workflow = _workflow("pr-review-evaluation.yml")
+
+    # Publishing to Braintrust/Kusto and the leaderboard is gated on `mock`.
+    assert "mock: ${{ inputs.test-run || inputs.modified-only }}" in workflow
+    # Requeue is disabled, so pin-commit must not leave an ephemeral tag behind.
+    assert "test-run: ${{ inputs.test-run || inputs.modified-only }}" in workflow
+    assert "!inputs.test-run && !inputs.modified-only" in workflow
+    assert "retention-days: ${{ (inputs.test-run || inputs.modified-only) && 1 || 30 }}" in workflow
+
+
 def test_agent_harness_action_pins_published_copilot_version() -> None:
     action = (ACTIONS / "install-agent-harnesses" / "action.yml").read_text(encoding="utf-8")
 

From 631f3d9dd0e4ffa6441d2098d9d698d4ae1b419c Mon Sep 17 00:00:00 2001
From: wenjiefan <wenjiefan@microsoft.com>
Date: Fri, 4 Sep 2026 15:53:35 +0200
Subject: [PATCH 3/4] Drop the modified-only flag from the requeue payload

Requeue is now disabled for a modified-only dispatch, so the payload key was unreachable. One run verifies the shape of the new entries; repeated runs are for the full corpus after those entries merge.
---
 .github/workflows/pr-review-evaluation.yml |  2 +-
 tests/test_review_workflows.py             | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml
index 250225124..49ed3a039 100644
--- a/.github/workflows/pr-review-evaluation.yml
+++ b/.github/workflows/pr-review-evaluation.yml
@@ -171,4 +171,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "modified-only": "${{ inputs.modified-only }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py
index b3792a2e3..7af3dbae5 100644
--- a/tests/test_review_workflows.py
+++ b/tests/test_review_workflows.py
@@ -58,8 +58,6 @@ def test_pr_review_workflow_propagates_modified_only() -> None:
 
     # Entry selection is delegated to get-entries.yml, which already implements --modified-only.
     assert "modified-only: ${{ inputs.modified-only }}" in workflow
-    # A requeue that dropped the flag would silently re-run the full corpus.
-    assert '"modified-only": "${{ inputs.modified-only }}"' in workflow
 
 
 def test_pr_review_workflow_treats_modified_only_as_a_partial_run() -> None:
@@ -68,10 +66,13 @@ def test_pr_review_workflow_treats_modified_only_as_a_partial_run() -> None:
 
     # Publishing to Braintrust/Kusto and the leaderboard is gated on `mock`.
     assert "mock: ${{ inputs.test-run || inputs.modified-only }}" in workflow
-    # Requeue is disabled, so pin-commit must not leave an ephemeral tag behind.
-    assert "test-run: ${{ inputs.test-run || inputs.modified-only }}" in workflow
-    assert "!inputs.test-run && !inputs.modified-only" in workflow
     assert "retention-days: ${{ (inputs.test-run || inputs.modified-only) && 1 || 30 }}" in workflow
+    # One run verifies the new entries; repeats are for the full corpus after merge.
+    assert "!inputs.test-run && !inputs.modified-only" in workflow
+    # With requeue disabled, pin-commit must not create a tag nothing would clean up.
+    assert "test-run: ${{ inputs.test-run || inputs.modified-only }}" in workflow
+    # A requeued run is always a full-corpus run, so the payload must not carry the flag.
+    assert "modified-only" not in workflow.split("workflow-inputs:")[1]
 
 
 def test_agent_harness_action_pins_published_copilot_version() -> None:

From e0eda507f3f4e1d9ad4ed4266e62842c66fc150f Mon Sep 17 00:00:00 2001
From: wenjiefan <wenjiefan@microsoft.com>
Date: Mon, 7 Sep 2026 11:24:29 +0200
Subject: [PATCH 4/4] Shorten the modified-only input description

The dispatch form wraps a long description across several lines, so keep the label to the two facts a caller needs: which entries run, and that the run does not publish. The precedence and requeue details are already recorded as comments at the wiring sites.
---
 .github/workflows/pr-review-evaluation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml
index 49ed3a039..cca6e13c8 100644
--- a/.github/workflows/pr-review-evaluation.yml
+++ b/.github/workflows/pr-review-evaluation.yml
@@ -26,7 +26,7 @@ on:
         default: true
         type: boolean
       modified-only:
-        description: "Only evaluate entries added or modified against origin/main (takes precedence over test-run). Scores a partial corpus, so the run never publishes results or requeues."
+        description: "Only evaluate added or modified entries (never publishes results)"
         required: false
         default: false
         type: boolean