From 09afd57351d2f3c5938c899d9f23ebce310b9b39 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Mon, 17 Aug 2026 16:14:09 -0700
Subject: [PATCH 01/44] Bridge netfilter enabled

---
 .../Validation.Infrastructure.Tests.yml       |  36 +++++++
 scripts/ci/prepare-linux-host.sh              |  24 +++++
 scripts/ci/validation-test-matrix.json        |  27 ++++-
 tests/scripts/run_ci_backend_tests.sh         |   3 +
 tests/scripts/run_wslc_state_aware_tests.ps1  | 101 +++++++++++++-----
 5 files changed, 162 insertions(+), 29 deletions(-)
 create mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
new file mode 100644
index 000000000..1785d2515
--- /dev/null
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -0,0 +1,36 @@
+name: Validation Infrastructure Testing
+
+on:
+  push:
+    branches:
+      - user/emichlin/validation-infra-2
+
+concurrency:
+  group: validation-infra-pr-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  dependency-feed-check:
+    uses: ./.github/workflows/Dependency.Feed.Check.Job.yml
+
+  windows:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Windows.Job.yml
+
+  linux:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Linux.Job.yml
+
+  macos:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.MacOS.Job.yml
+
+  test:
+    needs: [windows, linux, macos]
+    uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
+    with:
+      plan: enabled
\ No newline at end of file
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 16eafcfeb..062913762 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -129,6 +129,27 @@ start_lxc_bridge() {
     ip addr show "$bridge" || true
 }
 
+# Container network policy is programmed as iptables rules reached from
+# FORWARD, which only sees bridged traffic when br_netfilter is loaded and
+# bridge-nf-call-iptables is enabled. Neither is guaranteed on a fresh image,
+# and without them the backend refuses to report success for a policy it
+# cannot enforce.
+enable_bridge_netfilter() {
+    if ! sudo modprobe br_netfilter 2>/dev/null; then
+        echo "WARNING: could not load br_netfilter; bridged traffic may bypass iptables." >&2
+    fi
+
+    local knob
+    for knob in bridge-nf-call-iptables bridge-nf-call-ip6tables; do
+        if [[ -e "/proc/sys/net/bridge/$knob" ]]; then
+            sudo sysctl -w "net.bridge.$knob=1" >/dev/null ||
+                echo "WARNING: could not enable net.bridge.$knob." >&2
+        else
+            echo "WARNING: /proc/sys/net/bridge/$knob is absent; container network policy cannot be enforced." >&2
+        fi
+    done
+}
+
 # Report the host-side state that container networking depends on. Purely
 # diagnostic: never fails the job, so a networking problem still surfaces as
 # the backend test failure rather than as a prerequisite error.
@@ -139,6 +160,8 @@ report_lxc_network_diagnostics() {
 
     echo "net.ipv4.ip_forward: $(cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo unknown)"
 
+    echo "bridge-nf-call-iptables: $(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)"
+
     echo "Bridge $bridge:"
     ip -4 addr show "$bridge" 2>/dev/null | sed 's/^/  /' || echo "  (absent)"
 
@@ -203,6 +226,7 @@ case "$backend" in
             sudo apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null || true
         fi
         start_lxc_bridge
+        enable_bridge_netfilter
         report_lxc_network_diagnostics
         ;;
     microvm)
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index bf7b2fc62..96256003a 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -387,6 +387,31 @@
       }
     ],
     "weekly": [],
-    "enabled": []
+    "enabled": [
+      {
+        "os": "ubuntu-26.04",
+        "backends": [
+          "lxc"
+        ]
+      },
+      {
+        "os": "ubuntu-24.04",
+        "backends": [
+          "lxc"
+        ]
+      },
+      {
+        "os": "rhel-10",
+        "backends": [
+          "lxc"
+        ]
+      },
+      {
+        "os": "debian-13",
+        "backends": [
+          "lxc"
+        ]
+      }
+    ]
   }
 }
diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh
index dd03f1245..4e156ba21 100644
--- a/tests/scripts/run_ci_backend_tests.sh
+++ b/tests/scripts/run_ci_backend_tests.sh
@@ -48,6 +48,9 @@ case "$backend" in
         mkdir -p "$release_directory"
         cp -a "$binary_directory/." "$release_directory/"
         chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy"
+        # A skip here means a prerequisite disappeared on a runner provisioned
+        # to execute this suite, so turn it into a failure rather than a
+        # vacuously green gate.
         MXC_LXC_TESTS_REQUIRE_EXECUTION=1 bash "$script_root/run_lxc_all_tests.sh"
         ;;
     seatbelt)
diff --git a/tests/scripts/run_wslc_state_aware_tests.ps1 b/tests/scripts/run_wslc_state_aware_tests.ps1
index 2f2faa034..63967d9ac 100644
--- a/tests/scripts/run_wslc_state_aware_tests.ps1
+++ b/tests/scripts/run_wslc_state_aware_tests.ps1
@@ -283,6 +283,50 @@ function Envelope-Arm {
     '<unknown>'
 }
 
+# StrictMode-safe property read; returns $null when the object or property is
+# absent. run_ci_backend_tests.ps1 imposes Set-StrictMode -Version Latest, under
+# which touching a missing property on a PSCustomObject is a terminating error --
+# and an envelope carries exactly one of `result` / `error`, so the other arm is
+# always missing.
+function Get-JsonProperty {
+    param($Object, [Parameter(Mandatory)][string]$Name)
+    if ($null -eq $Object) { return $null }
+    $prop = $Object.PSObject.Properties[$Name]
+    if ($null -eq $prop) { return $null }
+    return $prop.Value
+}
+
+# The envelope's `error` object, or $null when the envelope is absent/succeeded.
+function Get-EnvelopeError {
+    param($Envelope)
+    Get-JsonProperty $Envelope 'error'
+}
+
+# The envelope's error code, or '<no envelope>' / '<no error>' when absent.
+function Get-EnvelopeErrorCode {
+    param($Envelope)
+    if ($null -eq $Envelope) { return '<no envelope>' }
+    $err = Get-JsonProperty $Envelope 'error'
+    if ($null -eq $err) { return '<no error>' }
+    $code = Get-JsonProperty $err 'code'
+    if ($null -eq $code) { return '<no code>' }
+    return [string]$code
+}
+
+# A property of the envelope's `result` object, or $null when absent.
+function Get-EnvelopeResultProperty {
+    param($Envelope, [Parameter(Mandatory)][string]$Name)
+    Get-JsonProperty (Get-JsonProperty $Envelope 'result') $Name
+}
+
+# The envelope's `result.sandboxId` as a string, or $null when absent.
+function Get-EnvelopeSandboxId {
+    param($Envelope)
+    $id = Get-EnvelopeResultProperty $Envelope 'sandboxId'
+    if ($null -eq $id) { return $null }
+    return [string]$id
+}
+
 # Is the daemon process currently running?
 function Test-DaemonRunning {
     $null -ne (Get-Process -Name $DaemonProcName -ErrorAction SilentlyContinue)
@@ -345,7 +389,7 @@ function Provision-Sandbox {
     $r = Invoke-StateAware -ConfigFile $ConfigFile
     $envObj = Parse-Envelope -Stdout $r.Stdout
     if ((Envelope-Arm $envObj) -ne 'result') { return $null }
-    [string]$envObj.result.sandboxId
+    Get-EnvelopeSandboxId $envObj
 }
 
 # ---------------- Backend-availability probe ----------------
@@ -363,16 +407,17 @@ if ($script:PreExistingDaemon) {
 
 $probe = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
 $probeEnv = Parse-Envelope -Stdout $probe.Stdout
-if ($null -ne $probeEnv -and $probeEnv.error.code -eq 'backend_unavailable') {
+$probeError = Get-EnvelopeError $probeEnv
+if ((Get-EnvelopeErrorCode $probeEnv) -eq 'backend_unavailable') {
     Write-Host "SKIPPED: wxc-exec reports backend_unavailable (built without --features wslc, or no WSLC runtime)" -ForegroundColor Yellow
     exit 0
 }
-if ($null -ne $probeEnv -and $null -ne $probeEnv.result -and $null -ne $probeEnv.result.sandboxId) {
-    $probeSandboxId = [string]$probeEnv.result.sandboxId
+$probeSandboxId = Get-EnvelopeSandboxId $probeEnv
+if ($null -ne $probeSandboxId) {
     Write-Host "Backend probe: provisioned $probeSandboxId, deprovisioning ..." -ForegroundColor DarkGray
     $null = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $probeSandboxId
-} elseif ($null -ne $probeEnv -and $null -ne $probeEnv.error) {
-    Write-Host "WARN: probe provision errored (code=$($probeEnv.error.code)): $($probeEnv.error.message)" -ForegroundColor Yellow
+} elseif ($null -ne $probeError) {
+    Write-Host "WARN: probe provision errored (code=$(Get-EnvelopeErrorCode $probeEnv)): $(Get-JsonProperty $probeError 'message')" -ForegroundColor Yellow
     Write-Host "  Continuing -- individual tests will report specific failures." -ForegroundColor Yellow
 }
 
@@ -386,7 +431,7 @@ try {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "provision"
         if ($envObj) {
-            $script:sandboxId = [string]$envObj.result.sandboxId
+            $script:sandboxId = Get-EnvelopeSandboxId $envObj
             Assert-True ($script:sandboxId -match '^wslc:[0-9a-f]{32}$') `
                 "sandbox_id matches wslc:<32-hex> ($script:sandboxId)"
         }
@@ -399,7 +444,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_start.json' -SandboxId $script:sandboxId
             $envObj = Assert-ResultEnvelope $r "start"
             if ($envObj) {
-                Assert-True ($null -eq $envObj.result.metadata) "result.metadata absent (no start metadata in v1)"
+                Assert-True ($null -eq (Get-EnvelopeResultProperty $envObj 'metadata')) "result.metadata absent (no start metadata in v1)"
             }
         }
     }
@@ -413,7 +458,7 @@ try {
             Assert-True ($r.Stdout -match 'wslc-state-aware-exec-marker') `
                 "stdout contains the script's output (relayed live, not enveloped)"
             $maybeEnv = Parse-Envelope -Stdout $r.Stdout
-            Assert-True ($null -eq $maybeEnv -or $null -eq $maybeEnv.error) `
+            Assert-True ($null -eq (Get-EnvelopeError $maybeEnv)) `
                 "stdout is not a state-aware error envelope on success"
         }
     }
@@ -500,7 +545,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_exec_rejected_filesystem.json' -SandboxId $script:sandboxId
             Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+            $code = Get-EnvelopeErrorCode $envObj
             Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
         } | Out-Null
     }
@@ -530,7 +575,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_stop.json' -SandboxId $script:sandboxId
             Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (stop on stale sandbox failed as expected)"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+            $code = Get-EnvelopeErrorCode $envObj
             Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')"
         } | Out-Null
     }
@@ -559,7 +604,7 @@ try {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_with_filesystem.json'
         $envObj = Assert-ResultEnvelope $r "filesystem provision"
         if ($envObj) {
-            $script:fsSandboxId = [string]$envObj.result.sandboxId
+            $script:fsSandboxId = Get-EnvelopeSandboxId $envObj
             Assert-True ($script:fsSandboxId -match '^wslc:[0-9a-f]{32}$') `
                 "sandbox_id matches wslc:<32-hex> ($script:fsSandboxId)"
         }
@@ -645,7 +690,7 @@ try {
     $netProvisionedOk = Run-StateAwareTest "C: provision (bridged network)" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_bridged.json'
         $envObj = Assert-ResultEnvelope $r "bridged provision"
-        if ($envObj) { $script:netSandboxId = [string]$envObj.result.sandboxId }
+        if ($envObj) { $script:netSandboxId = Get-EnvelopeSandboxId $envObj }
     }
 
     $netStartedOk = $false
@@ -696,7 +741,7 @@ Run-StateAwareTest "D: provision (deniedPaths nested under mount rejected)" {
     $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_denied.json'
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+    $code = Get-EnvelopeErrorCode $envObj
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -704,7 +749,7 @@ Run-StateAwareTest "D: provision (host filtering rejected)" {
     $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_hosts.json'
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+    $code = Get-EnvelopeErrorCode $envObj
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -712,7 +757,7 @@ Run-StateAwareTest "D: provision (proxy at provision rejected)" {
     $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_proxy.json'
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+    $code = Get-EnvelopeErrorCode $envObj
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -721,7 +766,7 @@ Run-StateAwareTest "D: start (filesystem policy rejected)" {
     $r = Invoke-StateAware -Request $req
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+    $code = Get-EnvelopeErrorCode $envObj
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -735,7 +780,7 @@ Run-StateAwareTest "D: start (network.proxy rejected post-provision)" {
     $r = Invoke-StateAware -Request $req
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+    $code = Get-EnvelopeErrorCode $envObj
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -744,7 +789,7 @@ Run-StateAwareTest "D: stop (network.proxy rejected post-provision)" {
     $r = Invoke-StateAware -Request $req
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+    $code = Get-EnvelopeErrorCode $envObj
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -763,7 +808,7 @@ try {
     $reProvisionedOk = Run-StateAwareTest "E: provision (restart cycle)" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "restart provision"
-        if ($envObj) { $script:reSandboxId = [string]$envObj.result.sandboxId }
+        if ($envObj) { $script:reSandboxId = Get-EnvelopeSandboxId $envObj }
     }
 
     $reStartedOk = $false
@@ -801,7 +846,7 @@ try {
                     $script:reRestartSucceeded = $true
                     Write-Host "  INFO: WSLc supports restart-after-stop on the same sandbox" -ForegroundColor DarkGray
                 } elseif ($arm -eq 'error') {
-                    Write-Host "  INFO: WSLc does NOT support restart-after-stop (error.code=$($envObj.error.code)) -- documented limitation" -ForegroundColor DarkGray
+                    Write-Host "  INFO: WSLc does NOT support restart-after-stop (error.code=$(Get-EnvelopeErrorCode $envObj)) -- documented limitation" -ForegroundColor DarkGray
                 }
             } | Out-Null
 
@@ -843,7 +888,7 @@ try {
     $edgeProvisionedOk = Run-StateAwareTest "F: provision (exec-edge sandbox)" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "edge provision"
-        if ($envObj) { $script:edgeSandboxId = [string]$envObj.result.sandboxId }
+        if ($envObj) { $script:edgeSandboxId = Get-EnvelopeSandboxId $envObj }
     }
 
     # F1: exec BEFORE start -> not_started (the daemon knows the id but the
@@ -854,7 +899,7 @@ try {
             $r = Invoke-StateAware -Request $req
             Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (exec before start rejected)"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+            $code = Get-EnvelopeErrorCode $envObj
             Assert-True ($code -eq 'not_started') "error.code is 'not_started' (got '$code')"
         } | Out-Null
     }
@@ -876,7 +921,7 @@ try {
             $r = Invoke-StateAware -Request $slow
             Assert-True ($r.ExitCode -ne 0) "timed-out exec exits non-zero"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+            $code = Get-EnvelopeErrorCode $envObj
             Assert-True ($code -eq 'backend_error') "timeout maps to 'backend_error' (got '$code')"
 
             $after = @{ phase = 'exec'; sandboxId = $script:edgeSandboxId; process = @{ commandLine = 'echo survived-timeout'; timeout = 30000 } }
@@ -912,7 +957,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $script:edgeSandboxId
             Assert-True ($r.ExitCode -ne 0) "second deprovision exits non-zero"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
+            $code = Get-EnvelopeErrorCode $envObj
             Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')"
         } | Out-Null
     }
@@ -937,12 +982,12 @@ try {
     $mcAProvOk = Run-StateAwareTest "G: provision sandbox A" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "multi-container A provision"
-        if ($envObj) { $script:mcSandboxA = [string]$envObj.result.sandboxId }
+        if ($envObj) { $script:mcSandboxA = Get-EnvelopeSandboxId $envObj }
     }
     $mcBProvOk = Run-StateAwareTest "G: provision sandbox B" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "multi-container B provision"
-        if ($envObj) { $script:mcSandboxB = [string]$envObj.result.sandboxId }
+        if ($envObj) { $script:mcSandboxB = Get-EnvelopeSandboxId $envObj }
     }
 
     if ($mcAProvOk -and $mcBProvOk) {
@@ -1057,7 +1102,7 @@ try {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "post-teardown provision"
         if ($envObj) {
-            $script:recSandboxId = [string]$envObj.result.sandboxId
+            $script:recSandboxId = Get-EnvelopeSandboxId $envObj
             Assert-True ($script:recSandboxId -match '^wslc:[0-9a-f]{32}$') `
                 "sandbox_id matches wslc:<32-hex> ($script:recSandboxId)"
         }

From 56a14028289d1515a97e6d182e9665062e1d71a5 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Mon, 17 Aug 2026 16:46:47 -0700
Subject: [PATCH 02/44] adding host forwarding

---
 scripts/ci/prepare-linux-host.sh | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 062913762..bec5aac98 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -150,6 +150,22 @@ enable_bridge_netfilter() {
     done
 }
 
+# Container-scoped rules hook egress (-i <veth>), so a reply arrives in the
+# opposite direction, matches nothing MXC installed, and falls through to the
+# host's FORWARD policy. A DROP policy (Docker sets one) therefore breaks
+# allowed destinations, and worse, makes the deny cases pass vacuously: a
+# container with no working hook at all is equally unreachable. Forwarding by
+# default leaves an MXC rule as the only thing that can block traffic.
+allow_host_forwarding() {
+    local command
+    for command in iptables ip6tables; do
+        if command -v "$command" >/dev/null 2>&1; then
+            sudo "$command" -P FORWARD ACCEPT ||
+                echo "WARNING: could not set the $command FORWARD policy to ACCEPT; deny-case tests may pass without enforcing anything." >&2
+        fi
+    done
+}
+
 # Report the host-side state that container networking depends on. Purely
 # diagnostic: never fails the job, so a networking problem still surfaces as
 # the backend test failure rather than as a prerequisite error.
@@ -172,8 +188,15 @@ report_lxc_network_diagnostics() {
     sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E '10\.0\.3|MASQUERADE' |
         sed 's/^/  /' || echo "  (none found)"
 
-    echo "FORWARD policy and bridge rules:"
-    sudo iptables -S FORWARD 2>/dev/null | grep -E "policy|$bridge" | sed 's/^/  /' ||
+    # A vacuous pass depends on this policy, so print it verbatim. grep reads
+    # the whole ruleset, avoiding the SIGPIPE an early-closing head would send
+    # back to iptables under pipefail.
+    echo "FORWARD policy (must be ACCEPT, or deny cases prove nothing):"
+    sudo iptables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/  /' || echo "  (unknown)"
+    sudo ip6tables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/  /' || echo "  (unknown)"
+
+    echo "FORWARD rules for the bridge:"
+    sudo iptables -S FORWARD 2>/dev/null | grep -E "$bridge" | sed 's/^/  /' ||
         echo "  (none found)"
 
     echo "Host /etc/resolv.conf nameservers:"
@@ -227,6 +250,7 @@ case "$backend" in
         fi
         start_lxc_bridge
         enable_bridge_netfilter
+        allow_host_forwarding
         report_lxc_network_diagnostics
         ;;
     microvm)

From a668c36134045ee0832a80dc2f418e445be4c9d4 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 10:49:32 -0700
Subject: [PATCH 03/44] NAT tests lxc

---
 scripts/ci/prepare-linux-host.sh | 77 +++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index bec5aac98..29c20b093 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -129,6 +129,37 @@ start_lxc_bridge() {
     ip addr show "$bridge" || true
 }
 
+# Outbound container traffic leaves the bridge subnet with a private source
+# address, so it needs a MASQUERADE rule to reach anything off-host. lxc-net
+# normally installs one, but it skips its firewall setup when it believes
+# another manager owns the ruleset, leaving a bridge that hands out leases the
+# container cannot use. The symptom is a name-resolution failure inside the
+# guest, which reads like a policy problem and is not one.
+ensure_bridge_nat() {
+    local bridge="${LXC_BRIDGE:-lxcbr0}"
+    local subnet
+
+    subnet="$(ip -4 -o addr show "$bridge" 2>/dev/null | awk '{print $4}' | head -n 1)"
+    if [[ -z "$subnet" ]]; then
+        echo "WARNING: $bridge has no IPv4 subnet; skipping NAT setup." >&2
+        return 0
+    fi
+
+    # Match on the source subnet rather than the rule text: lxc-net's own rule
+    # and ours are equivalent however they are spelled.
+    if sudo iptables -t nat -S POSTROUTING 2>/dev/null |
+        grep -q -- "-s ${subnet%%/*}"; then
+        echo "NAT for $subnet is already present."
+        return 0
+    fi
+
+    if sudo iptables -t nat -A POSTROUTING -s "$subnet" ! -d "$subnet" -j MASQUERADE; then
+        echo "Installed MASQUERADE for $subnet."
+    else
+        echo "WARNING: could not install MASQUERADE for $subnet; containers will not reach off-host destinations." >&2
+    fi
+}
+
 # Container network policy is programmed as iptables rules reached from
 # FORWARD, which only sees bridged traffic when br_netfilter is loaded and
 # bridge-nf-call-iptables is enabled. Neither is guaranteed on a fresh image,
@@ -185,7 +216,7 @@ report_lxc_network_diagnostics() {
     pgrep -af dnsmasq 2>/dev/null | sed 's/^/  /' || echo "  (none)"
 
     echo "NAT rules for the bridge subnet:"
-    sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E '10\.0\.3|MASQUERADE' |
+    sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E 'MASQUERADE|SNAT' |
         sed 's/^/  /' || echo "  (none found)"
 
     # A vacuous pass depends on this policy, so print it verbatim. grep reads
@@ -228,6 +259,48 @@ report_lxc_network_diagnostics() {
     echo "--- end diagnostics ---"
 }
 
+# Boot a throwaway container and confirm it gets an IPv4 lease and can reach
+# off-host. Every host-side check can pass while a container still comes up
+# IPv6-only, which surfaces as an unrelated-looking resolution failure much
+# later in the suite. Purely diagnostic: never fails the job.
+report_lxc_container_connectivity() {
+    local container="mxc-ci-netcheck-$$"
+
+    echo "--- LXC container connectivity probe ---"
+
+    if ! sudo lxc-create -n "$container" -t download -- \
+        --dist alpine --release 3.23 --arch "$(dpkg --print-architecture 2>/dev/null || arch)" \
+        >/dev/null 2>&1; then
+        echo "  could not create the probe container; skipping."
+        echo "--- end probe ---"
+        return 0
+    fi
+
+    if sudo lxc-start -n "$container" >/dev/null 2>&1; then
+        # The suite's own containers wait about this long for an address.
+        sudo lxc-wait -n "$container" -s RUNNING -t 15 >/dev/null 2>&1 || true
+        sleep 5
+
+        echo "  addresses:"
+        sudo lxc-info -n "$container" -iH 2>/dev/null | sed 's/^/    /' || echo "    (none)"
+
+        if sudo lxc-info -n "$container" -iH 2>/dev/null | grep -qE '^[0-9]+\.'; then
+            echo "  IPv4: present"
+        else
+            echo "  WARNING: no IPv4 address; DHCPv4 on the bridge is not working." >&2
+        fi
+
+        echo "  resolution from inside the container:"
+        sudo lxc-attach -n "$container" -- getent hosts api.github.com 2>&1 |
+            sed 's/^/    /' || echo "    FAILED - the container cannot resolve"
+    else
+        echo "  probe container failed to start."
+    fi
+
+    sudo lxc-destroy -n "$container" -f >/dev/null 2>&1 || true
+    echo "--- end probe ---"
+}
+
 chmod +x "$binary_directory/lxc-exec"
 case "$backend" in
     bubblewrap)
@@ -251,7 +324,9 @@ case "$backend" in
         start_lxc_bridge
         enable_bridge_netfilter
         allow_host_forwarding
+        ensure_bridge_nat
         report_lxc_network_diagnostics
+        report_lxc_container_connectivity
         ;;
     microvm)
         for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do

From f6a5af19b5c7c3f13f0580cc02745205beba106c Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 12:13:14 -0700
Subject: [PATCH 04/44] Remove strict mode issues ; made LXC more honest with
 failures

---
 scripts/ci/prepare-linux-host.sh             | 123 -------------------
 scripts/ci/validation-test-matrix.json       |  18 +++
 tests/scripts/run_ci_backend_tests.ps1       |   1 -
 tests/scripts/run_wslc_all_tests.ps1         |  20 +--
 tests/scripts/run_wslc_state_aware_tests.ps1 | 101 +++++----------
 5 files changed, 49 insertions(+), 214 deletions(-)

diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 29c20b093..f7afb2743 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -181,126 +181,6 @@ enable_bridge_netfilter() {
     done
 }
 
-# Container-scoped rules hook egress (-i <veth>), so a reply arrives in the
-# opposite direction, matches nothing MXC installed, and falls through to the
-# host's FORWARD policy. A DROP policy (Docker sets one) therefore breaks
-# allowed destinations, and worse, makes the deny cases pass vacuously: a
-# container with no working hook at all is equally unreachable. Forwarding by
-# default leaves an MXC rule as the only thing that can block traffic.
-allow_host_forwarding() {
-    local command
-    for command in iptables ip6tables; do
-        if command -v "$command" >/dev/null 2>&1; then
-            sudo "$command" -P FORWARD ACCEPT ||
-                echo "WARNING: could not set the $command FORWARD policy to ACCEPT; deny-case tests may pass without enforcing anything." >&2
-        fi
-    done
-}
-
-# Report the host-side state that container networking depends on. Purely
-# diagnostic: never fails the job, so a networking problem still surfaces as
-# the backend test failure rather than as a prerequisite error.
-report_lxc_network_diagnostics() {
-    local bridge="${LXC_BRIDGE:-lxcbr0}"
-
-    echo "--- LXC network diagnostics (host) ---"
-
-    echo "net.ipv4.ip_forward: $(cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo unknown)"
-
-    echo "bridge-nf-call-iptables: $(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)"
-
-    echo "Bridge $bridge:"
-    ip -4 addr show "$bridge" 2>/dev/null | sed 's/^/  /' || echo "  (absent)"
-
-    echo "dnsmasq processes:"
-    pgrep -af dnsmasq 2>/dev/null | sed 's/^/  /' || echo "  (none)"
-
-    echo "NAT rules for the bridge subnet:"
-    sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E 'MASQUERADE|SNAT' |
-        sed 's/^/  /' || echo "  (none found)"
-
-    # A vacuous pass depends on this policy, so print it verbatim. grep reads
-    # the whole ruleset, avoiding the SIGPIPE an early-closing head would send
-    # back to iptables under pipefail.
-    echo "FORWARD policy (must be ACCEPT, or deny cases prove nothing):"
-    sudo iptables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/  /' || echo "  (unknown)"
-    sudo ip6tables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/  /' || echo "  (unknown)"
-
-    echo "FORWARD rules for the bridge:"
-    sudo iptables -S FORWARD 2>/dev/null | grep -E "$bridge" | sed 's/^/  /' ||
-        echo "  (none found)"
-
-    echo "Host /etc/resolv.conf nameservers:"
-    grep '^nameserver' /etc/resolv.conf 2>/dev/null | sed 's/^/  /' || echo "  (none)"
-
-    echo "lxc-net configuration:"
-    grep -E '^(USE_LXC_BRIDGE|LXC_ADDR|LXC_NETMASK|LXC_DHCP_RANGE|LXC_DHCP_CONFILE)' \
-        /etc/default/lxc-net 2>/dev/null | sed 's/^/  /' || echo "  (no /etc/default/lxc-net)"
-
-    # Prove the host itself can resolve the name the network test uses. If this
-    # fails, the container was never going to succeed.
-    if command -v getent >/dev/null 2>&1; then
-        echo "Host resolution of api.github.com:"
-        getent ahostsv4 api.github.com 2>/dev/null | head -n 2 | sed 's/^/  /' ||
-            echo "  FAILED - the host cannot resolve it either"
-    fi
-
-    # Ask the bridge's own resolver, which is what a container is handed via
-    # DHCP. This isolates "dnsmasq is broken" from "the host is fine".
-    local bridge_ip
-    bridge_ip="$(ip -4 -o addr show "$bridge" 2>/dev/null |
-        awk '{print $4}' | cut -d/ -f1 | head -n 1)"
-    if [[ -n "$bridge_ip" ]] && command -v nslookup >/dev/null 2>&1; then
-        echo "Resolution via bridge resolver ($bridge_ip):"
-        nslookup api.github.com "$bridge_ip" 2>&1 | tail -n 4 | sed 's/^/  /' ||
-            echo "  FAILED - dnsmasq on $bridge is not answering"
-    fi
-
-    echo "--- end diagnostics ---"
-}
-
-# Boot a throwaway container and confirm it gets an IPv4 lease and can reach
-# off-host. Every host-side check can pass while a container still comes up
-# IPv6-only, which surfaces as an unrelated-looking resolution failure much
-# later in the suite. Purely diagnostic: never fails the job.
-report_lxc_container_connectivity() {
-    local container="mxc-ci-netcheck-$$"
-
-    echo "--- LXC container connectivity probe ---"
-
-    if ! sudo lxc-create -n "$container" -t download -- \
-        --dist alpine --release 3.23 --arch "$(dpkg --print-architecture 2>/dev/null || arch)" \
-        >/dev/null 2>&1; then
-        echo "  could not create the probe container; skipping."
-        echo "--- end probe ---"
-        return 0
-    fi
-
-    if sudo lxc-start -n "$container" >/dev/null 2>&1; then
-        # The suite's own containers wait about this long for an address.
-        sudo lxc-wait -n "$container" -s RUNNING -t 15 >/dev/null 2>&1 || true
-        sleep 5
-
-        echo "  addresses:"
-        sudo lxc-info -n "$container" -iH 2>/dev/null | sed 's/^/    /' || echo "    (none)"
-
-        if sudo lxc-info -n "$container" -iH 2>/dev/null | grep -qE '^[0-9]+\.'; then
-            echo "  IPv4: present"
-        else
-            echo "  WARNING: no IPv4 address; DHCPv4 on the bridge is not working." >&2
-        fi
-
-        echo "  resolution from inside the container:"
-        sudo lxc-attach -n "$container" -- getent hosts api.github.com 2>&1 |
-            sed 's/^/    /' || echo "    FAILED - the container cannot resolve"
-    else
-        echo "  probe container failed to start."
-    fi
-
-    sudo lxc-destroy -n "$container" -f >/dev/null 2>&1 || true
-    echo "--- end probe ---"
-}
-
 chmod +x "$binary_directory/lxc-exec"
 case "$backend" in
     bubblewrap)
@@ -323,10 +203,7 @@ case "$backend" in
         fi
         start_lxc_bridge
         enable_bridge_netfilter
-        allow_host_forwarding
         ensure_bridge_nat
-        report_lxc_network_diagnostics
-        report_lxc_container_connectivity
         ;;
     microvm)
         for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 96256003a..6b7fb8abc 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -411,6 +411,24 @@
         "backends": [
           "lxc"
         ]
+      },
+      {
+        "os": "windows-25h2",
+        "backends": [
+          "wslc"
+        ]
+      },
+      {
+        "os": "windows-24h2",
+        "backends": [
+          "wslc"
+        ]
+      },
+      {
+        "os": "windows-23h2",
+        "backends": [
+          "wslc"
+        ]
       }
     ]
   }
diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1
index adcb916a9..f72a0b6a6 100644
--- a/tests/scripts/run_ci_backend_tests.ps1
+++ b/tests/scripts/run_ci_backend_tests.ps1
@@ -28,7 +28,6 @@ param(
     [string]$Architecture
 )
 
-Set-StrictMode -Version Latest
 $ErrorActionPreference = 'Stop'
 
 $scriptRoot = Split-Path -Parent $MyInvocation.MyCommand.Path
diff --git a/tests/scripts/run_wslc_all_tests.ps1 b/tests/scripts/run_wslc_all_tests.ps1
index 694bdd0e0..9c2a6cb49 100644
--- a/tests/scripts/run_wslc_all_tests.ps1
+++ b/tests/scripts/run_wslc_all_tests.ps1
@@ -97,17 +97,6 @@ if (-not $SkipSetup) {
     }
 }
 
-# Helper: StrictMode-safe property read; returns $null when the property (or the
-# object) is absent. Lets the optional-config-field reads below work under the
-# Set-StrictMode -Version Latest that run_ci_backend_tests.ps1 imposes.
-function Get-JsonProperty {
-    param($Object, [Parameter(Mandatory)][string]$Name)
-    if ($null -eq $Object) { return $null }
-    $prop = $Object.PSObject.Properties[$Name]
-    if ($null -eq $prop) { return $null }
-    return $prop.Value
-}
-
 # Helper: run a single WSLC test config
 function Run-WslcTest {
     param(
@@ -125,12 +114,9 @@ function Run-WslcTest {
         return @{ Name = $ConfigFile; Pass = $true; Skipped = $true; Reason = "File not found" }
     }
 
-    # Skip if the config references a tar file that doesn't exist locally.
-    # Read the chain defensively: this suite inherits Set-StrictMode -Version
-    # Latest from run_ci_backend_tests.ps1, under which touching a missing
-    # property is a terminating error, and most configs have no wslc.imageTarPath.
+    # Skip if the config references a tar file that doesn't exist locally
     $configJson = Get-Content $configPath -Raw | ConvertFrom-Json
-    $tarPath = Get-JsonProperty (Get-JsonProperty (Get-JsonProperty $configJson 'experimental') 'wslc') 'imageTarPath'
+    $tarPath = $configJson.experimental.wslc.imageTarPath
     if ($tarPath -and -not (Test-Path $tarPath)) {
         Write-Host "  $ConfigFile ... " -NoNewline
         Write-Host "SKIP (tar not found: $tarPath)" -ForegroundColor Yellow
@@ -183,7 +169,7 @@ function Run-WslcTest {
     # PostExitCheck runs after exit/output gates pass. Receives ($id, $output)
     # and must return truthy. Use for externally-observable state assertions.
     if ($pass -and $PostExitCheck) {
-        $containerId = Get-JsonProperty $configJson 'containerId'
+        $containerId = $configJson.containerId
         try {
             $checkResult = & $PostExitCheck $containerId $output
             if (-not $checkResult) {
diff --git a/tests/scripts/run_wslc_state_aware_tests.ps1 b/tests/scripts/run_wslc_state_aware_tests.ps1
index 63967d9ac..2f2faa034 100644
--- a/tests/scripts/run_wslc_state_aware_tests.ps1
+++ b/tests/scripts/run_wslc_state_aware_tests.ps1
@@ -283,50 +283,6 @@ function Envelope-Arm {
     '<unknown>'
 }
 
-# StrictMode-safe property read; returns $null when the object or property is
-# absent. run_ci_backend_tests.ps1 imposes Set-StrictMode -Version Latest, under
-# which touching a missing property on a PSCustomObject is a terminating error --
-# and an envelope carries exactly one of `result` / `error`, so the other arm is
-# always missing.
-function Get-JsonProperty {
-    param($Object, [Parameter(Mandatory)][string]$Name)
-    if ($null -eq $Object) { return $null }
-    $prop = $Object.PSObject.Properties[$Name]
-    if ($null -eq $prop) { return $null }
-    return $prop.Value
-}
-
-# The envelope's `error` object, or $null when the envelope is absent/succeeded.
-function Get-EnvelopeError {
-    param($Envelope)
-    Get-JsonProperty $Envelope 'error'
-}
-
-# The envelope's error code, or '<no envelope>' / '<no error>' when absent.
-function Get-EnvelopeErrorCode {
-    param($Envelope)
-    if ($null -eq $Envelope) { return '<no envelope>' }
-    $err = Get-JsonProperty $Envelope 'error'
-    if ($null -eq $err) { return '<no error>' }
-    $code = Get-JsonProperty $err 'code'
-    if ($null -eq $code) { return '<no code>' }
-    return [string]$code
-}
-
-# A property of the envelope's `result` object, or $null when absent.
-function Get-EnvelopeResultProperty {
-    param($Envelope, [Parameter(Mandatory)][string]$Name)
-    Get-JsonProperty (Get-JsonProperty $Envelope 'result') $Name
-}
-
-# The envelope's `result.sandboxId` as a string, or $null when absent.
-function Get-EnvelopeSandboxId {
-    param($Envelope)
-    $id = Get-EnvelopeResultProperty $Envelope 'sandboxId'
-    if ($null -eq $id) { return $null }
-    return [string]$id
-}
-
 # Is the daemon process currently running?
 function Test-DaemonRunning {
     $null -ne (Get-Process -Name $DaemonProcName -ErrorAction SilentlyContinue)
@@ -389,7 +345,7 @@ function Provision-Sandbox {
     $r = Invoke-StateAware -ConfigFile $ConfigFile
     $envObj = Parse-Envelope -Stdout $r.Stdout
     if ((Envelope-Arm $envObj) -ne 'result') { return $null }
-    Get-EnvelopeSandboxId $envObj
+    [string]$envObj.result.sandboxId
 }
 
 # ---------------- Backend-availability probe ----------------
@@ -407,17 +363,16 @@ if ($script:PreExistingDaemon) {
 
 $probe = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
 $probeEnv = Parse-Envelope -Stdout $probe.Stdout
-$probeError = Get-EnvelopeError $probeEnv
-if ((Get-EnvelopeErrorCode $probeEnv) -eq 'backend_unavailable') {
+if ($null -ne $probeEnv -and $probeEnv.error.code -eq 'backend_unavailable') {
     Write-Host "SKIPPED: wxc-exec reports backend_unavailable (built without --features wslc, or no WSLC runtime)" -ForegroundColor Yellow
     exit 0
 }
-$probeSandboxId = Get-EnvelopeSandboxId $probeEnv
-if ($null -ne $probeSandboxId) {
+if ($null -ne $probeEnv -and $null -ne $probeEnv.result -and $null -ne $probeEnv.result.sandboxId) {
+    $probeSandboxId = [string]$probeEnv.result.sandboxId
     Write-Host "Backend probe: provisioned $probeSandboxId, deprovisioning ..." -ForegroundColor DarkGray
     $null = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $probeSandboxId
-} elseif ($null -ne $probeError) {
-    Write-Host "WARN: probe provision errored (code=$(Get-EnvelopeErrorCode $probeEnv)): $(Get-JsonProperty $probeError 'message')" -ForegroundColor Yellow
+} elseif ($null -ne $probeEnv -and $null -ne $probeEnv.error) {
+    Write-Host "WARN: probe provision errored (code=$($probeEnv.error.code)): $($probeEnv.error.message)" -ForegroundColor Yellow
     Write-Host "  Continuing -- individual tests will report specific failures." -ForegroundColor Yellow
 }
 
@@ -431,7 +386,7 @@ try {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "provision"
         if ($envObj) {
-            $script:sandboxId = Get-EnvelopeSandboxId $envObj
+            $script:sandboxId = [string]$envObj.result.sandboxId
             Assert-True ($script:sandboxId -match '^wslc:[0-9a-f]{32}$') `
                 "sandbox_id matches wslc:<32-hex> ($script:sandboxId)"
         }
@@ -444,7 +399,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_start.json' -SandboxId $script:sandboxId
             $envObj = Assert-ResultEnvelope $r "start"
             if ($envObj) {
-                Assert-True ($null -eq (Get-EnvelopeResultProperty $envObj 'metadata')) "result.metadata absent (no start metadata in v1)"
+                Assert-True ($null -eq $envObj.result.metadata) "result.metadata absent (no start metadata in v1)"
             }
         }
     }
@@ -458,7 +413,7 @@ try {
             Assert-True ($r.Stdout -match 'wslc-state-aware-exec-marker') `
                 "stdout contains the script's output (relayed live, not enveloped)"
             $maybeEnv = Parse-Envelope -Stdout $r.Stdout
-            Assert-True ($null -eq (Get-EnvelopeError $maybeEnv)) `
+            Assert-True ($null -eq $maybeEnv -or $null -eq $maybeEnv.error) `
                 "stdout is not a state-aware error envelope on success"
         }
     }
@@ -545,7 +500,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_exec_rejected_filesystem.json' -SandboxId $script:sandboxId
             Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = Get-EnvelopeErrorCode $envObj
+            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
             Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
         } | Out-Null
     }
@@ -575,7 +530,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_stop.json' -SandboxId $script:sandboxId
             Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (stop on stale sandbox failed as expected)"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = Get-EnvelopeErrorCode $envObj
+            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
             Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')"
         } | Out-Null
     }
@@ -604,7 +559,7 @@ try {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_with_filesystem.json'
         $envObj = Assert-ResultEnvelope $r "filesystem provision"
         if ($envObj) {
-            $script:fsSandboxId = Get-EnvelopeSandboxId $envObj
+            $script:fsSandboxId = [string]$envObj.result.sandboxId
             Assert-True ($script:fsSandboxId -match '^wslc:[0-9a-f]{32}$') `
                 "sandbox_id matches wslc:<32-hex> ($script:fsSandboxId)"
         }
@@ -690,7 +645,7 @@ try {
     $netProvisionedOk = Run-StateAwareTest "C: provision (bridged network)" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_bridged.json'
         $envObj = Assert-ResultEnvelope $r "bridged provision"
-        if ($envObj) { $script:netSandboxId = Get-EnvelopeSandboxId $envObj }
+        if ($envObj) { $script:netSandboxId = [string]$envObj.result.sandboxId }
     }
 
     $netStartedOk = $false
@@ -741,7 +696,7 @@ Run-StateAwareTest "D: provision (deniedPaths nested under mount rejected)" {
     $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_denied.json'
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = Get-EnvelopeErrorCode $envObj
+    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -749,7 +704,7 @@ Run-StateAwareTest "D: provision (host filtering rejected)" {
     $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_hosts.json'
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = Get-EnvelopeErrorCode $envObj
+    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -757,7 +712,7 @@ Run-StateAwareTest "D: provision (proxy at provision rejected)" {
     $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_proxy.json'
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = Get-EnvelopeErrorCode $envObj
+    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -766,7 +721,7 @@ Run-StateAwareTest "D: start (filesystem policy rejected)" {
     $r = Invoke-StateAware -Request $req
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = Get-EnvelopeErrorCode $envObj
+    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -780,7 +735,7 @@ Run-StateAwareTest "D: start (network.proxy rejected post-provision)" {
     $r = Invoke-StateAware -Request $req
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = Get-EnvelopeErrorCode $envObj
+    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -789,7 +744,7 @@ Run-StateAwareTest "D: stop (network.proxy rejected post-provision)" {
     $r = Invoke-StateAware -Request $req
     Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)"
     $envObj = Parse-Envelope -Stdout $r.Stdout
-    $code = Get-EnvelopeErrorCode $envObj
+    $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
     Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')"
 } | Out-Null
 
@@ -808,7 +763,7 @@ try {
     $reProvisionedOk = Run-StateAwareTest "E: provision (restart cycle)" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "restart provision"
-        if ($envObj) { $script:reSandboxId = Get-EnvelopeSandboxId $envObj }
+        if ($envObj) { $script:reSandboxId = [string]$envObj.result.sandboxId }
     }
 
     $reStartedOk = $false
@@ -846,7 +801,7 @@ try {
                     $script:reRestartSucceeded = $true
                     Write-Host "  INFO: WSLc supports restart-after-stop on the same sandbox" -ForegroundColor DarkGray
                 } elseif ($arm -eq 'error') {
-                    Write-Host "  INFO: WSLc does NOT support restart-after-stop (error.code=$(Get-EnvelopeErrorCode $envObj)) -- documented limitation" -ForegroundColor DarkGray
+                    Write-Host "  INFO: WSLc does NOT support restart-after-stop (error.code=$($envObj.error.code)) -- documented limitation" -ForegroundColor DarkGray
                 }
             } | Out-Null
 
@@ -888,7 +843,7 @@ try {
     $edgeProvisionedOk = Run-StateAwareTest "F: provision (exec-edge sandbox)" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "edge provision"
-        if ($envObj) { $script:edgeSandboxId = Get-EnvelopeSandboxId $envObj }
+        if ($envObj) { $script:edgeSandboxId = [string]$envObj.result.sandboxId }
     }
 
     # F1: exec BEFORE start -> not_started (the daemon knows the id but the
@@ -899,7 +854,7 @@ try {
             $r = Invoke-StateAware -Request $req
             Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (exec before start rejected)"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = Get-EnvelopeErrorCode $envObj
+            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
             Assert-True ($code -eq 'not_started') "error.code is 'not_started' (got '$code')"
         } | Out-Null
     }
@@ -921,7 +876,7 @@ try {
             $r = Invoke-StateAware -Request $slow
             Assert-True ($r.ExitCode -ne 0) "timed-out exec exits non-zero"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = Get-EnvelopeErrorCode $envObj
+            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
             Assert-True ($code -eq 'backend_error') "timeout maps to 'backend_error' (got '$code')"
 
             $after = @{ phase = 'exec'; sandboxId = $script:edgeSandboxId; process = @{ commandLine = 'echo survived-timeout'; timeout = 30000 } }
@@ -957,7 +912,7 @@ try {
             $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $script:edgeSandboxId
             Assert-True ($r.ExitCode -ne 0) "second deprovision exits non-zero"
             $envObj = Parse-Envelope -Stdout $r.Stdout
-            $code = Get-EnvelopeErrorCode $envObj
+            $code = if ($envObj) { $envObj.error.code } else { '<no envelope>' }
             Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')"
         } | Out-Null
     }
@@ -982,12 +937,12 @@ try {
     $mcAProvOk = Run-StateAwareTest "G: provision sandbox A" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "multi-container A provision"
-        if ($envObj) { $script:mcSandboxA = Get-EnvelopeSandboxId $envObj }
+        if ($envObj) { $script:mcSandboxA = [string]$envObj.result.sandboxId }
     }
     $mcBProvOk = Run-StateAwareTest "G: provision sandbox B" {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "multi-container B provision"
-        if ($envObj) { $script:mcSandboxB = Get-EnvelopeSandboxId $envObj }
+        if ($envObj) { $script:mcSandboxB = [string]$envObj.result.sandboxId }
     }
 
     if ($mcAProvOk -and $mcBProvOk) {
@@ -1102,7 +1057,7 @@ try {
         $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json'
         $envObj = Assert-ResultEnvelope $r "post-teardown provision"
         if ($envObj) {
-            $script:recSandboxId = Get-EnvelopeSandboxId $envObj
+            $script:recSandboxId = [string]$envObj.result.sandboxId
             Assert-True ($script:recSandboxId -match '^wslc:[0-9a-f]{32}$') `
                 "sandbox_id matches wslc:<32-hex> ($script:recSandboxId)"
         }

From 6c8c8339e5fdd1df4b0b45ec2a0bce9dd7e2f63d Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 12:16:29 -0700
Subject: [PATCH 05/44] testing scaleset

---
 .github/workflows/Validation.Tests.Matrix.Job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index 9a958d94a..bfd2ca10e 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -91,7 +91,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.resolve.outputs.linux) }}
-    runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ]
+    runs-on: ["${{ matrix.pool }}"]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

From fbef27dbee19c85ac9be8d14b171f2b9bc22c298 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 12:51:11 -0700
Subject: [PATCH 06/44] scaleset isn't supported yet

---
 .github/workflows/Validation.Tests.Matrix.Job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index bfd2ca10e..9a958d94a 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -91,7 +91,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.resolve.outputs.linux) }}
-    runs-on: ["${{ matrix.pool }}"]
+    runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

From cfaa550830f6f555fedee84df089eef15e60a119 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 12:57:09 -0700
Subject: [PATCH 07/44] create test file that wslc test script expects

---
 tests/scripts/run_wslc_all_tests.ps1 | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/scripts/run_wslc_all_tests.ps1 b/tests/scripts/run_wslc_all_tests.ps1
index 9c2a6cb49..993037bf1 100644
--- a/tests/scripts/run_wslc_all_tests.ps1
+++ b/tests/scripts/run_wslc_all_tests.ps1
@@ -215,6 +215,15 @@ $null = $results.Add((Run-WslcTest "wslc_stderr.json" -OutputContains "stdout me
 $null = $results.Add((Run-WslcTest "wslc_large_output.json"))
 
 Write-Host "`n--- Filesystem Tests ---" -ForegroundColor Cyan
+# The filesystem configs mount C:\workspace and read test.txt from it. A
+# developer machine usually has both already; a clean runner has neither, so
+# create them here rather than leaving the mount empty.
+$WorkspaceDir = "C:\workspace"
+New-Item -ItemType Directory -Path $WorkspaceDir -Force | Out-Null
+$WorkspaceFile = Join-Path $WorkspaceDir "test.txt"
+if (-not (Test-Path $WorkspaceFile)) {
+    Set-Content $WorkspaceFile "workspace fixture"
+}
 # wslc_filesystem.json also asserts cpuCount + memoryMb enforcement via nproc and /proc/meminfo.
 $null = $results.Add((Run-WslcTest "wslc_filesystem.json" `
     -OutputMatches "(?s)PASS: filesystem mount visible.*PASS: cpuCount enforced.*PASS: memoryMb enforced"))

From 1ce43fdae889a0fb0c6b102b00e2306c568e1629 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 13:46:20 -0700
Subject: [PATCH 08/44] stagger WSLC jobs

---
 .../workflows/Validation.Tests.Matrix.Job.yml | 14 ++++-
 docs/ci-validation-infrastructure.md          | 43 ++++++++++++-
 scripts/ci/resolve-validation-test-matrix.mjs | 63 +++++++++++++++++++
 scripts/ci/validation-test-matrix.json        |  6 ++
 4 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index 9a958d94a..fb702189a 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -38,7 +38,8 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.resolve.outputs.windows) }}
     runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ]
-    timeout-minutes: 60
+    # Includes any startup stagger, so a delayed entry keeps the full test budget.
+    timeout-minutes: ${{ 60 + (matrix.startup_delay_minutes || 0) }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
@@ -48,6 +49,17 @@ jobs:
           name: ${{ matrix.artifact }}
           path: artifacts/bin
 
+      - name: Delayed start
+        if: matrix.startup_delay_minutes > 0
+        shell: pwsh
+        run: |
+          # Spreads network-heavy setup across the pool's shared egress address
+          # instead of issuing every job's downloads at once. Configured by
+          # backendDelayedStart in scripts/ci/validation-test-matrix.json.
+          $minutes = ${{ matrix.startup_delay_minutes || 0 }}
+          Write-Host "Waiting $minutes minute(s) before downloading prerequisites."
+          Start-Sleep -Seconds ($minutes * 60)
+
       - name: Prepare backend prerequisites
         timeout-minutes: 15
         shell: pwsh
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index ec0e99486..c1baae81f 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -27,7 +27,7 @@ the individual local test scripts are documented in
 |------|------|
 | `.github/workflows/Validation.Tests.Scheduled.yml` | Scheduled entry point. Builds artifacts, then calls the matrix job. |
 | `.github/workflows/Validation.Tests.Matrix.Job.yml` | `workflow_call`-only. Resolves the plan and runs the per-family test jobs. |
-| `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers. |
+| `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers, job staggering. |
 | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. |
 | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. |
 | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). |
@@ -157,6 +157,40 @@ isolation-session, wslc; 25H2/24H2/23H2 × process-t3 + wslc) and
 8 Linux (each of the four distros × bubblewrap + lxc). macOS resolves empty
 because Seatbelt has no wired suite.
 
+### `backendDelayedStart`
+
+Optional. Staggers the start of jobs for a named backend instead of letting
+them all begin at once:
+
+```json
+"backendDelayedStart": [
+  { "backend": "wslc", "minutes": 5 }
+]
+```
+
+Every runner in a pool shares one egress address. A backend whose setup
+downloads a large runtime or several container images therefore concentrates
+that traffic into a burst when its jobs start together, which draws rate
+limiting from public registries and stalled downloads. 
+
+`minutes` is the gap between consecutive jobs of that backend, counted
+independently per backend, following the resolved job order. With the entry
+above, four WSLC jobs start at 0, 5, 10, and 15 minutes.
+
+The resolver emits the offset as `startup_delay_minutes` on each affected
+matrix entry. The job sleeps that long before its first network step, and its
+`timeout-minutes` grows by the same amount so a delayed entry keeps the full
+test budget. Entries for other backends carry no such field and never wait.
+
+Omit the section (or leave it empty) to have every job start as soon as its
+runner is ready. A backend id that no plan schedules is accepted and simply
+never applies.
+
+Note this holds the runner while it sleeps: Actions cannot defer allocating a
+matrix job, so the wait happens inside the job. Keep the value only as large as
+the contention requires. It spreads simultaneous load — it does not help a
+single download that stalls on its own.
+
 ## Backend status
 
 Snapshot of what the matrix actually proves today. Update this table as backends
@@ -282,6 +316,13 @@ Replace the explicit failure in the dispatcher with the suite invocation, add
 any host prerequisites, then add the OS/backend pair to a trigger. Always verify 
 by testing it ahead of time. 
 
+### Stagger a backend's job starts
+
+Add or edit its `backendDelayedStart` entry in the catalog, then resolve
+locally to confirm the offsets. Reach for this when a backend's setup is
+network-heavy enough that concurrent jobs hit rate limits or stalled
+downloads; remove the entry once that pressure is gone.
+
 ### Change the schedule
 
 Everything schedule-related lives in `Validation.Tests.Scheduled.yml`: the two
diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs
index 9c73382d2..79cae85b7 100644
--- a/scripts/ci/resolve-validation-test-matrix.mjs
+++ b/scripts/ci/resolve-validation-test-matrix.mjs
@@ -131,9 +131,41 @@ export function validateCatalog(catalog) {
     }
   }
 
+  validateBackendDelayedStart(catalog);
+
   return { platforms, plans };
 }
 
+// backendDelayedStart is optional: an absent or empty section means every job
+// starts as soon as its runner is ready.
+function validateBackendDelayedStart(catalog) {
+  const delays = catalog.backendDelayedStart;
+  if (delays == null) {
+    return;
+  }
+  if (!Array.isArray(delays)) {
+    throw new Error('catalog backendDelayedStart must be an array');
+  }
+
+  const seen = new Set();
+  for (const entry of delays) {
+    if (entry == null || typeof entry !== 'object' || Array.isArray(entry)) {
+      throw new Error('each backendDelayedStart entry must be an object');
+    }
+    assertNonEmptyString(entry.backend, 'backendDelayedStart backend');
+    if (seen.has(entry.backend)) {
+      throw new Error(`duplicate backendDelayedStart entry for ${entry.backend}`);
+    }
+    seen.add(entry.backend);
+
+    if (!Number.isInteger(entry.minutes) || entry.minutes < 0) {
+      throw new Error(
+        `backendDelayedStart ${entry.backend} minutes must be a non-negative integer`
+      );
+    }
+  }
+}
+
 export function expandPlan(catalog, plan) {
   const { platforms, plans } = validateCatalog(catalog);
   if (!plans.includes(plan)) {
@@ -186,9 +218,40 @@ export function resolvePlan(catalog, plan) {
 
   suppressNonMacArm64(matrices);
   sortMatrices(matrices);
+  applyDelayedStart(matrices, catalog.backendDelayedStart);
   return matrices;
 }
 
+// A backend named in backendDelayedStart has its jobs started at staggered
+// offsets rather than all at once. This exists for backends whose setup
+// downloads a large runtime or several container images: every runner in a
+// pool shares one egress address, so simultaneous starts concentrate that
+// traffic into a burst that draws rate limiting and stalled downloads.
+//
+// The entry's minutes value is the gap between consecutive jobs of that
+// backend, counted independently per backend. Applied after sorting so the
+// assignment follows the emitted order and stays reproducible.
+function applyDelayedStart(matrices, delays) {
+  if (!Array.isArray(delays) || delays.length === 0) {
+    return;
+  }
+
+  const stepMinutes = new Map(delays.map(entry => [entry.backend, entry.minutes]));
+  const scheduled = new Map();
+
+  for (const family of FAMILIES) {
+    for (const entry of matrices[family]) {
+      const step = stepMinutes.get(entry.backend);
+      if (step === undefined) {
+        continue;
+      }
+      const position = scheduled.get(entry.backend) ?? 0;
+      entry.startup_delay_minutes = position * step;
+      scheduled.set(entry.backend, position + 1);
+    }
+  }
+}
+
 // Windows and Linux ARM64 hosted VMs currently lack nested virtualization.
 // Keep their catalog entries intact for future enablement, but never emit them
 // until suitable test hosts are available. macOS remains ARM64-only.
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 6b7fb8abc..cacaba214 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -325,6 +325,12 @@
       }
     }
   ],
+  "backendDelayedStart": [
+    {
+      "backend": "wslc",
+      "minutes": 2
+    }
+  ],
   "triggers": {
     "pr": [],
     "nightly": [

From c6aad1655dc5839cb0f4a596aeb56ccaca0885c5 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 14:08:23 -0700
Subject: [PATCH 09/44] log upload should now include all windows logs

---
 .../workflows/Validation.Tests.Matrix.Job.yml |  22 ++--
 scripts/ci/validation-test-matrix.json        |  18 +--
 tests/scripts/run_ci_backend_tests.ps1        | 107 +++++++++++++-----
 3 files changed, 96 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index fb702189a..dc6726a57 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -83,16 +83,22 @@ jobs:
             Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 
-      - name: Upload failure logs
-        if: failure() || cancelled()
+      - name: Upload logs
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
         with:
           name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }}
           path: |
             ${{ runner.temp }}/mxc-ci.log
-            ${{ runner.temp }}/mxc-wpc-tests/logs
-            ${{ runner.temp }}/WinProcessContainer-Tests.results.*
+            ${{ runner.temp }}/mxc-wpc-tests
+            ${{ runner.temp }}/mxc-t3-workloads
             ${{ runner.temp }}/mxc_concurrent_oneshot
+            ${{ runner.temp }}/mxc_etw_test
+            ${{ runner.temp }}/wxc-wsb
+            ${{ runner.temp }}/wxc-sandbox-rendezvous
+            ${{ runner.temp }}/mxc-diagnostics-*
+            ${{ runner.temp }}/WinProcessContainer-Tests.results.*
+            ${{ runner.temp }}/WinProcessContainer-Tests.cargo.log
           if-no-files-found: ignore
           retention-days: 7
 
@@ -138,8 +144,8 @@ jobs:
               tee -a "$RUNNER_TEMP/mxc-ci.log"
           fi
 
-      - name: Upload failure logs
-        if: failure() || cancelled()
+      - name: Upload logs
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
         with:
           name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }}
@@ -175,8 +181,8 @@ jobs:
             '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 |
             tee "$RUNNER_TEMP/mxc-ci.log"
 
-      - name: Upload failure logs
-        if: failure() || cancelled()
+      - name: Upload logs
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
         with:
           name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }}
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index cacaba214..c1bbd3816 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -397,25 +397,15 @@
       {
         "os": "ubuntu-26.04",
         "backends": [
+          "bubblewrap",
           "lxc"
         ]
       },
       {
-        "os": "ubuntu-24.04",
-        "backends": [
-          "lxc"
-        ]
-      },
-      {
-        "os": "rhel-10",
-        "backends": [
-          "lxc"
-        ]
-      },
-      {
-        "os": "debian-13",
+        "os": "windows-prerelease-process-container",
         "backends": [
-          "lxc"
+          "wslc",
+          "process-t1"
         ]
       },
       {
diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1
index f72a0b6a6..572b2618c 100644
--- a/tests/scripts/run_ci_backend_tests.ps1
+++ b/tests/scripts/run_ci_backend_tests.ps1
@@ -42,6 +42,47 @@ function Assert-File {
     }
 }
 
+# The suites write their logs, configs, and transcripts under $env:TEMP, which
+# is not the directory CI uploads from. Several also refuse to run with those
+# paths pointed elsewhere, so copy the artifacts across afterwards instead of
+# redirecting them. Best-effort by design: losing a log must not turn a passing
+# run red, and a failing run keeps its original result.
+function Copy-TempArtifacts {
+    if (-not $env:RUNNER_TEMP -or -not $env:TEMP) {
+        return
+    }
+
+    $names = @(
+        'mxc-wpc-tests',
+        'mxc-t3-workloads',
+        'mxc_concurrent_oneshot',
+        'mxc_etw_test',
+        'wxc-wsb',
+        'wxc-sandbox-rendezvous',
+        'WinProcessContainer-Tests.results.txt',
+        'WinProcessContainer-Tests.results.json',
+        'WinProcessContainer-Tests.cargo.log'
+    )
+
+    foreach ($name in $names) {
+        $source = Join-Path $env:TEMP $name
+        if (-not (Test-Path -LiteralPath $source)) {
+            continue
+        }
+        Copy-Item -Recurse -Force -LiteralPath $source `
+            -Destination (Join-Path $env:RUNNER_TEMP $name) `
+            -ErrorAction SilentlyContinue
+    }
+
+    # MXC's own diagnostic dumps are timestamped per run, so match the family.
+    Get-ChildItem -Path $env:TEMP -Filter 'mxc-diagnostics-*' -ErrorAction SilentlyContinue |
+        ForEach-Object {
+            Copy-Item -Recurse -Force -LiteralPath $_.FullName `
+                -Destination (Join-Path $env:RUNNER_TEMP $_.Name) `
+                -ErrorAction SilentlyContinue
+        }
+}
+
 function Invoke-TestScript {
     param(
         [Parameter(Mandatory)][string]$Path,
@@ -77,6 +118,8 @@ function Invoke-ProcessContainerTests {
     Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force
 
     $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1'
+    # -KeepArtifacts stops the harness deleting its scratch tree on a clean
+    # run, so a passing job still uploads its per-test logs and configs.
     # Skip build and Cargo phases because this job consumes a previously
     # built artifact; retain the host and containment behavior phases.
     $phases = @(
@@ -96,45 +139,51 @@ function Invoke-ProcessContainerTests {
         -WxcRelease (Join-Path $releaseDirectory 'wxc-exec.exe') `
         -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') `
         -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') `
+        -KeepArtifacts `
         -Phases $phases
     if ($LASTEXITCODE -ne 0) {
         throw "Process Container tests failed with exit code $LASTEXITCODE."
     }
 }
 
-switch ($Backend) {
-    'process-t1' {
-        Invoke-ProcessContainerTests
-    }
-    'process-t3' {
-        Invoke-ProcessContainerTests
-    }
-    'isolation-session' {
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{
-            WxcExePath = $wxc
+try {
+    switch ($Backend) {
+        'process-t1' {
+            Invoke-ProcessContainerTests
         }
-    }
-    'windows-sandbox' {
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{
-            BinDir = $binaryDirectoryPath
+        'process-t3' {
+            Invoke-ProcessContainerTests
         }
-    }
-    'wslc' {
-        # The current WSLC helper hardcodes the x64 target when locating assets.
-        if ($Architecture -ne 'x64') {
-            throw 'The existing WSLC test harness is not architecture-portable yet.'
+        'isolation-session' {
+            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{
+                WxcExePath = $wxc
+            }
         }
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{
-            WxcExecPath = $wxc
+        'windows-sandbox' {
+            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{
+                BinDir = $binaryDirectoryPath
+            }
         }
-    }
-    'microvm' {
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{
-            BinDir = $binaryDirectoryPath
+        'wslc' {
+            # The current WSLC helper hardcodes the x64 target when locating assets.
+            if ($Architecture -ne 'x64') {
+                throw 'The existing WSLC test harness is not architecture-portable yet.'
+            }
+            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{
+                WxcExecPath = $wxc
+            }
+        }
+        'microvm' {
+            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{
+                BinDir = $binaryDirectoryPath
+            }
+        }
+        'hyperlight' {
+            # Keep unwired backends explicit so accidental activation fails loudly.
+            throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.'
         }
     }
-    'hyperlight' {
-        # Keep unwired backends explicit so accidental activation fails loudly.
-        throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.'
-    }
+} finally {
+    # Also runs when a suite throws, which is when these logs matter most.
+    Copy-TempArtifacts
 }

From a2bf450ff6581a9a5d8fda84ad21c892544b2f73 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 14:16:05 -0700
Subject: [PATCH 10/44] removed math in yml

---
 .../workflows/Validation.Tests.Matrix.Job.yml | 35 ++++++++++++++++---
 docs/ci-validation-infrastructure.md          | 14 ++++----
 scripts/ci/resolve-validation-test-matrix.mjs | 19 +++++++---
 3 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index dc6726a57..bcd424866 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -38,8 +38,9 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.resolve.outputs.windows) }}
     runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ]
-    # Includes any startup stagger, so a delayed entry keeps the full test budget.
-    timeout-minutes: ${{ 60 + (matrix.startup_delay_minutes || 0) }}
+    # Resolved per entry: the test budget plus any startup stagger, so a
+    # delayed entry keeps its full budget. Workflow expressions cannot add.
+    timeout-minutes: 180
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
@@ -56,7 +57,7 @@ jobs:
           # Spreads network-heavy setup across the pool's shared egress address
           # instead of issuing every job's downloads at once. Configured by
           # backendDelayedStart in scripts/ci/validation-test-matrix.json.
-          $minutes = ${{ matrix.startup_delay_minutes || 0 }}
+          $minutes = ${{ matrix.startup_delay_minutes }}
           Write-Host "Waiting $minutes minute(s) before downloading prerequisites."
           Start-Sleep -Seconds ($minutes * 60)
 
@@ -110,7 +111,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.resolve.outputs.linux) }}
     runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ]
-    timeout-minutes: 60
+    timeout-minutes: 180
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
@@ -120,6 +121,18 @@ jobs:
           name: ${{ matrix.artifact }}
           path: artifacts/bin
 
+      - name: Delayed start
+        if: matrix.startup_delay_minutes > 0
+        shell: bash
+        run: |
+          # Spreads network-heavy setup across the pool's shared egress address
+          # instead of issuing every job's downloads at once. Configured by
+          # backendDelayedStart in scripts/ci/validation-test-matrix.json.
+          set -euo pipefail
+          minutes=${{ matrix.startup_delay_minutes }}
+          echo "Waiting $minutes minute(s) before downloading prerequisites."
+          sleep $(( minutes * 60 ))
+
       - name: Prepare backend prerequisites
         timeout-minutes: 15
         shell: bash
@@ -161,7 +174,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.resolve.outputs.macos) }}
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 60
+    timeout-minutes: 180
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
@@ -171,6 +184,18 @@ jobs:
           name: ${{ matrix.artifact }}
           path: artifacts/bin
 
+      - name: Delayed start
+        if: matrix.startup_delay_minutes > 0
+        shell: bash
+        run: |
+          # Spreads network-heavy setup across the pool's shared egress address
+          # instead of issuing every job's downloads at once. Configured by
+          # backendDelayedStart in scripts/ci/validation-test-matrix.json.
+          set -euo pipefail
+          minutes=${{ matrix.startup_delay_minutes }}
+          echo "Waiting $minutes minute(s) before starting tests."
+          sleep $(( minutes * 60 ))
+
       - name: Run backend tests
         timeout-minutes: 60
         shell: bash
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index c1baae81f..366b939a9 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -75,9 +75,9 @@ Build artifacts are kept for 1 day — they exist only to feed these jobs.
 | `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `chmod +x` → `run_ci_backend_tests.sh <backend id>`. No host-prep step. |
 
 Per-job display name: `<platform id>, <architecture>, <backend>` (macOS omits
-the architecture). Job timeout 60 min; host prep 15 min; the test step 45 min
-(60 on macOS), so a hung backend fails while the log is still useful. On failure
-or cancellation the job uploads `mxc-ci.log` plus the Process Container log
+the architecture). Job timeout 180 min; host prep 15 min; the test step 45 min
+(60 on macOS), so a hung backend fails while the log is still useful. The job
+uploads `mxc-ci.log` plus the Process Container log
 directories as `logs-<plan>-<os>-<arch>-<backend>-<attempt>`, kept 7 days.
 
 ## The catalog
@@ -177,10 +177,10 @@ limiting from public registries and stalled downloads.
 independently per backend, following the resolved job order. With the entry
 above, four WSLC jobs start at 0, 5, 10, and 15 minutes.
 
-The resolver emits the offset as `startup_delay_minutes` on each affected
-matrix entry. The job sleeps that long before its first network step, and its
-`timeout-minutes` grows by the same amount so a delayed entry keeps the full
-test budget. Entries for other backends carry no such field and never wait.
+The resolver emits the offset as `startup_delay_minutes` on every matrix entry
+(`0` where no stagger applies). The job sleeps that long before its first
+network step; the job timeout is a fixed 180 minutes, wide enough to absorb any
+configured wait.
 
 Omit the section (or leave it empty) to have every job start as soon as its
 runner is ready. A backend id that no plan schedules is accepted and simply
diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs
index 79cae85b7..4383eb680 100644
--- a/scripts/ci/resolve-validation-test-matrix.mjs
+++ b/scripts/ci/resolve-validation-test-matrix.mjs
@@ -233,6 +233,7 @@ export function resolvePlan(catalog, plan) {
 // assignment follows the emitted order and stays reproducible.
 function applyDelayedStart(matrices, delays) {
   if (!Array.isArray(delays) || delays.length === 0) {
+    setDefaultDelays(matrices);
     return;
   }
 
@@ -243,11 +244,21 @@ function applyDelayedStart(matrices, delays) {
     for (const entry of matrices[family]) {
       const step = stepMinutes.get(entry.backend);
       if (step === undefined) {
-        continue;
+        entry.startup_delay_minutes = 0;
+      } else {
+        const position = scheduled.get(entry.backend) ?? 0;
+        entry.startup_delay_minutes = position * step;
+        scheduled.set(entry.backend, position + 1);
       }
-      const position = scheduled.get(entry.backend) ?? 0;
-      entry.startup_delay_minutes = position * step;
-      scheduled.set(entry.backend, position + 1);
+    }
+  }
+}
+
+// Every entry carries the field so the workflow's step condition is uniform.
+function setDefaultDelays(matrices) {
+  for (const family of FAMILIES) {
+    for (const entry of matrices[family]) {
+      entry.startup_delay_minutes = 0;
     }
   }
 }

From 87fc7bbb99dfe9d2468fccdd39fc1fc0e1616a57 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 15:10:34 -0700
Subject: [PATCH 11/44] Simpler way to collect logs

---
 .../workflows/Validation.Tests.Matrix.Job.yml |  23 ++--
 docs/ci-validation-infrastructure.md          |  78 ++++++++----
 tests/scripts/run_ci_backend_tests.ps1        | 117 +++++++-----------
 3 files changed, 111 insertions(+), 107 deletions(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index bcd424866..79e4a2dd1 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -89,17 +89,20 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
         with:
           name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }}
+          # The dispatcher points TEMP here, so suite logs and scratch trees are
+          # collected without enumerating each one. Exclude bulk binaries that
+          # tooling also drops in TEMP; they dwarf the logs and aid no triage.
           path: |
-            ${{ runner.temp }}/mxc-ci.log
-            ${{ runner.temp }}/mxc-wpc-tests
-            ${{ runner.temp }}/mxc-t3-workloads
-            ${{ runner.temp }}/mxc_concurrent_oneshot
-            ${{ runner.temp }}/mxc_etw_test
-            ${{ runner.temp }}/wxc-wsb
-            ${{ runner.temp }}/wxc-sandbox-rendezvous
-            ${{ runner.temp }}/mxc-diagnostics-*
-            ${{ runner.temp }}/WinProcessContainer-Tests.results.*
-            ${{ runner.temp }}/WinProcessContainer-Tests.cargo.log
+            ${{ runner.temp }}
+            !${{ runner.temp }}/**/*.vhd
+            !${{ runner.temp }}/**/*.vhdx
+            !${{ runner.temp }}/**/*.tar
+            !${{ runner.temp }}/**/*.msi
+            !${{ runner.temp }}/**/*.msix
+            !${{ runner.temp }}/**/*.appx
+            !${{ runner.temp }}/**/*.zip
+            !${{ runner.temp }}/**/*.exe
+            !${{ runner.temp }}/**/*.dll
           if-no-files-found: ignore
           retention-days: 7
 
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 366b939a9..0d44ec532 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -31,7 +31,7 @@ the individual local test scripts are documented in
 | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. |
 | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. |
 | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). |
-| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. |
+| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
 | `tests/scripts/run_ci_backend_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
 
 ### Flow
@@ -76,9 +76,8 @@ Build artifacts are kept for 1 day — they exist only to feed these jobs.
 
 Per-job display name: `<platform id>, <architecture>, <backend>` (macOS omits
 the architecture). Job timeout 180 min; host prep 15 min; the test step 45 min
-(60 on macOS), so a hung backend fails while the log is still useful. The job
-uploads `mxc-ci.log` plus the Process Container log
-directories as `logs-<plan>-<os>-<arch>-<backend>-<attempt>`, kept 7 days.
+(60 on macOS), so a hung backend fails while the log is still useful. Logs are
+uploaded either way — see [Log collection](#log-collection).
 
 ## The catalog
 
@@ -168,28 +167,27 @@ them all begin at once:
 ]
 ```
 
-Every runner in a pool shares one egress address. A backend whose setup
-downloads a large runtime or several container images therefore concentrates
-that traffic into a burst when its jobs start together, which draws rate
-limiting from public registries and stalled downloads. 
+Every runner in a pool shares one egress address, so a backend whose setup
+pulls down a large runtime or several container images concentrates all that
+traffic into a burst the moment its jobs start together. Public registries
+answer with rate limiting and stalled downloads.
 
-`minutes` is the gap between consecutive jobs of that backend, counted
-independently per backend, following the resolved job order. With the entry
-above, four WSLC jobs start at 0, 5, 10, and 15 minutes.
+`minutes` is the gap between consecutive jobs of that backend, counted per
+backend and following the resolved job order. With the entry above, four WSLC
+jobs start at 0, 5, 10, and 15 minutes.
 
-The resolver emits the offset as `startup_delay_minutes` on every matrix entry
-(`0` where no stagger applies). The job sleeps that long before its first
-network step; the job timeout is a fixed 180 minutes, wide enough to absorb any
-configured wait.
+The resolver puts the offset on every matrix entry as
+`startup_delay_minutes` — `0` where no stagger applies — and the job sleeps
+that long before its first network step. Job timeout is a flat 180 minutes,
+with plenty of room for any wait you'd reasonably configure.
 
-Omit the section (or leave it empty) to have every job start as soon as its
-runner is ready. A backend id that no plan schedules is accepted and simply
-never applies.
+Leave the section out (or empty) and every job starts as soon as its runner is
+ready. A backend id that no plan schedules is accepted; it just never applies.
 
-Note this holds the runner while it sleeps: Actions cannot defer allocating a
-matrix job, so the wait happens inside the job. Keep the value only as large as
-the contention requires. It spreads simultaneous load — it does not help a
-single download that stalls on its own.
+Do keep in mind that the runner is held while it sleeps — Actions can't defer
+allocating a matrix job, so the wait has to happen inside it. Use no more than
+the contention calls for. This spreads simultaneous load and nothing else; a
+single download that stalls on its own is unaffected.
 
 ## Backend status
 
@@ -238,13 +236,33 @@ message instead of surfacing later as an opaque backend error.
 - `bubblewrap` — installs `bwrap` (apt/dnf/yum/microdnf) and relaxes
   `kernel.apparmor_restrict_unprivileged_userns` (ephemeral CI hosts only).
 - `lxc` — installs the LXC stack, reloads the AppArmor profile, starts and waits
-  for `lxcbr0`, and prints network diagnostics. On RHEL-likes it first needs
-  EPEL, because Red Hat dropped LXC after RHEL 7 and ships no replacement.
+  for `lxcbr0`, enables bridge netfilter, and makes sure the bridge's NAT rule
+  is in place. On RHEL-likes it needs EPEL first, because Red Hat dropped LXC
+  after RHEL 7 and ships no replacement.
 - `microvm` — asserts the NanVix payload exists.
 - `hyperlight` — no-op.
 
 macOS has no preparation step.
 
+## Log collection
+
+Every job uploads its logs whether it passed or failed, as
+`logs-<plan>-<os>-<arch>-<backend>-<attempt>`, kept 7 days.
+
+The catch is that `$env:TEMP` is not `$RUNNER_TEMP`. The Windows suites write their
+scratch trees, transcripts, and results files under the user's temp directory
+(`C:\Users\<user>\AppData\Local\Temp`), but `upload-artifact` reads
+`${{ runner.temp }}` (`C:\a\_work\_temp`). Anything left in the former is simply
+never collected, which is why the artifact used to arrive nearly empty.
+
+So `run_ci_backend_tests.ps1` points `TEMP` and `TMP` at `$RUNNER_TEMP` before
+it dispatches. Parameter defaults, `[System.IO.Path]::GetTempPath()`, and child
+processes all read those variables, so everything temp-rooted lands in the
+upload directory without CI having to know a single filename.
+
+Linux and macOS need none of this — those suites log to stdout, and the run
+step tees that into `$RUNNER_TEMP/mxc-ci.log`.
+
 ## Runbook
 
 Always finish with a local resolve, which runs the full catalog validation:
@@ -319,9 +337,15 @@ by testing it ahead of time.
 ### Stagger a backend's job starts
 
 Add or edit its `backendDelayedStart` entry in the catalog, then resolve
-locally to confirm the offsets. Reach for this when a backend's setup is
-network-heavy enough that concurrent jobs hit rate limits or stalled
-downloads; remove the entry once that pressure is gone.
+locally to confirm the offsets. Worth reaching for when a backend's setup is
+network-heavy enough that concurrent jobs run into rate limits or stalled
+downloads — and worth removing again once that pressure is gone.
+
+### Collect a new log file
+
+Have the suite write it under `$env:TEMP`. The dispatcher redirects that to the
+upload directory, so nothing in CI needs to change. See
+[Log collection](#log-collection).
 
 ### Change the schedule
 
diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1
index 572b2618c..3770f6390 100644
--- a/tests/scripts/run_ci_backend_tests.ps1
+++ b/tests/scripts/run_ci_backend_tests.ps1
@@ -42,45 +42,25 @@ function Assert-File {
     }
 }
 
-# The suites write their logs, configs, and transcripts under $env:TEMP, which
-# is not the directory CI uploads from. Several also refuse to run with those
-# paths pointed elsewhere, so copy the artifacts across afterwards instead of
-# redirecting them. Best-effort by design: losing a log must not turn a passing
-# run red, and a failing run keeps its original result.
-function Copy-TempArtifacts {
-    if (-not $env:RUNNER_TEMP -or -not $env:TEMP) {
+# The suites, and MXC itself, write logs and scratch trees under $env:TEMP,
+# which is not the directory CI uploads from. Rather than enumerate every
+# artifact and copy it afterwards, point TEMP at the upload directory for the
+# duration of the run: parameter defaults, .NET GetTempPath(), and child
+# processes all follow it, so anything temp-rooted lands where CI collects it.
+#
+# The suites' scratch-root guards compare against $env:TEMP, so they stay
+# satisfied — this redirects what TEMP means rather than pointing a path
+# outside it.
+function Redirect-TempToRunnerTemp {
+    if (-not $env:RUNNER_TEMP) {
         return
     }
-
-    $names = @(
-        'mxc-wpc-tests',
-        'mxc-t3-workloads',
-        'mxc_concurrent_oneshot',
-        'mxc_etw_test',
-        'wxc-wsb',
-        'wxc-sandbox-rendezvous',
-        'WinProcessContainer-Tests.results.txt',
-        'WinProcessContainer-Tests.results.json',
-        'WinProcessContainer-Tests.cargo.log'
-    )
-
-    foreach ($name in $names) {
-        $source = Join-Path $env:TEMP $name
-        if (-not (Test-Path -LiteralPath $source)) {
-            continue
-        }
-        Copy-Item -Recurse -Force -LiteralPath $source `
-            -Destination (Join-Path $env:RUNNER_TEMP $name) `
-            -ErrorAction SilentlyContinue
+    if (-not (Test-Path -LiteralPath $env:RUNNER_TEMP)) {
+        New-Item -ItemType Directory -Force -Path $env:RUNNER_TEMP | Out-Null
     }
-
-    # MXC's own diagnostic dumps are timestamped per run, so match the family.
-    Get-ChildItem -Path $env:TEMP -Filter 'mxc-diagnostics-*' -ErrorAction SilentlyContinue |
-        ForEach-Object {
-            Copy-Item -Recurse -Force -LiteralPath $_.FullName `
-                -Destination (Join-Path $env:RUNNER_TEMP $_.Name) `
-                -ErrorAction SilentlyContinue
-        }
+    $env:TEMP = $env:RUNNER_TEMP
+    $env:TMP  = $env:RUNNER_TEMP
+    Write-Host "Redirected TEMP to $env:RUNNER_TEMP so test logs are collected."
 }
 
 function Invoke-TestScript {
@@ -146,44 +126,41 @@ function Invoke-ProcessContainerTests {
     }
 }
 
-try {
-    switch ($Backend) {
-        'process-t1' {
-            Invoke-ProcessContainerTests
-        }
-        'process-t3' {
-            Invoke-ProcessContainerTests
-        }
-        'isolation-session' {
-            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{
-                WxcExePath = $wxc
-            }
+Redirect-TempToRunnerTemp
+
+switch ($Backend) {
+    'process-t1' {
+        Invoke-ProcessContainerTests
+    }
+    'process-t3' {
+        Invoke-ProcessContainerTests
+    }
+    'isolation-session' {
+        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{
+            WxcExePath = $wxc
         }
-        'windows-sandbox' {
-            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{
-                BinDir = $binaryDirectoryPath
-            }
+    }
+    'windows-sandbox' {
+        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{
+            BinDir = $binaryDirectoryPath
         }
-        'wslc' {
-            # The current WSLC helper hardcodes the x64 target when locating assets.
-            if ($Architecture -ne 'x64') {
-                throw 'The existing WSLC test harness is not architecture-portable yet.'
-            }
-            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{
-                WxcExecPath = $wxc
-            }
+    }
+    'wslc' {
+        # The current WSLC helper hardcodes the x64 target when locating assets.
+        if ($Architecture -ne 'x64') {
+            throw 'The existing WSLC test harness is not architecture-portable yet.'
         }
-        'microvm' {
-            Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{
-                BinDir = $binaryDirectoryPath
-            }
+        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{
+            WxcExecPath = $wxc
         }
-        'hyperlight' {
-            # Keep unwired backends explicit so accidental activation fails loudly.
-            throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.'
+    }
+    'microvm' {
+        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{
+            BinDir = $binaryDirectoryPath
         }
     }
-} finally {
-    # Also runs when a suite throws, which is when these logs matter most.
-    Copy-TempArtifacts
+    'hyperlight' {
+        # Keep unwired backends explicit so accidental activation fails loudly.
+        throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.'
+    }
 }

From 4902c428f982a375bb4463872edf3cccd648c65e Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 15:41:58 -0700
Subject: [PATCH 12/44] reducing time between wslc jobs to 45 seconds to avoid
 idle agent from getting removed

---
 .../workflows/Validation.Tests.Matrix.Job.yml | 24 +++++++++----------
 docs/ci-validation-infrastructure.md          |  8 +++----
 scripts/ci/resolve-validation-test-matrix.mjs | 16 ++++++-------
 scripts/ci/validation-test-matrix.json        |  2 +-
 4 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index 79e4a2dd1..5fcaf425c 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -51,15 +51,15 @@ jobs:
           path: artifacts/bin
 
       - name: Delayed start
-        if: matrix.startup_delay_minutes > 0
+        if: matrix.startup_delay_seconds > 0
         shell: pwsh
         run: |
           # Spreads network-heavy setup across the pool's shared egress address
           # instead of issuing every job's downloads at once. Configured by
           # backendDelayedStart in scripts/ci/validation-test-matrix.json.
-          $minutes = ${{ matrix.startup_delay_minutes }}
-          Write-Host "Waiting $minutes minute(s) before downloading prerequisites."
-          Start-Sleep -Seconds ($minutes * 60)
+          $seconds = ${{ matrix.startup_delay_seconds }}
+          Write-Host "Waiting $seconds second(s) before downloading prerequisites."
+          Start-Sleep -Seconds $seconds
 
       - name: Prepare backend prerequisites
         timeout-minutes: 15
@@ -125,16 +125,16 @@ jobs:
           path: artifacts/bin
 
       - name: Delayed start
-        if: matrix.startup_delay_minutes > 0
+        if: matrix.startup_delay_seconds > 0
         shell: bash
         run: |
           # Spreads network-heavy setup across the pool's shared egress address
           # instead of issuing every job's downloads at once. Configured by
           # backendDelayedStart in scripts/ci/validation-test-matrix.json.
           set -euo pipefail
-          minutes=${{ matrix.startup_delay_minutes }}
-          echo "Waiting $minutes minute(s) before downloading prerequisites."
-          sleep $(( minutes * 60 ))
+          seconds=${{ matrix.startup_delay_seconds }}
+          echo "Waiting $seconds second(s) before downloading prerequisites."
+          sleep "$seconds"
 
       - name: Prepare backend prerequisites
         timeout-minutes: 15
@@ -188,16 +188,16 @@ jobs:
           path: artifacts/bin
 
       - name: Delayed start
-        if: matrix.startup_delay_minutes > 0
+        if: matrix.startup_delay_seconds > 0
         shell: bash
         run: |
           # Spreads network-heavy setup across the pool's shared egress address
           # instead of issuing every job's downloads at once. Configured by
           # backendDelayedStart in scripts/ci/validation-test-matrix.json.
           set -euo pipefail
-          minutes=${{ matrix.startup_delay_minutes }}
-          echo "Waiting $minutes minute(s) before starting tests."
-          sleep $(( minutes * 60 ))
+          seconds=${{ matrix.startup_delay_seconds }}
+          echo "Waiting $seconds second(s) before starting tests."
+          sleep "$seconds"
 
       - name: Run backend tests
         timeout-minutes: 60
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 0d44ec532..856cff73f 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -163,7 +163,7 @@ them all begin at once:
 
 ```json
 "backendDelayedStart": [
-  { "backend": "wslc", "minutes": 5 }
+  { "backend": "wslc", "seconds": 300 }
 ]
 ```
 
@@ -172,12 +172,12 @@ pulls down a large runtime or several container images concentrates all that
 traffic into a burst the moment its jobs start together. Public registries
 answer with rate limiting and stalled downloads.
 
-`minutes` is the gap between consecutive jobs of that backend, counted per
+`seconds` is the gap between consecutive jobs of that backend, counted per
 backend and following the resolved job order. With the entry above, four WSLC
-jobs start at 0, 5, 10, and 15 minutes.
+jobs start at 0, 300, 600, and 900 seconds.
 
 The resolver puts the offset on every matrix entry as
-`startup_delay_minutes` — `0` where no stagger applies — and the job sleeps
+`startup_delay_seconds` — `0` where no stagger applies — and the job sleeps
 that long before its first network step. Job timeout is a flat 180 minutes,
 with plenty of room for any wait you'd reasonably configure.
 
diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs
index 4383eb680..565fcec1d 100644
--- a/scripts/ci/resolve-validation-test-matrix.mjs
+++ b/scripts/ci/resolve-validation-test-matrix.mjs
@@ -158,9 +158,9 @@ function validateBackendDelayedStart(catalog) {
     }
     seen.add(entry.backend);
 
-    if (!Number.isInteger(entry.minutes) || entry.minutes < 0) {
+    if (!Number.isInteger(entry.seconds) || entry.seconds < 0) {
       throw new Error(
-        `backendDelayedStart ${entry.backend} minutes must be a non-negative integer`
+        `backendDelayedStart ${entry.backend} seconds must be a non-negative integer`
       );
     }
   }
@@ -228,7 +228,7 @@ export function resolvePlan(catalog, plan) {
 // pool shares one egress address, so simultaneous starts concentrate that
 // traffic into a burst that draws rate limiting and stalled downloads.
 //
-// The entry's minutes value is the gap between consecutive jobs of that
+// The entry's seconds value is the gap between consecutive jobs of that
 // backend, counted independently per backend. Applied after sorting so the
 // assignment follows the emitted order and stays reproducible.
 function applyDelayedStart(matrices, delays) {
@@ -237,17 +237,17 @@ function applyDelayedStart(matrices, delays) {
     return;
   }
 
-  const stepMinutes = new Map(delays.map(entry => [entry.backend, entry.minutes]));
+  const stepSeconds = new Map(delays.map(entry => [entry.backend, entry.seconds]));
   const scheduled = new Map();
 
   for (const family of FAMILIES) {
     for (const entry of matrices[family]) {
-      const step = stepMinutes.get(entry.backend);
+      const step = stepSeconds.get(entry.backend);
       if (step === undefined) {
-        entry.startup_delay_minutes = 0;
+        entry.startup_delay_seconds = 0;
       } else {
         const position = scheduled.get(entry.backend) ?? 0;
-        entry.startup_delay_minutes = position * step;
+        entry.startup_delay_seconds = position * step;
         scheduled.set(entry.backend, position + 1);
       }
     }
@@ -258,7 +258,7 @@ function applyDelayedStart(matrices, delays) {
 function setDefaultDelays(matrices) {
   for (const family of FAMILIES) {
     for (const entry of matrices[family]) {
-      entry.startup_delay_minutes = 0;
+      entry.startup_delay_seconds = 0;
     }
   }
 }
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index c1bbd3816..1a013f30a 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -328,7 +328,7 @@
   "backendDelayedStart": [
     {
       "backend": "wslc",
-      "minutes": 2
+      "seconds": 45
     }
   ],
   "triggers": {

From ecff4419b38de20d4add1ce0fdf614191d4ddcef Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 16:04:39 -0700
Subject: [PATCH 13/44] Making PR ready

---
 .../Validation.Infrastructure.Tests.yml       | 36 -------------------
 scripts/ci/validation-test-matrix.json        | 35 +-----------------
 2 files changed, 1 insertion(+), 70 deletions(-)
 delete mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
deleted file mode 100644
index 1785d2515..000000000
--- a/.github/workflows/Validation.Infrastructure.Tests.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-name: Validation Infrastructure Testing
-
-on:
-  push:
-    branches:
-      - user/emichlin/validation-infra-2
-
-concurrency:
-  group: validation-infra-pr-tests-${{ github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  actions: read
-  contents: read
-
-jobs:
-  dependency-feed-check:
-    uses: ./.github/workflows/Dependency.Feed.Check.Job.yml
-
-  windows:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.Windows.Job.yml
-
-  linux:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.Linux.Job.yml
-
-  macos:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.MacOS.Job.yml
-
-  test:
-    needs: [windows, linux, macos]
-    uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
-    with:
-      plan: enabled
\ No newline at end of file
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 1a013f30a..f0a2d46c3 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -393,39 +393,6 @@
       }
     ],
     "weekly": [],
-    "enabled": [
-      {
-        "os": "ubuntu-26.04",
-        "backends": [
-          "bubblewrap",
-          "lxc"
-        ]
-      },
-      {
-        "os": "windows-prerelease-process-container",
-        "backends": [
-          "wslc",
-          "process-t1"
-        ]
-      },
-      {
-        "os": "windows-25h2",
-        "backends": [
-          "wslc"
-        ]
-      },
-      {
-        "os": "windows-24h2",
-        "backends": [
-          "wslc"
-        ]
-      },
-      {
-        "os": "windows-23h2",
-        "backends": [
-          "wslc"
-        ]
-      }
-    ]
+    "enabled": []
   }
 }

From d6ebe7c9cd914168e0add6b5996b982bdf54b624 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 16:12:58 -0700
Subject: [PATCH 14/44] Document backendDelayedStart and the CI TEMP redirect

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .github/copilot-instructions.md | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 575f4e7f6..bddc4d518 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -72,9 +72,10 @@ workflows) before calling the matrix job.
 
 - `scripts/ci/validation-test-matrix.json` is the catalog: `platforms` (each
   with per-architecture target/artifact/1ES pool and the backends that platform
-  supports) and `triggers` (which OS/backend pairs each plan runs). The
-  `triggers` keys *are* the plan list — the resolver reads them at run time, so
-  adding a plan needs no script change.
+  supports), `triggers` (which OS/backend pairs each plan runs), and the
+  optional `backendDelayedStart` (per-backend job-start stagger, in seconds).
+  The `triggers` keys *are* the plan list — the resolver reads them at run time,
+  so adding a plan needs no script change.
 - `scripts/ci/resolve-validation-test-matrix.mjs` validates that catalog and
   expands a plan (currently `pr`, `nightly`, `weekly`, `enabled`) into GitHub
   Actions matrices. It rejects an invalid catalog before any specialized test
@@ -95,7 +96,9 @@ the matrix `backend` id to the repository's existing backend suite. Ids that
 share a suite get their own case (`process-t1` and `process-t3` both run
 `WinProcessContainer-Tests.ps1`, which derives the tier it expects from the
 host's own `--probe`). A backend with no wired suite fails loudly rather than
-reporting a false success.
+reporting a false success. The Windows dispatcher points `TEMP` at
+`$RUNNER_TEMP` before running a suite, so anything a test writes to the temp
+directory is picked up by the job's log upload without per-file CI wiring.
 
 ### Individual components
 

From 2d6df7ea3b427201d5048f346e624dc9e68a2291 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 16:33:19 -0700
Subject: [PATCH 15/44] removing old comments

---
 .github/workflows/Validation.Tests.Matrix.Job.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index 5fcaf425c..817ebf880 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -38,8 +38,6 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.resolve.outputs.windows) }}
     runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ]
-    # Resolved per entry: the test budget plus any startup stagger, so a
-    # delayed entry keeps its full budget. Workflow expressions cannot add.
     timeout-minutes: 180
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -72,8 +70,6 @@ jobs:
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 
       - name: Run backend tests
-        # A backend that hangs (rather than failing) would otherwise burn the
-        # full job timeout. Fail fast enough to keep the log useful.
         timeout-minutes: 45
         shell: pwsh
         run: |

From 233dc198c5db5ddc79a5ef49c62bd110a45ebe2e Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 18 Aug 2026 16:57:54 -0700
Subject: [PATCH 16/44] updating backend status in docs

---
 docs/ci-validation-infrastructure.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 856cff73f..821c5b67b 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -199,8 +199,8 @@ get fixed or wired.
 | Process T1 | ✅ Good | Prerelease Windows only. Remaining failures are genuine MXC bugs or harness limitations. |
 | Process T3 | ✅ Good | Non-prerelease Windows builds only, until the testing suite is updated. |
 | Bubblewrap | ✅ Good | |
-| LXC | ⚠️ Mostly good | Some networking tests fail on distros other than Ubuntu 24.04; host-vs-MXC cause not yet isolated. |
-| WSLC | ⚠️ Mostly good | Can hit a download rate limit while updating WSL / pulling container images. Planned fix: split into several jobs spaced ~15 min apart. |
+| LXC | ✅ Good | Some networking tests fail on distros other than Ubuntu 24.04; seems to be an issue with MXC. |
+| WSLC | ✅ Good | Might have to retry hung jobs - this is an issue with overzealous agent reclaiming. |
 | IsolationSession | ⚠️ Blocked | `Feature_AgentSessionsBaseSupport` is not enabled on the pool image yet. |
 | Windows Sandbox | ⛔ Not scheduled | Dispatcher case is wired; no trigger entry yet. |
 | MicroVM | ⛔ Not working | Windows cold and warm starts hang; no Linux suite. The artifact payload is currently commented out in the build jobs. |

From 02140c5feb62e917c6c9b254ff36e3697b353bc0 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 20 Aug 2026 13:20:10 -0700
Subject: [PATCH 17/44] isolation session test

---
 .../Validation.Infrastructure.Tests.yml       | 36 +++++++++++++++++++
 scripts/ci/validation-test-matrix.json        | 13 +++++--
 2 files changed, 46 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
new file mode 100644
index 000000000..87c8b937b
--- /dev/null
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -0,0 +1,36 @@
+name: Validation Infrastructure Testing
+
+on:
+  push:
+    branches:
+      - user/emichlin/validation-infra-3
+
+concurrency:
+  group: validation-infra-pr-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  dependency-feed-check:
+    uses: ./.github/workflows/Dependency.Feed.Check.Job.yml
+
+  windows:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Windows.Job.yml
+
+  linux:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Linux.Job.yml
+
+  macos:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.MacOS.Job.yml
+
+  test:
+    needs: [windows, linux, macos]
+    uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
+    with:
+      plan: enabled
\ No newline at end of file
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index f0a2d46c3..f567a0676 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -37,14 +37,14 @@
     },
     {
       "id": "windows-prerelease-isolation-session",
-      "displayName": "Windows Pre-release Isolation Session",
+      "displayName": "Windows Prerelease Isolation Session",
       "family": "windows",
       "prerelease": true,
       "architectures": {
         "x64": {
           "target": "x86_64-pc-windows-msvc",
           "artifact": "wxc-binaries-x86_64-pc-windows-msvc",
-          "pool": "",
+          "pool": "1es-mxc-e2e-win-prerelease-isolationsesh-x64",
           "backends": [
             "process-t1",
             "process-t3",
@@ -393,6 +393,13 @@
       }
     ],
     "weekly": [],
-    "enabled": []
+    "enabled": [
+      {
+        "os": "windows-prerelease-isolation-session",
+        "backends": [
+          "isolation-session"
+        ]
+      }
+    ]
   }
 }

From 24c7d2865708603fbad35b84b55126aef7dccff0 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 20 Aug 2026 13:20:10 -0700
Subject: [PATCH 18/44] isolation session test

---
 .../Validation.Infrastructure.Tests.yml       | 36 +++++++++++++++++++
 scripts/ci/validation-test-matrix.json        | 13 +++++--
 2 files changed, 46 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
new file mode 100644
index 000000000..87c8b937b
--- /dev/null
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -0,0 +1,36 @@
+name: Validation Infrastructure Testing
+
+on:
+  push:
+    branches:
+      - user/emichlin/validation-infra-3
+
+concurrency:
+  group: validation-infra-pr-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  dependency-feed-check:
+    uses: ./.github/workflows/Dependency.Feed.Check.Job.yml
+
+  windows:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Windows.Job.yml
+
+  linux:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Linux.Job.yml
+
+  macos:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.MacOS.Job.yml
+
+  test:
+    needs: [windows, linux, macos]
+    uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
+    with:
+      plan: enabled
\ No newline at end of file
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index f0a2d46c3..f567a0676 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -37,14 +37,14 @@
     },
     {
       "id": "windows-prerelease-isolation-session",
-      "displayName": "Windows Pre-release Isolation Session",
+      "displayName": "Windows Prerelease Isolation Session",
       "family": "windows",
       "prerelease": true,
       "architectures": {
         "x64": {
           "target": "x86_64-pc-windows-msvc",
           "artifact": "wxc-binaries-x86_64-pc-windows-msvc",
-          "pool": "",
+          "pool": "1es-mxc-e2e-win-prerelease-isolationsesh-x64",
           "backends": [
             "process-t1",
             "process-t3",
@@ -393,6 +393,13 @@
       }
     ],
     "weekly": [],
-    "enabled": []
+    "enabled": [
+      {
+        "os": "windows-prerelease-isolation-session",
+        "backends": [
+          "isolation-session"
+        ]
+      }
+    ]
   }
 }

From c613b26a21eca684d5d9bdd593b4dc9d3fab7098 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 20 Aug 2026 14:22:26 -0700
Subject: [PATCH 19/44] Added bubblewrap slirp4netns prereq

---
 .../Validation.Infrastructure.Tests.yml          |  2 +-
 docs/ci-validation-infrastructure.md             |  3 ++-
 scripts/ci/prepare-linux-host.sh                 | 14 ++++++++------
 scripts/ci/validation-test-matrix.json           | 16 +++++++---------
 4 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
index 87c8b937b..83dd18499 100644
--- a/.github/workflows/Validation.Infrastructure.Tests.yml
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -33,4 +33,4 @@ jobs:
     needs: [windows, linux, macos]
     uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
     with:
-      plan: enabled
\ No newline at end of file
+      plan: nightly
\ No newline at end of file
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 821c5b67b..839d2c5b4 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -233,7 +233,8 @@ message instead of surfacing later as an opaque backend error.
 
 `prepare-linux-host.sh`:
 
-- `bubblewrap` — installs `bwrap` (apt/dnf/yum/microdnf) and relaxes
+- `bubblewrap` — installs `bwrap` and `slirp4netns`
+  (apt/dnf/yum/microdnf) and relaxes
   `kernel.apparmor_restrict_unprivileged_userns` (ephemeral CI hosts only).
 - `lxc` — installs the LXC stack, reloads the AppArmor profile, starts and waits
   for `lxcbr0`, enables bridge netfilter, and makes sure the bridge's NAT rule
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index f7afb2743..7a89d5cb4 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -41,20 +41,21 @@ install_epel() {
 }
 
 install_bubblewrap() {
-    if command -v bwrap >/dev/null 2>&1; then
+    if command -v bwrap >/dev/null 2>&1 &&
+        command -v slirp4netns >/dev/null 2>&1; then
         return
     fi
     if command -v apt-get >/dev/null 2>&1; then
         apt_update
-        sudo apt-get install -y --no-install-recommends bubblewrap
+        sudo apt-get install -y --no-install-recommends bubblewrap slirp4netns
     elif command -v dnf >/dev/null 2>&1; then
-        sudo dnf install -y bubblewrap
+        sudo dnf install -y bubblewrap slirp4netns
     elif command -v yum >/dev/null 2>&1; then
-        sudo yum install -y bubblewrap
+        sudo yum install -y bubblewrap slirp4netns
     elif command -v microdnf >/dev/null 2>&1; then
-        sudo microdnf install -y bubblewrap
+        sudo microdnf install -y bubblewrap slirp4netns
     else
-        echo "No supported package manager found to install bubblewrap." >&2
+        echo "No supported package manager found to install Bubblewrap prerequisites." >&2
         exit 1
     fi
 }
@@ -186,6 +187,7 @@ case "$backend" in
     bubblewrap)
         install_bubblewrap
         command -v bwrap
+        command -v slirp4netns
         # disabled AppArmor restrictions on unprivileged user namespaces, which bubblewrap needs to create a new namespace.
         # should only be used on ephemeral CI runners, not on persistent hosts.
         if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index f567a0676..f5f7704af 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -338,10 +338,15 @@
         "os": "windows-prerelease-process-container",
         "backends": [
           "process-t1",
-          "isolation-session",
           "wslc"
         ]
       },
+      {
+        "os": "windows-prerelease-isolation-session",
+        "backends": [
+          "isolation-session"
+        ]
+      },
       {
         "os": "windows-25h2",
         "backends": [
@@ -393,13 +398,6 @@
       }
     ],
     "weekly": [],
-    "enabled": [
-      {
-        "os": "windows-prerelease-isolation-session",
-        "backends": [
-          "isolation-session"
-        ]
-      }
-    ]
+    "enabled": []
   }
 }

From b04cd3a212727acddf465eefad71c77478897879 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 20 Aug 2026 15:15:41 -0700
Subject: [PATCH 20/44] added additional bubblewrap prereqs

---
 docs/ci-validation-infrastructure.md |  4 ++--
 scripts/ci/prepare-linux-host.sh     | 19 ++++++++++++++-----
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 839d2c5b4..7d86032c9 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -233,8 +233,8 @@ message instead of surfacing later as an opaque backend error.
 
 `prepare-linux-host.sh`:
 
-- `bubblewrap` — installs `bwrap` and `slirp4netns`
-  (apt/dnf/yum/microdnf) and relaxes
+- `bubblewrap` — installs `bwrap`, `slirp4netns`, `util-linux`, and `iptables`
+  (apt/dnf/yum/microdnf), verifies their required commands, and relaxes
   `kernel.apparmor_restrict_unprivileged_userns` (ephemeral CI hosts only).
 - `lxc` — installs the LXC stack, reloads the AppArmor profile, starts and waits
   for `lxcbr0`, enables bridge netfilter, and makes sure the bridge's NAT rule
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 7a89d5cb4..28af94870 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -42,18 +42,23 @@ install_epel() {
 
 install_bubblewrap() {
     if command -v bwrap >/dev/null 2>&1 &&
-        command -v slirp4netns >/dev/null 2>&1; then
+        command -v slirp4netns >/dev/null 2>&1 &&
+        command -v unshare >/dev/null 2>&1 &&
+        command -v nsenter >/dev/null 2>&1 &&
+        command -v iptables >/dev/null 2>&1 &&
+        command -v ip6tables >/dev/null 2>&1; then
         return
     fi
     if command -v apt-get >/dev/null 2>&1; then
         apt_update
-        sudo apt-get install -y --no-install-recommends bubblewrap slirp4netns
+        sudo apt-get install -y --no-install-recommends \
+            bubblewrap slirp4netns util-linux iptables
     elif command -v dnf >/dev/null 2>&1; then
-        sudo dnf install -y bubblewrap slirp4netns
+        sudo dnf install -y bubblewrap slirp4netns util-linux iptables
     elif command -v yum >/dev/null 2>&1; then
-        sudo yum install -y bubblewrap slirp4netns
+        sudo yum install -y bubblewrap slirp4netns util-linux iptables
     elif command -v microdnf >/dev/null 2>&1; then
-        sudo microdnf install -y bubblewrap slirp4netns
+        sudo microdnf install -y bubblewrap slirp4netns util-linux iptables
     else
         echo "No supported package manager found to install Bubblewrap prerequisites." >&2
         exit 1
@@ -188,6 +193,10 @@ case "$backend" in
         install_bubblewrap
         command -v bwrap
         command -v slirp4netns
+        command -v unshare
+        command -v nsenter
+        command -v iptables
+        command -v ip6tables
         # disabled AppArmor restrictions on unprivileged user namespaces, which bubblewrap needs to create a new namespace.
         # should only be used on ephemeral CI runners, not on persistent hosts.
         if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then

From 14553bfafc90df34d7b68d542beada6a1fb817a4 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 20 Aug 2026 15:56:27 -0700
Subject: [PATCH 21/44] cleaning up for PR

---
 .../Validation.Infrastructure.Tests.yml       | 36 -------------------
 1 file changed, 36 deletions(-)
 delete mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
deleted file mode 100644
index 83dd18499..000000000
--- a/.github/workflows/Validation.Infrastructure.Tests.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-name: Validation Infrastructure Testing
-
-on:
-  push:
-    branches:
-      - user/emichlin/validation-infra-3
-
-concurrency:
-  group: validation-infra-pr-tests-${{ github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  actions: read
-  contents: read
-
-jobs:
-  dependency-feed-check:
-    uses: ./.github/workflows/Dependency.Feed.Check.Job.yml
-
-  windows:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.Windows.Job.yml
-
-  linux:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.Linux.Job.yml
-
-  macos:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.MacOS.Job.yml
-
-  test:
-    needs: [windows, linux, macos]
-    uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
-    with:
-      plan: nightly
\ No newline at end of file

From 7e09f7237e5b9c69d41152be7412bb44f179ca5a Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:49:23 -0700
Subject: [PATCH 22/44] testing wsl install during image provision

---
 .../Validation.Infrastructure.Tests.yml       |  36 +++
 scripts/ci/Setup.ps1                          | 260 ++++++++++++++++++
 scripts/ci/validation-test-matrix.json        |  29 +-
 3 files changed, 309 insertions(+), 16 deletions(-)
 create mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml
 create mode 100644 scripts/ci/Setup.ps1

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
new file mode 100644
index 000000000..87c8b937b
--- /dev/null
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -0,0 +1,36 @@
+name: Validation Infrastructure Testing
+
+on:
+  push:
+    branches:
+      - user/emichlin/validation-infra-3
+
+concurrency:
+  group: validation-infra-pr-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  dependency-feed-check:
+    uses: ./.github/workflows/Dependency.Feed.Check.Job.yml
+
+  windows:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Windows.Job.yml
+
+  linux:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.Linux.Job.yml
+
+  macos:
+    needs: dependency-feed-check
+    uses: ./.github/workflows/Build.MacOS.Job.yml
+
+  test:
+    needs: [windows, linux, macos]
+    uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
+    with:
+      plan: enabled
\ No newline at end of file
diff --git a/scripts/ci/Setup.ps1 b/scripts/ci/Setup.ps1
new file mode 100644
index 000000000..34dd71579
--- /dev/null
+++ b/scripts/ci/Setup.ps1
@@ -0,0 +1,260 @@
+#Requires -Version 7.0
+
+<#
+.SYNOPSIS
+    TEMP scratch script: runs only the WSLC host initialization steps.
+
+.DESCRIPTION
+    A trimmed copy of scripts/ci/prepare-windows-host.ps1 that keeps just the
+    WSLC path. Unlike the CI script this one is diagnostic-only: every failure
+    is reported and execution always ends with exit code 0.
+
+.PARAMETER BinaryDirectory
+    Optional directory holding a built/downloaded artifact. When supplied, the
+    WSLC binaries are checked for presence; otherwise that check is skipped.
+
+.EXAMPLE
+    ./scripts/ci/Setup.ps1
+    ./scripts/ci/Setup.ps1 -BinaryDirectory src/target/x86_64-pc-windows-msvc/release
+#>
+
+[CmdletBinding()]
+param(
+    [string]$BinaryDirectory
+)
+
+Set-StrictMode -Off
+$ErrorActionPreference = 'Continue'
+
+$script:Failed = $false
+
+function Write-Step {
+    param([Parameter(Mandatory)][string]$Message)
+    Write-Host ''
+    Write-Host "=== $Message ==="
+}
+
+function Write-Failure {
+    param([Parameter(Mandatory)][string]$Message)
+    $script:Failed = $true
+    Write-Host "FAILED: $Message"
+}
+
+function Test-RequiredFile {
+    param([Parameter(Mandatory)][string[]]$RelativePath)
+
+    if (-not $BinaryDirectory) {
+        Write-Host 'No -BinaryDirectory supplied; skipping artifact presence check.'
+        return
+    }
+
+    foreach ($relative in $RelativePath) {
+        $full = Join-Path $BinaryDirectory $relative
+        if (Test-Path $full) {
+            $item = Get-Item $full
+            Write-Host "  found $($item.FullName) ($($item.Length) bytes)"
+        } else {
+            Write-Failure "missing binary: $full"
+        }
+    }
+}
+
+# Read a Windows optional feature's state without throwing. A host that cannot
+# answer (querying needs elevation) reports the reason as its state.
+function Get-OptionalFeatureState {
+    param([Parameter(Mandatory)][string]$Name)
+
+    try {
+        $feature = Get-WindowsOptionalFeature -Online -FeatureName $Name -ErrorAction Stop
+    } catch {
+        return "query-failed: $($_.Exception.Message.Trim())"
+    }
+
+    if ($null -eq $feature) {
+        return 'unknown'
+    }
+    return [string]$feature.State
+}
+
+# Enabling an optional feature needs a reboot, so this reports rather than
+# installs.
+function Test-RequiredFeature {
+    param([Parameter(Mandatory)][string[]]$Name)
+
+    foreach ($feature in $Name) {
+        $state = Get-OptionalFeatureState -Name $feature
+        Write-Host "  $feature = $state"
+        if ($state -ne 'Enabled') {
+            Write-Failure "Windows optional feature not enabled: $feature ($state). WSL2 must be baked into the image; enabling it requires a reboot."
+        }
+    }
+}
+
+# wsl.exe emits UTF-16LE, which the default console encoding renders as
+# null-separated garbage. Returns @{ ExitCode; Output } with the output decoded.
+function Invoke-WslCapture {
+    param([Parameter(Mandatory)][string[]]$Arguments)
+
+    $previousEncoding = [Console]::OutputEncoding
+    try {
+        [Console]::OutputEncoding = [System.Text.Encoding]::Unicode
+        $output = & wsl.exe @Arguments 2>&1 | Out-String
+        return @{ ExitCode = $LASTEXITCODE; Output = $output }
+    } catch {
+        return @{ ExitCode = 1; Output = "wsl.exe could not be run: $($_.Exception.Message)" }
+    } finally {
+        [Console]::OutputEncoding = $previousEncoding
+    }
+}
+
+# Run wsl.exe and return its exit code. -Quiet suppresses output for probes,
+# where the legacy wsl.exe dumps its whole usage text on an unknown switch.
+function Invoke-Wsl {
+    param(
+        [Parameter(Mandatory)][string[]]$Arguments,
+        [switch]$Quiet
+    )
+
+    Write-Host "  > wsl.exe $($Arguments -join ' ')"
+    $result = Invoke-WslCapture -Arguments $Arguments
+    if (-not $Quiet -and $result.Output.Trim()) {
+        Write-Host $result.Output.Trim()
+    }
+    Write-Host "  exit code: $($result.ExitCode)"
+    return $result.ExitCode
+}
+
+# Minimum WSL runtime for WSLC, read from the pinned SDK version so the two
+# cannot drift. The SDK's own runtime error names this same version.
+function Get-RequiredWslVersion {
+    $buildScript = Join-Path $PSScriptRoot '..\..\src\backends\wslc\common\build.rs'
+    if (-not (Test-Path $buildScript)) {
+        Write-Host "WARNING: $buildScript not found; skipping the WSL version gate."
+        return $null
+    }
+
+    $match = [regex]::Match((Get-Content $buildScript -Raw), 'WSLC_SDK_VERSION:\s*&str\s*=\s*"([0-9]+(?:\.[0-9]+)+)"')
+    if (-not $match.Success) {
+        Write-Host 'WARNING: could not parse WSLC_SDK_VERSION; skipping the WSL version gate.'
+        return $null
+    }
+    return [version]$match.Groups[1].Value
+}
+
+# Installed modern-runtime version, or $null when wsl.exe is the legacy inbox
+# build (no --version) or otherwise unusable.
+function Get-InstalledWslVersion {
+    $result = Invoke-WslCapture -Arguments @('--version')
+    if ($result.ExitCode -ne 0) {
+        return $null
+    }
+
+    $match = [regex]::Match($result.Output, '(?im)^\s*WSL version:\s*([0-9]+(?:\.[0-9]+)+)')
+    if (-not $match.Success) {
+        return $null
+    }
+    return [version]$match.Groups[1].Value
+}
+
+function Initialize-WslcHost {
+    Write-Step 'WSLC artifact binaries'
+    # wslcsdk.dll ships beside wxc-exec.exe only in a --features wslc build.
+    Test-RequiredFile @('wxc-exec.exe', 'wslcsdk.dll')
+
+    Write-Step 'Required Windows optional features'
+    Test-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform'
+
+    Write-Step 'wsl.exe presence + status'
+    $wsl = Get-Command wsl.exe -ErrorAction SilentlyContinue
+    if ($wsl) {
+        Write-Host "wsl.exe: $($wsl.Source)"
+    } else {
+        Write-Failure 'wsl.exe NOT found on PATH; WSL2 is not installed on this host.'
+        return
+    }
+
+    $status = Invoke-WslCapture -Arguments @('--status')
+    Write-Host $status.Output.Trim()
+
+    if ($status.Output -match 'wsl.exe --install') {
+        Write-Failure 'WSL2 is not installed on this host (wsl --status advertises --install).'
+        return
+    }
+
+    Write-Step 'WSL runtime version'
+    if ((Invoke-Wsl @('--version') -Quiet) -ne 0) {
+        Write-Host 'wsl --version failed, so WSL2 is installed but not updated.'
+        Write-Host 'Updating inbox WSL to the modern runtime...'
+
+        if ((Invoke-Wsl @('--update', '--web-download') -Quiet) -ne 0 -and
+            (Invoke-Wsl @('--update')) -ne 0) {
+            Write-Failure 'wsl --update failed; the WSL2 runtime could not be installed on this host.'
+            return
+        }
+
+        if ((Invoke-Wsl @('--version') -Quiet) -ne 0) {
+            Write-Failure 'wsl --version failed after updating; the WSL2 runtime is not usable on this host.'
+            return
+        }
+
+        Write-Host 'WSL2 is installed and updated (not prerelease, yet).'
+    }
+
+    # WSLC needs a runtime at least as new as the pinned WSLC SDK, and those
+    # builds ship only on the pre-release ring - the stable ring lands well
+    # behind it. Without this the SDK fails at run time with
+    # "WSLC runtime unavailable. Missing components: WslPackage".
+    $required = Get-RequiredWslVersion
+    $installed = Get-InstalledWslVersion
+    Write-Host "Required WSL version: $(if ($null -eq $required) { '<unknown>' } else { $required })"
+    Write-Host "Installed WSL version: $(if ($null -eq $installed) { '<none>' } else { $installed })"
+
+    if ($null -ne $required -and ($null -eq $installed -or $installed -lt $required)) {
+        Write-Host "WSL $installed is older than the $required WSLC requires; updating to pre-release..."
+        if ((Invoke-Wsl @('--update', '--pre-release', '--web-download') -Quiet) -ne 0 -and
+            (Invoke-Wsl @('--update', '--pre-release')) -ne 0) {
+            Write-Failure "wsl --update --pre-release failed; WSLC requires WSL $required or newer."
+            return
+        }
+        $installed = Get-InstalledWslVersion
+        Write-Host "Installed WSL version after update: $(if ($null -eq $installed) { '<none>' } else { $installed })"
+    }
+
+    if ($null -eq $installed) {
+        Write-Failure 'wsl --version failed after updating; the WSL2 runtime is not usable on this host.'
+        return
+    }
+    if ($null -ne $required -and $installed -lt $required) {
+        Write-Failure "WSL $installed is installed, but WSLC requires $required or newer."
+        return
+    }
+
+    Write-Host "WSL runtime $installed is ready (WSLC requires $required or newer)."
+}
+
+Write-Host "WSLC host setup starting on $([System.Environment]::OSVersion)"
+
+if ($BinaryDirectory) {
+    if (Test-Path $BinaryDirectory) {
+        $BinaryDirectory = (Resolve-Path $BinaryDirectory).Path
+        Write-Host "Binary directory: $BinaryDirectory"
+    } else {
+        Write-Failure "Binary directory not found: $BinaryDirectory"
+        $BinaryDirectory = $null
+    }
+}
+
+try {
+    Initialize-WslcHost
+} catch {
+    Write-Failure "unexpected error: $($_.Exception.Message)"
+}
+
+Write-Step 'Result'
+if ($script:Failed) {
+    Write-Host 'WSLC host setup completed WITH failures (see FAILED lines above).'
+} else {
+    Write-Host 'WSLC host setup completed successfully.'
+}
+
+exit 0
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index f5f7704af..9cff37eeb 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -70,22 +70,16 @@
       }
     },
     {
-      "id": "windows-canary",
-      "displayName": "Windows canary",
+      "id": "windows-wslc",
+      "displayName": "Windows WSLc Test",
       "family": "windows",
       "architectures": {
         "x64": {
           "target": "x86_64-pc-windows-msvc",
           "artifact": "wxc-binaries-x86_64-pc-windows-msvc",
-          "pool": "",
+          "pool": "1es-mxc-e2e-windows-25h2-wsl2",
           "backends": [
-            "process-t1",
-            "process-t3",
-            "isolation-session",
-            "wslc",
-            "windows-sandbox",
-            "microvm",
-            "hyperlight"
+            "wslc"
           ]
         },
         "arm64": {
@@ -93,11 +87,7 @@
           "artifact": "wxc-binaries-aarch64-pc-windows-msvc",
           "pool": "",
           "backends": [
-            "process-t1",
-            "process-t3",
-            "isolation-session",
-            "wslc",
-            "windows-sandbox"
+            "wslc"
           ]
         }
       }
@@ -398,6 +388,13 @@
       }
     ],
     "weekly": [],
-    "enabled": []
+    "enabled": [
+      {
+        "os": "windows-wslc",
+        "backends": [
+          "wslc"
+        ]
+      }
+    ]
   }
 }

From ff8ff5fafd855a3695463e51e378c0f105ed47cd Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:11:09 -0700
Subject: [PATCH 23/44] migrate ci scripts into ci folder, and updated artifact
 script (to be removed)

---
 .github/copilot-instructions.md               | 10 +-
 .../workflows/Validation.Tests.Matrix.Job.yml |  8 +-
 docs/ci-validation-infrastructure.md          | 26 +++---
 scripts/ci/Setup.ps1                          | 93 +++++--------------
 .../ci/run_backend_validation_tests.ps1       | 12 ++-
 .../ci/run_backend_validation_tests.sh        |  5 +-
 tests/scripts/README.md                       |  8 +-
 7 files changed, 57 insertions(+), 105 deletions(-)
 rename tests/scripts/run_ci_backend_tests.ps1 => scripts/ci/run_backend_validation_tests.ps1 (89%)
 rename tests/scripts/run_ci_backend_tests.sh => scripts/ci/run_backend_validation_tests.sh (92%)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 1c08ab931..31b40ebf6 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -90,8 +90,8 @@ matrix `backend` id: `scripts/ci/prepare-windows-host.ps1` and
 `scripts/ci/prepare-linux-host.sh`. A backend with no prerequisites is an
 explicit no-op, so the step runs unconditionally for every entry.
 
-**Test dispatch** goes through `tests/scripts/run_ci_backend_tests.ps1`
-(Windows) and `tests/scripts/run_ci_backend_tests.sh` (Linux/macOS), which map
+**Test dispatch** goes through `scripts/ci/run_backend_validation_tests.ps1`
+(Windows) and `scripts/ci/run_backend_validation_tests.sh` (Linux/macOS), which map
 the matrix `backend` id to the repository's existing backend suite. Ids that
 share a suite get their own case (`process-t1` and `process-t3` both run
 `WinProcessContainer-Tests.ps1`, which derives the tier it expects from the
@@ -157,13 +157,13 @@ cargo test -p wxc_e2e_tests                 # Invokes MXC binaries directly
 cargo test -p wxc_e2e_tests -- --ignored    # Include stress tests (run_on_repeat)
 
 # WSLC has no cargo E2E suite — it is covered by tests\scripts\run_wslc_all_tests.ps1,
-# which the validation matrix runs via tests\scripts\run_ci_backend_tests.ps1.
+# which the validation matrix runs via scripts\ci\run_backend_validation_tests.ps1.
 
 # CI validation entry points — run a backend suite against a downloaded artifact
 # the way the validation matrix does. Take the matrix backend id exactly as it
 # appears in scripts/ci/validation-test-matrix.json.
-tests\scripts\run_ci_backend_tests.ps1 -Backend process-t1 -BinaryDirectory <dir> -Architecture x64
-tests\scripts\run_ci_backend_tests.sh <bubblewrap|lxc|seatbelt> <binary-directory>
+scripts\ci\run_backend_validation_tests.ps1 -Backend process-t1 -BinaryDirectory <dir> -Architecture x64
+scripts\ci\run_backend_validation_tests.sh <bubblewrap|lxc|seatbelt> <binary-directory>
 
 # Resolve a plan locally to see exactly what CI would schedule
 node scripts/ci/resolve-validation-test-matrix.mjs --plan nightly
diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index 817ebf880..f21ba3470 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -73,7 +73,7 @@ jobs:
         timeout-minutes: 45
         shell: pwsh
         run: |
-          & ./tests/scripts/run_ci_backend_tests.ps1 `
+          & ./scripts/ci/run_backend_validation_tests.ps1 `
             -Backend '${{ matrix.backend }}' `
             -BinaryDirectory (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') `
             -Architecture '${{ matrix.architecture }}' *>&1 |
@@ -147,11 +147,11 @@ jobs:
         run: |
           set -euo pipefail
           if [[ '${{ matrix.backend }}' == 'lxc' ]]; then
-            sudo --preserve-env=RUNNER_TEMP bash tests/scripts/run_ci_backend_tests.sh \
+            sudo --preserve-env=RUNNER_TEMP bash scripts/ci/run_backend_validation_tests.sh \
               '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 |
               tee -a "$RUNNER_TEMP/mxc-ci.log"
           else
-            bash tests/scripts/run_ci_backend_tests.sh \
+            bash scripts/ci/run_backend_validation_tests.sh \
               '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 |
               tee -a "$RUNNER_TEMP/mxc-ci.log"
           fi
@@ -201,7 +201,7 @@ jobs:
         run: |
           set -euo pipefail
           chmod +x artifacts/bin/mxc-exec-mac artifacts/bin/unix-test-proxy
-          bash tests/scripts/run_ci_backend_tests.sh \
+          bash scripts/ci/run_backend_validation_tests.sh \
             '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 |
             tee "$RUNNER_TEMP/mxc-ci.log"
 
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 7d86032c9..7b86002c8 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -31,8 +31,8 @@ the individual local test scripts are documented in
 | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. |
 | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. |
 | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). |
-| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
-| `tests/scripts/run_ci_backend_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
+| `scripts/ci/run_backend_validation_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
+| `scripts/ci/run_backend_validation_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
 
 ### Flow
 
@@ -42,9 +42,9 @@ Validation.Tests.Scheduled.yml
       ├─ windows / linux / macos    →  Build.*.Job.yml  (upload artifacts)
       └─ test-nightly / test-weekly →  Validation.Tests.Matrix.Job.yml
             └─ resolve  →  resolve-validation-test-matrix.mjs --plan <plan>
-                 ├─ windows job (matrix) → download artifact → prepare-windows-host.ps1 → run_ci_backend_tests.ps1
-                 ├─ linux   job (matrix) → download artifact → prepare-linux-host.sh   → run_ci_backend_tests.sh
-                 └─ macos   job (matrix) → download artifact →                            run_ci_backend_tests.sh
+                 ├─ windows job (matrix) → download artifact → prepare-windows-host.ps1 → run_backend_validation_tests.ps1
+                 ├─ linux   job (matrix) → download artifact → prepare-linux-host.sh   → run_backend_validation_tests.sh
+                 └─ macos   job (matrix) → download artifact →                            run_backend_validation_tests.sh
 ```
 
 An entry point **must** build the artifacts before calling the matrix job — the
@@ -70,9 +70,9 @@ Build artifacts are kept for 1 day — they exist only to feed these jobs.
 | Job | Runner | What it does |
 |-----|--------|--------------|
 | `resolve` | `ubuntu-latest` | Runs the resolver, emits one matrix per OS family plus `has_<family>` flags so an empty family is skipped rather than failing on an empty matrix. |
-| `windows` | `[self-hosted, 1ES.Pool=<pool>, JobId=mxc-e2e-…]` | Download artifact → `prepare-windows-host.ps1 -Backend <backend id>` → `run_ci_backend_tests.ps1 -Backend <backend id>`. |
-| `linux` | `[self-hosted, 1ES.Pool=<pool>, JobId=mxc-e2e-…]` | Download artifact → `prepare-linux-host.sh <backend id>` → `run_ci_backend_tests.sh <backend id>` (under `sudo` for LXC). |
-| `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `chmod +x` → `run_ci_backend_tests.sh <backend id>`. No host-prep step. |
+| `windows` | `[self-hosted, 1ES.Pool=<pool>, JobId=mxc-e2e-…]` | Download artifact → `prepare-windows-host.ps1 -Backend <backend id>` → `run_backend_validation_tests.ps1 -Backend <backend id>`. |
+| `linux` | `[self-hosted, 1ES.Pool=<pool>, JobId=mxc-e2e-…]` | Download artifact → `prepare-linux-host.sh <backend id>` → `run_backend_validation_tests.sh <backend id>` (under `sudo` for LXC). |
+| `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `chmod +x` → `run_backend_validation_tests.sh <backend id>`. No host-prep step. |
 
 Per-job display name: `<platform id>, <architecture>, <backend>` (macOS omits
 the architecture). Job timeout 180 min; host prep 15 min; the test step 45 min
@@ -201,8 +201,8 @@ get fixed or wired.
 | Bubblewrap | ✅ Good | |
 | LXC | ✅ Good | Some networking tests fail on distros other than Ubuntu 24.04; seems to be an issue with MXC. |
 | WSLC | ✅ Good | Might have to retry hung jobs - this is an issue with overzealous agent reclaiming. |
-| IsolationSession | ⚠️ Blocked | `Feature_AgentSessionsBaseSupport` is not enabled on the pool image yet. |
-| Windows Sandbox | ⛔ Not scheduled | Dispatcher case is wired; no trigger entry yet. |
+| IsolationSession | ✅ Good | |
+| Windows Sandbox | ⛔ Blocked | Images don't support `Containers-DisposableClientVM` opt. feature |
 | MicroVM | ⛔ Not working | Windows cold and warm starts hang; no Linux suite. The artifact payload is currently commented out in the build jobs. |
 | Hyperlight | ⛔ Not implemented | No suite on any platform. |
 | Seatbelt | ⛔ Not implemented | The backend itself is healthy; there is no official E2E suite to dispatch to. |
@@ -256,7 +256,7 @@ scratch trees, transcripts, and results files under the user's temp directory
 `${{ runner.temp }}` (`C:\a\_work\_temp`). Anything left in the former is simply
 never collected, which is why the artifact used to arrive nearly empty.
 
-So `run_ci_backend_tests.ps1` points `TEMP` and `TMP` at `$RUNNER_TEMP` before
+So `run_backend_validation_tests.ps1` points `TEMP` and `TMP` at `$RUNNER_TEMP` before
 it dispatches. Parameter defaults, `[System.IO.Path]::GetTempPath()`, and child
 processes all read those variables, so everything temp-rooted lands in the
 upload directory without CI having to know a single filename.
@@ -300,8 +300,8 @@ test runner is allocated.
 1. **Catalog:** add the id to the `backends` list of every platform/arch that
    can run it. There is no separate registration step — the id *is* the
    dispatcher argument.
-2. **Dispatcher:** add a case to `run_ci_backend_tests.ps1` (`ValidateSet` +
-   `switch`) or `run_ci_backend_tests.sh` (`usage` + `case`), pointing at the
+2. **Dispatcher:** add a case to `run_backend_validation_tests.ps1` (`ValidateSet` +
+   `switch`) or `run_backend_validation_tests.sh` (`usage` + `case`), pointing at the
    suite. Until a suite exists, leave the explicit throw / `exit 2` so
    accidental activation fails loudly.
 3. **Host prep:** add a branch to `prepare-windows-host.ps1` (`ValidateSet` +
diff --git a/scripts/ci/Setup.ps1 b/scripts/ci/Setup.ps1
index 34dd71579..55dbb4893 100644
--- a/scripts/ci/Setup.ps1
+++ b/scripts/ci/Setup.ps1
@@ -2,25 +2,30 @@
 
 <#
 .SYNOPSIS
-    TEMP scratch script: runs only the WSLC host initialization steps.
+    Installs or updates the WSL runtime, optionally from the pre-release ring, and checks for required Windows optional features.
 
 .DESCRIPTION
-    A trimmed copy of scripts/ci/prepare-windows-host.ps1 that keeps just the
-    WSLC path. Unlike the CI script this one is diagnostic-only: every failure
-    is reported and execution always ends with exit code 0.
+    Prepares and reports on a host's WSL2 installation: it verifies that the
+    Windows optional features WSL2 depends on are enabled, then inspects the
+    installed runtime, updating an inbox build to the modern runtime and
+    updating from the pre-release ring when -InstallPreRelease is set. It is
+    diagnostic-only in the sense that nothing aborts it: every problem is
+    reported as a FAILED line and the script always exits with code 0.
 
-.PARAMETER BinaryDirectory
-    Optional directory holding a built/downloaded artifact. When supplied, the
-    WSLC binaries are checked for presence; otherwise that check is skipped.
+.PARAMETER InstallPreRelease
+    Update WSL from the pre-release ring. Off by default, which leaves the host
+    on the stable ring.
 
 .EXAMPLE
     ./scripts/ci/Setup.ps1
-    ./scripts/ci/Setup.ps1 -BinaryDirectory src/target/x86_64-pc-windows-msvc/release
+
+.EXAMPLE
+    ./scripts/ci/Setup.ps1 -InstallPreRelease $true
 #>
 
 [CmdletBinding()]
 param(
-    [string]$BinaryDirectory
+    [bool]$InstallPreRelease = $false
 )
 
 Set-StrictMode -Off
@@ -40,25 +45,6 @@ function Write-Failure {
     Write-Host "FAILED: $Message"
 }
 
-function Test-RequiredFile {
-    param([Parameter(Mandatory)][string[]]$RelativePath)
-
-    if (-not $BinaryDirectory) {
-        Write-Host 'No -BinaryDirectory supplied; skipping artifact presence check.'
-        return
-    }
-
-    foreach ($relative in $RelativePath) {
-        $full = Join-Path $BinaryDirectory $relative
-        if (Test-Path $full) {
-            $item = Get-Item $full
-            Write-Host "  found $($item.FullName) ($($item.Length) bytes)"
-        } else {
-            Write-Failure "missing binary: $full"
-        }
-    }
-}
-
 # Read a Windows optional feature's state without throwing. A host that cannot
 # answer (querying needs elevation) reports the reason as its state.
 function Get-OptionalFeatureState {
@@ -124,23 +110,6 @@ function Invoke-Wsl {
     return $result.ExitCode
 }
 
-# Minimum WSL runtime for WSLC, read from the pinned SDK version so the two
-# cannot drift. The SDK's own runtime error names this same version.
-function Get-RequiredWslVersion {
-    $buildScript = Join-Path $PSScriptRoot '..\..\src\backends\wslc\common\build.rs'
-    if (-not (Test-Path $buildScript)) {
-        Write-Host "WARNING: $buildScript not found; skipping the WSL version gate."
-        return $null
-    }
-
-    $match = [regex]::Match((Get-Content $buildScript -Raw), 'WSLC_SDK_VERSION:\s*&str\s*=\s*"([0-9]+(?:\.[0-9]+)+)"')
-    if (-not $match.Success) {
-        Write-Host 'WARNING: could not parse WSLC_SDK_VERSION; skipping the WSL version gate.'
-        return $null
-    }
-    return [version]$match.Groups[1].Value
-}
-
 # Installed modern-runtime version, or $null when wsl.exe is the legacy inbox
 # build (no --version) or otherwise unusable.
 function Get-InstalledWslVersion {
@@ -158,8 +127,6 @@ function Get-InstalledWslVersion {
 
 function Initialize-WslcHost {
     Write-Step 'WSLC artifact binaries'
-    # wslcsdk.dll ships beside wxc-exec.exe only in a --features wslc build.
-    Test-RequiredFile @('wxc-exec.exe', 'wslcsdk.dll')
 
     Write-Step 'Required Windows optional features'
     Test-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform'
@@ -200,20 +167,16 @@ function Initialize-WslcHost {
         Write-Host 'WSL2 is installed and updated (not prerelease, yet).'
     }
 
-    # WSLC needs a runtime at least as new as the pinned WSLC SDK, and those
-    # builds ship only on the pre-release ring - the stable ring lands well
-    # behind it. Without this the SDK fails at run time with
-    # "WSLC runtime unavailable. Missing components: WslPackage".
-    $required = Get-RequiredWslVersion
+    # The pre-release ring carries builds the stable ring lands well behind, so
+    # it is opt-in rather than a version the script decides on its own.
     $installed = Get-InstalledWslVersion
-    Write-Host "Required WSL version: $(if ($null -eq $required) { '<unknown>' } else { $required })"
     Write-Host "Installed WSL version: $(if ($null -eq $installed) { '<none>' } else { $installed })"
 
-    if ($null -ne $required -and ($null -eq $installed -or $installed -lt $required)) {
-        Write-Host "WSL $installed is older than the $required WSLC requires; updating to pre-release..."
+    if ($InstallPreRelease) {
+        Write-Host 'Updating WSL to the latest pre-release build...'
         if ((Invoke-Wsl @('--update', '--pre-release', '--web-download') -Quiet) -ne 0 -and
             (Invoke-Wsl @('--update', '--pre-release')) -ne 0) {
-            Write-Failure "wsl --update --pre-release failed; WSLC requires WSL $required or newer."
+            Write-Failure 'wsl --update --pre-release failed; the pre-release WSL2 runtime could not be installed on this host.'
             return
         }
         $installed = Get-InstalledWslVersion
@@ -221,29 +184,15 @@ function Initialize-WslcHost {
     }
 
     if ($null -eq $installed) {
-        Write-Failure 'wsl --version failed after updating; the WSL2 runtime is not usable on this host.'
-        return
-    }
-    if ($null -ne $required -and $installed -lt $required) {
-        Write-Failure "WSL $installed is installed, but WSLC requires $required or newer."
+        Write-Failure 'wsl --version failed; the WSL2 runtime is not usable on this host.'
         return
     }
 
-    Write-Host "WSL runtime $installed is ready (WSLC requires $required or newer)."
+    Write-Host "WSL runtime $installed is ready."
 }
 
 Write-Host "WSLC host setup starting on $([System.Environment]::OSVersion)"
 
-if ($BinaryDirectory) {
-    if (Test-Path $BinaryDirectory) {
-        $BinaryDirectory = (Resolve-Path $BinaryDirectory).Path
-        Write-Host "Binary directory: $BinaryDirectory"
-    } else {
-        Write-Failure "Binary directory not found: $BinaryDirectory"
-        $BinaryDirectory = $null
-    }
-}
-
 try {
     Initialize-WslcHost
 } catch {
diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/scripts/ci/run_backend_validation_tests.ps1
similarity index 89%
rename from tests/scripts/run_ci_backend_tests.ps1
rename to scripts/ci/run_backend_validation_tests.ps1
index 3770f6390..81576e770 100644
--- a/tests/scripts/run_ci_backend_tests.ps1
+++ b/scripts/ci/run_backend_validation_tests.ps1
@@ -31,6 +31,8 @@ param(
 $ErrorActionPreference = 'Stop'
 
 $scriptRoot = Split-Path -Parent $MyInvocation.MyCommand.Path
+$repoRoot = Split-Path -Parent (Split-Path -Parent $scriptRoot)
+$testScriptRoot = Join-Path $repoRoot 'tests\scripts'
 $binaryDirectoryPath = (Resolve-Path -LiteralPath $BinaryDirectory).Path
 $wxc = Join-Path $binaryDirectoryPath 'wxc-exec.exe'
 
@@ -97,7 +99,7 @@ function Invoke-ProcessContainerTests {
     Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $debugDirectory 'wxc-ui-probe.exe') -Force
     Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force
 
-    $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1'
+    $script = Join-Path $testScriptRoot 'WinProcessContainer-Tests.ps1'
     # -KeepArtifacts stops the harness deleting its scratch tree on a clean
     # run, so a passing job still uploads its per-test logs and configs.
     # Skip build and Cargo phases because this job consumes a previously
@@ -136,12 +138,12 @@ switch ($Backend) {
         Invoke-ProcessContainerTests
     }
     'isolation-session' {
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{
+        Invoke-TestScript -Path (Join-Path $testScriptRoot 'run_isolation_session_tests.ps1') -Arguments @{
             WxcExePath = $wxc
         }
     }
     'windows-sandbox' {
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{
+        Invoke-TestScript -Path (Join-Path $testScriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{
             BinDir = $binaryDirectoryPath
         }
     }
@@ -150,12 +152,12 @@ switch ($Backend) {
         if ($Architecture -ne 'x64') {
             throw 'The existing WSLC test harness is not architecture-portable yet.'
         }
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{
+        Invoke-TestScript -Path (Join-Path $testScriptRoot 'run_wslc_all_tests.ps1') -Arguments @{
             WxcExecPath = $wxc
         }
     }
     'microvm' {
-        Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{
+        Invoke-TestScript -Path (Join-Path $testScriptRoot 'run_microvm_tests.ps1') -Arguments @{
             BinDir = $binaryDirectoryPath
         }
     }
diff --git a/tests/scripts/run_ci_backend_tests.sh b/scripts/ci/run_backend_validation_tests.sh
similarity index 92%
rename from tests/scripts/run_ci_backend_tests.sh
rename to scripts/ci/run_backend_validation_tests.sh
index 4e156ba21..f400b650b 100644
--- a/tests/scripts/run_ci_backend_tests.sh
+++ b/scripts/ci/run_backend_validation_tests.sh
@@ -18,6 +18,7 @@ backend="$1"
 binary_directory="$(cd "$2" && pwd)"
 script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 repo_root="$(cd "$script_root/../.." && pwd)"
+test_script_root="$repo_root/tests/scripts"
 release_directory="$repo_root/src/target/release"
 
 case "$backend" in
@@ -40,7 +41,7 @@ case "$backend" in
         mkdir -p "$release_directory"
         cp -a "$binary_directory/." "$release_directory/"
         chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy"
-        bash "$script_root/run_bwrap_all_tests.sh"
+        bash "$test_script_root/run_bwrap_all_tests.sh"
         ;;
     lxc)
         test -x "$binary_directory/lxc-exec"
@@ -51,7 +52,7 @@ case "$backend" in
         # A skip here means a prerequisite disappeared on a runner provisioned
         # to execute this suite, so turn it into a failure rather than a
         # vacuously green gate.
-        MXC_LXC_TESTS_REQUIRE_EXECUTION=1 bash "$script_root/run_lxc_all_tests.sh"
+        MXC_LXC_TESTS_REQUIRE_EXECUTION=1 bash "$test_script_root/run_lxc_all_tests.sh"
         ;;
     seatbelt)
         test -x "$binary_directory/mxc-exec-mac"
diff --git a/tests/scripts/README.md b/tests/scripts/README.md
index 62887713b..b77a2cc76 100644
--- a/tests/scripts/README.md
+++ b/tests/scripts/README.md
@@ -69,8 +69,8 @@ these dispatchers, which map a matrix backend id to the suites above:
 
 | Dispatcher | Platforms | Backend ids |
 |------------|-----------|-------------|
-| `run_ci_backend_tests.ps1` | Windows | `process-t1`, `process-t3`, `isolation-session`, `windows-sandbox`, `wslc`, `microvm`, `hyperlight` |
-| `run_ci_backend_tests.sh` | Linux, macOS | `bubblewrap`, `lxc`, `seatbelt`, `microvm`, `hyperlight` |
+| `scripts/ci/run_backend_validation_tests.ps1` | Windows | `process-t1`, `process-t3`, `isolation-session`, `windows-sandbox`, `wslc`, `microvm`, `hyperlight` |
+| `scripts/ci/run_backend_validation_tests.sh` | Linux, macOS | `bubblewrap`, `lxc`, `seatbelt`, `microvm`, `hyperlight` |
 
 Pass the backend id exactly as it appears in the catalog — there is no separate
 handler name. Ids that share a suite have their own case in the dispatcher:
@@ -78,12 +78,12 @@ handler name. Ids that share a suite have their own case in the dispatcher:
 determines the tier it expects from the host's own `wxc-exec --probe`.
 
 ```powershell
-tests\scripts\run_ci_backend_tests.ps1 -Backend process-t1 `
+scripts\ci\run_backend_validation_tests.ps1 -Backend process-t1 `
     -BinaryDirectory <dir> -Architecture x64
 ```
 
 ```bash
-tests/scripts/run_ci_backend_tests.sh bubblewrap <binary-directory>
+scripts/ci/run_backend_validation_tests.sh bubblewrap <binary-directory>
 ```
 
 A backend with no wired suite exits non-zero on purpose, so accidentally

From 1756ccff96e35d9d83ae13ddbe51027fa14744c1 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 26 Aug 2026 14:04:21 -0700
Subject: [PATCH 24/44] testing new wslc artifact on all windows versions

---
 scripts/ci/Setup.ps1                   | 209 -------------------------
 scripts/ci/validation-test-matrix.json |  32 +++-
 2 files changed, 26 insertions(+), 215 deletions(-)
 delete mode 100644 scripts/ci/Setup.ps1

diff --git a/scripts/ci/Setup.ps1 b/scripts/ci/Setup.ps1
deleted file mode 100644
index 55dbb4893..000000000
--- a/scripts/ci/Setup.ps1
+++ /dev/null
@@ -1,209 +0,0 @@
-#Requires -Version 7.0
-
-<#
-.SYNOPSIS
-    Installs or updates the WSL runtime, optionally from the pre-release ring, and checks for required Windows optional features.
-
-.DESCRIPTION
-    Prepares and reports on a host's WSL2 installation: it verifies that the
-    Windows optional features WSL2 depends on are enabled, then inspects the
-    installed runtime, updating an inbox build to the modern runtime and
-    updating from the pre-release ring when -InstallPreRelease is set. It is
-    diagnostic-only in the sense that nothing aborts it: every problem is
-    reported as a FAILED line and the script always exits with code 0.
-
-.PARAMETER InstallPreRelease
-    Update WSL from the pre-release ring. Off by default, which leaves the host
-    on the stable ring.
-
-.EXAMPLE
-    ./scripts/ci/Setup.ps1
-
-.EXAMPLE
-    ./scripts/ci/Setup.ps1 -InstallPreRelease $true
-#>
-
-[CmdletBinding()]
-param(
-    [bool]$InstallPreRelease = $false
-)
-
-Set-StrictMode -Off
-$ErrorActionPreference = 'Continue'
-
-$script:Failed = $false
-
-function Write-Step {
-    param([Parameter(Mandatory)][string]$Message)
-    Write-Host ''
-    Write-Host "=== $Message ==="
-}
-
-function Write-Failure {
-    param([Parameter(Mandatory)][string]$Message)
-    $script:Failed = $true
-    Write-Host "FAILED: $Message"
-}
-
-# Read a Windows optional feature's state without throwing. A host that cannot
-# answer (querying needs elevation) reports the reason as its state.
-function Get-OptionalFeatureState {
-    param([Parameter(Mandatory)][string]$Name)
-
-    try {
-        $feature = Get-WindowsOptionalFeature -Online -FeatureName $Name -ErrorAction Stop
-    } catch {
-        return "query-failed: $($_.Exception.Message.Trim())"
-    }
-
-    if ($null -eq $feature) {
-        return 'unknown'
-    }
-    return [string]$feature.State
-}
-
-# Enabling an optional feature needs a reboot, so this reports rather than
-# installs.
-function Test-RequiredFeature {
-    param([Parameter(Mandatory)][string[]]$Name)
-
-    foreach ($feature in $Name) {
-        $state = Get-OptionalFeatureState -Name $feature
-        Write-Host "  $feature = $state"
-        if ($state -ne 'Enabled') {
-            Write-Failure "Windows optional feature not enabled: $feature ($state). WSL2 must be baked into the image; enabling it requires a reboot."
-        }
-    }
-}
-
-# wsl.exe emits UTF-16LE, which the default console encoding renders as
-# null-separated garbage. Returns @{ ExitCode; Output } with the output decoded.
-function Invoke-WslCapture {
-    param([Parameter(Mandatory)][string[]]$Arguments)
-
-    $previousEncoding = [Console]::OutputEncoding
-    try {
-        [Console]::OutputEncoding = [System.Text.Encoding]::Unicode
-        $output = & wsl.exe @Arguments 2>&1 | Out-String
-        return @{ ExitCode = $LASTEXITCODE; Output = $output }
-    } catch {
-        return @{ ExitCode = 1; Output = "wsl.exe could not be run: $($_.Exception.Message)" }
-    } finally {
-        [Console]::OutputEncoding = $previousEncoding
-    }
-}
-
-# Run wsl.exe and return its exit code. -Quiet suppresses output for probes,
-# where the legacy wsl.exe dumps its whole usage text on an unknown switch.
-function Invoke-Wsl {
-    param(
-        [Parameter(Mandatory)][string[]]$Arguments,
-        [switch]$Quiet
-    )
-
-    Write-Host "  > wsl.exe $($Arguments -join ' ')"
-    $result = Invoke-WslCapture -Arguments $Arguments
-    if (-not $Quiet -and $result.Output.Trim()) {
-        Write-Host $result.Output.Trim()
-    }
-    Write-Host "  exit code: $($result.ExitCode)"
-    return $result.ExitCode
-}
-
-# Installed modern-runtime version, or $null when wsl.exe is the legacy inbox
-# build (no --version) or otherwise unusable.
-function Get-InstalledWslVersion {
-    $result = Invoke-WslCapture -Arguments @('--version')
-    if ($result.ExitCode -ne 0) {
-        return $null
-    }
-
-    $match = [regex]::Match($result.Output, '(?im)^\s*WSL version:\s*([0-9]+(?:\.[0-9]+)+)')
-    if (-not $match.Success) {
-        return $null
-    }
-    return [version]$match.Groups[1].Value
-}
-
-function Initialize-WslcHost {
-    Write-Step 'WSLC artifact binaries'
-
-    Write-Step 'Required Windows optional features'
-    Test-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform'
-
-    Write-Step 'wsl.exe presence + status'
-    $wsl = Get-Command wsl.exe -ErrorAction SilentlyContinue
-    if ($wsl) {
-        Write-Host "wsl.exe: $($wsl.Source)"
-    } else {
-        Write-Failure 'wsl.exe NOT found on PATH; WSL2 is not installed on this host.'
-        return
-    }
-
-    $status = Invoke-WslCapture -Arguments @('--status')
-    Write-Host $status.Output.Trim()
-
-    if ($status.Output -match 'wsl.exe --install') {
-        Write-Failure 'WSL2 is not installed on this host (wsl --status advertises --install).'
-        return
-    }
-
-    Write-Step 'WSL runtime version'
-    if ((Invoke-Wsl @('--version') -Quiet) -ne 0) {
-        Write-Host 'wsl --version failed, so WSL2 is installed but not updated.'
-        Write-Host 'Updating inbox WSL to the modern runtime...'
-
-        if ((Invoke-Wsl @('--update', '--web-download') -Quiet) -ne 0 -and
-            (Invoke-Wsl @('--update')) -ne 0) {
-            Write-Failure 'wsl --update failed; the WSL2 runtime could not be installed on this host.'
-            return
-        }
-
-        if ((Invoke-Wsl @('--version') -Quiet) -ne 0) {
-            Write-Failure 'wsl --version failed after updating; the WSL2 runtime is not usable on this host.'
-            return
-        }
-
-        Write-Host 'WSL2 is installed and updated (not prerelease, yet).'
-    }
-
-    # The pre-release ring carries builds the stable ring lands well behind, so
-    # it is opt-in rather than a version the script decides on its own.
-    $installed = Get-InstalledWslVersion
-    Write-Host "Installed WSL version: $(if ($null -eq $installed) { '<none>' } else { $installed })"
-
-    if ($InstallPreRelease) {
-        Write-Host 'Updating WSL to the latest pre-release build...'
-        if ((Invoke-Wsl @('--update', '--pre-release', '--web-download') -Quiet) -ne 0 -and
-            (Invoke-Wsl @('--update', '--pre-release')) -ne 0) {
-            Write-Failure 'wsl --update --pre-release failed; the pre-release WSL2 runtime could not be installed on this host.'
-            return
-        }
-        $installed = Get-InstalledWslVersion
-        Write-Host "Installed WSL version after update: $(if ($null -eq $installed) { '<none>' } else { $installed })"
-    }
-
-    if ($null -eq $installed) {
-        Write-Failure 'wsl --version failed; the WSL2 runtime is not usable on this host.'
-        return
-    }
-
-    Write-Host "WSL runtime $installed is ready."
-}
-
-Write-Host "WSLC host setup starting on $([System.Environment]::OSVersion)"
-
-try {
-    Initialize-WslcHost
-} catch {
-    Write-Failure "unexpected error: $($_.Exception.Message)"
-}
-
-Write-Step 'Result'
-if ($script:Failed) {
-    Write-Host 'WSLC host setup completed WITH failures (see FAILED lines above).'
-} else {
-    Write-Host 'WSLC host setup completed successfully.'
-}
-
-exit 0
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 9cff37eeb..a287d00ae 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -70,16 +70,20 @@
       }
     },
     {
-      "id": "windows-wslc",
-      "displayName": "Windows WSLc Test",
+      "id": "windows-canary",
+      "displayName": "Windows Canary",
       "family": "windows",
       "architectures": {
         "x64": {
           "target": "x86_64-pc-windows-msvc",
           "artifact": "wxc-binaries-x86_64-pc-windows-msvc",
-          "pool": "1es-mxc-e2e-windows-25h2-wsl2",
+          "pool": "",
           "backends": [
-            "wslc"
+            "process-t3",
+            "wslc",
+            "windows-sandbox",
+            "microvm",
+            "hyperlight"
           ]
         },
         "arm64": {
@@ -87,7 +91,11 @@
           "artifact": "wxc-binaries-aarch64-pc-windows-msvc",
           "pool": "",
           "backends": [
-            "wslc"
+            "process-t3",
+            "wslc",
+            "windows-sandbox",
+            "microvm",
+            "hyperlight"
           ]
         }
       }
@@ -390,7 +398,19 @@
     "weekly": [],
     "enabled": [
       {
-        "os": "windows-wslc",
+        "os": "windows-25h2",
+        "backends": [
+          "wslc"
+        ]
+      },
+      {
+        "os": "windows-24h2",
+        "backends": [
+          "wslc"
+        ]
+      },
+      {
+        "os": "windows-23h2",
         "backends": [
           "wslc"
         ]

From c5964a2315bc73a059ab1d02353d84734e690e33 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 26 Aug 2026 14:20:41 -0700
Subject: [PATCH 25/44] corrected arm64 support for canary

---
 scripts/ci/validation-test-matrix.json | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index a287d00ae..3a1a570f8 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -93,9 +93,7 @@
           "backends": [
             "process-t3",
             "wslc",
-            "windows-sandbox",
-            "microvm",
-            "hyperlight"
+            "windows-sandbox"
           ]
         }
       }

From 9ad0130befe75262d1be53f69dbf613d24989334 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 26 Aug 2026 15:24:55 -0700
Subject: [PATCH 26/44] Added T3-workloads.ps1 to tests

---
 .github/copilot-instructions.md             |   7 +-
 docs/ci-validation-infrastructure.md        |  44 +++++---
 scripts/ci/prepare-windows-host.ps1         |  45 ++++++++
 scripts/ci/run_backend_validation_tests.ps1 |  40 +++++--
 scripts/ci/validation-test-matrix.json      |  10 +-
 tests/scripts/README.md                     |   4 +
 tests/scripts/T3-Workloads.ps1              | 114 +++++++++++++++-----
 tests/scripts/WinProcessContainer-Tests.ps1 |  46 +++++---
 8 files changed, 243 insertions(+), 67 deletions(-)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index d9c42565b..197bbf737 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -73,7 +73,8 @@ workflows) before calling the matrix job.
 - `scripts/ci/validation-test-matrix.json` is the catalog: `platforms` (each
   with per-architecture target/artifact/1ES pool and the backends that platform
   supports), `triggers` (which OS/backend pairs each plan runs), and the
-  optional `backendDelayedStart` (per-backend job-start stagger, in seconds).
+  optional `backendDelayedStart` (per-backend job-start stagger, in seconds;
+  currently empty — nothing is staggered).
   The `triggers` keys *are* the plan list — the resolver reads them at run time,
   so adding a plan needs no script change.
 - `scripts/ci/resolve-validation-test-matrix.mjs` validates that catalog and
@@ -95,7 +96,9 @@ explicit no-op, so the step runs unconditionally for every entry.
 the matrix `backend` id to the repository's existing backend suite. Ids that
 share a suite get their own case (`process-t1` and `process-t3` both run
 `WinProcessContainer-Tests.ps1`, which derives the tier it expects from the
-host's own `--probe`). A backend with no wired suite fails loudly rather than
+host's own `--probe`; `process-t3` additionally runs `T3-Workloads.ps1` and
+reports both suites' exit codes together, so one failing suite never hides the
+other). A backend with no wired suite fails loudly rather than
 reporting a false success. The Windows dispatcher points `TEMP` at
 `$RUNNER_TEMP` before running a suite, so anything a test writes to the temp
 directory is picked up by the job's log upload without per-file CI wiring.
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 7b86002c8..a50354212 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -128,10 +128,14 @@ A backend id is passed straight through: the matrix job hands it to the host-pre
 script and then to the dispatcher, which has one `switch`/`case` per id. Ids that
 share a suite each keep their own case so they can diverge later without a
 mapping table — `process-t1` and `process-t3` both run
-`WinProcessContainer-Tests.ps1` today. Teaching the Process Container test suite to 
+`WinProcessContainer-Tests.ps1`, and `process-t3` additionally runs
+`T3-Workloads.ps1`. Teaching the Process Container test suite to
 accept an explicit tier (so a T1 host can also be exercised
-at the T3 fallback) is a worthwhile future improvement; see
-[Possible future improvements](#possible-future-improvements).
+at the T3 fallback) is a worthwhile future improvement.
+
+`process-t3` runs its two suites back to back and reports them together: a
+failure in the primitives suite does not skip the workloads suite, so one job
+run shows both results instead of costing a second run to triage.
 
 An unwired backend fails loudly on purpose: adding it to a trigger produces a
 red job ("write the tests or remove it"), never a green no-op. The dispatchers'
@@ -158,8 +162,9 @@ because Seatbelt has no wired suite.
 
 ### `backendDelayedStart`
 
-Optional. Staggers the start of jobs for a named backend instead of letting
-them all begin at once:
+Optional, and **currently empty** — no backend is staggered today, so every job
+starts as soon as its runner is ready. The section staggers the start of jobs
+for a named backend instead of letting them all begin at once:
 
 ```json
 "backendDelayedStart": [
@@ -173,13 +178,22 @@ traffic into a burst the moment its jobs start together. Public registries
 answer with rate limiting and stalled downloads.
 
 `seconds` is the gap between consecutive jobs of that backend, counted per
-backend and following the resolved job order. With the entry above, four WSLC
-jobs start at 0, 300, 600, and 900 seconds.
+backend and following the resolved job order. With the example entry above,
+four WSLC jobs would start at 0, 300, 600, and 900 seconds.
 
 The resolver puts the offset on every matrix entry as
-`startup_delay_seconds` — `0` where no stagger applies — and the job sleeps
-that long before its first network step. Job timeout is a flat 180 minutes,
-with plenty of room for any wait you'd reasonably configure.
+`startup_delay_seconds` — `0` where no stagger applies, which is every entry
+while the section is empty — and the job sleeps that long before its first
+network step. Job timeout is a flat 180 minutes, with plenty of room for any
+wait you'd reasonably configure.
+
+Leave the section out (or empty) and every job starts as soon as its runner is
+ready. A backend id that no plan schedules is accepted; it just never applies.
+
+Do keep in mind that the runner is held while it sleeps — Actions can't defer
+allocating a matrix job, so the wait has to happen inside it. Use no more than
+the contention calls for. This spreads simultaneous load and nothing else; a
+single download that stalls on its own is unaffected.
 
 Leave the section out (or empty) and every job starts as soon as its runner is
 ready. A backend id that no plan schedules is accepted; it just never applies.
@@ -196,8 +210,8 @@ get fixed or wired.
 
 | Backend | Status | Notes |
 |---------|--------|-------|
-| Process T1 | ✅ Good | Prerelease Windows only. Remaining failures are genuine MXC bugs or harness limitations. |
-| Process T3 | ✅ Good | Non-prerelease Windows builds only, until the testing suite is updated. |
+| Process T1 | ✅ Good | Prerelease Windows only. Runs the primitives suite. Remaining failures are genuine MXC bugs or harness limitations. |
+| Process T3 | ✅ Good | Non-prerelease Windows builds only. Runs the primitives suite plus `T3-Workloads.ps1` (real programs — pwsh, git, node, python, cmd — on top of the T3 primitives). |
 | Bubblewrap | ✅ Good | |
 | LXC | ✅ Good | Some networking tests fail on distros other than Ubuntu 24.04; seems to be an issue with MXC. |
 | WSLC | ✅ Good | Might have to retry hung jobs - this is an issue with overzealous agent reclaiming. |
@@ -216,7 +230,11 @@ every entry.
 `prepare-windows-host.ps1`:
 
 - `process-t3` — runs `wxc-host-prep.exe prepare-system-drive` and
-  `prepare-null-device --no-sacl`.
+  `prepare-null-device --no-sacl`, then verifies the workload interpreters:
+  `pwsh` and `git` are required (they gate 10 of the 19 `T3-Workloads.ps1`
+  cases, so their absence means a mis-imaged pool and fails the job), while
+  `node` and `python` only emit a warning because the suite reports their
+  cases as skipped.
 - `microvm` — asserts the NanVix payload is in the artifact, adds a Defender
   exclusion for the binary directory, and requires the Windows Hypervisor
   Platform feature *and* a running hypervisor.
diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index 76955478d..89f8d4342 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -145,6 +145,51 @@ function Initialize-ProcessContainerHost {
     if ($LASTEXITCODE -ne 0) {
         Exit-WithError "wxc-host-prep prepare-null-device failed with exit code $LASTEXITCODE"
     }
+
+    Assert-WorkloadInterpreters
+}
+
+# Checks for programs useful to the workload test suite. Missing optional interpreters are reported as warnings.
+function Assert-WorkloadInterpreters {
+    $required = @(
+        @{ Name = 'pwsh'; Remedy = 'install PowerShell 7 in the image' },
+        @{ Name = 'git'; Remedy = 'install Git for Windows in the image' }
+    )
+    $missing = @()
+    foreach ($tool in $required) {
+        $found = Get-Command $tool.Name -ErrorAction SilentlyContinue
+        if ($found) {
+            Write-Host "Workload interpreter '$($tool.Name)' found at $($found.Source)"
+        } else {
+            $missing += "$($tool.Name) ($($tool.Remedy))"
+        }
+    }
+    if ($missing) {
+        Exit-WithError "Workload interpreters missing from this image: $($missing -join '; ')"
+    }
+
+    # The suite tries `python` then `python3`, so mirror that candidate order.
+    $optional = @(
+        @{ Name = 'node'; Candidates = @('node') },
+        @{ Name = 'python'; Candidates = @('python', 'python3') }
+    )
+    foreach ($tool in $optional) {
+        $resolved = $null
+        foreach ($candidate in $tool.Candidates) {
+            $found = Get-Command $candidate -ErrorAction SilentlyContinue
+            # A `python` resolving into WindowsApps is the Microsoft Store
+            # AppExecutionAlias stub, which the suite deliberately ignores.
+            if ($found -and $found.Source -notlike '*\WindowsApps\*') {
+                $resolved = $found.Source
+                break
+            }
+        }
+        if ($resolved) {
+            Write-Host "Workload interpreter '$($tool.Name)' found at $resolved"
+        } else {
+            Write-Host "::warning::Workload interpreter '$($tool.Name)' is absent."
+        }
+    }
 }
 
 function Initialize-MicroVmHost {
diff --git a/scripts/ci/run_backend_validation_tests.ps1 b/scripts/ci/run_backend_validation_tests.ps1
index 81576e770..09aab6707 100644
--- a/scripts/ci/run_backend_validation_tests.ps1
+++ b/scripts/ci/run_backend_validation_tests.ps1
@@ -86,6 +86,14 @@ function Invoke-TestScript {
 Assert-File -Path $wxc
 
 function Invoke-ProcessContainerTests {
+    # Returns the harness exit code rather than throwing, so a caller running
+    # more than one suite can report both results instead of stopping at the
+    # first failure. The suite talks to the operator through Write-Host (which
+    # Out-Null does not touch), so discarding the success stream keeps the
+    # return value a scalar even if a phase leaks a stray object.
+    [OutputType([int])]
+    param()
+
     # The existing harness expects separate debug and release layouts. CI
     # intentionally tests one release artifact, so stage it in both slots.
     $debugDirectory = Join-Path $binaryDirectoryPath 'debug'
@@ -122,20 +130,40 @@ function Invoke-ProcessContainerTests {
         -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') `
         -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') `
         -KeepArtifacts `
-        -Phases $phases
-    if ($LASTEXITCODE -ne 0) {
-        throw "Process Container tests failed with exit code $LASTEXITCODE."
-    }
+        -Phases $phases | Out-Null
+    return $LASTEXITCODE
+}
+
+function Invoke-T3WorkloadTests {
+    [OutputType([int])]
+    param()
+
+    $script = Join-Path $testScriptRoot 'T3-Workloads.ps1'
+    # -Wxc is required: the script's default points at a debug build that does
+    # not exist in a CI artifact. -KeepArtifacts preserves the per-workload
+    # logs and configs on a clean run so a passing job still uploads them.
+    $global:LASTEXITCODE = 0
+    & $script -Wxc $wxc -KeepArtifacts | Out-Null
+    return $LASTEXITCODE
 }
 
 Redirect-TempToRunnerTemp
 
 switch ($Backend) {
     'process-t1' {
-        Invoke-ProcessContainerTests
+        $primitives = Invoke-ProcessContainerTests
+        if ($primitives -ne 0) {
+            throw "Process Container tests failed with exit code $primitives."
+        }
     }
     'process-t3' {
-        Invoke-ProcessContainerTests
+        # Run both suites before reporting. Stopping at the first failure would
+        # hide the other suite's result, costing an extra nightly run to triage.
+        $primitives = Invoke-ProcessContainerTests
+        $workloads = Invoke-T3WorkloadTests
+        if ($primitives -ne 0 -or $workloads -ne 0) {
+            throw "process-t3 tests failed (primitives exit=$primitives, workloads exit=$workloads)."
+        }
     }
     'isolation-session' {
         Invoke-TestScript -Path (Join-Path $testScriptRoot 'run_isolation_session_tests.ps1') -Arguments @{
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 3a1a570f8..0a2032a17 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -321,12 +321,7 @@
       }
     }
   ],
-  "backendDelayedStart": [
-    {
-      "backend": "wslc",
-      "seconds": 45
-    }
-  ],
+  "backendDelayedStart": [],
   "triggers": {
     "pr": [],
     "nightly": [
@@ -398,18 +393,21 @@
       {
         "os": "windows-25h2",
         "backends": [
+          "process-t3",
           "wslc"
         ]
       },
       {
         "os": "windows-24h2",
         "backends": [
+          "process-t3",
           "wslc"
         ]
       },
       {
         "os": "windows-23h2",
         "backends": [
+          "process-t3",
           "wslc"
         ]
       }
diff --git a/tests/scripts/README.md b/tests/scripts/README.md
index b77a2cc76..59b5c8bf6 100644
--- a/tests/scripts/README.md
+++ b/tests/scripts/README.md
@@ -41,6 +41,8 @@ Linux / macOS (`.sh`):
 | `run_windows_sandbox_one_shot_tests.ps1` | Windows Sandbox one-shot E2E suite (fresh disposable VM per test) | Windows Sandbox enabled |
 | `run_windows_sandbox_state_aware_tests.ps1` | Windows Sandbox state-aware lifecycle E2E (single VM held across provision/start/exec*/stop/deprovision) | Windows Sandbox enabled |
 | `run_processcontainer_proxy_tests.ps1` | Process container proxy tests | `wxc-exec.exe` |
+| `WinProcessContainer-Tests.ps1` | Process container (AppContainer / BaseContainer) primitives suite — tier probes, rw/ro/denied matrix, UI mitigations, DACL restore, crash recovery | `wxc-exec.exe`, `wxc-ui-probe.exe` |
+| `T3-Workloads.ps1` | Real workloads (pwsh, git, node, python, cmd) on top of the T3 primitives. A missing interpreter is reported as a skip, not a failure | `wxc-exec.exe`; `pwsh` and `git` for full coverage, `node` / `python` optional |
 | `run_on_repeat.ps1` | Stress test (loops core tests) | `wxc-exec.exe` |
 
 ### Linux suites
@@ -76,6 +78,8 @@ Pass the backend id exactly as it appears in the catalog — there is no separat
 handler name. Ids that share a suite have their own case in the dispatcher:
 `process-t1` and `process-t3` both run `WinProcessContainer-Tests.ps1`, which
 determines the tier it expects from the host's own `wxc-exec --probe`.
+`process-t3` additionally runs `T3-Workloads.ps1`; both suites run even if the
+first one fails, and the job reports their exit codes together.
 
 ```powershell
 scripts\ci\run_backend_validation_tests.ps1 -Backend process-t1 `
diff --git a/tests/scripts/T3-Workloads.ps1 b/tests/scripts/T3-Workloads.ps1
index 82e832029..75e028146 100644
--- a/tests/scripts/T3-Workloads.ps1
+++ b/tests/scripts/T3-Workloads.ps1
@@ -21,6 +21,9 @@ param(
     # -Wxc explicitly if the layout differs.
     [string]$Wxc          = (Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'src\target\debug\wxc-exec.exe'),
     [string]$ScratchRoot  = (Join-Path $env:TEMP 'mxc-t3-workloads'),
+    # Structured results. Lives in $env:TEMP but OUTSIDE $ScratchRoot so
+    # `Initialize-Scratch`'s recursive nuke can never take it with it.
+    [string]$ResultsJson  = (Join-Path $env:TEMP 'T3-Workloads.results.json'),
     # Subset of workloads to run. Default: all nineteen.
     [int[]] $Run          = @(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19),
     # Add a few extra "kitchen sink" RO grants (TEMP, LOCALAPPDATA, ...)
@@ -43,25 +46,40 @@ function Record-Workload {
     param(
         [Parameter(Mandatory)] [string]$Id,
         [Parameter(Mandatory)] [string]$Name,
-        [Parameter(Mandatory)] [bool]$Pass,
+        [bool]$Pass = $true,
+        # Visual/semantic status. When omitted, derived from -Pass for back-
+        # compat (pass/fail). 'skip' = the workload's interpreter isn't on this
+        # host, so nothing was proven either way. A skip does not fail the run,
+        # but it renders distinctly so absent coverage is never a green PASS.
+        [ValidateSet('pass', 'fail', 'skip')] [string]$Status,
         [int]$ExitCode = 0,
         [string]$Detail = '',
         [string]$Stderr = ''
     )
+    if (-not $PSBoundParameters.ContainsKey('Status')) {
+        $Status = if ($Pass) { 'pass' } else { 'fail' }
+    } else {
+        # Keep the boolean consistent for downstream logic: only 'fail' fails.
+        $Pass = ($Status -ne 'fail')
+    }
     $entry = [pscustomobject]@{
         Id        = $Id
         Name      = $Name
         Pass      = $Pass
+        Status    = $Status
         ExitCode  = $ExitCode
         Detail    = $Detail
         StderrTop = ($Stderr -split "`r?`n" | Select-Object -First 3) -join ' / '
     }
     $Script:Results.Add($entry) | Out-Null
-    $tag = if ($Pass) { '[PASS]' } else { '[FAIL]' }
-    $color = if ($Pass) { 'Green' } else { 'Red' }
+    $tag, $color = switch ($Status) {
+        'pass' { '[PASS]', 'Green' }
+        'fail' { '[FAIL]', 'Red' }
+        'skip' { '[SKIP]', 'Yellow' }
+    }
     Write-Host ("  {0} {1} :: {2} (exit={3})" -f $tag, $Id, $Name, $ExitCode) -ForegroundColor $color
     if ($Detail) { Write-Host ("        detail: {0}" -f $Detail) }
-    if (-not $Pass -and $entry.StderrTop) {
+    if ($Status -eq 'fail' -and $entry.StderrTop) {
         Write-Host ("        stderr: {0}" -f $entry.StderrTop) -ForegroundColor DarkRed
     }
 }
@@ -291,7 +309,7 @@ function W1-CmdTypeMarker {
 function W2-PwshReadFile {
     Section 'W2: pwsh -NoProfile -c "Get-Content marker.txt"'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W2' -Name 'pwsh Get-Content' -Pass $false -Detail 'pwsh not found on PATH'
+        Record-Workload -Id 'W2' -Name 'pwsh Get-Content' -Status 'skip' -Detail 'pwsh not found on PATH'
         return
     }
     # PowerShell's install dir already grants ReadAndExecute to
@@ -312,11 +330,11 @@ function W2-PwshReadFile {
 function W3-PwshGitVersion {
     Section 'W3: pwsh -NoProfile -c "git --version"'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W3' -Name 'pwsh git --version' -Pass $false -Detail 'pwsh not found'
+        Record-Workload -Id 'W3' -Name 'pwsh git --version' -Status 'skip' -Detail 'pwsh not found'
         return
     }
     if (-not $script:GitDir)  {
-        Record-Workload -Id 'W3' -Name 'pwsh git --version' -Pass $false -Detail 'git not found'
+        Record-Workload -Id 'W3' -Name 'pwsh git --version' -Status 'skip' -Detail 'git not found'
         return
     }
     # Git install dir also grants ReadAndExecute to ALL APPLICATION
@@ -358,7 +376,7 @@ function Initialize-Repo {
 function W4-PwshGitStatus {
     Section 'W4: pwsh -NoProfile -c "cd <repo>; git status"'
     if (-not $script:PwshDir -or -not $script:GitDir) {
-        Record-Workload -Id 'W4' -Name 'pwsh git status' -Pass $false -Detail 'pwsh or git not found'
+        Record-Workload -Id 'W4' -Name 'pwsh git status' -Status 'skip' -Detail 'pwsh or git not found'
         return
     }
     $repo = "$ScratchRoot\rw\repo"
@@ -385,7 +403,7 @@ function W4-PwshGitStatus {
 function W5-PwshGitLog {
     Section 'W5: pwsh -NoProfile -c "cd <repo>; git log --oneline -n 10"'
     if (-not $script:PwshDir -or -not $script:GitDir) {
-        Record-Workload -Id 'W5' -Name 'pwsh git log' -Pass $false -Detail 'pwsh or git not found'
+        Record-Workload -Id 'W5' -Name 'pwsh git log' -Status 'skip' -Detail 'pwsh or git not found'
         return
     }
     $repo = "$ScratchRoot\rw\repo"
@@ -412,7 +430,7 @@ function W5-PwshGitLog {
 function W6-PwshListDir {
     Section 'W6: pwsh Get-ChildItem on rw directory'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W6' -Name 'pwsh Get-ChildItem' -Pass $false -Detail 'pwsh not found'
+        Record-Workload -Id 'W6' -Name 'pwsh Get-ChildItem' -Status 'skip' -Detail 'pwsh not found'
         return
     }
     $cmd = "pwsh.exe -NoProfile -NoLogo -Command `"Get-ChildItem -LiteralPath '$ScratchRoot\rw' | Select-Object -ExpandProperty Name; exit 0`""
@@ -428,7 +446,7 @@ function W6-PwshListDir {
 function W7-PwshInProcEval {
     Section 'W7: pwsh in-process script eval (math, env, pipeline)'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W7' -Name 'pwsh in-proc eval' -Pass $false -Detail 'pwsh not found'
+        Record-Workload -Id 'W7' -Name 'pwsh in-proc eval' -Status 'skip' -Detail 'pwsh not found'
         return
     }
     $script = '$x = 6 * 7; Write-Output "answer=$x"; 1..3 | ForEach-Object { Write-Output "iter=$_" }; exit 0'
@@ -445,7 +463,7 @@ function W7-PwshInProcEval {
 function W8-PwshSpawnCmd {
     Section 'W8: pwsh spawning cmd /c (no NUL)'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W8' -Name 'pwsh spawn cmd' -Pass $false -Detail 'pwsh not found'
+        Record-Workload -Id 'W8' -Name 'pwsh spawn cmd' -Status 'skip' -Detail 'pwsh not found'
         return
     }
     # No NUL redirects in the child — just a one-line echo.
@@ -462,7 +480,7 @@ function W8-PwshSpawnCmd {
 function W9-PwshWriteReadRoundTrip {
     Section 'W9: pwsh Set-Content + Get-Content round-trip'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W9' -Name 'pwsh write+read' -Pass $false -Detail 'pwsh not found'
+        Record-Workload -Id 'W9' -Name 'pwsh write+read' -Status 'skip' -Detail 'pwsh not found'
         return
     }
     $target = "$ScratchRoot\rw\w9-out.txt"
@@ -480,7 +498,7 @@ function W9-PwshWriteReadRoundTrip {
 function W10-PwshDotNetIo {
     Section 'W10: pwsh .NET [System.IO.File]::ReadAllText'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W10' -Name 'pwsh .NET IO' -Pass $false -Detail 'pwsh not found'
+        Record-Workload -Id 'W10' -Name 'pwsh .NET IO' -Status 'skip' -Detail 'pwsh not found'
         return
     }
     $script = "Write-Output ([System.IO.File]::ReadAllText('$ScratchRoot\rw\marker.txt')); exit 0"
@@ -504,7 +522,7 @@ function W10-PwshDotNetIo {
 function W11-NodeReadFile {
     Section 'W11: node -e fs.readFileSync(marker)'
     if (-not $script:NodeDir) {
-        Record-Workload -Id 'W11' -Name 'node read file' -Pass $false -Detail 'node not found on PATH'
+        Record-Workload -Id 'W11' -Name 'node read file' -Status 'skip' -Detail 'node not found on PATH'
         return
     }
     $markerFwd = ("$ScratchRoot\rw\marker.txt") -replace '\\','/'
@@ -524,7 +542,7 @@ function W11-NodeReadFile {
 function W12-NodeEval {
     Section 'W12: node in-proc eval (math, loop)'
     if (-not $script:NodeDir) {
-        Record-Workload -Id 'W12' -Name 'node eval' -Pass $false -Detail 'node not found on PATH'
+        Record-Workload -Id 'W12' -Name 'node eval' -Status 'skip' -Detail 'node not found on PATH'
         return
     }
     $js  = "let x=6*7;console.log('answer='+x);for(let i=1;i<=3;i++)console.log('iter='+i);"
@@ -543,7 +561,7 @@ function W12-NodeEval {
 function W13-NodeRoundTrip {
     Section 'W13: node fs.writeFileSync + readFileSync round-trip'
     if (-not $script:NodeDir) {
-        Record-Workload -Id 'W13' -Name 'node write+read' -Pass $false -Detail 'node not found on PATH'
+        Record-Workload -Id 'W13' -Name 'node write+read' -Status 'skip' -Detail 'node not found on PATH'
         return
     }
     $targetFwd = ("$ScratchRoot\rw\w13-out.txt") -replace '\\','/'
@@ -570,7 +588,7 @@ function W13-NodeRoundTrip {
 function W14-PyReadFile {
     Section 'W14: python -c open(marker).read()'
     if (-not $script:PythonExe) {
-        Record-Workload -Id 'W14' -Name 'python read file' -Pass $false -Detail 'python not found on PATH'
+        Record-Workload -Id 'W14' -Name 'python read file' -Status 'skip' -Detail 'python not found on PATH'
         return
     }
     $py  = "import sys; sys.stdout.write(open(r'$ScratchRoot\rw\marker.txt').read())"
@@ -589,7 +607,7 @@ function W14-PyReadFile {
 function W15-PyEval {
     Section 'W15: python in-proc eval (math, loop)'
     if (-not $script:PythonExe) {
-        Record-Workload -Id 'W15' -Name 'python eval' -Pass $false -Detail 'python not found on PATH'
+        Record-Workload -Id 'W15' -Name 'python eval' -Status 'skip' -Detail 'python not found on PATH'
         return
     }
     # `python -c` accepts ';' between simple statements but requires
@@ -628,7 +646,7 @@ function Initialize-ChdirTarget {
 function W16-PyRoundTrip {
     Section 'W16: python open(w,write) + open(r,read) round-trip'
     if (-not $script:PythonExe) {
-        Record-Workload -Id 'W16' -Name 'python write+read' -Pass $false -Detail 'python not found on PATH'
+        Record-Workload -Id 'W16' -Name 'python write+read' -Status 'skip' -Detail 'python not found on PATH'
         return
     }
     $target = "$ScratchRoot\rw\w16-out.txt"
@@ -662,7 +680,7 @@ function W16-PyRoundTrip {
 function W17-PwshSetLocation {
     Section 'W17: pwsh Set-Location into rw subdir'
     if (-not $script:PwshDir) {
-        Record-Workload -Id 'W17' -Name 'pwsh Set-Location' -Pass $false -Detail 'pwsh not found'
+        Record-Workload -Id 'W17' -Name 'pwsh Set-Location' -Status 'skip' -Detail 'pwsh not found'
         return
     }
     $target = Initialize-ChdirTarget
@@ -679,7 +697,7 @@ function W17-PwshSetLocation {
 function W18-NodeChdir {
     Section 'W18: node process.chdir into rw subdir'
     if (-not $script:NodeDir) {
-        Record-Workload -Id 'W18' -Name 'node chdir' -Pass $false -Detail 'node not found'
+        Record-Workload -Id 'W18' -Name 'node chdir' -Status 'skip' -Detail 'node not found'
         return
     }
     $target = Initialize-ChdirTarget
@@ -700,7 +718,7 @@ function W18-NodeChdir {
 function W19-PyChdir {
     Section 'W19: python os.chdir into rw subdir'
     if (-not $script:PythonExe) {
-        Record-Workload -Id 'W19' -Name 'python chdir' -Pass $false -Detail 'python not found'
+        Record-Workload -Id 'W19' -Name 'python chdir' -Status 'skip' -Detail 'python not found'
         return
     }
     $target = Initialize-ChdirTarget
@@ -753,9 +771,18 @@ catch {
 }
 finally {
     Section 'Summary'
-    $passed = @($Script:Results | Where-Object { $_.Pass })
-    $failed = @($Script:Results | Where-Object { -not $_.Pass })
-    Write-Host ("Total: {0}    Passed: {1}    Failed: {2}" -f $Script:Results.Count, $passed.Count, $failed.Count)
+    $passed  = @($Script:Results | Where-Object { $_.Status -eq 'pass' })
+    $failed  = @($Script:Results | Where-Object { $_.Status -eq 'fail' })
+    $skipped = @($Script:Results | Where-Object { $_.Status -eq 'skip' })
+    Write-Host ("Total: {0}    Passed: {1}    Failed: {2}    Skipped: {3}" -f `
+        $Script:Results.Count, $passed.Count, $failed.Count, $skipped.Count)
+    if ($skipped.Count -gt 0) {
+        Write-Host ''
+        Write-Host 'Skipped (interpreter not available on this host):' -ForegroundColor Yellow
+        foreach ($r in $skipped) {
+            Write-Host ("  [{0}] {1} :: {2}" -f $r.Id, $r.Name, $r.Detail) -ForegroundColor Yellow
+        }
+    }
     if ($failed.Count -gt 0) {
         Write-Host ''
         Write-Host 'Failures:' -ForegroundColor Red
@@ -765,8 +792,40 @@ finally {
             if ($r.StderrTop) { Write-Host ("        stderr: {0}" -f $r.StderrTop) -ForegroundColor DarkRed }
         }
     }
+
+    # Structured results for programmatic consumption, mirroring the shape
+    # WinProcessContainer-Tests.ps1 writes. Every step here is guarded: this
+    # runs in `finally`, so an unhandled throw would skip the exit-code line
+    # below and report a bogus result. CIM in particular is unavailable on
+    # locked-down hosts.
+    try {
+        $osInfo = Get-CimInstance Win32_OperatingSystem -ErrorAction Stop
+        $osCaption = [string]$osInfo.Caption
+        $osBuild = [string]$osInfo.BuildNumber
+    } catch {
+        $osCaption = 'unknown'
+        $osBuild = 'unknown'
+    }
+    try {
+        $summary = [pscustomobject]@{
+            timestamp = (Get-Date).ToString('o')
+            host      = $env:COMPUTERNAME
+            os        = $osCaption
+            osBuild   = $osBuild
+            total     = $Script:Results.Count
+            passed    = $passed.Count
+            failed    = $failed.Count
+            skipped   = $skipped.Count
+            results   = $Script:Results
+        }
+        ($summary | ConvertTo-Json -Depth 6) | Out-File -LiteralPath $ResultsJson -Encoding utf8 -Force
+    } catch {
+        Write-Host "warning: could not write JSON results: $_" -ForegroundColor Yellow
+    }
+
     Write-Host ''
     Write-Host ("Scratch / logs: {0}" -f $ScratchRoot)
+    Write-Host ("JSON summary:   {0}" -f $ResultsJson)
     if (-not $KeepArtifacts -and $failed.Count -eq 0 -and $passed.Count -gt 0 -and (Test-Path $ScratchRoot)) {
         # Re-validate before deletion — `Assert-SafeScratchRoot` ran
         # at the start of the suite, but the variable could in
@@ -775,4 +834,7 @@ finally {
         Assert-SafeScratchRoot
         Remove-Item -Recurse -Force -LiteralPath $ScratchRoot -ErrorAction SilentlyContinue
     }
+    # A run that recorded nothing but skips proved nothing, so it is not a
+    # pass — otherwise a host missing every interpreter reports green.
+    if ($failed.Count -gt 0 -or $passed.Count -eq 0) { exit 1 } else { exit 0 }
 }
diff --git a/tests/scripts/WinProcessContainer-Tests.ps1 b/tests/scripts/WinProcessContainer-Tests.ps1
index 80598afff..957cc5c34 100644
--- a/tests/scripts/WinProcessContainer-Tests.ps1
+++ b/tests/scripts/WinProcessContainer-Tests.ps1
@@ -253,8 +253,15 @@ function Format-VerdictSummary {
 function Test-Preflight {
     Section 'Pre-flight'
 
-    $os = Get-CimInstance -ClassName Win32_OperatingSystem
-    Write-Host ("OS: {0} (build {1})" -f $os.Caption, $os.BuildNumber)
+    # Informational banner only — the load-bearing safety gate is the
+    # bfsCompiledIn check below. CIM is unavailable on some locked-down hosts,
+    # so don't let a cosmetic query abort the whole harness.
+    try {
+        $os = Get-CimInstance -ClassName Win32_OperatingSystem -ErrorAction Stop
+        Write-Host ("OS: {0} (build {1})" -f $os.Caption, $os.BuildNumber)
+    } catch {
+        Write-Host ("OS: unknown (CIM unavailable: {0})" -f $_.Exception.Message.Trim())
+    }
 
     $bfsPath = Join-Path $env:SystemRoot 'System32\bfscfg.exe'
     $bfsPresent = Test-Path $bfsPath
@@ -1688,20 +1695,31 @@ finally {
         }
     }
 
-    # Structured JSON for programmatic consumption.
-    $summary = [pscustomobject]@{
-        timestamp   = (Get-Date).ToString('o')
-        host        = $env:COMPUTERNAME
-        os          = (Get-CimInstance Win32_OperatingSystem).Caption
-        osBuild     = (Get-CimInstance Win32_OperatingSystem).BuildNumber
-        total       = $pass + $fail + $skip + $warn
-        passed      = $pass
-        failed      = $fail
-        skipped     = $skip
-        warnings    = $warn
-        results     = $Script:Results
+    # Structured JSON for programmatic consumption. Guarded end to end: this
+    # runs in `finally`, so an unhandled throw here would skip the exit-code
+    # line at the bottom and hand the CI dispatcher a bogus result. CIM is
+    # unavailable on locked-down hosts.
+    try {
+        $osInfo = Get-CimInstance Win32_OperatingSystem -ErrorAction Stop
+        $osCaption = [string]$osInfo.Caption
+        $osBuild = [string]$osInfo.BuildNumber
+    } catch {
+        $osCaption = 'unknown'
+        $osBuild = 'unknown'
     }
     try {
+        $summary = [pscustomobject]@{
+            timestamp   = (Get-Date).ToString('o')
+            host        = $env:COMPUTERNAME
+            os          = $osCaption
+            osBuild     = $osBuild
+            total       = $pass + $fail + $skip + $warn
+            passed      = $pass
+            failed      = $fail
+            skipped     = $skip
+            warnings    = $warn
+            results     = $Script:Results
+        }
         ($summary | ConvertTo-Json -Depth 6) | Out-File -LiteralPath $ResultsJson -Encoding utf8 -Force
     } catch {
         Write-Host "warning: could not write JSON results: $_" -ForegroundColor Yellow

From d46ab3665268c72b6b8f7f5f114d879a9dc4ad51 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 26 Aug 2026 16:21:18 -0700
Subject: [PATCH 27/44] fixed installed interpreter requirements

---
 docs/ci-validation-infrastructure.md   | 43 +++++++++++++++++++++---
 scripts/ci/prepare-windows-host.ps1    | 45 ++++++++++++--------------
 scripts/ci/validation-test-matrix.json |  9 ++----
 tests/scripts/README.md                |  2 +-
 4 files changed, 62 insertions(+), 37 deletions(-)

diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index a50354212..500bf16f5 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -230,11 +230,8 @@ every entry.
 `prepare-windows-host.ps1`:
 
 - `process-t3` — runs `wxc-host-prep.exe prepare-system-drive` and
-  `prepare-null-device --no-sacl`, then verifies the workload interpreters:
-  `pwsh` and `git` are required (they gate 10 of the 19 `T3-Workloads.ps1`
-  cases, so their absence means a mis-imaged pool and fails the job), while
-  `node` and `python` only emit a warning because the suite reports their
-  cases as skipped.
+  `prepare-null-device --no-sacl`, then calls the shared workload-interpreter
+  check ([below](#workload-interpreters)).
 - `microvm` — asserts the NanVix payload is in the artifact, adds a Defender
   exclusion for the binary directory, and requires the Windows Hypervisor
   Platform feature *and* a running hypervisor.
@@ -263,6 +260,42 @@ message instead of surfacing later as an opaque backend error.
 
 macOS has no preparation step.
 
+### Workload interpreters
+
+Some suites do not just exercise MXC's primitives — they run *real programs*
+(`pwsh`, `git`, `node`, `python`, `cmd`) inside the sandbox and assert on what
+those programs produce. `Assert-WorkloadInterpreters` in
+`prepare-windows-host.ps1` verifies that host-side inventory up front, so a
+missing tool is reported once as a preparation result rather than repeatedly as
+a confusing mid-suite failure.
+
+The check is **suite-agnostic by design**. It describes what a Windows
+validation *host* is expected to provide, not what any one suite consumes, so
+any current or future suite that shells out to these programs is served by the
+same list. `T3-Workloads.ps1` is simply the first caller; wiring up the next one
+needs no change here.
+
+Its inventory lives in a single table, each entry carrying:
+
+- `Candidates` — the command names to try, in order. Resolution mirrors what the
+  suites themselves do: `python` is tried before `python3`, and a match under
+  `WindowsApps` is ignored because that is a Microsoft Store alias stub rather
+  than a real interpreter.
+- `Required` — whether an absence fails the job or only warns. Today only
+  `pwsh` is required; its absence means a mis-imaged pool. `git`, `node`, and
+  `python` warn, because suites are expected to report their dependent cases as
+  skipped rather than failing.
+- `Remedy` — the image-level fix, quoted in whichever message is emitted. The
+  tools are **verified, never installed**, for the same reason the optional
+  features are: provisioning a toolchain mid-run would mask the image drift the
+  check exists to surface.
+
+Because a warning is deliberately not a failure, an absent optional interpreter
+silently shrinks coverage while the job still shows green. GitHub's pass/fail
+icon cannot express "passed, but with less coverage than yesterday" — a future
+test-analysis portal is intended to surface skip counts so that erosion is
+visible.
+
 ## Log collection
 
 Every job uploads its logs whether it passed or failed, as
diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index 89f8d4342..8327fe769 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -149,36 +149,25 @@ function Initialize-ProcessContainerHost {
     Assert-WorkloadInterpreters
 }
 
-# Checks for programs useful to the workload test suite. Missing optional interpreters are reported as warnings.
+# Verify the interpreters test suites drive inside the sandbox, following
+# the same verify-never-install rule as the optional-feature assertions above:
+# a missing one is an image problem, not something the job can fix mid-run.
 function Assert-WorkloadInterpreters {
-    $required = @(
-        @{ Name = 'pwsh'; Remedy = 'install PowerShell 7 in the image' },
-        @{ Name = 'git'; Remedy = 'install Git for Windows in the image' }
+    $interpreters = @(
+        @{ Name = 'pwsh';   Candidates = @('pwsh');              Required = $true;  Remedy = 'install PowerShell 7 in the image' },
+        @{ Name = 'git';    Candidates = @('git');               Required = $false; Remedy = 'install Git for Windows in the image' },
+        @{ Name = 'node';   Candidates = @('node');              Required = $false; Remedy = 'install Node.js in the image' },
+        @{ Name = 'python'; Candidates = @('python', 'python3'); Required = $false; Remedy = 'install Python in the image' }
     )
-    $missing = @()
-    foreach ($tool in $required) {
-        $found = Get-Command $tool.Name -ErrorAction SilentlyContinue
-        if ($found) {
-            Write-Host "Workload interpreter '$($tool.Name)' found at $($found.Source)"
-        } else {
-            $missing += "$($tool.Name) ($($tool.Remedy))"
-        }
-    }
-    if ($missing) {
-        Exit-WithError "Workload interpreters missing from this image: $($missing -join '; ')"
-    }
 
-    # The suite tries `python` then `python3`, so mirror that candidate order.
-    $optional = @(
-        @{ Name = 'node'; Candidates = @('node') },
-        @{ Name = 'python'; Candidates = @('python', 'python3') }
-    )
-    foreach ($tool in $optional) {
+    $missing = @()
+    foreach ($tool in $interpreters) {
         $resolved = $null
         foreach ($candidate in $tool.Candidates) {
             $found = Get-Command $candidate -ErrorAction SilentlyContinue
-            # A `python` resolving into WindowsApps is the Microsoft Store
-            # AppExecutionAlias stub, which the suite deliberately ignores.
+            # A command resolving into WindowsApps is a Microsoft Store
+            # AppExecutionAlias stub: a 0-byte redirect that opens the Store
+            # rather than running, which the suite deliberately ignores.
             if ($found -and $found.Source -notlike '*\WindowsApps\*') {
                 $resolved = $found.Source
                 break
@@ -186,10 +175,16 @@ function Assert-WorkloadInterpreters {
         }
         if ($resolved) {
             Write-Host "Workload interpreter '$($tool.Name)' found at $resolved"
+        } elseif ($tool.Required) {
+            $missing += "$($tool.Name) ($($tool.Remedy))"
         } else {
-            Write-Host "::warning::Workload interpreter '$($tool.Name)' is absent."
+            Write-Host "::warning::Workload interpreter '$($tool.Name)' is absent ($($tool.Remedy))"
         }
     }
+
+    if ($missing) {
+        Exit-WithError "Workload interpreters missing from this image: $($missing -join '; ')"
+    }
 }
 
 function Initialize-MicroVmHost {
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 0a2032a17..dc993aede 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -393,22 +393,19 @@
       {
         "os": "windows-25h2",
         "backends": [
-          "process-t3",
-          "wslc"
+          "process-t3"
         ]
       },
       {
         "os": "windows-24h2",
         "backends": [
-          "process-t3",
-          "wslc"
+          "process-t3"
         ]
       },
       {
         "os": "windows-23h2",
         "backends": [
-          "process-t3",
-          "wslc"
+          "process-t3"
         ]
       }
     ]
diff --git a/tests/scripts/README.md b/tests/scripts/README.md
index 59b5c8bf6..88302976c 100644
--- a/tests/scripts/README.md
+++ b/tests/scripts/README.md
@@ -42,7 +42,7 @@ Linux / macOS (`.sh`):
 | `run_windows_sandbox_state_aware_tests.ps1` | Windows Sandbox state-aware lifecycle E2E (single VM held across provision/start/exec*/stop/deprovision) | Windows Sandbox enabled |
 | `run_processcontainer_proxy_tests.ps1` | Process container proxy tests | `wxc-exec.exe` |
 | `WinProcessContainer-Tests.ps1` | Process container (AppContainer / BaseContainer) primitives suite — tier probes, rw/ro/denied matrix, UI mitigations, DACL restore, crash recovery | `wxc-exec.exe`, `wxc-ui-probe.exe` |
-| `T3-Workloads.ps1` | Real workloads (pwsh, git, node, python, cmd) on top of the T3 primitives. A missing interpreter is reported as a skip, not a failure | `wxc-exec.exe`; `pwsh` and `git` for full coverage, `node` / `python` optional |
+| `T3-Workloads.ps1` | Real workloads (pwsh, git, node, python, cmd) on top of the T3 primitives. A missing interpreter is reported as a skip, not a failure | `wxc-exec.exe`; `pwsh` / `git` / `node` / `python` each optional, gating their own cases |
 | `run_on_repeat.ps1` | Stress test (loops core tests) | `wxc-exec.exe` |
 
 ### Linux suites

From dd932a151dce2d3f864cfc82af85541c76ad620a Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 26 Aug 2026 17:06:03 -0700
Subject: [PATCH 28/44] added TEMPORARY path grant for tests that need $temp
 access. remove when pwsh 7.7 releases

---
 tests/scripts/T3-Workloads.ps1 | 57 +++++++++++++++++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/tests/scripts/T3-Workloads.ps1 b/tests/scripts/T3-Workloads.ps1
index 75e028146..ce74b9905 100644
--- a/tests/scripts/T3-Workloads.ps1
+++ b/tests/scripts/T3-Workloads.ps1
@@ -291,6 +291,56 @@ function Resolve-HostPaths {
     if ($script:PythonRoNeeded) { Write-Host ("  -> auto-grant ReadOnly: {0}" -f $script:PythonRoNeeded) }
 }
 
+# -----------------------------------------------------------------------
+# TEMPORARY: scratch-root ancestor grants for pwsh-driven workloads
+#
+# pwsh before 7.7, and git run underneath it, resolve the *whole ancestor
+# chain* of the working directory at startup -- not just the directory
+# itself. The policy only grants the leaf (`$ScratchRoot\rw`), so every
+# segment above it is denied and the interpreter fails before reaching
+# the behaviour these tests exist to check:
+#
+#   W4/W5  git: "Unable to read current working directory: Permission denied"
+#   W17    pwsh falls back to C:\ as its location, then Set-Location is
+#          denied at the first ungranted ancestor.
+#
+# MXC's own diagnostic suggests adding the drive root to `readonlyPaths`.
+# We deliberately do NOT do that: at T3 a policy path lands as an
+# inheritable ACE (`filesystem_dacl.rs` sets inheritable = is_dir()), so
+# naming C:\ would propagate an ACL rewrite across the entire system
+# drive on every run and again on teardown. Granting the ancestors of
+# $ScratchRoot achieves the same resolution with a far smaller blast
+# radius.
+#
+# The drive root itself is intentionally absent: `wxc-host-prep
+# prepare-system-drive` already grants sandbox tokens traverse access
+# there, which is all path resolution needs.
+#
+# REMOVE these grants and the warning once pwsh 7.7+ is out of preview
+# and baked into the validation images.
+# -----------------------------------------------------------------------
+
+# Ancestors of $ScratchRoot, deepest first, stopping *below* the drive
+# root. On a CI runner ($env:TEMP = C:\a\_temp) that is
+# mxc-t3-workloads, _temp, a.
+function Get-PwshAncestorGrants {
+    $grants = @()
+    $dir  = [System.IO.Path]::GetFullPath($ScratchRoot)
+    $root = [System.IO.Path]::GetPathRoot($dir).TrimEnd('\')
+    while ($dir -and $dir.TrimEnd('\') -ne $root) {
+        $grants += $dir
+        $parent = Split-Path $dir -Parent
+        if (-not $parent -or $parent -eq $dir) { break }
+        $dir = $parent
+    }
+    return $grants
+}
+
+function Write-PwshAncestorGrantWarning {
+    param([Parameter(Mandatory)] [string]$Id)
+    Write-Host ("  [{0}] WARNING: granting read-only {1} so pwsh/git can resolve the working directory's ancestor chain. TEMPORARY -- remove once pwsh 7.7 ships out of preview." -f $Id, ((Get-PwshAncestorGrants) -join ', ')) -ForegroundColor Yellow
+}
+
 # -----------------------------------------------------------------------
 # Workloads
 # -----------------------------------------------------------------------
@@ -388,9 +438,11 @@ function W4-PwshGitStatus {
     # `C:\Users\...` metadata-access checks that the AppContainer SID
     # doesn't have grants for.
     $cmd = "pwsh.exe -NoProfile -NoLogo -Command `"& git -C '$repo' status --porcelain; exit `$LASTEXITCODE`""
+    Write-PwshAncestorGrantWarning -Id 'W4'
     $cfg = New-Config -Name 'w4-git-status' `
         -CommandLine $cmd `
         -ReadWrite @("$ScratchRoot\rw") `
+        -ReadOnly (Get-PwshAncestorGrants) `
         -Cwd "$ScratchRoot\rw"
     $log = "$ScratchRoot\log\w4.log"
     $r = Invoke-Workload -ConfigPath $cfg -LogPath $log -TimeoutSec 90
@@ -412,9 +464,11 @@ function W5-PwshGitLog {
         Initialize-Repo -Dir $repo
     }
     $cmd = "pwsh.exe -NoProfile -NoLogo -Command `"& git -C '$repo' log --oneline -n 10; exit `$LASTEXITCODE`""
+    Write-PwshAncestorGrantWarning -Id 'W5'
     $cfg = New-Config -Name 'w5-git-log' `
         -CommandLine $cmd `
         -ReadWrite @("$ScratchRoot\rw") `
+        -ReadOnly (Get-PwshAncestorGrants) `
         -Cwd "$ScratchRoot\rw"
     $log = "$ScratchRoot\log\w5.log"
     $r = Invoke-Workload -ConfigPath $cfg -LogPath $log -TimeoutSec 90
@@ -685,8 +739,9 @@ function W17-PwshSetLocation {
     }
     $target = Initialize-ChdirTarget
     $cmd = "pwsh.exe -NoProfile -NoLogo -Command `"Set-Location -LiteralPath '$target'; Get-ChildItem -Name; exit 0`""
+    Write-PwshAncestorGrantWarning -Id 'W17'
     $cfg = New-Config -Name 'w17-pwsh-cd' -CommandLine $cmd `
-        -ReadWrite @("$ScratchRoot\rw") -Cwd "$ScratchRoot\rw"
+        -ReadWrite @("$ScratchRoot\rw") -ReadOnly (Get-PwshAncestorGrants) -Cwd "$ScratchRoot\rw"
     $log = "$ScratchRoot\log\w17.log"
     $r = Invoke-Workload -ConfigPath $cfg -LogPath $log -TimeoutSec 60
     $pass = ($r.ExitCode -eq 0) -and ($r.Stdout -match 'chdir-marker\.txt')

From dfed48bbfb3e28a86b289f0191096e86e58b65ef Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 27 Aug 2026 13:55:47 -0700
Subject: [PATCH 29/44] Add optional parameter to T3-Workloads script.
 Splitting macos and linux sysprep.

---
 .github/copilot-instructions.md               | 11 ++-
 .../workflows/Validation.Tests.Matrix.Job.yml | 11 ++-
 docs/ci-validation-infrastructure.md          | 76 ++++++++++++-------
 scripts/ci/assert-workload-interpreters.sh    | 60 +++++++++++++++
 scripts/ci/prepare-linux-host.sh              |  4 +
 scripts/ci/prepare-macos-host.sh              | 34 +++++++++
 scripts/ci/prepare-windows-host.ps1           |  5 +-
 scripts/ci/run_backend_validation_tests.ps1   |  6 +-
 tests/scripts/T3-Workloads.ps1                | 69 +++++++++--------
 9 files changed, 208 insertions(+), 68 deletions(-)
 create mode 100644 scripts/ci/assert-workload-interpreters.sh
 create mode 100644 scripts/ci/prepare-macos-host.sh

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 197bbf737..b4087373f 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -87,9 +87,14 @@ workflows) before calling the matrix job.
   GitHub-hosted `runner` instead of a 1ES `pool`.
 
 **Host preparation** happens in the matrix job before the tests, keyed by the
-matrix `backend` id: `scripts/ci/prepare-windows-host.ps1` and
-`scripts/ci/prepare-linux-host.sh`. A backend with no prerequisites is an
-explicit no-op, so the step runs unconditionally for every entry.
+matrix `backend` id: `scripts/ci/prepare-windows-host.ps1`,
+`scripts/ci/prepare-linux-host.sh`, and `scripts/ci/prepare-macos-host.sh`. A
+backend with no prerequisites is an explicit no-op, so the step runs
+unconditionally for every entry. Independent of the backend id, all three also
+verify the host's workload interpreters (`pwsh`, `git`, `node`, `python`) —
+Windows in `Assert-WorkloadInterpreters`, Linux and macOS via the shared
+bash-3.2-compatible `scripts/ci/assert-workload-interpreters.sh`. Interpreters
+are verified, never installed.
 
 **Test dispatch** goes through `scripts/ci/run_backend_validation_tests.ps1`
 (Windows) and `scripts/ci/run_backend_validation_tests.sh` (Linux/macOS), which map
diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml
index f21ba3470..a108628ff 100644
--- a/.github/workflows/Validation.Tests.Matrix.Job.yml
+++ b/.github/workflows/Validation.Tests.Matrix.Job.yml
@@ -195,6 +195,15 @@ jobs:
           echo "Waiting $seconds second(s) before starting tests."
           sleep "$seconds"
 
+      - name: Prepare backend prerequisites
+        timeout-minutes: 15
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash scripts/ci/prepare-macos-host.sh \
+            '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 |
+            tee "$RUNNER_TEMP/mxc-ci.log"
+
       - name: Run backend tests
         timeout-minutes: 60
         shell: bash
@@ -203,7 +212,7 @@ jobs:
           chmod +x artifacts/bin/mxc-exec-mac artifacts/bin/unix-test-proxy
           bash scripts/ci/run_backend_validation_tests.sh \
             '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 |
-            tee "$RUNNER_TEMP/mxc-ci.log"
+            tee -a "$RUNNER_TEMP/mxc-ci.log"
 
       - name: Upload logs
         if: always()
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 500bf16f5..6ae6692f8 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -31,6 +31,8 @@ the individual local test scripts are documented in
 | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. |
 | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. |
 | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). |
+| `scripts/ci/prepare-macos-host.sh` | Per-backend macOS host preparation / prerequisite assertions. |
+| `scripts/ci/assert-workload-interpreters.sh` | Shared Linux/macOS workload-interpreter inventory check. |
 | `scripts/ci/run_backend_validation_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
 | `scripts/ci/run_backend_validation_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
 
@@ -44,7 +46,7 @@ Validation.Tests.Scheduled.yml
             └─ resolve  →  resolve-validation-test-matrix.mjs --plan <plan>
                  ├─ windows job (matrix) → download artifact → prepare-windows-host.ps1 → run_backend_validation_tests.ps1
                  ├─ linux   job (matrix) → download artifact → prepare-linux-host.sh   → run_backend_validation_tests.sh
-                 └─ macos   job (matrix) → download artifact →                            run_backend_validation_tests.sh
+                 └─ macos   job (matrix) → download artifact → prepare-macos-host.sh  → run_backend_validation_tests.sh
 ```
 
 An entry point **must** build the artifacts before calling the matrix job — the
@@ -72,7 +74,7 @@ Build artifacts are kept for 1 day — they exist only to feed these jobs.
 | `resolve` | `ubuntu-latest` | Runs the resolver, emits one matrix per OS family plus `has_<family>` flags so an empty family is skipped rather than failing on an empty matrix. |
 | `windows` | `[self-hosted, 1ES.Pool=<pool>, JobId=mxc-e2e-…]` | Download artifact → `prepare-windows-host.ps1 -Backend <backend id>` → `run_backend_validation_tests.ps1 -Backend <backend id>`. |
 | `linux` | `[self-hosted, 1ES.Pool=<pool>, JobId=mxc-e2e-…]` | Download artifact → `prepare-linux-host.sh <backend id>` → `run_backend_validation_tests.sh <backend id>` (under `sudo` for LXC). |
-| `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `chmod +x` → `run_backend_validation_tests.sh <backend id>`. No host-prep step. |
+| `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `prepare-macos-host.sh <backend id>` → `chmod +x` → `run_backend_validation_tests.sh <backend id>`. |
 
 Per-job display name: `<platform id>, <architecture>, <backend>` (macOS omits
 the architecture). Job timeout 180 min; host prep 15 min; the test step 45 min
@@ -137,6 +139,15 @@ at the T3 fallback) is a worthwhile future improvement.
 failure in the primitives suite does not skip the workloads suite, so one job
 run shows both results instead of costing a second run to triage.
 
+The dispatcher passes `T3-Workloads.ps1` its `-GrantDriveRoot` switch. The
+pwsh- and git-driven workloads resolve the whole ancestor chain of their working
+directory at startup, so granting only the scratch leaf leaves them failing
+before they reach the behaviour under test. Granting the drive root covers the
+chain, but at T3 a policy path becomes an inheritable ACE, so it rewrites ACLs
+across the system drive on every run and again on teardown — acceptable on a
+disposable runner, which is why the switch is off by default and only CI opts
+in. Temporary until pwsh 7.7 leaves preview.
+
 An unwired backend fails loudly on purpose: adding it to a trigger produces a
 red job ("write the tests or remove it"), never a green no-op. The dispatchers'
 accepted-id lists (`ValidateSet` on Windows, the `case` arms on Unix) are what
@@ -230,8 +241,7 @@ every entry.
 `prepare-windows-host.ps1`:
 
 - `process-t3` — runs `wxc-host-prep.exe prepare-system-drive` and
-  `prepare-null-device --no-sacl`, then calls the shared workload-interpreter
-  check ([below](#workload-interpreters)).
+  `prepare-null-device --no-sacl`.
 - `microvm` — asserts the NanVix payload is in the artifact, adds a Defender
   exclusion for the binary directory, and requires the Windows Hypervisor
   Platform feature *and* a running hypervisor.
@@ -258,37 +268,51 @@ message instead of surfacing later as an opaque backend error.
 - `microvm` — asserts the NanVix payload exists.
 - `hyperlight` — no-op.
 
-macOS has no preparation step.
+`prepare-macos-host.sh`:
+
+- `seatbelt` — asserts `mxc-exec-mac` shipped. The sandbox is part of the OS, so
+  there is nothing to install; the script exists so macOS has the same shape as
+  the other two and a future prerequisite has an obvious home.
+
+Every one of the three scripts also runs the workload-interpreter check
+([below](#workload-interpreters)) before it dispatches on the backend id.
 
 ### Workload interpreters
 
 Some suites do not just exercise MXC's primitives — they run *real programs*
 (`pwsh`, `git`, `node`, `python`, `cmd`) inside the sandbox and assert on what
-those programs produce. `Assert-WorkloadInterpreters` in
-`prepare-windows-host.ps1` verifies that host-side inventory up front, so a
-missing tool is reported once as a preparation result rather than repeatedly as
-a confusing mid-suite failure.
+those programs produce. Each preparation script verifies that host-side
+inventory up front, so a missing tool is reported once as a preparation result
+rather than repeatedly as a confusing mid-suite failure.
 
-The check is **suite-agnostic by design**. It describes what a Windows
+The check is **suite-agnostic by design**, and runs for *every* backend rather
+than only the ones whose suites happen to need it today. It describes what a
 validation *host* is expected to provide, not what any one suite consumes, so
 any current or future suite that shells out to these programs is served by the
 same list. `T3-Workloads.ps1` is simply the first caller; wiring up the next one
 needs no change here.
 
-Its inventory lives in a single table, each entry carrying:
-
-- `Candidates` — the command names to try, in order. Resolution mirrors what the
-  suites themselves do: `python` is tried before `python3`, and a match under
-  `WindowsApps` is ignored because that is a Microsoft Store alias stub rather
-  than a real interpreter.
-- `Required` — whether an absence fails the job or only warns. Today only
-  `pwsh` is required; its absence means a mis-imaged pool. `git`, `node`, and
-  `python` warn, because suites are expected to report their dependent cases as
-  skipped rather than failing.
-- `Remedy` — the image-level fix, quoted in whichever message is emitted. The
-  tools are **verified, never installed**, for the same reason the optional
-  features are: provisioning a toolchain mid-run would mask the image drift the
-  check exists to surface.
+There are two implementations, because the two host families share no shell:
+`Assert-WorkloadInterpreters` in `prepare-windows-host.ps1`, and
+`assert-workload-interpreters.sh`, which Linux and macOS both invoke. The Unix
+script is written to bash 3.2 — no associative arrays — because that is what
+macOS still ships.
+
+Each inventory entry carries:
+
+- the command names to try, in order. Resolution mirrors what the suites
+  themselves do: on Windows `python` is tried before `python3` and a match under
+  `WindowsApps` is ignored (that is a Microsoft Store alias stub, not a real
+  interpreter), while on Unix `python3` is tried first.
+- whether an absence fails the job or only warns. On Windows only `pwsh` is
+  required; its absence means a mis-imaged pool. Everything else warns, because
+  suites are expected to report their dependent cases as skipped rather than
+  failing. Nothing is required on Unix yet — no Unix suite drives these
+  interpreters — so that check currently runs purely as host inventory.
+- the image-level fix, quoted in whichever message is emitted. The tools are
+  **verified, never installed**, for the same reason the optional features are:
+  provisioning a toolchain mid-run would mask the image drift the check exists
+  to surface.
 
 Because a warning is deliberately not a failure, an absent optional interpreter
 silently shrinks coverage while the job still shows green. GitHub's pass/fail
@@ -356,8 +380,8 @@ test runner is allocated.
    suite. Until a suite exists, leave the explicit throw / `exit 2` so
    accidental activation fails loudly.
 3. **Host prep:** add a branch to `prepare-windows-host.ps1` (`ValidateSet` +
-   `switch`) or `prepare-linux-host.sh` (`usage` + `case`). Skip only if there
-   is genuinely nothing to install or assert.
+   `switch`), `prepare-linux-host.sh`, or `prepare-macos-host.sh` (`usage` +
+   `case`). Skip only if there is genuinely nothing to install or assert.
 4. **Artifact:** make sure everything the suite needs is in the
    `Upload binaries` list of the relevant `Build.*.Job.yml`, and that the build
    enables the backend's cargo feature.
diff --git a/scripts/ci/assert-workload-interpreters.sh b/scripts/ci/assert-workload-interpreters.sh
new file mode 100644
index 000000000..16f17b21a
--- /dev/null
+++ b/scripts/ci/assert-workload-interpreters.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Verifies the interpreters test suites drive inside the sandbox, following the
+# same verify-never-install rule as the rest of host preparation: a missing one
+# is an image problem, not something a job can fix mid-run.
+#
+# Shared by prepare-linux-host.sh and prepare-macos-host.sh so the two cannot
+# drift. The Windows twin is Assert-WorkloadInterpreters in
+# prepare-windows-host.ps1, which stays separate because it has to filter the
+# Microsoft Store alias stubs that only exist there.
+#
+# The check is suite-agnostic: it describes what a validation host is expected
+# to provide, not what any one suite consumes, so a future suite that shells out
+# to these programs needs no change here.
+#
+# Written for bash 3.2, which is what macOS still ships -- hence the delimited
+# string table rather than associative arrays.
+
+# name|candidates (tried in order)|required|remedy
+#
+# Nothing is required on Unix today: no Linux or macOS suite drives these
+# interpreters yet, so this runs as host inventory and reports absences as
+# warnings. Flip a `false` to `true` when a suite starts depending on one --
+# that is the whole change.
+interpreters=(
+    "pwsh|pwsh|false|install PowerShell 7 in the image"
+    "git|git|false|install Git in the image"
+    "node|node|false|install Node.js in the image"
+    "python|python3,python|false|install Python in the image"
+)
+
+missing=""
+for entry in "${interpreters[@]}"; do
+    IFS='|' read -r name candidates required remedy <<<"$entry"
+
+    resolved=""
+    IFS=',' read -r -a candidate_list <<<"$candidates"
+    for candidate in "${candidate_list[@]}"; do
+        # python3 before python: on Unix a bare `python` is usually absent, and
+        # where it does exist it can still be Python 2.
+        if resolved="$(command -v "$candidate" 2>/dev/null)"; then
+            break
+        fi
+        resolved=""
+    done
+
+    if [[ -n "$resolved" ]]; then
+        echo "Workload interpreter '$name' found at $resolved"
+    elif [[ "$required" == "true" ]]; then
+        missing="${missing:+$missing; }$name ($remedy)"
+    else
+        echo "::warning::Workload interpreter '$name' is absent ($remedy)"
+    fi
+done
+
+if [[ -n "$missing" ]]; then
+    echo "::error::Workload interpreters missing from this image: $missing"
+    exit 1
+fi
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 72eb55232..1ecb2640a 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -207,6 +207,10 @@ enable_bridge_netfilter() {
 }
 
 chmod +x "$binary_directory/lxc-exec"
+
+# Runs for every backend: this is host inventory, not a backend prerequisite.
+bash "$(dirname "$0")/assert-workload-interpreters.sh"
+
 case "$backend" in
     bubblewrap)
         install_bubblewrap
diff --git a/scripts/ci/prepare-macos-host.sh b/scripts/ci/prepare-macos-host.sh
new file mode 100644
index 000000000..3eb898226
--- /dev/null
+++ b/scripts/ci/prepare-macos-host.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Prepares a macOS host for a backend's artifact-only test suite.
+#
+# Seatbelt needs nothing installed -- the sandbox is part of the OS -- so this
+# currently only takes the host's workload-interpreter inventory. It exists so
+# macOS has the same shape as the Windows and Linux preparation scripts, giving
+# a future prerequisite an obvious home instead of another workflow-inline step.
+
+usage() {
+    echo "Usage: $0 <seatbelt> <binary-directory>" >&2
+}
+
+if [[ $# -ne 2 ]]; then
+    usage
+    exit 2
+fi
+
+backend="$1"
+binary_directory="$2"
+
+# Runs for every backend: this is host inventory, not a backend prerequisite.
+bash "$(dirname "$0")/assert-workload-interpreters.sh"
+
+case "$backend" in
+    seatbelt)
+        test -f "$binary_directory/mxc-exec-mac"
+        ;;
+    *)
+        usage
+        exit 2
+        ;;
+esac
diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index 8327fe769..78c9035dc 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -145,8 +145,6 @@ function Initialize-ProcessContainerHost {
     if ($LASTEXITCODE -ne 0) {
         Exit-WithError "wxc-host-prep prepare-null-device failed with exit code $LASTEXITCODE"
     }
-
-    Assert-WorkloadInterpreters
 }
 
 # Verify the interpreters test suites drive inside the sandbox, following
@@ -354,6 +352,9 @@ $BinaryDirectory = (Resolve-Path $BinaryDirectory).Path
 
 Write-Host "Preparing Windows host for backend '$Backend' using $BinaryDirectory"
 
+# Runs for every backend: this is host inventory, not a backend prerequisite.
+Assert-WorkloadInterpreters
+
 switch ($Backend) {
     'process-t3' { Initialize-ProcessContainerHost }
     'microvm' { Initialize-MicroVmHost }
diff --git a/scripts/ci/run_backend_validation_tests.ps1 b/scripts/ci/run_backend_validation_tests.ps1
index 09aab6707..c3e33b980 100644
--- a/scripts/ci/run_backend_validation_tests.ps1
+++ b/scripts/ci/run_backend_validation_tests.ps1
@@ -142,8 +142,12 @@ function Invoke-T3WorkloadTests {
     # -Wxc is required: the script's default points at a debug build that does
     # not exist in a CI artifact. -KeepArtifacts preserves the per-workload
     # logs and configs on a clean run so a passing job still uploads them.
+    # -GrantDriveRoot lets the pwsh/git workloads resolve their working
+    # directory's ancestor chain; it rewrites ACLs across the system drive,
+    # which is why the script leaves it off by default and only a disposable
+    # CI runner opts in. Temporary until pwsh 7.7 leaves preview.
     $global:LASTEXITCODE = 0
-    & $script -Wxc $wxc -KeepArtifacts | Out-Null
+    & $script -Wxc $wxc -KeepArtifacts -GrantDriveRoot | Out-Null
     return $LASTEXITCODE
 }
 
diff --git a/tests/scripts/T3-Workloads.ps1 b/tests/scripts/T3-Workloads.ps1
index ce74b9905..fee690476 100644
--- a/tests/scripts/T3-Workloads.ps1
+++ b/tests/scripts/T3-Workloads.ps1
@@ -21,6 +21,12 @@ param(
     # -Wxc explicitly if the layout differs.
     [string]$Wxc          = (Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'src\target\debug\wxc-exec.exe'),
     [string]$ScratchRoot  = (Join-Path $env:TEMP 'mxc-t3-workloads'),
+    # Grant read-only access to the drive root (C:\) for the pwsh/git
+    # workloads (W4, W5, W17). Those interpreters resolve the ENTIRE
+    # ancestor chain of the working directory at startup, so granting the
+    # leaf alone is not enough. Off by default -- see the TEMPORARY note
+    # above `Get-DriveRootGrant` for the cost of turning it on.
+    [switch]$GrantDriveRoot,
     # Structured results. Lives in $env:TEMP but OUTSIDE $ScratchRoot so
     # `Initialize-Scratch`'s recursive nuke can never take it with it.
     [string]$ResultsJson  = (Join-Path $env:TEMP 'T3-Workloads.results.json'),
@@ -292,7 +298,9 @@ function Resolve-HostPaths {
 }
 
 # -----------------------------------------------------------------------
-# TEMPORARY: scratch-root ancestor grants for pwsh-driven workloads
+# TEMPORARY: drive-root read-only grant for pwsh-driven workloads
+#
+# Opt-in via -GrantDriveRoot; off by default.
 #
 # pwsh before 7.7, and git run underneath it, resolve the *whole ancestor
 # chain* of the working directory at startup -- not just the directory
@@ -304,41 +312,32 @@ function Resolve-HostPaths {
 #   W17    pwsh falls back to C:\ as its location, then Set-Location is
 #          denied at the first ungranted ancestor.
 #
-# MXC's own diagnostic suggests adding the drive root to `readonlyPaths`.
-# We deliberately do NOT do that: at T3 a policy path lands as an
-# inheritable ACE (`filesystem_dacl.rs` sets inheritable = is_dir()), so
-# naming C:\ would propagate an ACL rewrite across the entire system
-# drive on every run and again on teardown. Granting the ancestors of
-# $ScratchRoot achieves the same resolution with a far smaller blast
-# radius.
+# Granting the drive root covers every segment at once, and is what MXC's
+# own launch diagnostic recommends.
 #
-# The drive root itself is intentionally absent: `wxc-host-prep
-# prepare-system-drive` already grants sandbox tokens traverse access
-# there, which is all path resolution needs.
+# COST, and why this is a switch rather than the default: at T3 a policy
+# path becomes an inheritable ACE (`filesystem_dacl.rs` sets
+# inheritable = is_dir()), so naming C:\ propagates an ACL rewrite across
+# the whole system drive on every run, and again in reverse on teardown.
+# That is acceptable on a disposable CI runner and unpleasant on a
+# developer box, which is why the caller has to ask for it.
 #
-# REMOVE these grants and the warning once pwsh 7.7+ is out of preview
-# and baked into the validation images.
+# REMOVE the switch, the grant, and the warning once pwsh 7.7+ is out of
+# preview and baked into the validation images.
 # -----------------------------------------------------------------------
 
-# Ancestors of $ScratchRoot, deepest first, stopping *below* the drive
-# root. On a CI runner ($env:TEMP = C:\a\_temp) that is
-# mxc-t3-workloads, _temp, a.
-function Get-PwshAncestorGrants {
-    $grants = @()
-    $dir  = [System.IO.Path]::GetFullPath($ScratchRoot)
-    $root = [System.IO.Path]::GetPathRoot($dir).TrimEnd('\')
-    while ($dir -and $dir.TrimEnd('\') -ne $root) {
-        $grants += $dir
-        $parent = Split-Path $dir -Parent
-        if (-not $parent -or $parent -eq $dir) { break }
-        $dir = $parent
-    }
-    return $grants
+# The readonlyPaths entry for the pwsh/git workloads: the drive root when
+# -GrantDriveRoot is set, otherwise nothing.
+function Get-DriveRootGrant {
+    if (-not $GrantDriveRoot) { return @() }
+    $root = if ($env:SystemDrive) { "$env:SystemDrive\" } else { 'C:\' }
+    return @($root)
 }
 
-function Write-PwshAncestorGrantWarning {
+function Write-DriveRootGrantWarning {
     param([Parameter(Mandatory)] [string]$Id)
-    Write-Host ("  [{0}] WARNING: granting read-only {1} so pwsh/git can resolve the working directory's ancestor chain. TEMPORARY -- remove once pwsh 7.7 ships out of preview." -f $Id, ((Get-PwshAncestorGrants) -join ', ')) -ForegroundColor Yellow
+    if (-not $GrantDriveRoot) { return }
+    Write-Host ("  [{0}] WARNING: granting read-only {1} so pwsh/git can resolve the working directory's ancestor chain. TEMPORARY -- remove once pwsh 7.7 ships out of preview." -f $Id, ((Get-DriveRootGrant) -join ', ')) -ForegroundColor Yellow
 }
 
 # -----------------------------------------------------------------------
@@ -438,11 +437,11 @@ function W4-PwshGitStatus {
     # `C:\Users\...` metadata-access checks that the AppContainer SID
     # doesn't have grants for.
     $cmd = "pwsh.exe -NoProfile -NoLogo -Command `"& git -C '$repo' status --porcelain; exit `$LASTEXITCODE`""
-    Write-PwshAncestorGrantWarning -Id 'W4'
+    Write-DriveRootGrantWarning -Id 'W4'
     $cfg = New-Config -Name 'w4-git-status' `
         -CommandLine $cmd `
         -ReadWrite @("$ScratchRoot\rw") `
-        -ReadOnly (Get-PwshAncestorGrants) `
+        -ReadOnly (Get-DriveRootGrant) `
         -Cwd "$ScratchRoot\rw"
     $log = "$ScratchRoot\log\w4.log"
     $r = Invoke-Workload -ConfigPath $cfg -LogPath $log -TimeoutSec 90
@@ -464,11 +463,11 @@ function W5-PwshGitLog {
         Initialize-Repo -Dir $repo
     }
     $cmd = "pwsh.exe -NoProfile -NoLogo -Command `"& git -C '$repo' log --oneline -n 10; exit `$LASTEXITCODE`""
-    Write-PwshAncestorGrantWarning -Id 'W5'
+    Write-DriveRootGrantWarning -Id 'W5'
     $cfg = New-Config -Name 'w5-git-log' `
         -CommandLine $cmd `
         -ReadWrite @("$ScratchRoot\rw") `
-        -ReadOnly (Get-PwshAncestorGrants) `
+        -ReadOnly (Get-DriveRootGrant) `
         -Cwd "$ScratchRoot\rw"
     $log = "$ScratchRoot\log\w5.log"
     $r = Invoke-Workload -ConfigPath $cfg -LogPath $log -TimeoutSec 90
@@ -739,9 +738,9 @@ function W17-PwshSetLocation {
     }
     $target = Initialize-ChdirTarget
     $cmd = "pwsh.exe -NoProfile -NoLogo -Command `"Set-Location -LiteralPath '$target'; Get-ChildItem -Name; exit 0`""
-    Write-PwshAncestorGrantWarning -Id 'W17'
+    Write-DriveRootGrantWarning -Id 'W17'
     $cfg = New-Config -Name 'w17-pwsh-cd' -CommandLine $cmd `
-        -ReadWrite @("$ScratchRoot\rw") -ReadOnly (Get-PwshAncestorGrants) -Cwd "$ScratchRoot\rw"
+        -ReadWrite @("$ScratchRoot\rw") -ReadOnly (Get-DriveRootGrant) -Cwd "$ScratchRoot\rw"
     $log = "$ScratchRoot\log\w17.log"
     $r = Invoke-Workload -ConfigPath $cfg -LogPath $log -TimeoutSec 60
     $pass = ($r.ExitCode -eq 0) -and ($r.Stdout -match 'chdir-marker\.txt')

From 9cfaf8a20f5141ba521b4e7416bd841e0ccb096b Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 27 Aug 2026 14:27:59 -0700
Subject: [PATCH 30/44] fix git repo ownership errors

---
 docs/ci-validation-infrastructure.md |  9 +++++++
 docs/playground-limitations.md       | 13 ++++++++++
 tests/scripts/T3-Workloads.ps1       | 37 ++++++++++++++++++++++++++++
 3 files changed, 59 insertions(+)

diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 6ae6692f8..352e83ed4 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -148,6 +148,15 @@ across the system drive on every run and again on teardown — acceptable on a
 disposable runner, which is why the switch is off by default and only CI opts
 in. Temporary until pwsh 7.7 leaves preview.
 
+The suite's git workloads also restamp the ownership of the repo they build.
+The 1ES agent runs elevated, and an elevated token's *default owner* is
+`BUILTIN\Administrators`, so a fixture created there is Administrators-owned;
+git refuses such a repo ("detected dubious ownership") unless the caller is
+itself an elevated administrator, which a contained process never is. That is a
+property of the agent rather than of containment — the identical fixture is
+user-owned on a dev box — so the suite reowns it to the current user and keeps
+the two environments testing the same thing.
+
 An unwired backend fails loudly on purpose: adding it to a trigger produces a
 red job ("write the tests or remove it"), never a green no-op. The dispatchers'
 accepted-id lists (`ValidateSet` on Windows, the `case` arms on Unix) are what
diff --git a/docs/playground-limitations.md b/docs/playground-limitations.md
index 7d272a528..d08b386a5 100644
--- a/docs/playground-limitations.md
+++ b/docs/playground-limitations.md
@@ -46,6 +46,19 @@ work if the ACL is set:
 icacls C:\Python314 /grant "ALL APPLICATION PACKAGES:(OI)(CI)(RX)" /T
 ```
 
+### git and files created by an elevated process
+
+git refuses a repository whose owner is not the caller (`fatal: detected dubious
+ownership`). It makes one exception — a repo owned by `BUILTIN\Administrators` is
+accepted if the caller is *itself* an elevated administrator — and a contained
+process can never qualify, because the AppContainer token drops that membership.
+
+An elevated process's token hands out `BUILTIN\Administrators` as the default
+owner of everything it creates, so **any repo cloned or created while elevated is
+unusable from inside a container**, even though the same repo works fine on the
+host. Create it unelevated, reassign the owner, or add it to `safe.directory` in
+protected (system/global) git config.
+
 This is a Windows security model limitation, not an MXC bug.
 
 ## Network Limitations
diff --git a/tests/scripts/T3-Workloads.ps1 b/tests/scripts/T3-Workloads.ps1
index fee690476..888ae7d03 100644
--- a/tests/scripts/T3-Workloads.ps1
+++ b/tests/scripts/T3-Workloads.ps1
@@ -402,6 +402,42 @@ function W3-PwshGitVersion {
         -ExitCode $r.ExitCode -Detail "stdout=$($r.Stdout.Trim())" -Stderr $r.Stderr
 }
 
+function Repair-FixtureOwnership {
+    param([string]$Dir)
+    # An elevated process's token hands out BUILTIN\Administrators as the
+    # *default owner* of everything it creates, so on a CI agent the repo this
+    # harness just built is owned by Administrators rather than by the user.
+    #
+    # git then refuses it: `detected dubious ownership`. git accepts an
+    # Administrators-owned repo only when the caller is *itself* an elevated
+    # administrator, and a contained process never is -- the AppContainer token
+    # drops that membership by construction. So the check fires inside the
+    # sandbox and cannot be satisfied there.
+    #
+    # That is an artifact of who built the fixture, not of containment: on a
+    # non-elevated dev box the same repo is user-owned and W4/W5 pass. Restamp
+    # the tree so the fixture is identical in both places and the workloads
+    # test git-in-a-sandbox rather than git's host ownership heuristic.
+    $user = [System.Security.Principal.WindowsIdentity]::GetCurrent().User
+    $items = @(Get-Item -LiteralPath $Dir -Force) +
+             @(Get-ChildItem -LiteralPath $Dir -Recurse -Force -ErrorAction SilentlyContinue)
+    $restamped = 0
+    foreach ($item in $items) {
+        try {
+            $acl = Get-Acl -LiteralPath $item.FullName
+            if ($acl.GetOwner([System.Security.Principal.SecurityIdentifier]) -eq $user) { continue }
+            $acl.SetOwner($user)
+            Set-Acl -LiteralPath $item.FullName -AclObject $acl
+            $restamped++
+        } catch {
+            Write-Host ("  WARNING: could not set owner on {0}: {1}" -f $item.FullName, $_.Exception.Message) -ForegroundColor Yellow
+        }
+    }
+    if ($restamped -gt 0) {
+        Write-Host ("  reowned {0} fixture object(s) to {1} (harness is running elevated)" -f $restamped, $user.Value)
+    }
+}
+
 function Initialize-Repo {
     param([string]$Dir)
     # Build a tiny standalone repo via real git, then we'll exercise
@@ -420,6 +456,7 @@ function Initialize-Repo {
     } finally {
         Pop-Location
     }
+    Repair-FixtureOwnership -Dir $Dir
 }
 
 function W4-PwshGitStatus {

From 2c2253e5824895c5a6218b2341740ae88e4e7d2b Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 27 Aug 2026 15:26:54 -0700
Subject: [PATCH 31/44] expanded list of asserted interpreters

---
 .github/copilot-instructions.md            | 10 ++--
 docs/ci-validation-infrastructure.md       | 36 ++++++++-----
 scripts/ci/assert-workload-interpreters.sh | 60 ----------------------
 scripts/ci/prepare-linux-host.sh           | 56 +++++++++++++++++++-
 scripts/ci/prepare-macos-host.sh           | 58 ++++++++++++++++++++-
 scripts/ci/prepare-windows-host.ps1        | 31 ++++++++---
 6 files changed, 166 insertions(+), 85 deletions(-)
 delete mode 100644 scripts/ci/assert-workload-interpreters.sh

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index b4087373f..ed9797f00 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -91,10 +91,12 @@ matrix `backend` id: `scripts/ci/prepare-windows-host.ps1`,
 `scripts/ci/prepare-linux-host.sh`, and `scripts/ci/prepare-macos-host.sh`. A
 backend with no prerequisites is an explicit no-op, so the step runs
 unconditionally for every entry. Independent of the backend id, all three also
-verify the host's workload interpreters (`pwsh`, `git`, `node`, `python`) —
-Windows in `Assert-WorkloadInterpreters`, Linux and macOS via the shared
-bash-3.2-compatible `scripts/ci/assert-workload-interpreters.sh`. Interpreters
-are verified, never installed.
+verify the host's workload interpreters and CLIs (`pwsh`, `git`, `node`, `npm`,
+`npx`, `python`, `pip`, `dotnet`, `az`, `gh`, `openssl`, plus `nuget`, `winapp`,
+`winget`, `scoop`, and `choco` on Windows only, and `brew` on macOS only) —
+Windows in `Assert-WorkloadInterpreters`, Linux and macOS in a deliberately
+duplicated bash-3.2-compatible `assert_workload_interpreters` function so each
+platform's list can diverge. These are verified, never installed.
 
 **Test dispatch** goes through `scripts/ci/run_backend_validation_tests.ps1`
 (Windows) and `scripts/ci/run_backend_validation_tests.sh` (Linux/macOS), which map
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 352e83ed4..e636f38a5 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -32,7 +32,6 @@ the individual local test scripts are documented in
 | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. |
 | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). |
 | `scripts/ci/prepare-macos-host.sh` | Per-backend macOS host preparation / prerequisite assertions. |
-| `scripts/ci/assert-workload-interpreters.sh` | Shared Linux/macOS workload-interpreter inventory check. |
 | `scripts/ci/run_backend_validation_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
 | `scripts/ci/run_backend_validation_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
 
@@ -289,10 +288,17 @@ Every one of the three scripts also runs the workload-interpreter check
 ### Workload interpreters
 
 Some suites do not just exercise MXC's primitives — they run *real programs*
-(`pwsh`, `git`, `node`, `python`, `cmd`) inside the sandbox and assert on what
-those programs produce. Each preparation script verifies that host-side
-inventory up front, so a missing tool is reported once as a preparation result
-rather than repeatedly as a confusing mid-suite failure.
+inside the sandbox and assert on what those programs produce. Each preparation
+script verifies that host-side inventory up front, so a missing tool is reported
+once as a preparation result rather than repeatedly as a confusing mid-suite
+failure.
+
+The list is `pwsh`, `git`, `node`, `npm`, `npx`, `python`, `pip`, `dotnet`, `az`,
+`gh`, and `openssl` on every OS, plus `nuget`, `winapp` (the Windows App
+Development CLI), `winget`, `scoop`, and `choco` on Windows only, and `brew` on
+macOS only. The Windows five have no Unix counterpart — except NuGet, which Unix
+reaches through `dotnet nuget` rather than a standalone binary, so checking for
+one there would warn forever.
 
 The check is **suite-agnostic by design**, and runs for *every* backend rather
 than only the ones whose suites happen to need it today. It describes what a
@@ -301,18 +307,22 @@ any current or future suite that shells out to these programs is served by the
 same list. `T3-Workloads.ps1` is simply the first caller; wiring up the next one
 needs no change here.
 
-There are two implementations, because the two host families share no shell:
-`Assert-WorkloadInterpreters` in `prepare-windows-host.ps1`, and
-`assert-workload-interpreters.sh`, which Linux and macOS both invoke. The Unix
-script is written to bash 3.2 — no associative arrays — because that is what
-macOS still ships.
+Each preparation script carries its own copy — `Assert-WorkloadInterpreters` in
+`prepare-windows-host.ps1`, `assert_workload_interpreters` in the two `.sh`
+scripts.
 
 Each inventory entry carries:
 
 - the command names to try, in order. Resolution mirrors what the suites
-  themselves do: on Windows `python` is tried before `python3` and a match under
-  `WindowsApps` is ignored (that is a Microsoft Store alias stub, not a real
-  interpreter), while on Unix `python3` is tried first.
+  themselves do: on Windows `python` is tried before `python3`, while on Unix
+  `python3` is tried first.
+- on Windows, whether a match under `WindowsApps` counts. By default it does
+  not: that is usually a Microsoft Store `AppExecutionAlias` stub, a 0-byte
+  redirect that opens the Store rather than running. But `WindowsApps` is also
+  how App Installer legitimately ships `winget`, and a working alias is
+  indistinguishable from a stub by path or size — both are 0-byte reparse
+  points — so entries delivered that way set `AllowStoreAlias` and opt out.
+  Without it the check reports an installed `winget` as missing.
 - whether an absence fails the job or only warns. On Windows only `pwsh` is
   required; its absence means a mis-imaged pool. Everything else warns, because
   suites are expected to report their dependent cases as skipped rather than
diff --git a/scripts/ci/assert-workload-interpreters.sh b/scripts/ci/assert-workload-interpreters.sh
deleted file mode 100644
index 16f17b21a..000000000
--- a/scripts/ci/assert-workload-interpreters.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Verifies the interpreters test suites drive inside the sandbox, following the
-# same verify-never-install rule as the rest of host preparation: a missing one
-# is an image problem, not something a job can fix mid-run.
-#
-# Shared by prepare-linux-host.sh and prepare-macos-host.sh so the two cannot
-# drift. The Windows twin is Assert-WorkloadInterpreters in
-# prepare-windows-host.ps1, which stays separate because it has to filter the
-# Microsoft Store alias stubs that only exist there.
-#
-# The check is suite-agnostic: it describes what a validation host is expected
-# to provide, not what any one suite consumes, so a future suite that shells out
-# to these programs needs no change here.
-#
-# Written for bash 3.2, which is what macOS still ships -- hence the delimited
-# string table rather than associative arrays.
-
-# name|candidates (tried in order)|required|remedy
-#
-# Nothing is required on Unix today: no Linux or macOS suite drives these
-# interpreters yet, so this runs as host inventory and reports absences as
-# warnings. Flip a `false` to `true` when a suite starts depending on one --
-# that is the whole change.
-interpreters=(
-    "pwsh|pwsh|false|install PowerShell 7 in the image"
-    "git|git|false|install Git in the image"
-    "node|node|false|install Node.js in the image"
-    "python|python3,python|false|install Python in the image"
-)
-
-missing=""
-for entry in "${interpreters[@]}"; do
-    IFS='|' read -r name candidates required remedy <<<"$entry"
-
-    resolved=""
-    IFS=',' read -r -a candidate_list <<<"$candidates"
-    for candidate in "${candidate_list[@]}"; do
-        # python3 before python: on Unix a bare `python` is usually absent, and
-        # where it does exist it can still be Python 2.
-        if resolved="$(command -v "$candidate" 2>/dev/null)"; then
-            break
-        fi
-        resolved=""
-    done
-
-    if [[ -n "$resolved" ]]; then
-        echo "Workload interpreter '$name' found at $resolved"
-    elif [[ "$required" == "true" ]]; then
-        missing="${missing:+$missing; }$name ($remedy)"
-    else
-        echo "::warning::Workload interpreter '$name' is absent ($remedy)"
-    fi
-done
-
-if [[ -n "$missing" ]]; then
-    echo "::error::Workload interpreters missing from this image: $missing"
-    exit 1
-fi
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 1ecb2640a..88eb8f741 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -206,10 +206,64 @@ enable_bridge_netfilter() {
     done
 }
 
+# Verifies the interpreters test suites drive inside the sandbox, following the
+# same verify-never-install rule as the rest of host preparation: a missing one
+# is an image problem, not something a job can fix mid-run.
+#
+# The check is suite-agnostic: it describes what a validation host is expected
+# to provide, not what any one suite consumes, so a future suite that shells out
+# to these programs needs no change here.
+assert_workload_interpreters() {
+    # name|candidates (tried in order)|required|remedy
+    local interpreters=(
+        "pwsh|pwsh|false|install PowerShell 7 in the image"
+        "git|git|false|install Git in the image"
+        "node|node|false|install Node.js in the image"
+        "npm|npm|false|install Node.js in the image (npm ships with it)"
+        "npx|npx|false|install Node.js in the image (npx ships with it)"
+        "python|python3,python|false|install Python in the image"
+        "pip|pip3,pip|false|install Python in the image (pip ships with it)"
+        "dotnet|dotnet|false|install the .NET SDK in the image"
+        "az|az|false|install the Azure CLI in the image"
+        "gh|gh|false|install the GitHub CLI in the image"
+        "openssl|openssl|false|install OpenSSL in the image"
+    )
+
+    local missing="" entry name candidates required remedy resolved candidate
+    local candidate_list
+    for entry in "${interpreters[@]}"; do
+        IFS='|' read -r name candidates required remedy <<<"$entry"
+
+        resolved=""
+        IFS=',' read -r -a candidate_list <<<"$candidates"
+        for candidate in "${candidate_list[@]}"; do
+            # python3 before python: on Unix a bare `python` is usually absent,
+            # and where it does exist it can still be Python 2.
+            if resolved="$(command -v "$candidate" 2>/dev/null)"; then
+                break
+            fi
+            resolved=""
+        done
+
+        if [[ -n "$resolved" ]]; then
+            echo "Workload interpreter '$name' found at $resolved"
+        elif [[ "$required" == "true" ]]; then
+            missing="${missing:+$missing; }$name ($remedy)"
+        else
+            echo "::warning::Workload interpreter '$name' is absent ($remedy)"
+        fi
+    done
+
+    if [[ -n "$missing" ]]; then
+        echo "::error::Workload interpreters missing from this image: $missing"
+        exit 1
+    fi
+}
+
 chmod +x "$binary_directory/lxc-exec"
 
 # Runs for every backend: this is host inventory, not a backend prerequisite.
-bash "$(dirname "$0")/assert-workload-interpreters.sh"
+assert_workload_interpreters
 
 case "$backend" in
     bubblewrap)
diff --git a/scripts/ci/prepare-macos-host.sh b/scripts/ci/prepare-macos-host.sh
index 3eb898226..a3e7042c9 100644
--- a/scripts/ci/prepare-macos-host.sh
+++ b/scripts/ci/prepare-macos-host.sh
@@ -20,8 +20,64 @@ fi
 backend="$1"
 binary_directory="$2"
 
+# Verifies the interpreters test suites drive inside the sandbox, following the
+# same verify-never-install rule as the rest of host preparation: a missing one
+# is an image problem, not something a job can fix mid-run.
+#
+# The check is suite-agnostic: it describes what a validation host is expected
+# to provide, not what any one suite consumes, so a future suite that shells out
+# to these programs needs no change here.
+assert_workload_interpreters() {
+    # name|candidates (tried in order)|required|remedy
+    local interpreters=(
+        "pwsh|pwsh|false|install PowerShell 7 in the image"
+        "git|git|false|install Git in the image"
+        "node|node|false|install Node.js in the image"
+        "npm|npm|false|install Node.js in the image (npm ships with it)"
+        "npx|npx|false|install Node.js in the image (npx ships with it)"
+        "python|python3,python|false|install Python in the image"
+        "pip|pip3,pip|false|install Python in the image (pip ships with it)"
+        "dotnet|dotnet|false|install the .NET SDK in the image"
+        "az|az|false|install the Azure CLI in the image"
+        "gh|gh|false|install the GitHub CLI in the image"
+        "openssl|openssl|false|install OpenSSL in the image"
+        # macOS-only
+        "brew|brew|false|install Homebrew in the image"
+    )
+
+    local missing="" entry name candidates required remedy resolved candidate
+    local candidate_list
+    for entry in "${interpreters[@]}"; do
+        IFS='|' read -r name candidates required remedy <<<"$entry"
+
+        resolved=""
+        IFS=',' read -r -a candidate_list <<<"$candidates"
+        for candidate in "${candidate_list[@]}"; do
+            # python3 before python: on Unix a bare `python` is usually absent,
+            # and where it does exist it can still be Python 2.
+            if resolved="$(command -v "$candidate" 2>/dev/null)"; then
+                break
+            fi
+            resolved=""
+        done
+
+        if [[ -n "$resolved" ]]; then
+            echo "Workload interpreter '$name' found at $resolved"
+        elif [[ "$required" == "true" ]]; then
+            missing="${missing:+$missing; }$name ($remedy)"
+        else
+            echo "::warning::Workload interpreter '$name' is absent ($remedy)"
+        fi
+    done
+
+    if [[ -n "$missing" ]]; then
+        echo "::error::Workload interpreters missing from this image: $missing"
+        exit 1
+    fi
+}
+
 # Runs for every backend: this is host inventory, not a backend prerequisite.
-bash "$(dirname "$0")/assert-workload-interpreters.sh"
+assert_workload_interpreters
 
 case "$backend" in
     seatbelt)
diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index 78c9035dc..d5b16a0c6 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -152,10 +152,23 @@ function Initialize-ProcessContainerHost {
 # a missing one is an image problem, not something the job can fix mid-run.
 function Assert-WorkloadInterpreters {
     $interpreters = @(
-        @{ Name = 'pwsh';   Candidates = @('pwsh');              Required = $true;  Remedy = 'install PowerShell 7 in the image' },
-        @{ Name = 'git';    Candidates = @('git');               Required = $false; Remedy = 'install Git for Windows in the image' },
-        @{ Name = 'node';   Candidates = @('node');              Required = $false; Remedy = 'install Node.js in the image' },
-        @{ Name = 'python'; Candidates = @('python', 'python3'); Required = $false; Remedy = 'install Python in the image' }
+        @{ Name = 'pwsh';    Candidates = @('pwsh');              Required = $true;  Remedy = 'install PowerShell 7 in the image' },
+        @{ Name = 'git';     Candidates = @('git');               Required = $false; Remedy = 'install Git for Windows in the image' },
+        @{ Name = 'node';    Candidates = @('node');              Required = $false; Remedy = 'install Node.js in the image' },
+        @{ Name = 'npm';     Candidates = @('npm');               Required = $false; Remedy = 'install Node.js in the image (npm ships with it)' },
+        @{ Name = 'npx';     Candidates = @('npx');               Required = $false; Remedy = 'install Node.js in the image (npx ships with it)' },
+        @{ Name = 'python';  Candidates = @('python', 'python3'); Required = $false; Remedy = 'install Python in the image' },
+        @{ Name = 'pip';     Candidates = @('pip', 'pip3');       Required = $false; Remedy = 'install Python in the image (pip ships with it)' },
+        @{ Name = 'dotnet';  Candidates = @('dotnet');            Required = $false; Remedy = 'install the .NET SDK in the image' },
+        @{ Name = 'az';      Candidates = @('az');                Required = $false; Remedy = 'install the Azure CLI in the image' },
+        @{ Name = 'gh';      Candidates = @('gh');                Required = $false; Remedy = 'install the GitHub CLI in the image' },
+        @{ Name = 'openssl'; Candidates = @('openssl');           Required = $false; Remedy = 'install OpenSSL in the image' },
+        # Windows-only
+        @{ Name = 'nuget';   Candidates = @('nuget');             Required = $false; Remedy = 'install the NuGet CLI in the image' },
+        @{ Name = 'winapp';  Candidates = @('winapp');            Required = $false; Remedy = 'install the Windows App Development CLI (winget install Microsoft.WinAppCli) in the image' },
+        @{ Name = 'winget';  Candidates = @('winget');            Required = $false; AllowStoreAlias = $true; Remedy = 'install the Windows Package Manager (App Installer) in the image' },
+        @{ Name = 'scoop';   Candidates = @('scoop');             Required = $false; Remedy = 'install Scoop in the image' },
+        @{ Name = 'choco';   Candidates = @('choco');             Required = $false; Remedy = 'install Chocolatey in the image' }
     )
 
     $missing = @()
@@ -163,10 +176,16 @@ function Assert-WorkloadInterpreters {
         $resolved = $null
         foreach ($candidate in $tool.Candidates) {
             $found = Get-Command $candidate -ErrorAction SilentlyContinue
-            # A command resolving into WindowsApps is a Microsoft Store
+            # A command resolving into WindowsApps is normally a Microsoft Store
             # AppExecutionAlias stub: a 0-byte redirect that opens the Store
             # rather than running, which the suite deliberately ignores.
-            if ($found -and $found.Source -notlike '*\WindowsApps\*') {
+            #
+            # WindowsApps is also how App Installer legitimately delivers winget,
+            # and a working alias is indistinguishable from a stub by path or by
+            # size (both are 0-byte reparse points). So entries that ship that
+            # way opt out of the filter via AllowStoreAlias; blanket-filtering
+            # them reports an installed tool as missing.
+            if ($found -and ($tool['AllowStoreAlias'] -or $found.Source -notlike '*\WindowsApps\*')) {
                 $resolved = $found.Source
                 break
             }

From 26fba6f6b16f48378082f295489c7a35f0cdd3c3 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 27 Aug 2026 17:04:47 -0700
Subject: [PATCH 32/44] preparing windows/linux by now installing packages on
 linux and winget on windows

---
 .github/copilot-instructions.md               |  15 +-
 .../Validation.Infrastructure.Tests.yml       |   2 +-
 docs/ci-validation-infrastructure.md          |  88 +++++-
 scripts/ci/prepare-linux-host.sh              | 297 ++++++++++++++++--
 scripts/ci/prepare-windows-host.ps1           |  85 ++++-
 scripts/ci/validation-test-matrix.json        |  21 +-
 6 files changed, 443 insertions(+), 65 deletions(-)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index ed9797f00..0bf7a9c29 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -96,7 +96,20 @@ verify the host's workload interpreters and CLIs (`pwsh`, `git`, `node`, `npm`,
 `winget`, `scoop`, and `choco` on Windows only, and `brew` on macOS only) —
 Windows in `Assert-WorkloadInterpreters`, Linux and macOS in a deliberately
 duplicated bash-3.2-compatible `assert_workload_interpreters` function so each
-platform's list can diverge. These are verified, never installed.
+platform's list can diverge. macOS verifies only; Linux first runs
+`install_workload_interpreters`, which installs whatever the stock distribution
+image lacks (distribution repositories, plus Microsoft's feed for `pwsh`/`az`
+and GitHub's for `gh`). That pass is best-effort and can never fail the job —
+the inventory that follows reports the outcome. Its package-manager access goes
+through `resolve_package_manager` and `install_packages`, the same helpers the
+backend prerequisite installers use, so a new distribution family is one arm in
+`install_packages` rather than a branch in every installer. Windows provisions
+one entry, also best-effort and also ahead of the inventory: `Repair-Winget`
+re-registers the App Installer package (`Add-AppxPackage -RegisterByFamilyName`)
+when `winget` resolves on `PATH` but fails to run, the symptom of a package the
+image shipped but never registered for the account the job runs as. It decides
+by invoking `winget --version`, not by resolving the command, since a resolvable
+alias is the broken case.
 
 **Test dispatch** goes through `scripts/ci/run_backend_validation_tests.ps1`
 (Windows) and `scripts/ci/run_backend_validation_tests.sh` (Linux/macOS), which map
diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
index 87c8b937b..83dd18499 100644
--- a/.github/workflows/Validation.Infrastructure.Tests.yml
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -33,4 +33,4 @@ jobs:
     needs: [windows, linux, macos]
     uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
     with:
-      plan: enabled
\ No newline at end of file
+      plan: nightly
\ No newline at end of file
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index e636f38a5..07309f7ee 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -29,8 +29,8 @@ the individual local test scripts are documented in
 | `.github/workflows/Validation.Tests.Matrix.Job.yml` | `workflow_call`-only. Resolves the plan and runs the per-family test jobs. |
 | `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers, job staggering. |
 | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. |
-| `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. |
-| `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). |
+| `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions, plus the `winget` repair. |
+| `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware), plus the workload-interpreter install pass. |
 | `scripts/ci/prepare-macos-host.sh` | Per-backend macOS host preparation / prerequisite assertions. |
 | `scripts/ci/run_backend_validation_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
 | `scripts/ci/run_backend_validation_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
@@ -264,6 +264,11 @@ Windows optional features are **verified, never enabled**: turning one on needs
 reboot the job cannot take, so a mis-imaged pool fails here with a pointed
 message instead of surfacing later as an opaque backend error.
 
+The script does provision one thing, for every backend rather than a particular
+one: `Repair-Winget` re-registers the App Installer package when `winget` is on
+`PATH` but cannot run
+([below](#verified-everywhere-installed-on-linux-repaired-on-windows)).
+
 `prepare-linux-host.sh`:
 
 - `bubblewrap` — installs `bwrap`, `slirp4netns`, `util-linux`, and `iptables`
@@ -276,22 +281,33 @@ message instead of surfacing later as an opaque backend error.
 - `microvm` — asserts the NanVix payload exists.
 - `hyperlight` — no-op.
 
+Every install above goes through two shared helpers rather than its own
+package-manager chain: `resolve_package_manager` picks the first of `apt-get`,
+`dnf`, `yum`, or `microdnf` on the host and caches it, and `install_packages`
+holds the single `case` that knows how each one is invoked. Supporting a new
+distribution family is therefore one new arm in `install_packages`, not another
+branch in every installer. `install_packages` returns the package manager's own
+status rather than acting on it, which is what lets a backend prerequisite treat
+a failure as fatal while a workload interpreter only warns.
+
 `prepare-macos-host.sh`:
 
 - `seatbelt` — asserts `mxc-exec-mac` shipped. The sandbox is part of the OS, so
   there is nothing to install; the script exists so macOS has the same shape as
   the other two and a future prerequisite has an obvious home.
 
-Every one of the three scripts also runs the workload-interpreter check
+Every one of the three scripts also takes the workload-interpreter inventory
 ([below](#workload-interpreters)) before it dispatches on the backend id.
 
 ### Workload interpreters
 
 Some suites do not just exercise MXC's primitives — they run *real programs*
 inside the sandbox and assert on what those programs produce. Each preparation
-script verifies that host-side inventory up front, so a missing tool is reported
+script takes that host-side inventory up front, so a missing tool is reported
 once as a preparation result rather than repeatedly as a confusing mid-suite
-failure.
+failure. On Linux the inventory is preceded by an install pass, and on Windows
+by a narrow `winget` repair
+([below](#verified-everywhere-installed-on-linux-repaired-on-windows)).
 
 The list is `pwsh`, `git`, `node`, `npm`, `npx`, `python`, `pip`, `dotnet`, `az`,
 `gh`, and `openssl` on every OS, plus `nuget`, `winapp` (the Windows App
@@ -328,10 +344,7 @@ Each inventory entry carries:
   suites are expected to report their dependent cases as skipped rather than
   failing. Nothing is required on Unix yet — no Unix suite drives these
   interpreters — so that check currently runs purely as host inventory.
-- the image-level fix, quoted in whichever message is emitted. The tools are
-  **verified, never installed**, for the same reason the optional features are:
-  provisioning a toolchain mid-run would mask the image drift the check exists
-  to surface.
+- the image-level fix, quoted in whichever message is emitted.
 
 Because a warning is deliberately not a failure, an absent optional interpreter
 silently shrinks coverage while the job still shows green. GitHub's pass/fail
@@ -339,6 +352,58 @@ icon cannot express "passed, but with less coverage than yesterday" — a future
 test-analysis portal is intended to surface skip counts so that erosion is
 visible.
 
+#### Verified everywhere, installed on Linux, repaired on Windows
+
+On Windows and macOS the tools are, with one exception, **verified, never
+installed**, for the same reason the optional features are: provisioning a
+toolchain mid-run would mask the image drift the check exists to surface.
+Neither platform needs it anyway — Windows runs on a 1ES image whose contents we
+control, and the GitHub-hosted macOS runners already ship all twelve.
+
+The Windows exception is `winget`, which `Repair-Winget` provisions immediately
+before the inventory. It is the narrowest form of install there is: it downloads
+nothing and adds nothing to the image, it only re-registers for the running
+account a package the image already shipped. That is worth doing because the
+failure it fixes is not image drift at all — App Installer is routinely present
+but unregistered for the account the job runs as, which leaves an
+`AppExecutionAlias` that resolves on `PATH` and then fails with "The file cannot
+be accessed by the system". Reporting a tool the image *does* carry as missing
+would surface nothing anyone could act on.
+
+The repair is written to be indistinguishable from a no-op when it is not
+needed. It decides by *running* `winget --version` rather than by resolving the
+command, because a resolvable alias is precisely the broken case; an operational
+host returns before touching Appx at all. Like the Linux install pass, nothing
+it does can fail the job — an absent package, an unreachable `Appx` module, and
+a failed registration all warn and fall through to the inventory, which then
+reports `winget` in the usual way.
+
+Linux is the exception. `prepare-linux-host.sh` runs
+`install_workload_interpreters` immediately before taking the inventory,
+installing whatever the image did not already provide. The Linux pools run stock
+distribution images that MXC does not bake, so there is no curated image to
+drift *from*; refusing to install would not surface a provisioning mistake, it
+would only cost coverage. Most of the list comes from the distribution's own
+repositories, `pwsh` and `az` from Microsoft's feed and `gh` from GitHub's since
+no distribution carries them, and `npx` from nowhere at all — it arrives with
+`npm`.
+
+Two properties make that safe to run on every job:
+
+- **Nothing it does can fail the job.** Every step warns and continues, and a
+  host with no recognized package manager is skipped outright. The inventory
+  immediately afterwards is what reports the outcome, so a tool that could not
+  be installed stays visible instead of becoming a silent absence.
+- **A failed batch degrades to individual installs.** apt and dnf abort the
+  *entire* transaction over a single unavailable package, so one name missing on
+  one distribution would otherwise cost that host every other interpreter as
+  well. After a batch failure each package is retried on its own.
+
+A host that already has everything short-circuits before touching the package
+manager, so the common case costs one `command -v` per entry. Vendor feeds are
+added at most once and a failure is remembered, so the second tool wanting a
+broken feed does not retry it.
+
 ## Log collection
 
 Every job uploads its logs whether it passed or failed, as
@@ -420,7 +485,10 @@ passing a distinguishing argument later without touching the matrix.
 3. For a Windows prerelease image set `"prerelease": true` and use a neutral
    `windows-prerelease-<name>` id — the id appears in public job names.
 4. For a new Linux distro, check that `prepare-linux-host.sh` handles its
-   package manager and service layout.
+   package manager and service layout. A new package-manager family needs one
+   arm in `install_packages` and one entry in `resolve_package_manager`; a
+   family whose package *names* differ also needs its column in the tables in
+   `install_lxc`, `install_bubblewrap`, and `install_workload_interpreters`.
 5. Add it to a plan's `triggers`, then resolve locally.
 
 ### Wire an unwired backend to a suite
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 88eb8f741..19e441898 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -25,18 +25,69 @@ apt_update() {
     fi
 }
 
+# Resolves the host's package manager once, so supporting a new distribution
+# family is a case in install_packages rather than another branch in every
+# installer below.
+package_manager=""
+resolve_package_manager() {
+    if [[ -n "$package_manager" ]]; then
+        return 0
+    fi
+
+    local candidate
+    for candidate in apt-get dnf yum microdnf; do
+        if command -v "$candidate" >/dev/null 2>&1; then
+            package_manager="$candidate"
+            return 0
+        fi
+    done
+    return 1
+}
+
+# install_packages <package>...
+# Installs from the feeds the host already has configured, and returns the
+# package manager's own status so each caller decides what a failure means: a
+# missing backend prerequisite is fatal, a missing workload interpreter is not.
+install_packages() {
+    case "$package_manager" in
+        apt-get)
+            # sudo resets the environment, so the frontend setting has to be
+            # applied on the far side of it rather than exported here.
+            sudo env DEBIAN_FRONTEND=noninteractive \
+                apt-get install -y --no-install-recommends "$@"
+            ;;
+        dnf | yum | microdnf)
+            sudo "$package_manager" install -y "$@"
+            ;;
+        *)
+            echo "Unsupported package manager: '$package_manager'." >&2
+            return 1
+            ;;
+    esac
+}
+
+# Returns 0 when any of the comma-separated candidates is on PATH.
+have_command() {
+    local candidate
+    local IFS=','
+    for candidate in $1; do
+        if command -v "$candidate" >/dev/null 2>&1; then
+            return 0
+        fi
+    done
+    return 1
+}
+
 # Red Hat ships no third-party content, so epel-release is not in RHEL's own
 # repos; the documented install is the release RPM straight from Fedora. 
 install_epel() {
-    local package_manager="$1"
-
     if command -v subscription-manager >/dev/null 2>&1; then
         sudo subscription-manager repos \
             --enable "codeready-builder-for-rhel-10-$(arch)-rpms" ||
             echo "WARNING: could not enable the CRB repository; EPEL packages that depend on it may fail to install." >&2
     fi
 
-    sudo "$package_manager" install -y \
+    install_packages \
         https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm
 }
 
@@ -51,20 +102,20 @@ install_bubblewrap() {
         command -v ip >/dev/null 2>&1; then
         return
     fi
-    if command -v apt-get >/dev/null 2>&1; then
-        apt_update
-        sudo apt-get install -y --no-install-recommends \
-            bubblewrap slirp4netns util-linux iptables iproute2
-    elif command -v dnf >/dev/null 2>&1; then
-        sudo dnf install -y bubblewrap slirp4netns util-linux iptables iproute
-    elif command -v yum >/dev/null 2>&1; then
-        sudo yum install -y bubblewrap slirp4netns util-linux iptables iproute
-    elif command -v microdnf >/dev/null 2>&1; then
-        sudo microdnf install -y bubblewrap slirp4netns util-linux iptables iproute
-    else
+
+    if ! resolve_package_manager; then
         echo "No supported package manager found to install Bubblewrap prerequisites." >&2
         exit 1
     fi
+
+    local packages=(bubblewrap slirp4netns util-linux iptables)
+    if [[ "$package_manager" == "apt-get" ]]; then
+        apt_update
+        packages+=(iproute2)
+    else
+        packages+=(iproute)
+    fi
+    install_packages "${packages[@]}"
 }
 
 # The ingress chain matches on connection state, which iptables can only
@@ -88,27 +139,25 @@ install_lxc() {
     if command -v lxc-start >/dev/null 2>&1; then
         return
     fi
-    if command -v apt-get >/dev/null 2>&1; then
+
+    if ! resolve_package_manager; then
+        echo "No supported package manager found to install LXC." >&2
+        exit 1
+    fi
+
+    local packages
+    if [[ "$package_manager" == "apt-get" ]]; then
         apt_update
+        packages=(lxc dnsmasq-base iptables bridge-utils)
         # Debian dropped lxc-utils; Ubuntu still ships it.
-        local packages=(lxc dnsmasq-base iptables bridge-utils)
         if apt-cache show lxc-utils >/dev/null 2>&1; then
             packages+=(lxc-utils)
         fi
-        sudo apt-get install -y --no-install-recommends "${packages[@]}"
-    elif command -v dnf >/dev/null 2>&1; then
-        install_epel dnf
-        sudo dnf install -y lxc lxc-templates dnsmasq iptables
-    elif command -v yum >/dev/null 2>&1; then
-        install_epel yum
-        sudo yum install -y lxc lxc-templates dnsmasq iptables
-    elif command -v microdnf >/dev/null 2>&1; then
-        install_epel microdnf
-        sudo microdnf install -y lxc lxc-templates dnsmasq iptables
     else
-        echo "No supported package manager found to install LXC." >&2
-        exit 1
+        install_epel
+        packages=(lxc lxc-templates dnsmasq iptables)
     fi
+    install_packages "${packages[@]}"
 }
 
 # Start the LXC bridge and wait until it can actually serve containers.
@@ -206,9 +255,192 @@ enable_bridge_netfilter() {
     done
 }
 
-# Verifies the interpreters test suites drive inside the sandbox, following the
-# same verify-never-install rule as the rest of host preparation: a missing one
-# is an image problem, not something a job can fix mid-run.
+# PowerShell, the Azure CLI, and the GitHub CLI are in no distribution's own
+# repositories, so each comes from its vendor's feed. A feed is added at most
+# once and a failure is remembered, so the second tool wanting a broken feed
+# does not retry it.
+microsoft_feed_state=""
+add_microsoft_feed() {
+    case "$microsoft_feed_state" in
+        added) return 0 ;;
+        failed) return 1 ;;
+    esac
+    microsoft_feed_state="failed"
+
+    local id="" version_id=""
+    if [[ -r /etc/os-release ]]; then
+        # shellcheck disable=SC1091
+        . /etc/os-release
+        id="${ID:-}"
+        version_id="${VERSION_ID:-}"
+    fi
+    if [[ -z "$id" || -z "$version_id" ]]; then
+        echo "WARNING: could not read the distribution from /etc/os-release; skipping the Microsoft package feed." >&2
+        return 1
+    fi
+
+    if [[ "$package_manager" == "apt-get" ]]; then
+        local package="/tmp/packages-microsoft-prod.deb"
+        if ! curl -fsSL \
+            "https://packages.microsoft.com/config/${id}/${version_id}/packages-microsoft-prod.deb" \
+            -o "$package"; then
+            echo "WARNING: no Microsoft package feed published for ${id} ${version_id}." >&2
+            return 1
+        fi
+        if ! sudo dpkg -i "$package"; then
+            rm -f "$package"
+            echo "WARNING: could not install the Microsoft package feed." >&2
+            return 1
+        fi
+        rm -f "$package"
+        apt_update
+    else
+        # The RPM feed is keyed on the major version alone.
+        if ! install_packages \
+            "https://packages.microsoft.com/config/rhel/${version_id%%.*}/packages-microsoft-prod.rpm"; then
+            echo "WARNING: could not install the Microsoft package feed." >&2
+            return 1
+        fi
+    fi
+
+    microsoft_feed_state="added"
+}
+
+add_github_feed() {
+    if [[ "$package_manager" == "apt-get" ]]; then
+        local keyring="/usr/share/keyrings/githubcli-archive-keyring.gpg"
+        if ! curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg |
+            sudo dd of="$keyring" status=none; then
+            echo "WARNING: could not download the GitHub CLI signing key." >&2
+            return 1
+        fi
+        if ! sudo chmod go+r "$keyring"; then
+            echo "WARNING: could not make the GitHub CLI signing key readable." >&2
+            return 1
+        fi
+        if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=$keyring] https://cli.github.com/packages stable main" |
+            sudo tee /etc/apt/sources.list.d/github-cli.list >/dev/null; then
+            echo "WARNING: could not write the GitHub CLI package source." >&2
+            return 1
+        fi
+        apt_update
+        return 0
+    fi
+
+    # config-manager is a separate package on some releases and is what adds the
+    # feed, so it is requested before it is used.
+    install_packages "dnf-command(config-manager)" >/dev/null 2>&1 || true
+    if ! sudo "$package_manager" config-manager \
+        --add-repo https://cli.github.com/packages/rpm/gh-cli.repo; then
+        echo "WARNING: could not add the GitHub CLI package feed." >&2
+        return 1
+    fi
+}
+
+# Installs the workload interpreters the image did not already provide.
+#
+# Host preparation verifies rather than installs wherever it can, because a
+# missing backend prerequisite means the image is wrong for the job. The
+# workload interpreters are the exception: they are ordinary developer tools the
+# package manager can supply in one transaction, which is cheaper than losing a
+# suite's coverage to a tool the image happened not to bake.
+#
+# Every step here is best effort and none of them fails the job. What the host
+# actually ended up with is reported by assert_workload_interpreters below.
+install_workload_interpreters() {
+    if ! resolve_package_manager; then
+        echo "WARNING: no supported package manager on this host; skipping workload interpreter installation." >&2
+        return 0
+    fi
+
+    # name|command candidates|apt package(s)|rpm package(s)
+    # npx has no package of its own; it arrives with npm. pwsh, az, and gh need
+    # a vendor feed and are handled separately below.
+    local packaged=(
+        "git|git|git|git"
+        "openssl|openssl|openssl|openssl"
+        "node|node|nodejs|nodejs"
+        "npm|npm|npm|npm"
+        "python|python3,python|python3|python3"
+        "pip|pip3,pip|python3-pip|python3-pip"
+        "dotnet|dotnet|dotnet-sdk-8.0|dotnet-sdk-8.0"
+    )
+
+    local entry name candidates apt_packages rpm_packages
+    local wanted=()
+    for entry in "${packaged[@]}"; do
+        IFS='|' read -r name candidates apt_packages rpm_packages <<<"$entry"
+        if have_command "$candidates"; then
+            continue
+        fi
+        # Deliberately unquoted: an entry may name more than one package.
+        if [[ "$package_manager" == "apt-get" ]]; then
+            wanted+=($apt_packages)
+        else
+            wanted+=($rpm_packages)
+        fi
+    done
+
+    local need_pwsh="false" need_az="false" need_gh="false"
+    have_command pwsh || need_pwsh="true"
+    have_command az || need_az="true"
+    have_command gh || need_gh="true"
+
+    if [[ ${#wanted[@]} -eq 0 && "$need_pwsh" == "false" &&
+        "$need_az" == "false" && "$need_gh" == "false" ]]; then
+        echo "All workload interpreters are already present; nothing to install."
+        return 0
+    fi
+
+    if [[ "$package_manager" == "apt-get" ]]; then
+        apt_update
+    fi
+
+    if [[ "$need_pwsh" == "true" || "$need_az" == "true" || "$need_gh" == "true" ]]; then
+        # Both feeds are fetched over HTTPS and verified against a signing key,
+        # so these have to be present before either can be added.
+        install_packages ca-certificates curl gnupg ||
+            echo "WARNING: could not install the prerequisites for adding vendor package feeds." >&2
+    fi
+
+    if [[ "$need_pwsh" == "true" || "$need_az" == "true" ]]; then
+        if add_microsoft_feed; then
+            if [[ "$need_pwsh" == "true" ]]; then
+                wanted+=(powershell)
+            fi
+            if [[ "$need_az" == "true" ]]; then
+                wanted+=(azure-cli)
+            fi
+        fi
+    fi
+
+    if [[ "$need_gh" == "true" ]] && add_github_feed; then
+        wanted+=(gh)
+    fi
+
+    if [[ ${#wanted[@]} -eq 0 ]]; then
+        return 0
+    fi
+
+    echo "Installing workload interpreters: ${wanted[*]}"
+    if install_packages "${wanted[@]}"; then
+        return 0
+    fi
+
+    # A single unavailable package fails the whole transaction, so retry them
+    # one at a time rather than leaving the host with none of them.
+    echo "WARNING: the combined install failed; retrying each package on its own." >&2
+    local package
+    for package in "${wanted[@]}"; do
+        install_packages "$package" ||
+            echo "WARNING: could not install package '$package'." >&2
+    done
+}
+
+# Verifies the interpreters test suites drive inside the sandbox. This runs
+# after install_workload_interpreters and reports what the host ended up with,
+# so a tool that could not be installed stays visible instead of silently
+# absent.
 #
 # The check is suite-agnostic: it describes what a validation host is expected
 # to provide, not what any one suite consumes, so a future suite that shells out
@@ -262,7 +494,8 @@ assert_workload_interpreters() {
 
 chmod +x "$binary_directory/lxc-exec"
 
-# Runs for every backend: this is host inventory, not a backend prerequisite.
+# Run for every backend: this is host inventory, not a backend prerequisite.
+install_workload_interpreters
 assert_workload_interpreters
 
 case "$backend" in
diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index d5b16a0c6..f2930eef3 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -204,6 +204,87 @@ function Assert-WorkloadInterpreters {
     }
 }
 
+# Returns whether winget can actually run, which is not the same as being on
+# PATH: an unregistered App Installer leaves an alias that resolves and then
+# fails to launch. The explicit alias path is a second candidate because
+# PowerShell caches command lookups, so a repair inside this process is not
+# guaranteed to be visible through Get-Command.
+function Test-WingetOperational {
+    $candidates = @()
+    $command = Get-Command winget -ErrorAction SilentlyContinue
+    if ($command) { $candidates += $command.Source }
+    $candidates += Join-Path $env:LOCALAPPDATA 'Microsoft\WindowsApps\winget.exe'
+
+    foreach ($candidate in $candidates) {
+        if (-not $candidate) { continue }
+        try {
+            $version = & $candidate --version 2>&1
+            if ($LASTEXITCODE -eq 0 -and $version) {
+                Write-Host "winget is operational ($($version | Select-Object -First 1)) at $candidate"
+                return $true
+            }
+        } catch {
+            # A broken alias throws rather than returning an exit code.
+        }
+    }
+    return $false
+}
+
+# winget ships as an AppExecutionAlias belonging to the App Installer package.
+# CI images routinely carry the package while leaving it unregistered for the
+# account the job runs as, which produces an alias that resolves on PATH but
+# fails with "The file cannot be accessed by the system". Registering the
+# package already on the image is the documented repair and downloads nothing.
+#
+# This is the one exception to the verify-never-install rule above, and only
+# barely: it installs nothing, it re-registers what the image already shipped.
+# It is best effort — winget is optional, so nothing here fails the job.
+function Repair-Winget {
+    if (Test-WingetOperational) {
+        return
+    }
+
+    Write-Host "winget did not run; checking whether the App Installer package is present."
+
+    $package = $null
+    try {
+        $package = Get-AppxPackage -Name Microsoft.DesktopAppInstaller -ErrorAction Stop |
+            Select-Object -First 1
+    } catch {
+        # PowerShell 7 builds without the native Appx binary module have to
+        # reach these cmdlets through Windows PowerShell.
+        try {
+            Import-Module Appx -UseWindowsPowerShell -ErrorAction Stop -WarningAction SilentlyContinue
+            $package = Get-AppxPackage -Name Microsoft.DesktopAppInstaller -ErrorAction Stop |
+                Select-Object -First 1
+        } catch {
+            Write-Host "::warning::Could not query Appx packages, so winget cannot be repaired: $($_.Exception.Message)"
+            return
+        }
+    }
+
+    if (-not $package) {
+        Write-Host "::warning::winget is unavailable and the App Installer package is absent; install it in the image."
+        return
+    }
+
+    Write-Host "App Installer $($package.Version) is present (status $($package.Status)); registering it for the current user."
+    try {
+        # PackageFamilyName is Microsoft.DesktopAppInstaller_8wekyb3d8bbwe, read
+        # off the package rather than hard-coded.
+        Add-AppxPackage -RegisterByFamilyName -MainPackage $package.PackageFamilyName -ErrorAction Stop
+    } catch {
+        Write-Host "::warning::Could not register the App Installer package: $($_.Exception.Message)"
+        return
+    }
+
+    if (Test-WingetOperational) {
+        Write-Host "winget is operational after registering App Installer."
+    } else {
+        Write-Host "::warning::winget is still not operational after registering App Installer."
+    }
+}
+
 function Initialize-MicroVmHost {
     # Staged next to wxc-exec.exe by the --features microvm build, so their
     # absence means a broken artifact rather than a host problem. Snapshots are
@@ -371,7 +452,9 @@ $BinaryDirectory = (Resolve-Path $BinaryDirectory).Path
 
 Write-Host "Preparing Windows host for backend '$Backend' using $BinaryDirectory"
 
-# Runs for every backend: this is host inventory, not a backend prerequisite.
+# Run for every backend: this is host inventory, not a backend prerequisite.
+# The winget repair comes first so the inventory reports the repaired state.
+Repair-Winget
 Assert-WorkloadInterpreters
 
 switch ($Backend) {
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index dc993aede..d2f00a454 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -389,25 +389,6 @@
       }
     ],
     "weekly": [],
-    "enabled": [
-      {
-        "os": "windows-25h2",
-        "backends": [
-          "process-t3"
-        ]
-      },
-      {
-        "os": "windows-24h2",
-        "backends": [
-          "process-t3"
-        ]
-      },
-      {
-        "os": "windows-23h2",
-        "backends": [
-          "process-t3"
-        ]
-      }
-    ]
+    "enabled": []
   }
 }

From 8aec1889062bc4cfb9ef629def1da6a5d636b5aa Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Fri, 28 Aug 2026 11:27:25 -0700
Subject: [PATCH 33/44] fix bwrap script regression and azure cli not resolving
 on debian systems

---
 .github/copilot-instructions.md            |  11 ++-
 docs/ci-validation-infrastructure.md       |  19 +++-
 scripts/ci/prepare-linux-host.sh           | 105 +++++++++++++++++++--
 scripts/ci/run_backend_validation_tests.sh |   4 +-
 4 files changed, 125 insertions(+), 14 deletions(-)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 0bf7a9c29..cb7e3dc09 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -98,8 +98,13 @@ Windows in `Assert-WorkloadInterpreters`, Linux and macOS in a deliberately
 duplicated bash-3.2-compatible `assert_workload_interpreters` function so each
 platform's list can diverge. macOS verifies only; Linux first runs
 `install_workload_interpreters`, which installs whatever the stock distribution
-image lacks (distribution repositories, plus Microsoft's feed for `pwsh`/`az`
-and GitHub's for `gh`). That pass is best-effort and can never fail the job —
+image lacks (distribution repositories, plus Microsoft's feeds for `pwsh`/`az`
+and GitHub's for `gh`). Microsoft splits those two: `packages-microsoft-prod`
+carries `powershell` everywhere and `azure-cli` on RPM only, while on apt the
+Azure CLI has its own codename-keyed feed added by `add_azure_cli_apt_feed`,
+which probes for the suite before writing it and falls back to the newest
+published one for the family (Debian 13 `trixie` is not published). That pass is
+best-effort and can never fail the job —
 the inventory that follows reports the outcome. Its package-manager access goes
 through `resolve_package_manager` and `install_packages`, the same helpers the
 backend prerequisite installers use, so a new distribution family is one arm in
@@ -174,7 +179,7 @@ tests\scripts\run_windows_sandbox_one_shot_tests.ps1       # Windows Sandbox one
 tests\scripts\run_windows_sandbox_state_aware_tests.ps1     # Windows Sandbox state-aware lifecycle E2E (provision/start/exec*/stop/deprovision; requires the Windows Sandbox optional feature; skips if absent)
 tests\scripts\run_lxc_all_tests.sh            # All LXC tests (Linux)
 tests\scripts\run_bwrap_all_tests.sh          # All Bubblewrap tests (Linux, requires bwrap). Must NOT run as root — several tests assert the sandbox drops capabilities, which cannot hold under a root launcher; the script refuses root explicitly.
-sudo tests\scripts\run_bwrap_inbound_deny_test.sh  # Bubblewrap inbound default-deny E2E (root-only: needs host CAP_NET_ADMIN to read the sandbox netns and inject a peer). Reported as skipped by the suite above; CI runs it separately from run_ci_backend_tests.sh.
+sudo tests\scripts\run_bwrap_inbound_deny_test.sh  # Bubblewrap inbound default-deny E2E (root-only: needs host CAP_NET_ADMIN to read the sandbox netns and inject a peer). Reported as skipped by the suite above; CI runs it separately from run_backend_validation_tests.sh.
 
 # E2E test crate — Rust executor integration tests (from src/)
 cargo test -p wxc_e2e_tests                 # Invokes MXC binaries directly
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 07309f7ee..611d5b88b 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -384,10 +384,27 @@ installing whatever the image did not already provide. The Linux pools run stock
 distribution images that MXC does not bake, so there is no curated image to
 drift *from*; refusing to install would not surface a provisioning mistake, it
 would only cost coverage. Most of the list comes from the distribution's own
-repositories, `pwsh` and `az` from Microsoft's feed and `gh` from GitHub's since
+repositories, `pwsh` and `az` from Microsoft's feeds and `gh` from GitHub's since
 no distribution carries them, and `npx` from nowhere at all — it arrives with
 `npm`.
 
+Microsoft publishes those two tools to *different* feeds, and only on the RPM
+side do they coincide. `packages-microsoft-prod` carries `powershell` on both
+families and `azure-cli` on the RPM side only; on apt the Azure CLI has a
+repository of its own, keyed by distribution codename rather than version. Asking
+the prod feed for `azure-cli` on a Debian-family host resolves nothing and fails
+with `E: Unable to locate package azure-cli`, so `add_azure_cli_apt_feed` adds
+that second feed separately.
+
+That feed lags new distribution releases, so a host's own codename may not be
+published yet — Debian 13 (`trixie`) is not. The function probes for the suite
+before writing it and falls back to the newest suite the vendor does publish for
+the family (`bookworm` for Debian, `noble` for Ubuntu), warning when it
+substitutes. The probe is what makes the fallback safe to skip entirely when
+nothing matches: an unpublished suite written into a source list makes *every*
+later `apt-get update` fail, which would cost the host the packages that were
+otherwise going to install.
+
 Two properties make that safe to run on every job:
 
 - **Nothing it does can fail the job.** Every step warns and continues, and a
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index 19e441898..c180307f6 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -306,6 +306,90 @@ add_microsoft_feed() {
     microsoft_feed_state="added"
 }
 
+# On apt the Azure CLI is not in packages-microsoft-prod. It is published to a
+# repository of its own, keyed by distribution codename rather than version, so
+# adding the prod feed and then asking for azure-cli resolves nothing and fails
+# with "E: Unable to locate package azure-cli". The RPM side needs none of this:
+# the prod feed carries azure-cli directly, which is why only Debian-family
+# hosts were affected.
+azure_cli_feed_state=""
+add_azure_cli_apt_feed() {
+    case "$azure_cli_feed_state" in
+        added) return 0 ;;
+        failed) return 1 ;;
+    esac
+    azure_cli_feed_state="failed"
+
+    local id="" codename=""
+    if [[ -r /etc/os-release ]]; then
+        # shellcheck disable=SC1091
+        . /etc/os-release
+        id="${ID:-}"
+        codename="${VERSION_CODENAME:-}"
+    fi
+    if [[ -z "$codename" ]] && command -v lsb_release >/dev/null 2>&1; then
+        codename="$(lsb_release -cs 2>/dev/null || true)"
+    fi
+    if [[ -z "$codename" ]]; then
+        echo "WARNING: could not read the distribution codename; skipping the Azure CLI package feed." >&2
+        return 1
+    fi
+
+    # The vendor publishes a suite per codename and lags new releases, so the
+    # host's own codename may not exist yet. An unpublished suite must never
+    # reach a source list: apt then fails every later refresh against it, which
+    # would cost this host the packages that were going to install fine. Probe
+    # first, and fall back to the newest suite the vendor does publish for the
+    # family. The package vendors its own Python interpreter, so it does not
+    # bind to the release it was built against.
+    local suites=("$codename")
+    case "$id" in
+        debian) [[ "$codename" == "bookworm" ]] || suites+=(bookworm) ;;
+        ubuntu) [[ "$codename" == "noble" ]] || suites+=(noble) ;;
+    esac
+
+    local suite="" candidate
+    for candidate in "${suites[@]}"; do
+        if curl -fsL --head -o /dev/null \
+            "https://packages.microsoft.com/repos/azure-cli/dists/$candidate/Release"; then
+            suite="$candidate"
+            break
+        fi
+        echo "The Azure CLI feed publishes no '$candidate' suite." >&2
+    done
+
+    if [[ -z "$suite" ]]; then
+        echo "WARNING: the Azure CLI feed publishes no suite usable on ${id:-this distribution} '$codename'." >&2
+        return 1
+    fi
+    if [[ "$suite" != "$codename" ]]; then
+        echo "WARNING: the Azure CLI feed has no '$codename' suite; using '$suite' instead." >&2
+    fi
+
+    local keyring="/etc/apt/keyrings/microsoft.gpg"
+    if ! sudo install -d -m 0755 /etc/apt/keyrings; then
+        echo "WARNING: could not create the apt keyring directory." >&2
+        return 1
+    fi
+    if ! curl -fsSL https://packages.microsoft.com/keys/microsoft.asc |
+        gpg --dearmor | sudo dd of="$keyring" status=none; then
+        echo "WARNING: could not install the Microsoft signing key." >&2
+        return 1
+    fi
+    if ! sudo chmod go+r "$keyring"; then
+        echo "WARNING: could not make the Microsoft signing key readable." >&2
+        return 1
+    fi
+    if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=$keyring] https://packages.microsoft.com/repos/azure-cli/ $suite main" |
+        sudo tee /etc/apt/sources.list.d/azure-cli.list >/dev/null; then
+        echo "WARNING: could not write the Azure CLI package source." >&2
+        return 1
+    fi
+    apt_update
+
+    azure_cli_feed_state="added"
+}
+
 add_github_feed() {
     if [[ "$package_manager" == "apt-get" ]]; then
         local keyring="/usr/share/keyrings/githubcli-archive-keyring.gpg"
@@ -397,20 +481,25 @@ install_workload_interpreters() {
     fi
 
     if [[ "$need_pwsh" == "true" || "$need_az" == "true" || "$need_gh" == "true" ]]; then
-        # Both feeds are fetched over HTTPS and verified against a signing key,
-        # so these have to be present before either can be added.
+        # Every feed is fetched over HTTPS and verified against a signing key,
+        # so these have to be present before any of them can be added.
         install_packages ca-certificates curl gnupg ||
             echo "WARNING: could not install the prerequisites for adding vendor package feeds." >&2
     fi
 
-    if [[ "$need_pwsh" == "true" || "$need_az" == "true" ]]; then
-        if add_microsoft_feed; then
-            if [[ "$need_pwsh" == "true" ]]; then
-                wanted+=(powershell)
-            fi
-            if [[ "$need_az" == "true" ]]; then
+    if [[ "$need_pwsh" == "true" ]] && add_microsoft_feed; then
+        wanted+=(powershell)
+    fi
+
+    # Two different feeds: the prod feed carries azure-cli on the RPM side, but
+    # on apt it lives in the Azure CLI's own repository.
+    if [[ "$need_az" == "true" ]]; then
+        if [[ "$package_manager" == "apt-get" ]]; then
+            if add_azure_cli_apt_feed; then
                 wanted+=(azure-cli)
             fi
+        elif add_microsoft_feed; then
+            wanted+=(azure-cli)
         fi
     fi
 
diff --git a/scripts/ci/run_backend_validation_tests.sh b/scripts/ci/run_backend_validation_tests.sh
index d2ee7c3d1..a82734198 100644
--- a/scripts/ci/run_backend_validation_tests.sh
+++ b/scripts/ci/run_backend_validation_tests.sh
@@ -44,7 +44,7 @@ case "$backend" in
         # Strict: a skip on a provisioned runner means a prerequisite vanished,
         # not that the assertion held. Without this the job goes green having
         # verified none of the directional enforcement.
-        MXC_BWRAP_TESTS_REQUIRE_EXECUTION=1 bash "$script_root/run_bwrap_all_tests.sh"
+        MXC_BWRAP_TESTS_REQUIRE_EXECUTION=1 bash "$test_script_root/run_bwrap_all_tests.sh"
         # The inbound chain test needs real host CAP_NET_ADMIN to read the
         # sandbox's network namespace and inject a peer into it, so the non-root
         # suite above can only report it as skipped. Invoking it separately here
@@ -60,7 +60,7 @@ case "$backend" in
         # the assertion passed. Translate it into an explicit failure so the
         # inbound guarantee can never be reported as verified without running.
         inbound_status=0
-        sudo -n bash "$script_root/run_bwrap_inbound_deny_test.sh" || inbound_status=$?
+        sudo -n bash "$test_script_root/run_bwrap_inbound_deny_test.sh" || inbound_status=$?
         if [[ $inbound_status -eq 77 ]]; then
             echo "The Bubblewrap inbound default-deny test skipped for a missing prerequisite;" \
                  "prepare-linux-host.sh should have installed slirp4netns, nsenter, iptables," \

From 76d547f28eef6a81f69e46dd9b8ba6423464f9a1 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Sun, 30 Aug 2026 17:25:50 -0700
Subject: [PATCH 34/44] testing pre-provisioning linux packages

---
 .../workflows/Validation.Tests.Scheduled.yml  |   2 +-
 scripts/ci/Setup-rhel.sh                      | 404 ++++++++++++++
 scripts/ci/Setup.sh                           | 496 ++++++++++++++++++
 scripts/ci/validation-test-matrix.json        |  17 +-
 4 files changed, 917 insertions(+), 2 deletions(-)
 create mode 100755 scripts/ci/Setup-rhel.sh
 create mode 100755 scripts/ci/Setup.sh

diff --git a/.github/workflows/Validation.Tests.Scheduled.yml b/.github/workflows/Validation.Tests.Scheduled.yml
index 9bc3bf9ad..ae67b110b 100644
--- a/.github/workflows/Validation.Tests.Scheduled.yml
+++ b/.github/workflows/Validation.Tests.Scheduled.yml
@@ -51,4 +51,4 @@ jobs:
     if: (github.event_name == 'schedule' && github.event.schedule == '0 8 * * 0') || inputs.plan == 'weekly'
     uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
     with:
-      plan: weekly
\ No newline at end of file
+      plan: enabled
\ No newline at end of file
diff --git a/scripts/ci/Setup-rhel.sh b/scripts/ci/Setup-rhel.sh
new file mode 100755
index 000000000..35b88c025
--- /dev/null
+++ b/scripts/ci/Setup-rhel.sh
@@ -0,0 +1,404 @@
+#!/usr/bin/env bash
+#
+# Installation script for MXC Linux machines (RHEL and compatible).
+#
+# Installs the sandboxing runtimes, kernel prerequisites and workload
+# interpreters an MXC machine is expected to provide, and persists the kernel
+# settings so they survive a reboot.
+#
+# Every step is best effort. Nothing here aborts the run and the script always
+# exits 0; what the machine actually ended up with is printed in the summary at
+# the end.
+
+set -o pipefail
+
+# Guarantee the exit status regardless of how the script leaves.
+trap 'exit 0' EXIT
+
+failures=()
+notes=()
+
+step() {
+    echo ""
+    echo "=== $* ==="
+}
+
+ok() {
+    echo "OK: $*"
+}
+
+note() {
+    echo "NOTE: $*"
+    notes+=("$*")
+}
+
+fail() {
+    echo "FAILED: $*"
+    failures+=("$*")
+}
+
+# Image builders run as root without sudo installed; interactive machines have
+# sudo and an unprivileged user. Resolve once and use the same wrapper for
+# every privileged call.
+if [ "$(id -u)" -eq 0 ]; then
+    priv() { "$@"; }
+elif command -v sudo >/dev/null 2>&1; then
+    priv() { sudo -n "$@"; }
+else
+    priv() { return 1; }
+fi
+
+have() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+# ---------------------------------------------------------------------------
+# Distribution identity
+# ---------------------------------------------------------------------------
+
+distro_id=""
+distro_version=""
+if [ -r /etc/os-release ]; then
+    # shellcheck disable=SC1091
+    . /etc/os-release
+    distro_id="${ID:-}"
+    distro_version="${VERSION_ID:-}"
+fi
+major_version="${distro_version%%.*}"
+architecture="$(uname -m 2>/dev/null)"
+
+step "Machine"
+echo "Distribution : ${distro_id:-unknown} ${distro_version:-unknown}"
+echo "Kernel       : $(uname -r 2>/dev/null)"
+echo "Architecture : ${architecture:-unknown}"
+
+package_manager=""
+for candidate in dnf yum microdnf; do
+    if have "$candidate"; then
+        package_manager="$candidate"
+        break
+    fi
+done
+
+if [ -z "$package_manager" ]; then
+    fail "no dnf, yum or microdnf on this machine; this script targets RHEL and compatible machines."
+    step "Summary"
+    echo "Nothing was installed."
+    exit 0
+fi
+echo "Package tool : $package_manager"
+
+if ! priv true 2>/dev/null; then
+    fail "cannot obtain root privileges; no packages can be installed."
+    step "Summary"
+    echo "Nothing was installed."
+    exit 0
+fi
+
+if [ -z "$major_version" ]; then
+    major_version="10"
+    note "could not read the release version; assuming ${major_version} for versioned feeds."
+fi
+
+# ---------------------------------------------------------------------------
+# Package helpers
+# ---------------------------------------------------------------------------
+
+# install <package-or-url>...
+install() {
+    priv "$package_manager" install -y "$@"
+}
+
+# install_group <description> <package>...
+# Installs as one transaction, then retries individually so a single package
+# that is unavailable on this release cannot cost the machine the rest.
+install_group() {
+    description="$1"
+    shift
+    [ "$#" -eq 0 ] && return 0
+
+    if install "$@"; then
+        ok "$description: $*"
+        return 0
+    fi
+
+    note "combined install for $description failed; retrying each package on its own."
+    for package in "$@"; do
+        if install "$package"; then
+            ok "$description: $package"
+        else
+            fail "could not install '$package' ($description)."
+        fi
+    done
+}
+
+# ---------------------------------------------------------------------------
+# Repositories
+# ---------------------------------------------------------------------------
+
+step "Enabling supplementary repositories"
+
+# Several of the packages below are built against content that only the
+# CodeReady Builder repository provides.
+if have subscription-manager; then
+    if priv subscription-manager repos \
+        --enable "codeready-builder-for-rhel-${major_version}-${architecture}-rpms" >/dev/null 2>&1; then
+        ok "enabled the CodeReady Builder repository."
+    else
+        note "could not enable the CodeReady Builder repository; some packages may be unavailable."
+    fi
+else
+    # Rebuilds carry the same content under their own repository name.
+    for repo in crb powertools; do
+        if priv "$package_manager" config-manager --set-enabled "$repo" >/dev/null 2>&1; then
+            ok "enabled the '${repo}' repository."
+            break
+        fi
+    done
+fi
+
+# This family ships no third-party content, so the community repository comes
+# from its own release package.
+if rpm -q epel-release >/dev/null 2>&1; then
+    ok "the EPEL repository is already configured."
+elif install "https://dl.fedoraproject.org/pub/epel/epel-release-latest-${major_version}.noarch.rpm"; then
+    ok "added the EPEL repository."
+else
+    fail "could not add the EPEL repository; packages that live there will be unavailable."
+fi
+
+step "Adding vendor package feeds"
+
+# Carries both PowerShell and the Azure CLI on this family.
+if rpm -q packages-microsoft-prod >/dev/null 2>&1; then
+    ok "the Microsoft package feed is already configured."
+else
+    priv rpm --import https://packages.microsoft.com/keys/microsoft.asc >/dev/null 2>&1 ||
+        note "could not import the Microsoft signing key; the feed may still supply it."
+    # Newer releases are signed with a separate key.
+    priv rpm --import https://packages.microsoft.com/keys/microsoft-2025.asc >/dev/null 2>&1 || true
+
+    if install "https://packages.microsoft.com/config/rhel/${major_version}/packages-microsoft-prod.rpm"; then
+        ok "added the Microsoft package feed."
+    else
+        fail "could not add the Microsoft package feed."
+    fi
+fi
+
+if [ -f /etc/yum.repos.d/gh-cli.repo ]; then
+    ok "the GitHub CLI package feed is already configured."
+else
+    # config-manager is a separate package on some releases and is what adds
+    # the feed, so it is requested before it is used.
+    install "dnf-command(config-manager)" >/dev/null 2>&1 || true
+    if priv "$package_manager" config-manager \
+        --add-repo https://cli.github.com/packages/rpm/gh-cli.repo >/dev/null 2>&1; then
+        ok "added the GitHub CLI package feed."
+    else
+        fail "could not add the GitHub CLI package feed."
+    fi
+fi
+
+# ---------------------------------------------------------------------------
+# Sandboxing runtimes
+# ---------------------------------------------------------------------------
+
+step "Installing unprivileged sandboxing prerequisites"
+install_group "unprivileged sandboxing" \
+    bubblewrap slirp4netns util-linux iproute iptables
+
+# The packet-filter command moved into a separate package on newer releases.
+if have iptables; then
+    ok "the packet filter command is present."
+else
+    install_group "the packet filter" iptables-nft
+fi
+
+step "Installing container prerequisites"
+install_group "containers" lxc lxc-templates dnsmasq
+
+# ---------------------------------------------------------------------------
+# Workload interpreters
+# ---------------------------------------------------------------------------
+
+step "Installing workload interpreters"
+install_group "interpreters" \
+    git openssl nodejs npm python3 python3-pip
+
+# Named separately so an unavailable release does not take the rest with it.
+install_group "the .NET SDK" dotnet-sdk-8.0
+install_group "PowerShell" powershell
+install_group "the Azure CLI" azure-cli
+install_group "the GitHub CLI" gh
+
+# ---------------------------------------------------------------------------
+# Kernel configuration
+# ---------------------------------------------------------------------------
+
+# Written to disk rather than only applied live, so the machine comes back the
+# same way after a reboot.
+persist_module() {
+    module="$1"
+    if echo "$module" | priv tee "/etc/modules-load.d/mxc-${module}.conf" >/dev/null; then
+        ok "${module} will load at boot."
+    else
+        fail "could not configure ${module} to load at boot."
+    fi
+    priv modprobe "$module" 2>/dev/null ||
+        note "could not load ${module} now; it may be built in or unavailable until reboot."
+}
+
+step "Configuring kernel modules"
+# Connection-state matching in the packet filter needs conntrack present.
+persist_module nf_conntrack
+# Bridged traffic is only visible to the packet filter with this loaded.
+persist_module br_netfilter
+
+step "Configuring kernel settings"
+sysctl_file="/etc/sysctl.d/99-mxc.conf"
+sysctl_lines=""
+
+# Unprivileged user namespaces are what unprivileged sandboxing is built on.
+if [ -e /proc/sys/user/max_user_namespaces ]; then
+    sysctl_lines="${sysctl_lines}user.max_user_namespaces = 28633
+"
+fi
+# Bridged traffic must traverse the packet filter for container network policy
+# to apply to it.
+sysctl_lines="${sysctl_lines}net.bridge.bridge-nf-call-iptables = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward = 1
+"
+
+if printf '%s' "$sysctl_lines" | priv tee "$sysctl_file" >/dev/null; then
+    ok "wrote ${sysctl_file}."
+else
+    fail "could not write ${sysctl_file}."
+fi
+
+if priv sysctl --system >/dev/null 2>&1; then
+    ok "applied the kernel settings."
+else
+    note "could not apply the kernel settings now; they will take effect after a reboot."
+fi
+
+# ---------------------------------------------------------------------------
+# Container bridge
+# ---------------------------------------------------------------------------
+
+step "Configuring the container bridge"
+if have systemctl; then
+    for unit in lxc-net lxc; do
+        if priv systemctl enable "$unit" >/dev/null 2>&1; then
+            ok "'${unit}' will start at boot."
+        else
+            note "could not enable '${unit}'."
+        fi
+    done
+    if priv systemctl restart lxc-net >/dev/null 2>&1; then
+        ok "started the container bridge."
+    else
+        note "could not start the container bridge; it should come up on the next boot."
+    fi
+else
+    note "no systemctl on this machine; skipping the bridge service."
+fi
+
+# The default container profile is not created by the package install on every
+# release, and container startup needs one.
+if [ ! -f /etc/lxc/default.conf ] && [ -d /etc/lxc ]; then
+    if printf 'lxc.net.0.type = veth\nlxc.net.0.link = lxcbr0\nlxc.net.0.flags = up\n' |
+        priv tee /etc/lxc/default.conf >/dev/null; then
+        ok "wrote the default container profile."
+    else
+        note "could not write the default container profile."
+    fi
+fi
+
+# ---------------------------------------------------------------------------
+# Inventory
+# ---------------------------------------------------------------------------
+
+step "Inventory"
+
+# name|candidates tried in order
+inventory="bwrap|bwrap
+slirp4netns|slirp4netns
+unshare|unshare
+nsenter|nsenter
+ip|ip
+iptables|iptables
+ip6tables|ip6tables
+lxc-start|lxc-start
+git|git
+openssl|openssl
+node|node,nodejs
+npm|npm
+npx|npx
+python|python3,python
+pip|pip3,pip
+dotnet|dotnet
+pwsh|pwsh
+az|az
+gh|gh"
+
+absent=""
+while IFS='|' read -r name candidates; do
+    [ -z "$name" ] && continue
+    resolved=""
+    old_ifs="$IFS"
+    IFS=','
+    for candidate in $candidates; do
+        if resolved="$(command -v "$candidate" 2>/dev/null)"; then
+            break
+        fi
+        resolved=""
+    done
+    IFS="$old_ifs"
+
+    if [ -n "$resolved" ]; then
+        printf '  %-14s %s\n' "$name" "$resolved"
+    else
+        printf '  %-14s ABSENT\n' "$name"
+        absent="${absent:+$absent }$name"
+    fi
+done <<EOF
+$inventory
+EOF
+
+# ---------------------------------------------------------------------------
+# Cleanup and summary
+# ---------------------------------------------------------------------------
+
+step "Cleaning up"
+priv "$package_manager" clean all >/dev/null 2>&1 && ok "cleared the package cache."
+
+step "Summary"
+if [ -n "$absent" ]; then
+    echo "Absent after installation: $absent"
+else
+    echo "Every expected program is present."
+fi
+
+if [ "${#notes[@]}" -gt 0 ]; then
+    echo ""
+    echo "Notes:"
+    for entry in "${notes[@]}"; do
+        echo "  - $entry"
+    done
+fi
+
+if [ "${#failures[@]}" -gt 0 ]; then
+    echo ""
+    echo "Failures:"
+    for entry in "${failures[@]}"; do
+        echo "  - $entry"
+    done
+else
+    echo ""
+    echo "No failures."
+fi
+
+echo ""
+echo "Setup finished."
+exit 0
diff --git a/scripts/ci/Setup.sh b/scripts/ci/Setup.sh
new file mode 100755
index 000000000..a35d6c00c
--- /dev/null
+++ b/scripts/ci/Setup.sh
@@ -0,0 +1,496 @@
+#!/usr/bin/env bash
+#
+# Installation script for MXC Linux machines (Debian / Ubuntu).
+#
+# Installs the sandboxing runtimes, kernel prerequisites and workload
+# interpreters an MXC machine is expected to provide, and persists the kernel
+# settings so they survive a reboot.
+#
+# Every step is best effort. Nothing here aborts the run and the script always
+# exits 0; what the machine actually ended up with is printed in the summary at
+# the end.
+
+set -o pipefail
+
+# Guarantee the exit status regardless of how the script leaves.
+trap 'exit 0' EXIT
+
+export DEBIAN_FRONTEND=noninteractive
+
+failures=()
+notes=()
+
+step() {
+    echo ""
+    echo "=== $* ==="
+}
+
+ok() {
+    echo "OK: $*"
+}
+
+note() {
+    echo "NOTE: $*"
+    notes+=("$*")
+}
+
+fail() {
+    echo "FAILED: $*"
+    failures+=("$*")
+}
+
+# Image builders run as root without sudo installed; interactive machines have
+# sudo and an unprivileged user. Resolve once and use the same wrapper for
+# every privileged call.
+if [ "$(id -u)" -eq 0 ]; then
+    priv() { "$@"; }
+elif command -v sudo >/dev/null 2>&1; then
+    priv() { sudo -n "$@"; }
+else
+    priv() { return 1; }
+fi
+
+have() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+# ---------------------------------------------------------------------------
+# Distribution identity
+# ---------------------------------------------------------------------------
+
+distro_id=""
+distro_version=""
+distro_codename=""
+if [ -r /etc/os-release ]; then
+    # shellcheck disable=SC1091
+    . /etc/os-release
+    distro_id="${ID:-}"
+    distro_version="${VERSION_ID:-}"
+    distro_codename="${VERSION_CODENAME:-}"
+fi
+if [ -z "$distro_codename" ] && have lsb_release; then
+    distro_codename="$(lsb_release -cs 2>/dev/null)"
+fi
+
+step "Machine"
+echo "Distribution : ${distro_id:-unknown} ${distro_version:-} (${distro_codename:-unknown codename})"
+echo "Kernel       : $(uname -r 2>/dev/null)"
+echo "Architecture : $(uname -m 2>/dev/null)"
+
+if ! have apt-get; then
+    fail "apt-get is not available; this script targets Debian and Ubuntu machines."
+    step "Summary"
+    echo "Nothing was installed."
+    exit 0
+fi
+
+if ! priv true 2>/dev/null; then
+    fail "cannot obtain root privileges; no packages can be installed."
+    step "Summary"
+    echo "Nothing was installed."
+    exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Package helpers
+# ---------------------------------------------------------------------------
+
+apt_refresh() {
+    # A broken third-party feed makes the whole refresh non-zero; the installs
+    # that follow still decide success against whatever indexes did update.
+    if priv apt-get update; then
+        return 0
+    fi
+    note "apt-get update reported repository errors; continuing with the available package indexes."
+    return 0
+}
+
+# install <package>...
+install() {
+    priv env DEBIAN_FRONTEND=noninteractive \
+        apt-get install -y --no-install-recommends "$@"
+}
+
+# install_group <description> <package>...
+# Installs as one transaction, then retries individually so a single package
+# that is unavailable on this release cannot cost the machine the rest.
+install_group() {
+    description="$1"
+    shift
+    [ "$#" -eq 0 ] && return 0
+
+    if install "$@"; then
+        ok "$description: $*"
+        return 0
+    fi
+
+    note "combined install for $description failed; retrying each package on its own."
+    for package in "$@"; do
+        if install "$package"; then
+            ok "$description: $package"
+        else
+            fail "could not install '$package' ($description)."
+        fi
+    done
+}
+
+# available <package>
+available() {
+    apt-cache show "$1" >/dev/null 2>&1
+}
+
+step "Refreshing package indexes"
+apt_refresh
+
+step "Installing feed prerequisites"
+install_group "feed prerequisites" ca-certificates curl gnupg apt-transport-https
+
+# ---------------------------------------------------------------------------
+# Vendor package feeds
+# ---------------------------------------------------------------------------
+
+# Carries PowerShell on this family. It does not carry the Azure CLI, which is
+# published separately below.
+add_microsoft_prod_feed() {
+    if [ -f /etc/apt/sources.list.d/microsoft-prod.list ] ||
+        [ -f /etc/apt/sources.list.d/microsoft-prod.sources ]; then
+        ok "the Microsoft package feed is already configured."
+        return 0
+    fi
+
+    if [ -z "$distro_id" ] || [ -z "$distro_version" ]; then
+        fail "could not read the distribution; skipping the Microsoft package feed."
+        return 1
+    fi
+
+    package="/tmp/packages-microsoft-prod.deb"
+    if ! curl -fsSL \
+        "https://packages.microsoft.com/config/${distro_id}/${distro_version}/packages-microsoft-prod.deb" \
+        -o "$package"; then
+        fail "no Microsoft package feed is published for ${distro_id} ${distro_version}."
+        return 1
+    fi
+
+    if ! priv dpkg -i "$package"; then
+        rm -f "$package"
+        fail "could not install the Microsoft package feed."
+        return 1
+    fi
+
+    rm -f "$package"
+    apt_refresh
+    ok "added the Microsoft package feed."
+}
+
+# The Azure CLI has a repository of its own, keyed by distribution codename
+# rather than version. It lags new releases, so the codename this machine
+# reports may not be published yet.
+add_azure_cli_feed() {
+    if [ -f /etc/apt/sources.list.d/azure-cli.list ] ||
+        [ -f /etc/apt/sources.list.d/azure-cli.sources ]; then
+        ok "the Azure CLI package feed is already configured."
+        return 0
+    fi
+
+    if [ -z "$distro_codename" ]; then
+        fail "could not read the distribution codename; skipping the Azure CLI feed."
+        return 1
+    fi
+
+    candidates="$distro_codename"
+    case "$distro_id" in
+        debian) [ "$distro_codename" = "bookworm" ] || candidates="$candidates bookworm" ;;
+        ubuntu) [ "$distro_codename" = "noble" ] || candidates="$candidates noble" ;;
+    esac
+
+    # Probe before writing. An unpublished suite in a source list makes every
+    # later refresh fail, which would cost this machine the packages that were
+    # otherwise going to install.
+    suite=""
+    for candidate in $candidates; do
+        if curl -fsL --head -o /dev/null \
+            "https://packages.microsoft.com/repos/azure-cli/dists/${candidate}/Release"; then
+            suite="$candidate"
+            break
+        fi
+        echo "The Azure CLI feed publishes no '$candidate' suite."
+    done
+
+    if [ -z "$suite" ]; then
+        fail "the Azure CLI feed publishes no suite usable on '${distro_codename}'."
+        return 1
+    fi
+    if [ "$suite" != "$distro_codename" ]; then
+        note "the Azure CLI feed has no '${distro_codename}' suite; using '${suite}'."
+    fi
+
+    keyring="/etc/apt/keyrings/microsoft.gpg"
+    if ! priv install -d -m 0755 /etc/apt/keyrings; then
+        fail "could not create the apt keyring directory."
+        return 1
+    fi
+    if ! curl -fsSL https://packages.microsoft.com/keys/microsoft.asc |
+        gpg --dearmor | priv dd of="$keyring" status=none; then
+        fail "could not install the Microsoft signing key."
+        return 1
+    fi
+    priv chmod go+r "$keyring"
+
+    if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=${keyring}] https://packages.microsoft.com/repos/azure-cli/ ${suite} main" |
+        priv tee /etc/apt/sources.list.d/azure-cli.list >/dev/null; then
+        fail "could not write the Azure CLI package source."
+        return 1
+    fi
+
+    apt_refresh
+    ok "added the Azure CLI package feed (${suite})."
+}
+
+add_github_cli_feed() {
+    if [ -f /etc/apt/sources.list.d/github-cli.list ]; then
+        ok "the GitHub CLI package feed is already configured."
+        return 0
+    fi
+
+    keyring="/usr/share/keyrings/githubcli-archive-keyring.gpg"
+    if ! curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg |
+        priv dd of="$keyring" status=none; then
+        fail "could not download the GitHub CLI signing key."
+        return 1
+    fi
+    priv chmod go+r "$keyring"
+
+    if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=${keyring}] https://cli.github.com/packages stable main" |
+        priv tee /etc/apt/sources.list.d/github-cli.list >/dev/null; then
+        fail "could not write the GitHub CLI package source."
+        return 1
+    fi
+
+    apt_refresh
+    ok "added the GitHub CLI package feed."
+}
+
+step "Adding vendor package feeds"
+add_microsoft_prod_feed
+add_azure_cli_feed
+add_github_cli_feed
+
+# ---------------------------------------------------------------------------
+# Sandboxing runtimes
+# ---------------------------------------------------------------------------
+
+step "Installing unprivileged sandboxing prerequisites"
+install_group "unprivileged sandboxing" \
+    bubblewrap slirp4netns util-linux iproute2 iptables
+
+step "Installing container prerequisites"
+container_packages="lxc dnsmasq-base bridge-utils iptables"
+# Debian folded the tools into the lxc package; Ubuntu still ships them apart.
+if available lxc-utils; then
+    container_packages="$container_packages lxc-utils"
+fi
+if available lxc-templates; then
+    container_packages="$container_packages lxc-templates"
+fi
+# shellcheck disable=SC2086
+install_group "containers" $container_packages
+
+# ---------------------------------------------------------------------------
+# Workload interpreters
+# ---------------------------------------------------------------------------
+
+step "Installing workload interpreters"
+install_group "interpreters" \
+    git openssl nodejs npm python3 python3-pip
+
+# Named separately so an unavailable release does not take the rest with it.
+install_group "the .NET SDK" dotnet-sdk-8.0
+install_group "PowerShell" powershell
+install_group "the Azure CLI" azure-cli
+install_group "the GitHub CLI" gh
+
+# ---------------------------------------------------------------------------
+# Kernel configuration
+# ---------------------------------------------------------------------------
+
+# Written to disk rather than only applied live, so the machine comes back the
+# same way after a reboot.
+persist_module() {
+    module="$1"
+    if echo "$module" | priv tee "/etc/modules-load.d/mxc-${module}.conf" >/dev/null; then
+        ok "${module} will load at boot."
+    else
+        fail "could not configure ${module} to load at boot."
+    fi
+    priv modprobe "$module" 2>/dev/null ||
+        note "could not load ${module} now; it may be built in or unavailable until reboot."
+}
+
+step "Configuring kernel modules"
+# Connection-state matching in the packet filter needs conntrack present.
+persist_module nf_conntrack
+# Bridged traffic is only visible to the packet filter with this loaded.
+persist_module br_netfilter
+
+step "Configuring kernel settings"
+sysctl_file="/etc/sysctl.d/99-mxc.conf"
+sysctl_lines=""
+
+# Unprivileged user namespaces are what unprivileged sandboxing is built on.
+# Recent releases restrict them by default.
+if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then
+    sysctl_lines="${sysctl_lines}kernel.apparmor_restrict_unprivileged_userns = 0
+"
+fi
+if [ -e /proc/sys/kernel/unprivileged_userns_clone ]; then
+    sysctl_lines="${sysctl_lines}kernel.unprivileged_userns_clone = 1
+"
+fi
+# Bridged traffic must traverse the packet filter for container network policy
+# to apply to it.
+sysctl_lines="${sysctl_lines}net.bridge.bridge-nf-call-iptables = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward = 1
+"
+
+if printf '%s' "$sysctl_lines" | priv tee "$sysctl_file" >/dev/null; then
+    ok "wrote ${sysctl_file}."
+else
+    fail "could not write ${sysctl_file}."
+fi
+
+if priv sysctl --system >/dev/null 2>&1; then
+    ok "applied the kernel settings."
+else
+    note "could not apply the kernel settings now; they will take effect after a reboot."
+fi
+
+# ---------------------------------------------------------------------------
+# Container bridge
+# ---------------------------------------------------------------------------
+
+step "Configuring the container bridge"
+# Debian ships the bridge disabled; Ubuntu enables it.
+if [ -f /etc/default/lxc-net ]; then
+    if priv sed -i 's/^\s*#\?\s*USE_LXC_BRIDGE\s*=.*/USE_LXC_BRIDGE="true"/' /etc/default/lxc-net &&
+        grep -q 'USE_LXC_BRIDGE="true"' /etc/default/lxc-net; then
+        ok "enabled the container bridge in /etc/default/lxc-net."
+    else
+        note "could not confirm the bridge setting in /etc/default/lxc-net."
+    fi
+else
+    note "/etc/default/lxc-net is absent; leaving the bridge at its packaged default."
+fi
+
+if have systemctl; then
+    if priv systemctl enable lxc-net >/dev/null 2>&1; then
+        ok "the container bridge will start at boot."
+    else
+        note "could not enable the container bridge service."
+    fi
+    if priv systemctl restart lxc-net >/dev/null 2>&1; then
+        ok "started the container bridge."
+    else
+        note "could not start the container bridge; it should come up on the next boot."
+    fi
+else
+    note "no systemctl on this machine; skipping the bridge service."
+fi
+
+# Package installs do not always activate the confinement profiles the
+# container tooling relies on.
+if have apparmor_parser && [ -d /etc/apparmor.d ]; then
+    if priv apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null; then
+        ok "reloaded the container confinement profiles."
+    else
+        note "could not reload the container confinement profiles."
+    fi
+fi
+
+# ---------------------------------------------------------------------------
+# Inventory
+# ---------------------------------------------------------------------------
+
+step "Inventory"
+
+# name|candidates tried in order
+inventory="bwrap|bwrap
+slirp4netns|slirp4netns
+unshare|unshare
+nsenter|nsenter
+ip|ip
+iptables|iptables
+ip6tables|ip6tables
+lxc-start|lxc-start
+git|git
+openssl|openssl
+node|node,nodejs
+npm|npm
+npx|npx
+python|python3,python
+pip|pip3,pip
+dotnet|dotnet
+pwsh|pwsh
+az|az
+gh|gh"
+
+absent=""
+while IFS='|' read -r name candidates; do
+    [ -z "$name" ] && continue
+    resolved=""
+    old_ifs="$IFS"
+    IFS=','
+    for candidate in $candidates; do
+        if resolved="$(command -v "$candidate" 2>/dev/null)"; then
+            break
+        fi
+        resolved=""
+    done
+    IFS="$old_ifs"
+
+    if [ -n "$resolved" ]; then
+        printf '  %-14s %s\n' "$name" "$resolved"
+    else
+        printf '  %-14s ABSENT\n' "$name"
+        absent="${absent:+$absent }$name"
+    fi
+done <<EOF
+$inventory
+EOF
+
+# ---------------------------------------------------------------------------
+# Cleanup and summary
+# ---------------------------------------------------------------------------
+
+step "Cleaning up"
+priv apt-get clean >/dev/null 2>&1 && ok "cleared the package cache."
+
+step "Summary"
+if [ -n "$absent" ]; then
+    echo "Absent after installation: $absent"
+else
+    echo "Every expected program is present."
+fi
+
+if [ "${#notes[@]}" -gt 0 ]; then
+    echo ""
+    echo "Notes:"
+    for entry in "${notes[@]}"; do
+        echo "  - $entry"
+    done
+fi
+
+if [ "${#failures[@]}" -gt 0 ]; then
+    echo ""
+    echo "Failures:"
+    for entry in "${failures[@]}"; do
+        echo "  - $entry"
+    done
+else
+    echo ""
+    echo "No failures."
+fi
+
+echo ""
+echo "Setup finished."
+exit 0
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index d2f00a454..9f6394907 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -389,6 +389,21 @@
       }
     ],
     "weekly": [],
-    "enabled": []
+    "enabled": [
+      {
+        "os": "rhel-10",
+        "backends": [
+          "bubblewrap",
+          "lxc"
+        ]
+      },
+      {
+        "os": "debian-13",
+        "backends": [
+          "bubblewrap",
+          "lxc"
+        ]
+      }
+    ]
   }
 }

From e8a7b340c221adaf5371f2c8917e7d1f713370f8 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Mon, 31 Aug 2026 14:16:56 -0700
Subject: [PATCH 35/44] testing macos image pre-provision status

---
 .github/copilot-instructions.md               |  31 +-
 .../Validation.Infrastructure.Tests.yml       |   2 +-
 .../workflows/Validation.Tests.Scheduled.yml  |   2 +-
 docs/ci-validation-infrastructure.md          | 191 +++----
 scripts/ci/Setup.ps1                          | 534 ++++++++++++++++++
 scripts/ci/prepare-windows-host.ps1           | 127 ++++-
 scripts/ci/validation-test-matrix.json        |  12 +-
 7 files changed, 756 insertions(+), 143 deletions(-)
 create mode 100644 scripts/ci/Setup.ps1

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index cb7e3dc09..c9fe27037 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -96,10 +96,13 @@ verify the host's workload interpreters and CLIs (`pwsh`, `git`, `node`, `npm`,
 `winget`, `scoop`, and `choco` on Windows only, and `brew` on macOS only) —
 Windows in `Assert-WorkloadInterpreters`, Linux and macOS in a deliberately
 duplicated bash-3.2-compatible `assert_workload_interpreters` function so each
-platform's list can diverge. macOS verifies only; Linux first runs
-`install_workload_interpreters`, which installs whatever the stock distribution
-image lacks (distribution repositories, plus Microsoft's feeds for `pwsh`/`az`
-and GitHub's for `gh`). Microsoft splits those two: `packages-microsoft-prod`
+platform's list can diverge. Every pool runs an image provisioned ahead of time
+by `scripts/ci/Setup.sh` / `Setup-rhel.sh` (Linux) and `scripts/ci/Setup.ps1`
+(Windows), so the whole list is normally present before a job starts. macOS
+verifies only; Linux first runs `install_workload_interpreters`, which re-runs
+the install as a top-up and is a no-op on a correctly built image (distribution
+repositories, plus Microsoft's feeds for `pwsh`/`az` and GitHub's for `gh`).
+Microsoft splits those two: `packages-microsoft-prod`
 carries `powershell` everywhere and `azure-cli` on RPM only, while on apt the
 Azure CLI has its own codename-keyed feed added by `add_azure_cli_apt_feed`,
 which probes for the suite before writing it and falls back to the newest
@@ -108,13 +111,27 @@ best-effort and can never fail the job —
 the inventory that follows reports the outcome. Its package-manager access goes
 through `resolve_package_manager` and `install_packages`, the same helpers the
 backend prerequisite installers use, so a new distribution family is one arm in
-`install_packages` rather than a branch in every installer. Windows provisions
-one entry, also best-effort and also ahead of the inventory: `Repair-Winget`
+`install_packages` rather than a branch in every installer. `Setup.ps1` installs
+no packaged application and never uses winget, because neither is available
+during image provisioning; it takes only `-Architecture` (`x64`/`arm64`, which
+selects the `gh` MSI — the Azure CLI has no ARM64 build and is used emulated)
+and `-ScoopRoot`. Its scope is `choco`, `scoop`, `az`, `gh` and `nuget`: the
+language runtimes (`dotnet`, `node`, `python`, `pwsh`, `git`) arrive from
+separate image artifacts, and `Setup.ps1` only inventories them. Windows
+therefore provisions
+two entries at job time, also best-effort and also ahead of the inventory.
+`Repair-Winget`
 re-registers the App Installer package (`Add-AppxPackage -RegisterByFamilyName`)
 when `winget` resolves on `PATH` but fails to run, the symptom of a package the
 image shipped but never registered for the account the job runs as. It decides
 by invoking `winget --version`, not by resolving the command, since a resolvable
-alias is the broken case.
+alias is the broken case. `Install-PackagedTooling` then installs `winapp` and
+`openssl`, the two interpreters that cannot be baked into an image at all
+because both ship only as packaged applications; `winapp` is requested as
+`--installer-type zip` so the portable build lands on `PATH` instead of behind a
+`WindowsApps` alias the inventory's store-alias filter would reject, and
+`openssl`'s `bin` directory is appended to `PATH` afterwards because its
+installer publishes none.
 
 **Test dispatch** goes through `scripts/ci/run_backend_validation_tests.ps1`
 (Windows) and `scripts/ci/run_backend_validation_tests.sh` (Linux/macOS), which map
diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
index 83dd18499..87c8b937b 100644
--- a/.github/workflows/Validation.Infrastructure.Tests.yml
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -33,4 +33,4 @@ jobs:
     needs: [windows, linux, macos]
     uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
     with:
-      plan: nightly
\ No newline at end of file
+      plan: enabled
\ No newline at end of file
diff --git a/.github/workflows/Validation.Tests.Scheduled.yml b/.github/workflows/Validation.Tests.Scheduled.yml
index ae67b110b..9bc3bf9ad 100644
--- a/.github/workflows/Validation.Tests.Scheduled.yml
+++ b/.github/workflows/Validation.Tests.Scheduled.yml
@@ -51,4 +51,4 @@ jobs:
     if: (github.event_name == 'schedule' && github.event.schedule == '0 8 * * 0') || inputs.plan == 'weekly'
     uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
     with:
-      plan: enabled
\ No newline at end of file
+      plan: weekly
\ No newline at end of file
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 611d5b88b..94bdbf7fd 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -29,9 +29,11 @@ the individual local test scripts are documented in
 | `.github/workflows/Validation.Tests.Matrix.Job.yml` | `workflow_call`-only. Resolves the plan and runs the per-family test jobs. |
 | `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers, job staggering. |
 | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. |
-| `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions, plus the `winget` repair. |
-| `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware), plus the workload-interpreter install pass. |
+| `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions, plus the `winget` repair and the packaged-tooling install. |
+| `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware), plus the workload-interpreter top-up pass. |
 | `scripts/ci/prepare-macos-host.sh` | Per-backend macOS host preparation / prerequisite assertions. |
+| `scripts/ci/Setup.sh` / `scripts/ci/Setup-rhel.sh` | Image-provisioning scripts. Run when the Linux images are built, not per job. |
+| `scripts/ci/Setup.ps1` | Image-provisioning script. Runs when the Windows image is built, not per job. |
 | `scripts/ci/run_backend_validation_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
 | `scripts/ci/run_backend_validation_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
 
@@ -264,10 +266,11 @@ Windows optional features are **verified, never enabled**: turning one on needs
 reboot the job cannot take, so a mis-imaged pool fails here with a pointed
 message instead of surfacing later as an opaque backend error.
 
-The script does provision one thing, for every backend rather than a particular
+The script does provision two things, for every backend rather than a particular
 one: `Repair-Winget` re-registers the App Installer package when `winget` is on
-`PATH` but cannot run
-([below](#verified-everywhere-installed-on-linux-repaired-on-windows)).
+`PATH` but cannot run, and `Install-PackagedTooling` then installs `winapp` and
+`openssl`
+([below](#who-installs-what)).
 
 `prepare-linux-host.sh`:
 
@@ -303,123 +306,69 @@ Every one of the three scripts also takes the workload-interpreter inventory
 
 Some suites do not just exercise MXC's primitives — they run *real programs*
 inside the sandbox and assert on what those programs produce. Each preparation
-script takes that host-side inventory up front, so a missing tool is reported
-once as a preparation result rather than repeatedly as a confusing mid-suite
-failure. On Linux the inventory is preceded by an install pass, and on Windows
-by a narrow `winget` repair
-([below](#verified-everywhere-installed-on-linux-repaired-on-windows)).
-
-The list is `pwsh`, `git`, `node`, `npm`, `npx`, `python`, `pip`, `dotnet`, `az`,
-`gh`, and `openssl` on every OS, plus `nuget`, `winapp` (the Windows App
-Development CLI), `winget`, `scoop`, and `choco` on Windows only, and `brew` on
-macOS only. The Windows five have no Unix counterpart — except NuGet, which Unix
-reaches through `dotnet nuget` rather than a standalone binary, so checking for
-one there would warn forever.
-
-The check is **suite-agnostic by design**, and runs for *every* backend rather
-than only the ones whose suites happen to need it today. It describes what a
-validation *host* is expected to provide, not what any one suite consumes, so
-any current or future suite that shells out to these programs is served by the
-same list. `T3-Workloads.ps1` is simply the first caller; wiring up the next one
-needs no change here.
-
-Each preparation script carries its own copy — `Assert-WorkloadInterpreters` in
+script inventories those programs on the host up front, so a missing one is
+reported once, as a preparation result, rather than repeatedly as a confusing
+mid-suite failure.
+
+#### The list
+
+| Interpreter | Platforms | Version | Notes |
+|-------------|-----------|---------|-------|
+| `pwsh` | all | Latest 7.x | The only entry whose absence fails a job, and only on Windows. |
+| `git` | all | Latest | |
+| `node`, `npm`, `npx` | all | 24.x | `npm` and `npx` arrive with Node. |
+| `python`, `pip` | all | Latest | Windows tries `python` first, Unix `python3`. |
+| `dotnet` | all | Latest LTS | |
+| `az` | all | Latest | No ARM64 Windows build exists; ARM64 images get the x64 one under emulation. |
+| `gh` | all | Latest | |
+| `openssl` | all | Latest | On Windows, installed per job — see below. |
+| `nuget` | Windows | Latest | Unix reaches NuGet through `dotnet nuget`, so looking for a binary there would warn forever. |
+| `winapp` | Windows | Latest | The Windows App Development CLI. Installed per job — see below. |
+| `winget` | Windows | Image | The only entry allowed to resolve inside `WindowsApps`. |
+| `scoop`, `choco` | Windows | Latest | |
+| `brew` | macOS | Latest | |
+
+Nothing is pinned: every entry is whatever was current when the image was built,
+within the constraint in the Version column. Only Node is held to a major
+version, because the SDK targets it.
+
+The list is **suite-agnostic by design** and is checked for every backend, not
+only the ones whose suites need it today: it describes what a validation *host*
+provides, not what any one suite consumes. `T3-Workloads.ps1` is simply the
+first caller, and wiring up the next one needs no change here.
+
+Each script carries its own copy — `Assert-WorkloadInterpreters` in
 `prepare-windows-host.ps1`, `assert_workload_interpreters` in the two `.sh`
-scripts.
-
-Each inventory entry carries:
-
-- the command names to try, in order. Resolution mirrors what the suites
-  themselves do: on Windows `python` is tried before `python3`, while on Unix
-  `python3` is tried first.
-- on Windows, whether a match under `WindowsApps` counts. By default it does
-  not: that is usually a Microsoft Store `AppExecutionAlias` stub, a 0-byte
-  redirect that opens the Store rather than running. But `WindowsApps` is also
-  how App Installer legitimately ships `winget`, and a working alias is
-  indistinguishable from a stub by path or size — both are 0-byte reparse
-  points — so entries delivered that way set `AllowStoreAlias` and opt out.
-  Without it the check reports an installed `winget` as missing.
-- whether an absence fails the job or only warns. On Windows only `pwsh` is
-  required; its absence means a mis-imaged pool. Everything else warns, because
-  suites are expected to report their dependent cases as skipped rather than
-  failing. Nothing is required on Unix yet — no Unix suite drives these
-  interpreters — so that check currently runs purely as host inventory.
-- the image-level fix, quoted in whichever message is emitted.
-
-Because a warning is deliberately not a failure, an absent optional interpreter
-silently shrinks coverage while the job still shows green. GitHub's pass/fail
-icon cannot express "passed, but with less coverage than yesterday" — a future
-test-analysis portal is intended to surface skip counts so that erosion is
-visible.
-
-#### Verified everywhere, installed on Linux, repaired on Windows
-
-On Windows and macOS the tools are, with one exception, **verified, never
-installed**, for the same reason the optional features are: provisioning a
-toolchain mid-run would mask the image drift the check exists to surface.
-Neither platform needs it anyway — Windows runs on a 1ES image whose contents we
-control, and the GitHub-hosted macOS runners already ship all twelve.
-
-The Windows exception is `winget`, which `Repair-Winget` provisions immediately
-before the inventory. It is the narrowest form of install there is: it downloads
-nothing and adds nothing to the image, it only re-registers for the running
-account a package the image already shipped. That is worth doing because the
-failure it fixes is not image drift at all — App Installer is routinely present
-but unregistered for the account the job runs as, which leaves an
-`AppExecutionAlias` that resolves on `PATH` and then fails with "The file cannot
-be accessed by the system". Reporting a tool the image *does* carry as missing
-would surface nothing anyone could act on.
-
-The repair is written to be indistinguishable from a no-op when it is not
-needed. It decides by *running* `winget --version` rather than by resolving the
-command, because a resolvable alias is precisely the broken case; an operational
-host returns before touching Appx at all. Like the Linux install pass, nothing
-it does can fail the job — an absent package, an unreachable `Appx` module, and
-a failed registration all warn and fall through to the inventory, which then
-reports `winget` in the usual way.
-
-Linux is the exception. `prepare-linux-host.sh` runs
-`install_workload_interpreters` immediately before taking the inventory,
-installing whatever the image did not already provide. The Linux pools run stock
-distribution images that MXC does not bake, so there is no curated image to
-drift *from*; refusing to install would not surface a provisioning mistake, it
-would only cost coverage. Most of the list comes from the distribution's own
-repositories, `pwsh` and `az` from Microsoft's feeds and `gh` from GitHub's since
-no distribution carries them, and `npx` from nowhere at all — it arrives with
-`npm`.
-
-Microsoft publishes those two tools to *different* feeds, and only on the RPM
-side do they coincide. `packages-microsoft-prod` carries `powershell` on both
-families and `azure-cli` on the RPM side only; on apt the Azure CLI has a
-repository of its own, keyed by distribution codename rather than version. Asking
-the prod feed for `azure-cli` on a Debian-family host resolves nothing and fails
-with `E: Unable to locate package azure-cli`, so `add_azure_cli_apt_feed` adds
-that second feed separately.
-
-That feed lags new distribution releases, so a host's own codename may not be
-published yet — Debian 13 (`trixie`) is not. The function probes for the suite
-before writing it and falls back to the newest suite the vendor does publish for
-the family (`bookworm` for Debian, `noble` for Ubuntu), warning when it
-substitutes. The probe is what makes the fallback safe to skip entirely when
-nothing matches: an unpublished suite written into a source list makes *every*
-later `apt-get update` fail, which would cost the host the packages that were
-otherwise going to install.
-
-Two properties make that safe to run on every job:
-
-- **Nothing it does can fail the job.** Every step warns and continues, and a
-  host with no recognized package manager is skipped outright. The inventory
-  immediately afterwards is what reports the outcome, so a tool that could not
-  be installed stays visible instead of becoming a silent absence.
-- **A failed batch degrades to individual installs.** apt and dnf abort the
-  *entire* transaction over a single unavailable package, so one name missing on
-  one distribution would otherwise cost that host every other interpreter as
-  well. After a batch failure each package is retried on its own.
-
-A host that already has everything short-circuits before touching the package
-manager, so the common case costs one `command -v` per entry. Vendor feeds are
-added at most once and a failure is remembered, so the second tool wanting a
-broken feed does not retry it.
+scripts — so the platform lists can diverge.
+
+#### Missing means a warning, not a failure
+
+Only `pwsh` on Windows is required; everything else warns, because suites are
+expected to report their dependent cases as skipped rather than failing. Nothing
+is required on Unix yet.
+
+The cost is that an absent interpreter silently shrinks coverage while the job
+still shows green. GitHub's pass/fail icon cannot express "passed, but with less
+coverage than yesterday" — a future test-analysis portal is intended to surface
+skip counts so that erosion is visible.
+
+On Windows a command resolving under `WindowsApps` does not count. That is
+normally a Microsoft Store `AppExecutionAlias` stub: a 0-byte redirect that
+opens the Store rather than running. App Installer legitimately ships `winget`
+that way, and a working alias is indistinguishable from a stub — both are 0-byte
+reparse points — so `winget` alone opts out of the filter.
+
+#### Who installs what
+
+Every pool runs images pre-provisioned with programs installed by
+`ubuntu-debian-provision.sh` / `rhel-provision.sh` for Linux and 
+`windows-provision.ps1` for Windows. These scripts are located in the
+`validation-provision-artifacts` branch in the ADO repo. On Windows that
+script covers `choco`, `scoop`, `az`, `gh` and `nuget`; the language runtimes
+(`dotnet`, `node`, `python`, `pwsh`, `git`) come from separate image artifacts.
+
+During the start of a job, the installed programs are inventoried. Windows also
+installs OpenSSL and WinApp via the repaired WinGet. 
 
 ## Log collection
 
diff --git a/scripts/ci/Setup.ps1 b/scripts/ci/Setup.ps1
new file mode 100644
index 000000000..902b6376c
--- /dev/null
+++ b/scripts/ci/Setup.ps1
@@ -0,0 +1,534 @@
+#Requires -Version 5.1
+
+<#
+.SYNOPSIS
+    Installation script for MXC Windows machines.
+
+.DESCRIPTION
+    Installs the command-line tooling an MXC machine is expected to provide:
+    Chocolatey, Scoop, the Azure CLI, the GitHub CLI and the NuGet CLI.
+
+    Everything is installed from an official installer or install script that
+    is downloaded directly. The Windows Package Manager is deliberately not
+    used and no packaged application is registered, because neither is
+    available while an image is being provisioned. Tooling that is only
+    published as a packaged application is installed later, when the machine
+    is running, rather than here.
+
+    The .NET SDK, Node.js, Python, PowerShell 7 and Git arrive with the image
+    from other provisioning artifacts rather than being installed here, but
+    they are still inventoried at the end so the image is reported on as a
+    whole.
+
+    Every step is best effort. Nothing here aborts the run and the script always
+    exits 0; what the machine actually ended up with is printed in the summary
+    at the end.
+
+    Installs are machine-scoped and every directory this script adds to PATH is
+    added to the machine PATH, because the account that runs workloads later is
+    usually not the account that runs this script.
+
+.EXAMPLE
+    ./Setup.ps1
+
+.EXAMPLE
+    ./Setup.ps1 -Architecture arm64
+#>
+
+[CmdletBinding()]
+param(
+    # The machine's processor architecture, which selects the installer variant
+    # to download. Defaults to what this machine reports.
+    [ValidateSet('x64', 'arm64')]
+    [string]$Architecture = $(if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'x64' }),
+
+    # Where Scoop is installed. Deliberately outside a user profile so every
+    # account on the machine can reach it.
+    [string]$ScoopRoot = 'C:\Scoop'
+)
+
+Set-StrictMode -Off
+$ErrorActionPreference = 'Continue'
+$ProgressPreference = 'SilentlyContinue'
+
+# Provisioning is best effort: an unexpected terminating error reports what went
+# wrong and still leaves the image in a usable state, so the exit code is always 0.
+trap {
+    Write-Host ''
+    Write-Host "Setup stopped early: $($_.Exception.Message)"
+    Write-Host "  at $($_.InvocationInfo.ScriptLineNumber): $($_.InvocationInfo.Line.Trim())"
+    Write-Host 'Setup finished.'
+    exit 0
+}
+
+# TLS 1.2 is not the default on older hosts and every download below is HTTPS.
+try {
+    [Net.ServicePointManager]::SecurityProtocol =
+        [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
+} catch {
+    # A host that pins its own protocols is left alone.
+}
+
+$script:Notes = New-Object System.Collections.Generic.List[string]
+$script:Failures = New-Object System.Collections.Generic.List[string]
+$script:Scratch = Join-Path ([IO.Path]::GetTempPath()) ("mxc-setup-" + [Guid]::NewGuid().ToString('N').Substring(0, 8))
+
+function Write-Step {
+    param([Parameter(Mandatory)][string]$Message)
+    Write-Host ''
+    Write-Host "=== $Message ==="
+}
+
+function Write-Ok {
+    param([Parameter(Mandatory)][string]$Message)
+    Write-Host "OK: $Message"
+}
+
+function Write-Note {
+    param([Parameter(Mandatory)][string]$Message)
+    # An exception message can span lines; the summary reads better on one.
+    $flat = ($Message -replace '\s+', ' ').Trim()
+    Write-Host "NOTE: $flat"
+    $script:Notes.Add($flat)
+}
+
+function Write-Failure {
+    param([Parameter(Mandatory)][string]$Message)
+    $flat = ($Message -replace '\s+', ' ').Trim()
+    Write-Host "FAILED: $flat"
+    $script:Failures.Add($flat)
+}
+
+# ---------------------------------------------------------------------------
+# Environment helpers
+# ---------------------------------------------------------------------------
+
+# Installers edit the stored PATH, not this process's copy of it. Re-reading it
+# is what lets a tool installed a moment ago be found by the step after it.
+function Update-ProcessPath {
+    $parts = @()
+    foreach ($scope in 'Machine', 'User') {
+        $value = [Environment]::GetEnvironmentVariable('Path', $scope)
+        if ($value) { $parts += $value }
+    }
+    if ($parts.Count -gt 0) {
+        $env:Path = ($parts -join ';')
+    }
+}
+
+# Adds a directory to the machine PATH if it exists and is not already there.
+function Add-MachinePath {
+    param([Parameter(Mandatory)][string]$Directory)
+
+    if (-not (Test-Path -LiteralPath $Directory)) {
+        return $false
+    }
+
+    $normalized = $Directory.TrimEnd('\')
+    $current = [Environment]::GetEnvironmentVariable('Path', 'Machine')
+    $entries = @()
+    if ($current) {
+        $entries = $current -split ';' | Where-Object { $_ -and $_.Trim() }
+    }
+
+    foreach ($entry in $entries) {
+        if ($entry.TrimEnd('\') -ieq $normalized) {
+            Update-ProcessPath
+            return $true
+        }
+    }
+
+    try {
+        $updated = (@($entries) + $normalized) -join ';'
+        [Environment]::SetEnvironmentVariable('Path', $updated, 'Machine')
+        Update-ProcessPath
+        Write-Ok "added '$normalized' to the machine PATH."
+        return $true
+    } catch {
+        Write-Failure "could not add '$normalized' to the machine PATH: $($_.Exception.Message.Trim())"
+        return $false
+    }
+}
+
+# Resolves a command the same way the workload inventory does. A command that
+# resolves inside WindowsApps is normally an execution-alias stub that opens the
+# Store rather than running, so it does not count as present unless the tool is
+# known to ship that way.
+function Resolve-Tool {
+    param(
+        [Parameter(Mandatory)][string[]]$Candidates,
+        [switch]$AllowStoreAlias
+    )
+
+    foreach ($candidate in $Candidates) {
+        $found = Get-Command $candidate -ErrorAction SilentlyContinue |
+            Select-Object -First 1
+        if ($found -and ($AllowStoreAlias -or $found.Source -notlike '*\WindowsApps\*')) {
+            return $found.Source
+        }
+    }
+    return $null
+}
+
+function Test-Tool {
+    param([Parameter(Mandatory)][string[]]$Candidates)
+    return [bool](Resolve-Tool -Candidates $Candidates)
+}
+
+# Runs a program and reports only whether it succeeded, keeping installer chatter
+# out of the transcript unless something goes wrong.
+function Invoke-Program {
+    param(
+        [Parameter(Mandatory)][string]$FilePath,
+        [string[]]$Arguments = @(),
+        [int[]]$SuccessCodes = @(0)
+    )
+
+    try {
+        $output = & $FilePath @Arguments 2>&1
+        $code = $LASTEXITCODE
+        if ($null -eq $code) { $code = 0 }
+        if ($SuccessCodes -contains $code) {
+            return $true
+        }
+        $detail = ($output | Select-Object -Last 3 | Out-String).Trim()
+        if ($detail) {
+            Write-Host "  $($detail -replace "`r?`n", "`n  ")"
+        }
+        Write-Host "  exit code $code"
+        return $false
+    } catch {
+        # A native launch failure embeds a position trace after the first line.
+        $reason = ($_.Exception.Message -split "`r?`n" | Select-Object -First 1).Trim()
+        Write-Host "  $reason"
+        return $false
+    }
+}
+
+function Get-Download {
+    param(
+        [Parameter(Mandatory)][string]$Uri,
+        [Parameter(Mandatory)][string]$OutFile
+    )
+
+    try {
+        if (-not (Test-Path -LiteralPath $script:Scratch)) {
+            New-Item -ItemType Directory -Path $script:Scratch -Force | Out-Null
+        }
+        Invoke-WebRequest -Uri $Uri -OutFile $OutFile -UseBasicParsing -TimeoutSec 600
+        return (Test-Path -LiteralPath $OutFile)
+    } catch {
+        Write-Host "  $($_.Exception.Message.Trim())"
+        return $false
+    }
+}
+
+function Install-Msi {
+    param(
+        [Parameter(Mandatory)][string]$Uri,
+        [Parameter(Mandatory)][string]$Name
+    )
+
+    $package = Join-Path $script:Scratch $Name
+    if (-not (Get-Download -Uri $Uri -OutFile $package)) {
+        return $false
+    }
+    return (Invoke-Program -FilePath 'msiexec.exe' -Arguments @('/i', $package, '/quiet', '/norestart') -SuccessCodes @(0, 3010))
+}
+
+# ---------------------------------------------------------------------------
+# Machine
+# ---------------------------------------------------------------------------
+
+Write-Step 'Machine'
+try {
+    $os = Get-CimInstance Win32_OperatingSystem -ErrorAction Stop
+    Write-Host "Windows      : $($os.Caption) $($os.Version)"
+} catch {
+    Write-Host "Windows      : $([Environment]::OSVersion.VersionString)"
+}
+Write-Host "Architecture : $env:PROCESSOR_ARCHITECTURE"
+Write-Host "PowerShell   : $($PSVersionTable.PSVersion)"
+Write-Host "User         : $([Environment]::UserName)"
+
+$isAdministrator = $false
+try {
+    $identity = [Security.Principal.WindowsIdentity]::GetCurrent()
+    $principal = New-Object Security.Principal.WindowsPrincipal($identity)
+    $isAdministrator = $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
+} catch {
+    # Left false; the warning below covers it.
+}
+Write-Host "Elevated     : $isAdministrator"
+
+if (-not $isAdministrator) {
+    Write-Note 'this session is not elevated; machine-wide installs and PATH changes will not take effect.'
+}
+
+Update-ProcessPath
+
+# Install-Tool orchestrates one tool: skip if already present, otherwise work
+# through the attempts in order, re-checking after each. PATH is refreshed
+# between attempts because an installer's PATH edit is not visible to this
+# process until it is re-read.
+function Install-Tool {
+    param(
+        [Parameter(Mandatory)][string]$Name,
+        [Parameter(Mandatory)][string[]]$Candidates,
+        [Parameter(Mandatory)][hashtable[]]$Attempts,
+        [string[]]$PathCandidates = @()
+    )
+
+    Write-Step "Installing $Name"
+
+    if (Test-Tool -Candidates $Candidates) {
+        Write-Ok "$Name is already present at $(Resolve-Tool -Candidates $Candidates)."
+        return
+    }
+
+    foreach ($attempt in $Attempts) {
+        Write-Host "Trying $($attempt.Description)..."
+        $succeeded = $false
+        try {
+            $succeeded = [bool](& $attempt.Action)
+        } catch {
+            Write-Host "  $($_.Exception.Message.Trim())"
+        }
+
+        Update-ProcessPath
+        foreach ($directory in $PathCandidates) {
+            if (Test-Path -LiteralPath $directory) {
+                Add-MachinePath -Directory $directory | Out-Null
+            }
+        }
+
+        $resolved = Resolve-Tool -Candidates $Candidates
+        if ($resolved) {
+            Write-Ok "$Name is available at $resolved."
+            return
+        }
+        if ($succeeded) {
+            Write-Host "  reported success but '$($Candidates[0])' still does not resolve."
+        }
+    }
+
+    Write-Failure "could not install $Name."
+}
+
+# ---------------------------------------------------------------------------
+# Package managers
+# ---------------------------------------------------------------------------
+
+# Installed first so the tools below can fall back to them.
+
+Install-Tool -Name 'Chocolatey' -Candidates @('choco') -PathCandidates @(
+    (Join-Path $env:ProgramData 'chocolatey\bin')
+) -Attempts @(
+    @{
+        Description = 'the official install script'
+        Action      = {
+            $installer = Join-Path $script:Scratch 'install-chocolatey.ps1'
+            if (-not (Get-Download -Uri 'https://community.chocolatey.org/install.ps1' -OutFile $installer)) {
+                return $false
+            }
+            $env:ChocolateyUseWindowsCompression = 'false'
+            & $installer | Out-Null
+            return $true
+        }
+    }
+)
+
+Install-Tool -Name 'Scoop' -Candidates @('scoop') -PathCandidates @(
+    (Join-Path $ScoopRoot 'shims')
+) -Attempts @(
+    @{
+        Description = 'the official install script'
+        Action      = {
+            # Scoop is normally per-user, which would hide it from the account
+            # that runs workloads later. Pointing it at a fixed directory and
+            # publishing that location machine-wide is what makes it shared.
+            try {
+                [Environment]::SetEnvironmentVariable('SCOOP', $ScoopRoot, 'Machine')
+            } catch {
+                # Without machine scope the install still succeeds; only the
+                # variable is missing for other accounts.
+                Write-Host '  could not publish SCOOP machine-wide.'
+            }
+            $env:SCOOP = $ScoopRoot
+
+            $installer = Join-Path $script:Scratch 'install-scoop.ps1'
+            if (-not (Get-Download -Uri 'https://get.scoop.sh' -OutFile $installer)) {
+                return $false
+            }
+            # The installer refuses to run elevated unless told that is intended.
+            $arguments = @{ ScoopDir = $ScoopRoot; ScoopGlobalDir = (Join-Path $ScoopRoot 'global') }
+            if ($isAdministrator) { $arguments['RunAsAdmin'] = $true }
+            & $installer @arguments | Out-Null
+            return $true
+        }
+    }
+)
+
+# ---------------------------------------------------------------------------
+# Tooling
+# ---------------------------------------------------------------------------
+
+Install-Tool -Name 'the Azure CLI' -Candidates @('az') -PathCandidates @(
+    (Join-Path $env:ProgramFiles 'Microsoft SDKs\Azure\CLI2\wbin'),
+    (Join-Path ${env:ProgramFiles(x86)} 'Microsoft SDKs\Azure\CLI2\wbin')
+) -Attempts @(
+    @{
+        Description = 'the official installer'
+        # The Azure CLI publishes no ARM64 build, so an ARM64 machine gets the
+        # x64 one and runs it emulated.
+        Action      = { Install-Msi -Uri 'https://aka.ms/installazurecliwindowsx64' -Name 'azure-cli.msi' }
+    },
+    @{
+        Description = 'Chocolatey'
+        Action      = {
+            if (-not (Test-Tool -Candidates @('choco'))) { return $false }
+            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'azure-cli', '-y', '--no-progress'))
+        }
+    }
+)
+
+Install-Tool -Name 'the GitHub CLI' -Candidates @('gh') -PathCandidates @(
+    (Join-Path $env:ProgramFiles 'GitHub CLI')
+) -Attempts @(
+    @{
+        Description = 'the latest published installer'
+        Action      = {
+            try {
+                $release = Invoke-RestMethod -Uri 'https://api.github.com/repos/cli/cli/releases/latest' `
+                    -UseBasicParsing -Headers @{ 'User-Agent' = 'mxc-setup' } -TimeoutSec 120
+            } catch {
+                Write-Host "  $($_.Exception.Message.Trim())"
+                return $false
+            }
+            # Assets are named by Go's architecture, not Windows'.
+            $suffix = if ($Architecture -eq 'arm64') { 'windows_arm64.msi' } else { 'windows_amd64.msi' }
+            $asset = $release.assets |
+                Where-Object { $_.name -like "*$suffix" } |
+                Select-Object -First 1
+            if (-not $asset) {
+                Write-Host "  no $suffix asset in release $($release.tag_name)."
+                return $false
+            }
+            return (Install-Msi -Uri $asset.browser_download_url -Name 'gh.msi')
+        }
+    },
+    @{
+        Description = 'Chocolatey'
+        Action      = {
+            if (-not (Test-Tool -Candidates @('choco'))) { return $false }
+            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'gh', '-y', '--no-progress'))
+        }
+    }
+)
+
+Install-Tool -Name 'the NuGet CLI' -Candidates @('nuget') -PathCandidates @(
+    (Join-Path $env:ProgramData 'MXC\bin')
+) -Attempts @(
+    @{
+        Description = 'a direct download of the standalone executable'
+        Action      = {
+            # Published as a bare executable rather than a package, so it needs
+            # a directory of its own and an entry on PATH.
+            $target = Join-Path $env:ProgramData 'MXC\bin'
+            if (-not (Test-Path -LiteralPath $target)) {
+                New-Item -ItemType Directory -Path $target -Force -ErrorAction SilentlyContinue | Out-Null
+            }
+            if (-not (Test-Path -LiteralPath $target)) {
+                Write-Host "  could not create $target."
+                return $false
+            }
+            return (Get-Download -Uri 'https://dist.nuget.org/win-x86-commandline/latest/nuget.exe' `
+                    -OutFile (Join-Path $target 'nuget.exe'))
+        }
+    }
+)
+
+# ---------------------------------------------------------------------------
+# Inventory
+# ---------------------------------------------------------------------------
+
+Update-ProcessPath
+
+Write-Step 'Inventory'
+
+# Everything the finished image is expected to provide, whoever put it there.
+# openssl, winapp and winget are the exception: the first two are only published
+# as packaged applications and are installed once the machine is running, and
+# the third cannot be added to an image at all. They are listed so the log shows
+# the whole picture, but they are not counted as missing.
+$deferred = @('openssl', 'winapp', 'winget')
+
+$inventory = @(
+    @{ Name = 'pwsh';    Candidates = @('pwsh') },
+    @{ Name = 'git';     Candidates = @('git') },
+    @{ Name = 'node';    Candidates = @('node') },
+    @{ Name = 'npm';     Candidates = @('npm') },
+    @{ Name = 'npx';     Candidates = @('npx') },
+    @{ Name = 'python';  Candidates = @('python', 'python3') },
+    @{ Name = 'pip';     Candidates = @('pip', 'pip3') },
+    @{ Name = 'dotnet';  Candidates = @('dotnet') },
+    @{ Name = 'az';      Candidates = @('az') },
+    @{ Name = 'gh';      Candidates = @('gh') },
+    @{ Name = 'nuget';   Candidates = @('nuget') },
+    @{ Name = 'scoop';   Candidates = @('scoop') },
+    @{ Name = 'choco';   Candidates = @('choco') },
+    @{ Name = 'openssl'; Candidates = @('openssl') },
+    @{ Name = 'winapp';  Candidates = @('winapp') },
+    @{ Name = 'winget';  Candidates = @('winget'); AllowStoreAlias = $true }
+)
+
+$absent = @()
+foreach ($tool in $inventory) {
+    $resolved = Resolve-Tool -Candidates $tool.Candidates -AllowStoreAlias:([bool]$tool['AllowStoreAlias'])
+    if ($resolved) {
+        Write-Host ("  {0,-10} {1}" -f $tool.Name, $resolved)
+    } elseif ($deferred -contains $tool.Name) {
+        Write-Host ("  {0,-10} absent (installed once the machine is running)" -f $tool.Name)
+    } else {
+        Write-Host ("  {0,-10} ABSENT" -f $tool.Name)
+        $absent += $tool.Name
+    }
+}
+
+# ---------------------------------------------------------------------------
+# Cleanup and summary
+# ---------------------------------------------------------------------------
+
+Write-Step 'Cleaning up'
+if (Test-Path -LiteralPath $script:Scratch) {
+    Remove-Item -LiteralPath $script:Scratch -Recurse -Force -ErrorAction SilentlyContinue
+}
+Write-Ok 'removed the temporary download directory.'
+
+Write-Step 'Summary'
+if ($absent.Count -gt 0) {
+    Write-Host "Absent after installation: $($absent -join ', ')"
+} else {
+    Write-Host 'Every expected program is present.'
+}
+
+if ($script:Notes.Count -gt 0) {
+    Write-Host ''
+    Write-Host 'Notes:'
+    foreach ($entry in $script:Notes) { Write-Host "  - $entry" }
+}
+
+if ($script:Failures.Count -gt 0) {
+    Write-Host ''
+    Write-Host 'Failures:'
+    foreach ($entry in $script:Failures) { Write-Host "  - $entry" }
+} else {
+    Write-Host ''
+    Write-Host 'No failures.'
+}
+
+Write-Host ''
+Write-Host 'A new session is needed for the PATH changes to be visible.'
+Write-Host 'Setup finished.'
+
+exit 0
diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index f2930eef3..c62ed8ca4 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -162,10 +162,10 @@ function Assert-WorkloadInterpreters {
         @{ Name = 'dotnet';  Candidates = @('dotnet');            Required = $false; Remedy = 'install the .NET SDK in the image' },
         @{ Name = 'az';      Candidates = @('az');                Required = $false; Remedy = 'install the Azure CLI in the image' },
         @{ Name = 'gh';      Candidates = @('gh');                Required = $false; Remedy = 'install the GitHub CLI in the image' },
-        @{ Name = 'openssl'; Candidates = @('openssl');           Required = $false; Remedy = 'install OpenSSL in the image' },
+        @{ Name = 'openssl'; Candidates = @('openssl');           Required = $false; Remedy = 'only published as a packaged application, so it is installed by Install-PackagedTooling rather than baked into the image' },
         # Windows-only
         @{ Name = 'nuget';   Candidates = @('nuget');             Required = $false; Remedy = 'install the NuGet CLI in the image' },
-        @{ Name = 'winapp';  Candidates = @('winapp');            Required = $false; Remedy = 'install the Windows App Development CLI (winget install Microsoft.WinAppCli) in the image' },
+        @{ Name = 'winapp';  Candidates = @('winapp');            Required = $false; Remedy = 'only published as a packaged application, so it is installed by Install-PackagedTooling rather than baked into the image' },
         @{ Name = 'winget';  Candidates = @('winget');            Required = $false; AllowStoreAlias = $true; Remedy = 'install the Windows Package Manager (App Installer) in the image' },
         @{ Name = 'scoop';   Candidates = @('scoop');             Required = $false; Remedy = 'install Scoop in the image' },
         @{ Name = 'choco';   Candidates = @('choco');             Required = $false; Remedy = 'install Chocolatey in the image' }
@@ -221,6 +221,7 @@ function Test-WingetOperational {
             $version = & $candidate --version 2>&1
             if ($LASTEXITCODE -eq 0 -and $version) {
                 Write-Host "winget is operational ($($version | Select-Object -First 1)) at $candidate"
+                $script:WingetPath = $candidate
                 return $true
             }
         } catch {
@@ -285,6 +286,124 @@ function Repair-Winget {
     }
 }
 
+# Re-reads PATH from the registry so a directory an installer just published is
+# visible to this process, which otherwise keeps the PATH it started with.
+function Update-ProcessPath {
+    $parts = foreach ($scope in 'Machine', 'User') {
+        $value = [Environment]::GetEnvironmentVariable('Path', $scope)
+        if ($value) { $value -split ';' }
+    }
+    $seen = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
+    $env:Path = (($parts | Where-Object { $_ -and $seen.Add($_) }) -join ';')
+}
+
+# Resolves a command the way Assert-WorkloadInterpreters does, so a tool this
+# step installs is judged by the same rule that reports it later.
+function Resolve-Interpreter {
+    param([Parameter(Mandatory)][string[]]$Candidates)
+
+    foreach ($candidate in $Candidates) {
+        $found = Get-Command $candidate -ErrorAction SilentlyContinue
+        if ($found -and $found.Source -notlike '*\WindowsApps\*') {
+            return $found.Source
+        }
+    }
+    return $null
+}
+
+# openssl and the Windows App Development CLI are the two workload interpreters
+# that cannot be baked into an image: both are published only as packaged
+# applications, and no packaged application can be registered while an image is
+# being provisioned. They are installed here instead, on the running machine,
+# which is the first point at which winget works.
+#
+# This is the second exception to the verify-never-install rule above. It is
+# best effort: both tools are optional, so a failure warns here and the
+# inventory that follows reports what the machine actually ended up with.
+function Install-PackagedTooling {
+    # 0 is success; the other two are "already installed" and "no applicable
+    # upgrade", which both mean the tool is present and are equally fine.
+    $success = @(0, -1978335135, -1978335189)
+
+    $packages = @(
+        @{
+            Name       = 'winapp'
+            Candidates = @('winapp')
+            Id         = 'Microsoft.WinAppCli'
+            # This package publishes a packaged build and a portable one. The
+            # packaged build installs behind an execution alias in WindowsApps,
+            # which is indistinguishable from a Store stub and is therefore not
+            # counted as present. The portable build puts a real executable on
+            # PATH instead, so it is the one to ask for.
+            Extra      = @('--installer-type', 'zip')
+            PathHints  = @(
+                (Join-Path $env:ProgramFiles 'WinGet\Links'),
+                (Join-Path $env:LOCALAPPDATA 'Microsoft\WinGet\Links')
+            )
+        },
+        @{
+            Name       = 'openssl'
+            Candidates = @('openssl')
+            Id         = 'ShiningLight.OpenSSL.Light'
+            Extra      = @()
+            # This installer does not publish its own bin directory.
+            PathHints  = @(
+                (Join-Path $env:ProgramFiles 'OpenSSL-Win64\bin'),
+                (Join-Path $env:ProgramFiles 'OpenSSL\bin')
+            )
+        }
+    )
+
+    $wanted = $packages | Where-Object { -not (Resolve-Interpreter -Candidates $_.Candidates) }
+    if (-not $wanted) {
+        Write-Host "Packaged workload tooling is already present."
+        return
+    }
+
+    if (-not $script:WingetPath) {
+        Write-Host "::warning::winget is unavailable, so $(($wanted.Name) -join ' and ') cannot be installed."
+        return
+    }
+
+    foreach ($package in $wanted) {
+        Write-Host "Installing $($package.Name) ($($package.Id))."
+        $arguments = @(
+            'install', '--id', $package.Id, '--exact', '--silent',
+            '--disable-interactivity', '--accept-source-agreements',
+            '--accept-package-agreements'
+        ) + $package.Extra
+
+        try {
+            $output = & $script:WingetPath @arguments 2>&1
+            $code = $LASTEXITCODE
+        } catch {
+            Write-Host "::warning::Could not install $($package.Name): $(($_.Exception.Message -split "`r?`n" | Select-Object -First 1).Trim())"
+            continue
+        }
+
+        if ($success -notcontains $code) {
+            $detail = ($output | Select-Object -Last 3 | Out-String).Trim()
+            if ($detail) { Write-Host $detail }
+            Write-Host "::warning::Installing $($package.Name) reported exit code $code."
+            continue
+        }
+
+        Update-ProcessPath
+        foreach ($hint in $package.PathHints) {
+            if ((Test-Path -LiteralPath $hint) -and (($env:Path -split ';') -notcontains $hint)) {
+                $env:Path = "$env:Path;$hint"
+            }
+        }
+
+        $resolved = Resolve-Interpreter -Candidates $package.Candidates
+        if ($resolved) {
+            Write-Host "$($package.Name) is available at $resolved"
+        } else {
+            Write-Host "::warning::$($package.Name) installed but still does not resolve on PATH."
+        }
+    }
+}
+
 function Initialize-MicroVmHost {
     # Staged next to wxc-exec.exe by the --features microvm build, so their
     # absence means a broken artifact rather than a host problem. Snapshots are
@@ -453,8 +572,10 @@ $BinaryDirectory = (Resolve-Path $BinaryDirectory).Path
 Write-Host "Preparing Windows host for backend '$Backend' using $BinaryDirectory"
 
 # Run for every backend: this is host inventory, not a backend prerequisite.
-# The winget repair comes first so the inventory reports the repaired state.
+# The winget repair comes first so the packaged-tooling install below can use
+# it, and the inventory reports the state after both.
 Repair-Winget
+Install-PackagedTooling
 Assert-WorkloadInterpreters
 
 switch ($Backend) {
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 9f6394907..83bc0396e 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -391,17 +391,9 @@
     "weekly": [],
     "enabled": [
       {
-        "os": "rhel-10",
-        "backends": [
-          "bubblewrap",
-          "lxc"
-        ]
-      },
-      {
-        "os": "debian-13",
+        "os": "macos-26",
         "backends": [
-          "bubblewrap",
-          "lxc"
+          "seatbelt"
         ]
       }
     ]

From 1523def79600410ee402ea1961dda7edd104bcc3 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 1 Sep 2026 14:14:45 -0700
Subject: [PATCH 36/44] test new windows provisioning script

---
 scripts/ci/validation-test-matrix.json | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 83bc0396e..9f7c93d32 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -391,9 +391,10 @@
     "weekly": [],
     "enabled": [
       {
-        "os": "macos-26",
+        "os": "windows-24h2",
         "backends": [
-          "seatbelt"
+          "process-t3",
+          "wslc"
         ]
       }
     ]

From 760c9a0fb95b9aad6ab99704c174fd37f2e00a13 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 1 Sep 2026 17:13:29 -0700
Subject: [PATCH 37/44] testing 25H2 fully provisioned

---
 .github/copilot-instructions.md        |  34 +++--
 docs/ci-validation-infrastructure.md   |   8 +-
 scripts/ci/Setup.ps1                   | 176 ++++++++++++++++++++++---
 scripts/ci/validation-test-matrix.json |   2 +-
 4 files changed, 184 insertions(+), 36 deletions(-)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index c9fe27037..ed2521487 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -96,9 +96,12 @@ verify the host's workload interpreters and CLIs (`pwsh`, `git`, `node`, `npm`,
 `winget`, `scoop`, and `choco` on Windows only, and `brew` on macOS only) —
 Windows in `Assert-WorkloadInterpreters`, Linux and macOS in a deliberately
 duplicated bash-3.2-compatible `assert_workload_interpreters` function so each
-platform's list can diverge. Every pool runs an image provisioned ahead of time
-by `scripts/ci/Setup.sh` / `Setup-rhel.sh` (Linux) and `scripts/ci/Setup.ps1`
-(Windows), so the whole list is normally present before a job starts. macOS
+platform's list can diverge. Every pool runs an image provisioned ahead of time,
+so the whole list is normally present before a job starts. The provisioning
+scripts are **not in this repository** — they live in the
+`validation-provision-artifacts` branch of the ADO repo as
+`ubuntu-debian-provision.sh` / `rhel-provision.sh` (Linux) and
+`windows-provision.ps1` (Windows). macOS
 verifies only; Linux first runs `install_workload_interpreters`, which re-runs
 the install as a top-up and is a no-op on a correctly built image (distribution
 repositories, plus Microsoft's feeds for `pwsh`/`az` and GitHub's for `gh`).
@@ -111,14 +114,27 @@ best-effort and can never fail the job —
 the inventory that follows reports the outcome. Its package-manager access goes
 through `resolve_package_manager` and `install_packages`, the same helpers the
 backend prerequisite installers use, so a new distribution family is one arm in
-`install_packages` rather than a branch in every installer. `Setup.ps1` installs
+`install_packages` rather than a branch in every installer. The Windows
+provisioning script installs
 no packaged application and never uses winget, because neither is available
 during image provisioning; it takes only `-Architecture` (`x64`/`arm64`, which
-selects the `gh` MSI — the Azure CLI has no ARM64 build and is used emulated)
-and `-ScoopRoot`. Its scope is `choco`, `scoop`, `az`, `gh` and `nuget`: the
-language runtimes (`dotnet`, `node`, `python`, `pwsh`, `git`) arrive from
-separate image artifacts, and `Setup.ps1` only inventories them. Windows
-therefore provisions
+selects the `gh` MSI and the .NET SDK bundle — the Azure CLI has no ARM64 build
+and is used emulated) and `-ScoopRoot`. It launches every installer through
+`Start-Process` rather
+than the call operator, because `msiexec` is a GUI-subsystem program for which
+the call operator leaves `$LASTEXITCODE` unset — which previously read as
+success and turned a failed install into a silent one. Windows Installer
+serializes machine-wide installs behind the `Global\_MSIExecute` mutex and
+returns 1618 to whoever arrives while another provisioning artifact holds it, so
+every MSI-touching call waits for that mutex to clear before each attempt and
+retries a 1618 up to three times. Its scope is the .NET SDK, `choco`, `scoop`,
+`az`, `gh` and `nuget`. The .NET SDK comes from the arch-matched
+`aka.ms/dotnet/LTS/dotnet-sdk-win-<arch>.exe` bundle rather than
+`dotnet-install.ps1`, because that script's only PATH write is to `$env:path` —
+it persists nothing, so an SDK it installs is on disk but resolves from nowhere
+in any later process. The remaining runtimes (`node`, `python`, `pwsh`, `git`)
+arrive from separate image artifacts, which it only inventories.
+Windows therefore provisions
 two entries at job time, also best-effort and also ahead of the inventory.
 `Repair-Winget`
 re-registers the App Installer package (`Add-AppxPackage -RegisterByFamilyName`)
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 94bdbf7fd..ddc49d7fe 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -32,8 +32,6 @@ the individual local test scripts are documented in
 | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions, plus the `winget` repair and the packaged-tooling install. |
 | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware), plus the workload-interpreter top-up pass. |
 | `scripts/ci/prepare-macos-host.sh` | Per-backend macOS host preparation / prerequisite assertions. |
-| `scripts/ci/Setup.sh` / `scripts/ci/Setup-rhel.sh` | Image-provisioning scripts. Run when the Linux images are built, not per job. |
-| `scripts/ci/Setup.ps1` | Image-provisioning script. Runs when the Windows image is built, not per job. |
 | `scripts/ci/run_backend_validation_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
 | `scripts/ci/run_backend_validation_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
 
@@ -318,7 +316,7 @@ mid-suite failure.
 | `git` | all | Latest | |
 | `node`, `npm`, `npx` | all | 24.x | `npm` and `npx` arrive with Node. |
 | `python`, `pip` | all | Latest | Windows tries `python` first, Unix `python3`. |
-| `dotnet` | all | Latest LTS | |
+| `dotnet` | all | Latest LTS | Currently 10.x, from the `LTS` channel — resolved at image-build time, not pinned. |
 | `az` | all | Latest | No ARM64 Windows build exists; ARM64 images get the x64 one under emulation. |
 | `gh` | all | Latest | |
 | `openssl` | all | Latest | On Windows, installed per job — see below. |
@@ -364,8 +362,8 @@ Every pool runs images pre-provisioned with programs installed by
 `ubuntu-debian-provision.sh` / `rhel-provision.sh` for Linux and 
 `windows-provision.ps1` for Windows. These scripts are located in the
 `validation-provision-artifacts` branch in the ADO repo. On Windows that
-script covers `choco`, `scoop`, `az`, `gh` and `nuget`; the language runtimes
-(`dotnet`, `node`, `python`, `pwsh`, `git`) come from separate image artifacts.
+script covers `dotnet`, `choco`, `scoop`, `az`, `gh` and `nuget`; the remaining
+runtimes (`node`, `python`, `pwsh`, `git`) come from separate image artifacts.
 
 During the start of a job, the installed programs are inventoried. Windows also
 installs OpenSSL and WinApp via the repaired WinGet. 
diff --git a/scripts/ci/Setup.ps1 b/scripts/ci/Setup.ps1
index 902b6376c..ba41bf7db 100644
--- a/scripts/ci/Setup.ps1
+++ b/scripts/ci/Setup.ps1
@@ -6,7 +6,8 @@
 
 .DESCRIPTION
     Installs the command-line tooling an MXC machine is expected to provide:
-    Chocolatey, Scoop, the Azure CLI, the GitHub CLI and the NuGet CLI.
+    Chocolatey, Scoop, the .NET SDK, the Azure CLI, the GitHub CLI and the
+    NuGet CLI.
 
     Everything is installed from an official installer or install script that
     is downloaded directly. The Windows Package Manager is deliberately not
@@ -15,10 +16,9 @@
     published as a packaged application is installed later, when the machine
     is running, rather than here.
 
-    The .NET SDK, Node.js, Python, PowerShell 7 and Git arrive with the image
-    from other provisioning artifacts rather than being installed here, but
-    they are still inventoried at the end so the image is reported on as a
-    whole.
+    Node.js, Python, PowerShell 7 and Git arrive with the image from other
+    provisioning artifacts rather than being installed here, but they are still
+    inventoried at the end so the image is reported on as a whole.
 
     Every step is best effort. Nothing here aborts the run and the script always
     exits 0; what the machine actually ended up with is printed in the summary
@@ -175,34 +175,135 @@ function Test-Tool {
     return [bool](Resolve-Tool -Candidates $Candidates)
 }
 
+# Windows Installer serializes machine-wide installs behind a single mutex and
+# hands 1618 to anyone who arrives while it is held. While an image is being
+# provisioned that is routine rather than exceptional, because other artifacts
+# install at the same time, so a 1618 is waited out and retried.
+$script:InstallerBusy = 1618
+
+function Test-WindowsInstallerBusy {
+    $mutex = $null
+    try {
+        $mutex = [System.Threading.Mutex]::OpenExisting('Global\_MSIExecute')
+        return $true
+    } catch [System.Threading.WaitHandleCannotBeOpenedException] {
+        # The mutex does not exist, so no installation is in progress.
+        return $false
+    } catch {
+        # Any other failure means the mutex exists but could not be opened,
+        # which still means an installation is in progress.
+        return $true
+    } finally {
+        if ($mutex) { $mutex.Dispose() }
+    }
+}
+
+function Wait-ForWindowsInstaller {
+    param([int]$TimeoutSeconds = 900)
+
+    if (-not (Test-WindowsInstallerBusy)) { return $true }
+
+    Write-Host '  another installation is in progress; waiting for it to finish...'
+    $deadline = (Get-Date).AddSeconds($TimeoutSeconds)
+    while ((Get-Date) -lt $deadline) {
+        Start-Sleep -Seconds 10
+        if (-not (Test-WindowsInstallerBusy)) {
+            Write-Host '  the other installation finished.'
+            return $true
+        }
+    }
+
+    Write-Host "  still in progress after $TimeoutSeconds seconds; going ahead anyway."
+    return $false
+}
+
+# Launches a program, waits for it, and returns its exit code alongside whatever
+# it wrote. Everything goes through Start-Process rather than the call operator
+# because msiexec is a GUI-subsystem program: the call operator leaves
+# $LASTEXITCODE unset for it, which would read as success and turn a failed
+# install into a silent one.
+function Start-ProgramOnce {
+    param(
+        [Parameter(Mandatory)][string]$FilePath,
+        [string[]]$Arguments = @()
+    )
+
+    $stdout = Join-Path $script:Scratch ('run-' + [Guid]::NewGuid().ToString('N') + '.out')
+    $stderr = [IO.Path]::ChangeExtension($stdout, 'err')
+    try {
+        if (-not (Test-Path -LiteralPath $script:Scratch)) {
+            New-Item -ItemType Directory -Path $script:Scratch -Force -ErrorAction SilentlyContinue | Out-Null
+        }
+
+        $launch = @{
+            FilePath               = $FilePath
+            Wait                   = $true
+            PassThru               = $true
+            NoNewWindow            = $true
+            RedirectStandardOutput = $stdout
+            RedirectStandardError  = $stderr
+        }
+        if ($Arguments.Count -gt 0) { $launch['ArgumentList'] = $Arguments }
+
+        $process = Start-Process @launch
+
+        $written = @()
+        foreach ($file in @($stdout, $stderr)) {
+            if (Test-Path -LiteralPath $file) {
+                $written += @(Get-Content -LiteralPath $file -ErrorAction SilentlyContinue)
+            }
+        }
+
+        return [pscustomobject]@{ Code = $process.ExitCode; Output = $written }
+    } catch {
+        # A launch failure embeds a position trace after the first line.
+        $reason = ($_.Exception.Message -split "`r?`n" | Select-Object -First 1).Trim()
+        Write-Host "  $reason"
+        return $null
+    } finally {
+        Remove-Item -LiteralPath $stdout, $stderr -Force -ErrorAction SilentlyContinue
+    }
+}
+
 # Runs a program and reports only whether it succeeded, keeping installer chatter
-# out of the transcript unless something goes wrong.
+# out of the transcript unless something goes wrong. Naming 1618 in RetryCodes
+# also makes every attempt wait for Windows Installer to go idle first, so the
+# common case is avoiding the collision rather than recovering from it.
 function Invoke-Program {
     param(
         [Parameter(Mandatory)][string]$FilePath,
         [string[]]$Arguments = @(),
-        [int[]]$SuccessCodes = @(0)
+        [int[]]$SuccessCodes = @(0),
+        [int[]]$RetryCodes = @(),
+        [int]$MaxAttempts = 1
     )
 
-    try {
-        $output = & $FilePath @Arguments 2>&1
-        $code = $LASTEXITCODE
+    $serialized = $RetryCodes -contains $script:InstallerBusy
+
+    for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) {
+        if ($serialized) { Wait-ForWindowsInstaller | Out-Null }
+
+        $result = Start-ProgramOnce -FilePath $FilePath -Arguments $Arguments
+        if ($null -eq $result) { return $false }
+
+        $code = $result.Code
         if ($null -eq $code) { $code = 0 }
-        if ($SuccessCodes -contains $code) {
-            return $true
+        if ($SuccessCodes -contains $code) { return $true }
+
+        if (($RetryCodes -contains $code) -and ($attempt -lt $MaxAttempts)) {
+            Write-Host "  exit code $code; retrying ($attempt of $($MaxAttempts - 1))."
+            continue
         }
-        $detail = ($output | Select-Object -Last 3 | Out-String).Trim()
+
+        $detail = ($result.Output | Select-Object -Last 3 | Out-String).Trim()
         if ($detail) {
             Write-Host "  $($detail -replace "`r?`n", "`n  ")"
         }
         Write-Host "  exit code $code"
         return $false
-    } catch {
-        # A native launch failure embeds a position trace after the first line.
-        $reason = ($_.Exception.Message -split "`r?`n" | Select-Object -First 1).Trim()
-        Write-Host "  $reason"
-        return $false
     }
+
+    return $false
 }
 
 function Get-Download {
@@ -233,7 +334,25 @@ function Install-Msi {
     if (-not (Get-Download -Uri $Uri -OutFile $package)) {
         return $false
     }
-    return (Invoke-Program -FilePath 'msiexec.exe' -Arguments @('/i', $package, '/quiet', '/norestart') -SuccessCodes @(0, 3010))
+    return (Invoke-Program -FilePath 'msiexec.exe' -Arguments @('/i', $package, '/quiet', '/norestart') `
+        -SuccessCodes @(0, 3010) -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
+}
+
+# Some installers ship as a WiX bundle rather than a bare MSI and so take their
+# own switches instead of msiexec's. A bundle still wraps MSIs underneath, so it
+# can report 1618 just the same.
+function Install-Bundle {
+    param(
+        [Parameter(Mandatory)][string]$Uri,
+        [Parameter(Mandatory)][string]$Name
+    )
+
+    $package = Join-Path $script:Scratch $Name
+    if (-not (Get-Download -Uri $Uri -OutFile $package)) {
+        return $false
+    }
+    return (Invoke-Program -FilePath $package -Arguments @('/install', '/quiet', '/norestart') `
+        -SuccessCodes @(0, 3010) -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
 }
 
 # ---------------------------------------------------------------------------
@@ -373,6 +492,19 @@ Install-Tool -Name 'Scoop' -Candidates @('scoop') -PathCandidates @(
 # Tooling
 # ---------------------------------------------------------------------------
 
+Install-Tool -Name 'the .NET SDK' -Candidates @('dotnet') -PathCandidates @(
+    (Join-Path $env:ProgramFiles 'dotnet')
+) -Attempts @(
+    @{
+        Description = 'the official installer'
+        # The LTS channel link always points at the current long-term-support
+        # release, and there is a native build for each architecture.
+        Action      = {
+            Install-Bundle -Uri "https://aka.ms/dotnet/LTS/dotnet-sdk-win-$Architecture.exe" -Name 'dotnet-sdk.exe'
+        }
+    }
+)
+
 Install-Tool -Name 'the Azure CLI' -Candidates @('az') -PathCandidates @(
     (Join-Path $env:ProgramFiles 'Microsoft SDKs\Azure\CLI2\wbin'),
     (Join-Path ${env:ProgramFiles(x86)} 'Microsoft SDKs\Azure\CLI2\wbin')
@@ -387,7 +519,8 @@ Install-Tool -Name 'the Azure CLI' -Candidates @('az') -PathCandidates @(
         Description = 'Chocolatey'
         Action      = {
             if (-not (Test-Tool -Candidates @('choco'))) { return $false }
-            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'azure-cli', '-y', '--no-progress'))
+            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'azure-cli', '-y', '--no-progress') `
+                -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
         }
     }
 )
@@ -421,7 +554,8 @@ Install-Tool -Name 'the GitHub CLI' -Candidates @('gh') -PathCandidates @(
         Description = 'Chocolatey'
         Action      = {
             if (-not (Test-Tool -Candidates @('choco'))) { return $false }
-            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'gh', '-y', '--no-progress'))
+            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'gh', '-y', '--no-progress') `
+                -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
         }
     }
 )
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index 9f7c93d32..aa6448abf 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -391,7 +391,7 @@
     "weekly": [],
     "enabled": [
       {
-        "os": "windows-24h2",
+        "os": "windows-25h2",
         "backends": [
           "process-t3",
           "wslc"

From f18d4ddacf5bd8251a2839a668a649cfdbfcd914 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 2 Sep 2026 13:17:10 -0700
Subject: [PATCH 38/44] linux no longer installs workload interpreters during
 the job.

---
 .github/copilot-instructions.md        |  23 +-
 docs/ci-validation-infrastructure.md   |  16 +-
 scripts/ci/Setup-rhel.sh               | 404 ---------------
 scripts/ci/Setup.ps1                   | 668 -------------------------
 scripts/ci/Setup.sh                    | 496 ------------------
 scripts/ci/prepare-linux-host.sh       | 296 +----------
 scripts/ci/validation-test-matrix.json |  10 +-
 7 files changed, 25 insertions(+), 1888 deletions(-)
 delete mode 100755 scripts/ci/Setup-rhel.sh
 delete mode 100644 scripts/ci/Setup.ps1
 delete mode 100755 scripts/ci/Setup.sh

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index c64ad311a..50d91877b 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -101,21 +101,14 @@ so the whole list is normally present before a job starts. The provisioning
 scripts are **not in this repository** — they live in the
 `validation-provision-artifacts` branch of the ADO repo as
 `ubuntu-debian-provision.sh` / `rhel-provision.sh` (Linux) and
-`windows-provision.ps1` (Windows). macOS
-verifies only; Linux first runs `install_workload_interpreters`, which re-runs
-the install as a top-up and is a no-op on a correctly built image (distribution
-repositories, plus Microsoft's feeds for `pwsh`/`az` and GitHub's for `gh`).
-Microsoft splits those two: `packages-microsoft-prod`
-carries `powershell` everywhere and `azure-cli` on RPM only, while on apt the
-Azure CLI has its own codename-keyed feed added by `add_azure_cli_apt_feed`,
-which probes for the suite before writing it and falls back to the newest
-published one for the family (Debian 13 `trixie` is not published). That pass is
-best-effort and can never fail the job —
-the inventory that follows reports the outcome. Its package-manager access goes
-through `resolve_package_manager` and `install_packages`, the same helpers the
-backend prerequisite installers use, so a new distribution family is one arm in
-`install_packages` rather than a branch in every installer. The Windows
-provisioning script installs
+`windows-provision.ps1` (Windows). No job installs a workload interpreter on any
+platform: all three scripts verify and report only, and a missing one warns
+rather than failing (except `pwsh` on Windows). A backend's own prerequisites
+are separate and are still installed per job — `install_bubblewrap` /
+`install_lxc` in `prepare-linux-host.sh` reach the package manager through
+`resolve_package_manager` and `install_packages`, so a new distribution family
+is one arm in `install_packages` rather than a branch in every installer. The
+Windows provisioning script installs
 no packaged application and never uses winget, because neither is available
 during image provisioning; it takes only `-Architecture` (`x64`/`arm64`, which
 selects the `gh` MSI and the .NET SDK bundle — the Azure CLI has no ARM64 build
diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index ddc49d7fe..bf22f3fe8 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -30,7 +30,7 @@ the individual local test scripts are documented in
 | `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers, job staggering. |
 | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. |
 | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions, plus the `winget` repair and the packaged-tooling install. |
-| `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware), plus the workload-interpreter top-up pass. |
+| `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware), plus the workload-interpreter inventory. |
 | `scripts/ci/prepare-macos-host.sh` | Per-backend macOS host preparation / prerequisite assertions. |
 | `scripts/ci/run_backend_validation_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. |
 | `scripts/ci/run_backend_validation_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. |
@@ -288,8 +288,7 @@ package-manager chain: `resolve_package_manager` picks the first of `apt-get`,
 holds the single `case` that knows how each one is invoked. Supporting a new
 distribution family is therefore one new arm in `install_packages`, not another
 branch in every installer. `install_packages` returns the package manager's own
-status rather than acting on it, which is what lets a backend prerequisite treat
-a failure as fatal while a workload interpreter only warns.
+status rather than acting on it, so each caller decides what a failure means.
 
 `prepare-macos-host.sh`:
 
@@ -365,8 +364,13 @@ Every pool runs images pre-provisioned with programs installed by
 script covers `dotnet`, `choco`, `scoop`, `az`, `gh` and `nuget`; the remaining
 runtimes (`node`, `python`, `pwsh`, `git`) come from separate image artifacts.
 
-During the start of a job, the installed programs are inventoried. Windows also
-installs OpenSSL and WinApp via the repaired WinGet. 
+During the start of a job, the installed programs are inventoried; no job
+installs a workload interpreter. Windows is the one exception, installing
+OpenSSL and WinApp via the repaired WinGet, because both are published as
+packaged applications.
+
+A backend's own prerequisites are separate and are still installed per job by
+`prepare-linux-host.sh` — see [Host preparation](#host-preparation).
 
 ## Log collection
 
@@ -452,7 +456,7 @@ passing a distinguishing argument later without touching the matrix.
    package manager and service layout. A new package-manager family needs one
    arm in `install_packages` and one entry in `resolve_package_manager`; a
    family whose package *names* differ also needs its column in the tables in
-   `install_lxc`, `install_bubblewrap`, and `install_workload_interpreters`.
+   `install_lxc` and `install_bubblewrap`.
 5. Add it to a plan's `triggers`, then resolve locally.
 
 ### Wire an unwired backend to a suite
diff --git a/scripts/ci/Setup-rhel.sh b/scripts/ci/Setup-rhel.sh
deleted file mode 100755
index 35b88c025..000000000
--- a/scripts/ci/Setup-rhel.sh
+++ /dev/null
@@ -1,404 +0,0 @@
-#!/usr/bin/env bash
-#
-# Installation script for MXC Linux machines (RHEL and compatible).
-#
-# Installs the sandboxing runtimes, kernel prerequisites and workload
-# interpreters an MXC machine is expected to provide, and persists the kernel
-# settings so they survive a reboot.
-#
-# Every step is best effort. Nothing here aborts the run and the script always
-# exits 0; what the machine actually ended up with is printed in the summary at
-# the end.
-
-set -o pipefail
-
-# Guarantee the exit status regardless of how the script leaves.
-trap 'exit 0' EXIT
-
-failures=()
-notes=()
-
-step() {
-    echo ""
-    echo "=== $* ==="
-}
-
-ok() {
-    echo "OK: $*"
-}
-
-note() {
-    echo "NOTE: $*"
-    notes+=("$*")
-}
-
-fail() {
-    echo "FAILED: $*"
-    failures+=("$*")
-}
-
-# Image builders run as root without sudo installed; interactive machines have
-# sudo and an unprivileged user. Resolve once and use the same wrapper for
-# every privileged call.
-if [ "$(id -u)" -eq 0 ]; then
-    priv() { "$@"; }
-elif command -v sudo >/dev/null 2>&1; then
-    priv() { sudo -n "$@"; }
-else
-    priv() { return 1; }
-fi
-
-have() {
-    command -v "$1" >/dev/null 2>&1
-}
-
-# ---------------------------------------------------------------------------
-# Distribution identity
-# ---------------------------------------------------------------------------
-
-distro_id=""
-distro_version=""
-if [ -r /etc/os-release ]; then
-    # shellcheck disable=SC1091
-    . /etc/os-release
-    distro_id="${ID:-}"
-    distro_version="${VERSION_ID:-}"
-fi
-major_version="${distro_version%%.*}"
-architecture="$(uname -m 2>/dev/null)"
-
-step "Machine"
-echo "Distribution : ${distro_id:-unknown} ${distro_version:-unknown}"
-echo "Kernel       : $(uname -r 2>/dev/null)"
-echo "Architecture : ${architecture:-unknown}"
-
-package_manager=""
-for candidate in dnf yum microdnf; do
-    if have "$candidate"; then
-        package_manager="$candidate"
-        break
-    fi
-done
-
-if [ -z "$package_manager" ]; then
-    fail "no dnf, yum or microdnf on this machine; this script targets RHEL and compatible machines."
-    step "Summary"
-    echo "Nothing was installed."
-    exit 0
-fi
-echo "Package tool : $package_manager"
-
-if ! priv true 2>/dev/null; then
-    fail "cannot obtain root privileges; no packages can be installed."
-    step "Summary"
-    echo "Nothing was installed."
-    exit 0
-fi
-
-if [ -z "$major_version" ]; then
-    major_version="10"
-    note "could not read the release version; assuming ${major_version} for versioned feeds."
-fi
-
-# ---------------------------------------------------------------------------
-# Package helpers
-# ---------------------------------------------------------------------------
-
-# install <package-or-url>...
-install() {
-    priv "$package_manager" install -y "$@"
-}
-
-# install_group <description> <package>...
-# Installs as one transaction, then retries individually so a single package
-# that is unavailable on this release cannot cost the machine the rest.
-install_group() {
-    description="$1"
-    shift
-    [ "$#" -eq 0 ] && return 0
-
-    if install "$@"; then
-        ok "$description: $*"
-        return 0
-    fi
-
-    note "combined install for $description failed; retrying each package on its own."
-    for package in "$@"; do
-        if install "$package"; then
-            ok "$description: $package"
-        else
-            fail "could not install '$package' ($description)."
-        fi
-    done
-}
-
-# ---------------------------------------------------------------------------
-# Repositories
-# ---------------------------------------------------------------------------
-
-step "Enabling supplementary repositories"
-
-# Several of the packages below are built against content that only the
-# CodeReady Builder repository provides.
-if have subscription-manager; then
-    if priv subscription-manager repos \
-        --enable "codeready-builder-for-rhel-${major_version}-${architecture}-rpms" >/dev/null 2>&1; then
-        ok "enabled the CodeReady Builder repository."
-    else
-        note "could not enable the CodeReady Builder repository; some packages may be unavailable."
-    fi
-else
-    # Rebuilds carry the same content under their own repository name.
-    for repo in crb powertools; do
-        if priv "$package_manager" config-manager --set-enabled "$repo" >/dev/null 2>&1; then
-            ok "enabled the '${repo}' repository."
-            break
-        fi
-    done
-fi
-
-# This family ships no third-party content, so the community repository comes
-# from its own release package.
-if rpm -q epel-release >/dev/null 2>&1; then
-    ok "the EPEL repository is already configured."
-elif install "https://dl.fedoraproject.org/pub/epel/epel-release-latest-${major_version}.noarch.rpm"; then
-    ok "added the EPEL repository."
-else
-    fail "could not add the EPEL repository; packages that live there will be unavailable."
-fi
-
-step "Adding vendor package feeds"
-
-# Carries both PowerShell and the Azure CLI on this family.
-if rpm -q packages-microsoft-prod >/dev/null 2>&1; then
-    ok "the Microsoft package feed is already configured."
-else
-    priv rpm --import https://packages.microsoft.com/keys/microsoft.asc >/dev/null 2>&1 ||
-        note "could not import the Microsoft signing key; the feed may still supply it."
-    # Newer releases are signed with a separate key.
-    priv rpm --import https://packages.microsoft.com/keys/microsoft-2025.asc >/dev/null 2>&1 || true
-
-    if install "https://packages.microsoft.com/config/rhel/${major_version}/packages-microsoft-prod.rpm"; then
-        ok "added the Microsoft package feed."
-    else
-        fail "could not add the Microsoft package feed."
-    fi
-fi
-
-if [ -f /etc/yum.repos.d/gh-cli.repo ]; then
-    ok "the GitHub CLI package feed is already configured."
-else
-    # config-manager is a separate package on some releases and is what adds
-    # the feed, so it is requested before it is used.
-    install "dnf-command(config-manager)" >/dev/null 2>&1 || true
-    if priv "$package_manager" config-manager \
-        --add-repo https://cli.github.com/packages/rpm/gh-cli.repo >/dev/null 2>&1; then
-        ok "added the GitHub CLI package feed."
-    else
-        fail "could not add the GitHub CLI package feed."
-    fi
-fi
-
-# ---------------------------------------------------------------------------
-# Sandboxing runtimes
-# ---------------------------------------------------------------------------
-
-step "Installing unprivileged sandboxing prerequisites"
-install_group "unprivileged sandboxing" \
-    bubblewrap slirp4netns util-linux iproute iptables
-
-# The packet-filter command moved into a separate package on newer releases.
-if have iptables; then
-    ok "the packet filter command is present."
-else
-    install_group "the packet filter" iptables-nft
-fi
-
-step "Installing container prerequisites"
-install_group "containers" lxc lxc-templates dnsmasq
-
-# ---------------------------------------------------------------------------
-# Workload interpreters
-# ---------------------------------------------------------------------------
-
-step "Installing workload interpreters"
-install_group "interpreters" \
-    git openssl nodejs npm python3 python3-pip
-
-# Named separately so an unavailable release does not take the rest with it.
-install_group "the .NET SDK" dotnet-sdk-8.0
-install_group "PowerShell" powershell
-install_group "the Azure CLI" azure-cli
-install_group "the GitHub CLI" gh
-
-# ---------------------------------------------------------------------------
-# Kernel configuration
-# ---------------------------------------------------------------------------
-
-# Written to disk rather than only applied live, so the machine comes back the
-# same way after a reboot.
-persist_module() {
-    module="$1"
-    if echo "$module" | priv tee "/etc/modules-load.d/mxc-${module}.conf" >/dev/null; then
-        ok "${module} will load at boot."
-    else
-        fail "could not configure ${module} to load at boot."
-    fi
-    priv modprobe "$module" 2>/dev/null ||
-        note "could not load ${module} now; it may be built in or unavailable until reboot."
-}
-
-step "Configuring kernel modules"
-# Connection-state matching in the packet filter needs conntrack present.
-persist_module nf_conntrack
-# Bridged traffic is only visible to the packet filter with this loaded.
-persist_module br_netfilter
-
-step "Configuring kernel settings"
-sysctl_file="/etc/sysctl.d/99-mxc.conf"
-sysctl_lines=""
-
-# Unprivileged user namespaces are what unprivileged sandboxing is built on.
-if [ -e /proc/sys/user/max_user_namespaces ]; then
-    sysctl_lines="${sysctl_lines}user.max_user_namespaces = 28633
-"
-fi
-# Bridged traffic must traverse the packet filter for container network policy
-# to apply to it.
-sysctl_lines="${sysctl_lines}net.bridge.bridge-nf-call-iptables = 1
-net.bridge.bridge-nf-call-ip6tables = 1
-net.ipv4.ip_forward = 1
-"
-
-if printf '%s' "$sysctl_lines" | priv tee "$sysctl_file" >/dev/null; then
-    ok "wrote ${sysctl_file}."
-else
-    fail "could not write ${sysctl_file}."
-fi
-
-if priv sysctl --system >/dev/null 2>&1; then
-    ok "applied the kernel settings."
-else
-    note "could not apply the kernel settings now; they will take effect after a reboot."
-fi
-
-# ---------------------------------------------------------------------------
-# Container bridge
-# ---------------------------------------------------------------------------
-
-step "Configuring the container bridge"
-if have systemctl; then
-    for unit in lxc-net lxc; do
-        if priv systemctl enable "$unit" >/dev/null 2>&1; then
-            ok "'${unit}' will start at boot."
-        else
-            note "could not enable '${unit}'."
-        fi
-    done
-    if priv systemctl restart lxc-net >/dev/null 2>&1; then
-        ok "started the container bridge."
-    else
-        note "could not start the container bridge; it should come up on the next boot."
-    fi
-else
-    note "no systemctl on this machine; skipping the bridge service."
-fi
-
-# The default container profile is not created by the package install on every
-# release, and container startup needs one.
-if [ ! -f /etc/lxc/default.conf ] && [ -d /etc/lxc ]; then
-    if printf 'lxc.net.0.type = veth\nlxc.net.0.link = lxcbr0\nlxc.net.0.flags = up\n' |
-        priv tee /etc/lxc/default.conf >/dev/null; then
-        ok "wrote the default container profile."
-    else
-        note "could not write the default container profile."
-    fi
-fi
-
-# ---------------------------------------------------------------------------
-# Inventory
-# ---------------------------------------------------------------------------
-
-step "Inventory"
-
-# name|candidates tried in order
-inventory="bwrap|bwrap
-slirp4netns|slirp4netns
-unshare|unshare
-nsenter|nsenter
-ip|ip
-iptables|iptables
-ip6tables|ip6tables
-lxc-start|lxc-start
-git|git
-openssl|openssl
-node|node,nodejs
-npm|npm
-npx|npx
-python|python3,python
-pip|pip3,pip
-dotnet|dotnet
-pwsh|pwsh
-az|az
-gh|gh"
-
-absent=""
-while IFS='|' read -r name candidates; do
-    [ -z "$name" ] && continue
-    resolved=""
-    old_ifs="$IFS"
-    IFS=','
-    for candidate in $candidates; do
-        if resolved="$(command -v "$candidate" 2>/dev/null)"; then
-            break
-        fi
-        resolved=""
-    done
-    IFS="$old_ifs"
-
-    if [ -n "$resolved" ]; then
-        printf '  %-14s %s\n' "$name" "$resolved"
-    else
-        printf '  %-14s ABSENT\n' "$name"
-        absent="${absent:+$absent }$name"
-    fi
-done <<EOF
-$inventory
-EOF
-
-# ---------------------------------------------------------------------------
-# Cleanup and summary
-# ---------------------------------------------------------------------------
-
-step "Cleaning up"
-priv "$package_manager" clean all >/dev/null 2>&1 && ok "cleared the package cache."
-
-step "Summary"
-if [ -n "$absent" ]; then
-    echo "Absent after installation: $absent"
-else
-    echo "Every expected program is present."
-fi
-
-if [ "${#notes[@]}" -gt 0 ]; then
-    echo ""
-    echo "Notes:"
-    for entry in "${notes[@]}"; do
-        echo "  - $entry"
-    done
-fi
-
-if [ "${#failures[@]}" -gt 0 ]; then
-    echo ""
-    echo "Failures:"
-    for entry in "${failures[@]}"; do
-        echo "  - $entry"
-    done
-else
-    echo ""
-    echo "No failures."
-fi
-
-echo ""
-echo "Setup finished."
-exit 0
diff --git a/scripts/ci/Setup.ps1 b/scripts/ci/Setup.ps1
deleted file mode 100644
index ba41bf7db..000000000
--- a/scripts/ci/Setup.ps1
+++ /dev/null
@@ -1,668 +0,0 @@
-#Requires -Version 5.1
-
-<#
-.SYNOPSIS
-    Installation script for MXC Windows machines.
-
-.DESCRIPTION
-    Installs the command-line tooling an MXC machine is expected to provide:
-    Chocolatey, Scoop, the .NET SDK, the Azure CLI, the GitHub CLI and the
-    NuGet CLI.
-
-    Everything is installed from an official installer or install script that
-    is downloaded directly. The Windows Package Manager is deliberately not
-    used and no packaged application is registered, because neither is
-    available while an image is being provisioned. Tooling that is only
-    published as a packaged application is installed later, when the machine
-    is running, rather than here.
-
-    Node.js, Python, PowerShell 7 and Git arrive with the image from other
-    provisioning artifacts rather than being installed here, but they are still
-    inventoried at the end so the image is reported on as a whole.
-
-    Every step is best effort. Nothing here aborts the run and the script always
-    exits 0; what the machine actually ended up with is printed in the summary
-    at the end.
-
-    Installs are machine-scoped and every directory this script adds to PATH is
-    added to the machine PATH, because the account that runs workloads later is
-    usually not the account that runs this script.
-
-.EXAMPLE
-    ./Setup.ps1
-
-.EXAMPLE
-    ./Setup.ps1 -Architecture arm64
-#>
-
-[CmdletBinding()]
-param(
-    # The machine's processor architecture, which selects the installer variant
-    # to download. Defaults to what this machine reports.
-    [ValidateSet('x64', 'arm64')]
-    [string]$Architecture = $(if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'x64' }),
-
-    # Where Scoop is installed. Deliberately outside a user profile so every
-    # account on the machine can reach it.
-    [string]$ScoopRoot = 'C:\Scoop'
-)
-
-Set-StrictMode -Off
-$ErrorActionPreference = 'Continue'
-$ProgressPreference = 'SilentlyContinue'
-
-# Provisioning is best effort: an unexpected terminating error reports what went
-# wrong and still leaves the image in a usable state, so the exit code is always 0.
-trap {
-    Write-Host ''
-    Write-Host "Setup stopped early: $($_.Exception.Message)"
-    Write-Host "  at $($_.InvocationInfo.ScriptLineNumber): $($_.InvocationInfo.Line.Trim())"
-    Write-Host 'Setup finished.'
-    exit 0
-}
-
-# TLS 1.2 is not the default on older hosts and every download below is HTTPS.
-try {
-    [Net.ServicePointManager]::SecurityProtocol =
-        [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
-} catch {
-    # A host that pins its own protocols is left alone.
-}
-
-$script:Notes = New-Object System.Collections.Generic.List[string]
-$script:Failures = New-Object System.Collections.Generic.List[string]
-$script:Scratch = Join-Path ([IO.Path]::GetTempPath()) ("mxc-setup-" + [Guid]::NewGuid().ToString('N').Substring(0, 8))
-
-function Write-Step {
-    param([Parameter(Mandatory)][string]$Message)
-    Write-Host ''
-    Write-Host "=== $Message ==="
-}
-
-function Write-Ok {
-    param([Parameter(Mandatory)][string]$Message)
-    Write-Host "OK: $Message"
-}
-
-function Write-Note {
-    param([Parameter(Mandatory)][string]$Message)
-    # An exception message can span lines; the summary reads better on one.
-    $flat = ($Message -replace '\s+', ' ').Trim()
-    Write-Host "NOTE: $flat"
-    $script:Notes.Add($flat)
-}
-
-function Write-Failure {
-    param([Parameter(Mandatory)][string]$Message)
-    $flat = ($Message -replace '\s+', ' ').Trim()
-    Write-Host "FAILED: $flat"
-    $script:Failures.Add($flat)
-}
-
-# ---------------------------------------------------------------------------
-# Environment helpers
-# ---------------------------------------------------------------------------
-
-# Installers edit the stored PATH, not this process's copy of it. Re-reading it
-# is what lets a tool installed a moment ago be found by the step after it.
-function Update-ProcessPath {
-    $parts = @()
-    foreach ($scope in 'Machine', 'User') {
-        $value = [Environment]::GetEnvironmentVariable('Path', $scope)
-        if ($value) { $parts += $value }
-    }
-    if ($parts.Count -gt 0) {
-        $env:Path = ($parts -join ';')
-    }
-}
-
-# Adds a directory to the machine PATH if it exists and is not already there.
-function Add-MachinePath {
-    param([Parameter(Mandatory)][string]$Directory)
-
-    if (-not (Test-Path -LiteralPath $Directory)) {
-        return $false
-    }
-
-    $normalized = $Directory.TrimEnd('\')
-    $current = [Environment]::GetEnvironmentVariable('Path', 'Machine')
-    $entries = @()
-    if ($current) {
-        $entries = $current -split ';' | Where-Object { $_ -and $_.Trim() }
-    }
-
-    foreach ($entry in $entries) {
-        if ($entry.TrimEnd('\') -ieq $normalized) {
-            Update-ProcessPath
-            return $true
-        }
-    }
-
-    try {
-        $updated = (@($entries) + $normalized) -join ';'
-        [Environment]::SetEnvironmentVariable('Path', $updated, 'Machine')
-        Update-ProcessPath
-        Write-Ok "added '$normalized' to the machine PATH."
-        return $true
-    } catch {
-        Write-Failure "could not add '$normalized' to the machine PATH: $($_.Exception.Message.Trim())"
-        return $false
-    }
-}
-
-# Resolves a command the same way the workload inventory does. A command that
-# resolves inside WindowsApps is normally an execution-alias stub that opens the
-# Store rather than running, so it does not count as present unless the tool is
-# known to ship that way.
-function Resolve-Tool {
-    param(
-        [Parameter(Mandatory)][string[]]$Candidates,
-        [switch]$AllowStoreAlias
-    )
-
-    foreach ($candidate in $Candidates) {
-        $found = Get-Command $candidate -ErrorAction SilentlyContinue |
-            Select-Object -First 1
-        if ($found -and ($AllowStoreAlias -or $found.Source -notlike '*\WindowsApps\*')) {
-            return $found.Source
-        }
-    }
-    return $null
-}
-
-function Test-Tool {
-    param([Parameter(Mandatory)][string[]]$Candidates)
-    return [bool](Resolve-Tool -Candidates $Candidates)
-}
-
-# Windows Installer serializes machine-wide installs behind a single mutex and
-# hands 1618 to anyone who arrives while it is held. While an image is being
-# provisioned that is routine rather than exceptional, because other artifacts
-# install at the same time, so a 1618 is waited out and retried.
-$script:InstallerBusy = 1618
-
-function Test-WindowsInstallerBusy {
-    $mutex = $null
-    try {
-        $mutex = [System.Threading.Mutex]::OpenExisting('Global\_MSIExecute')
-        return $true
-    } catch [System.Threading.WaitHandleCannotBeOpenedException] {
-        # The mutex does not exist, so no installation is in progress.
-        return $false
-    } catch {
-        # Any other failure means the mutex exists but could not be opened,
-        # which still means an installation is in progress.
-        return $true
-    } finally {
-        if ($mutex) { $mutex.Dispose() }
-    }
-}
-
-function Wait-ForWindowsInstaller {
-    param([int]$TimeoutSeconds = 900)
-
-    if (-not (Test-WindowsInstallerBusy)) { return $true }
-
-    Write-Host '  another installation is in progress; waiting for it to finish...'
-    $deadline = (Get-Date).AddSeconds($TimeoutSeconds)
-    while ((Get-Date) -lt $deadline) {
-        Start-Sleep -Seconds 10
-        if (-not (Test-WindowsInstallerBusy)) {
-            Write-Host '  the other installation finished.'
-            return $true
-        }
-    }
-
-    Write-Host "  still in progress after $TimeoutSeconds seconds; going ahead anyway."
-    return $false
-}
-
-# Launches a program, waits for it, and returns its exit code alongside whatever
-# it wrote. Everything goes through Start-Process rather than the call operator
-# because msiexec is a GUI-subsystem program: the call operator leaves
-# $LASTEXITCODE unset for it, which would read as success and turn a failed
-# install into a silent one.
-function Start-ProgramOnce {
-    param(
-        [Parameter(Mandatory)][string]$FilePath,
-        [string[]]$Arguments = @()
-    )
-
-    $stdout = Join-Path $script:Scratch ('run-' + [Guid]::NewGuid().ToString('N') + '.out')
-    $stderr = [IO.Path]::ChangeExtension($stdout, 'err')
-    try {
-        if (-not (Test-Path -LiteralPath $script:Scratch)) {
-            New-Item -ItemType Directory -Path $script:Scratch -Force -ErrorAction SilentlyContinue | Out-Null
-        }
-
-        $launch = @{
-            FilePath               = $FilePath
-            Wait                   = $true
-            PassThru               = $true
-            NoNewWindow            = $true
-            RedirectStandardOutput = $stdout
-            RedirectStandardError  = $stderr
-        }
-        if ($Arguments.Count -gt 0) { $launch['ArgumentList'] = $Arguments }
-
-        $process = Start-Process @launch
-
-        $written = @()
-        foreach ($file in @($stdout, $stderr)) {
-            if (Test-Path -LiteralPath $file) {
-                $written += @(Get-Content -LiteralPath $file -ErrorAction SilentlyContinue)
-            }
-        }
-
-        return [pscustomobject]@{ Code = $process.ExitCode; Output = $written }
-    } catch {
-        # A launch failure embeds a position trace after the first line.
-        $reason = ($_.Exception.Message -split "`r?`n" | Select-Object -First 1).Trim()
-        Write-Host "  $reason"
-        return $null
-    } finally {
-        Remove-Item -LiteralPath $stdout, $stderr -Force -ErrorAction SilentlyContinue
-    }
-}
-
-# Runs a program and reports only whether it succeeded, keeping installer chatter
-# out of the transcript unless something goes wrong. Naming 1618 in RetryCodes
-# also makes every attempt wait for Windows Installer to go idle first, so the
-# common case is avoiding the collision rather than recovering from it.
-function Invoke-Program {
-    param(
-        [Parameter(Mandatory)][string]$FilePath,
-        [string[]]$Arguments = @(),
-        [int[]]$SuccessCodes = @(0),
-        [int[]]$RetryCodes = @(),
-        [int]$MaxAttempts = 1
-    )
-
-    $serialized = $RetryCodes -contains $script:InstallerBusy
-
-    for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) {
-        if ($serialized) { Wait-ForWindowsInstaller | Out-Null }
-
-        $result = Start-ProgramOnce -FilePath $FilePath -Arguments $Arguments
-        if ($null -eq $result) { return $false }
-
-        $code = $result.Code
-        if ($null -eq $code) { $code = 0 }
-        if ($SuccessCodes -contains $code) { return $true }
-
-        if (($RetryCodes -contains $code) -and ($attempt -lt $MaxAttempts)) {
-            Write-Host "  exit code $code; retrying ($attempt of $($MaxAttempts - 1))."
-            continue
-        }
-
-        $detail = ($result.Output | Select-Object -Last 3 | Out-String).Trim()
-        if ($detail) {
-            Write-Host "  $($detail -replace "`r?`n", "`n  ")"
-        }
-        Write-Host "  exit code $code"
-        return $false
-    }
-
-    return $false
-}
-
-function Get-Download {
-    param(
-        [Parameter(Mandatory)][string]$Uri,
-        [Parameter(Mandatory)][string]$OutFile
-    )
-
-    try {
-        if (-not (Test-Path -LiteralPath $script:Scratch)) {
-            New-Item -ItemType Directory -Path $script:Scratch -Force | Out-Null
-        }
-        Invoke-WebRequest -Uri $Uri -OutFile $OutFile -UseBasicParsing -TimeoutSec 600
-        return (Test-Path -LiteralPath $OutFile)
-    } catch {
-        Write-Host "  $($_.Exception.Message.Trim())"
-        return $false
-    }
-}
-
-function Install-Msi {
-    param(
-        [Parameter(Mandatory)][string]$Uri,
-        [Parameter(Mandatory)][string]$Name
-    )
-
-    $package = Join-Path $script:Scratch $Name
-    if (-not (Get-Download -Uri $Uri -OutFile $package)) {
-        return $false
-    }
-    return (Invoke-Program -FilePath 'msiexec.exe' -Arguments @('/i', $package, '/quiet', '/norestart') `
-        -SuccessCodes @(0, 3010) -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
-}
-
-# Some installers ship as a WiX bundle rather than a bare MSI and so take their
-# own switches instead of msiexec's. A bundle still wraps MSIs underneath, so it
-# can report 1618 just the same.
-function Install-Bundle {
-    param(
-        [Parameter(Mandatory)][string]$Uri,
-        [Parameter(Mandatory)][string]$Name
-    )
-
-    $package = Join-Path $script:Scratch $Name
-    if (-not (Get-Download -Uri $Uri -OutFile $package)) {
-        return $false
-    }
-    return (Invoke-Program -FilePath $package -Arguments @('/install', '/quiet', '/norestart') `
-        -SuccessCodes @(0, 3010) -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
-}
-
-# ---------------------------------------------------------------------------
-# Machine
-# ---------------------------------------------------------------------------
-
-Write-Step 'Machine'
-try {
-    $os = Get-CimInstance Win32_OperatingSystem -ErrorAction Stop
-    Write-Host "Windows      : $($os.Caption) $($os.Version)"
-} catch {
-    Write-Host "Windows      : $([Environment]::OSVersion.VersionString)"
-}
-Write-Host "Architecture : $env:PROCESSOR_ARCHITECTURE"
-Write-Host "PowerShell   : $($PSVersionTable.PSVersion)"
-Write-Host "User         : $([Environment]::UserName)"
-
-$isAdministrator = $false
-try {
-    $identity = [Security.Principal.WindowsIdentity]::GetCurrent()
-    $principal = New-Object Security.Principal.WindowsPrincipal($identity)
-    $isAdministrator = $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
-} catch {
-    # Left false; the warning below covers it.
-}
-Write-Host "Elevated     : $isAdministrator"
-
-if (-not $isAdministrator) {
-    Write-Note 'this session is not elevated; machine-wide installs and PATH changes will not take effect.'
-}
-
-Update-ProcessPath
-
-# Install-Tool orchestrates one tool: skip if already present, otherwise work
-# through the attempts in order, re-checking after each. PATH is refreshed
-# between attempts because an installer's PATH edit is not visible to this
-# process until it is re-read.
-function Install-Tool {
-    param(
-        [Parameter(Mandatory)][string]$Name,
-        [Parameter(Mandatory)][string[]]$Candidates,
-        [Parameter(Mandatory)][hashtable[]]$Attempts,
-        [string[]]$PathCandidates = @()
-    )
-
-    Write-Step "Installing $Name"
-
-    if (Test-Tool -Candidates $Candidates) {
-        Write-Ok "$Name is already present at $(Resolve-Tool -Candidates $Candidates)."
-        return
-    }
-
-    foreach ($attempt in $Attempts) {
-        Write-Host "Trying $($attempt.Description)..."
-        $succeeded = $false
-        try {
-            $succeeded = [bool](& $attempt.Action)
-        } catch {
-            Write-Host "  $($_.Exception.Message.Trim())"
-        }
-
-        Update-ProcessPath
-        foreach ($directory in $PathCandidates) {
-            if (Test-Path -LiteralPath $directory) {
-                Add-MachinePath -Directory $directory | Out-Null
-            }
-        }
-
-        $resolved = Resolve-Tool -Candidates $Candidates
-        if ($resolved) {
-            Write-Ok "$Name is available at $resolved."
-            return
-        }
-        if ($succeeded) {
-            Write-Host "  reported success but '$($Candidates[0])' still does not resolve."
-        }
-    }
-
-    Write-Failure "could not install $Name."
-}
-
-# ---------------------------------------------------------------------------
-# Package managers
-# ---------------------------------------------------------------------------
-
-# Installed first so the tools below can fall back to them.
-
-Install-Tool -Name 'Chocolatey' -Candidates @('choco') -PathCandidates @(
-    (Join-Path $env:ProgramData 'chocolatey\bin')
-) -Attempts @(
-    @{
-        Description = 'the official install script'
-        Action      = {
-            $installer = Join-Path $script:Scratch 'install-chocolatey.ps1'
-            if (-not (Get-Download -Uri 'https://community.chocolatey.org/install.ps1' -OutFile $installer)) {
-                return $false
-            }
-            $env:ChocolateyUseWindowsCompression = 'false'
-            & $installer | Out-Null
-            return $true
-        }
-    }
-)
-
-Install-Tool -Name 'Scoop' -Candidates @('scoop') -PathCandidates @(
-    (Join-Path $ScoopRoot 'shims')
-) -Attempts @(
-    @{
-        Description = 'the official install script'
-        Action      = {
-            # Scoop is normally per-user, which would hide it from the account
-            # that runs workloads later. Pointing it at a fixed directory and
-            # publishing that location machine-wide is what makes it shared.
-            try {
-                [Environment]::SetEnvironmentVariable('SCOOP', $ScoopRoot, 'Machine')
-            } catch {
-                # Without machine scope the install still succeeds; only the
-                # variable is missing for other accounts.
-                Write-Host '  could not publish SCOOP machine-wide.'
-            }
-            $env:SCOOP = $ScoopRoot
-
-            $installer = Join-Path $script:Scratch 'install-scoop.ps1'
-            if (-not (Get-Download -Uri 'https://get.scoop.sh' -OutFile $installer)) {
-                return $false
-            }
-            # The installer refuses to run elevated unless told that is intended.
-            $arguments = @{ ScoopDir = $ScoopRoot; ScoopGlobalDir = (Join-Path $ScoopRoot 'global') }
-            if ($isAdministrator) { $arguments['RunAsAdmin'] = $true }
-            & $installer @arguments | Out-Null
-            return $true
-        }
-    }
-)
-
-# ---------------------------------------------------------------------------
-# Tooling
-# ---------------------------------------------------------------------------
-
-Install-Tool -Name 'the .NET SDK' -Candidates @('dotnet') -PathCandidates @(
-    (Join-Path $env:ProgramFiles 'dotnet')
-) -Attempts @(
-    @{
-        Description = 'the official installer'
-        # The LTS channel link always points at the current long-term-support
-        # release, and there is a native build for each architecture.
-        Action      = {
-            Install-Bundle -Uri "https://aka.ms/dotnet/LTS/dotnet-sdk-win-$Architecture.exe" -Name 'dotnet-sdk.exe'
-        }
-    }
-)
-
-Install-Tool -Name 'the Azure CLI' -Candidates @('az') -PathCandidates @(
-    (Join-Path $env:ProgramFiles 'Microsoft SDKs\Azure\CLI2\wbin'),
-    (Join-Path ${env:ProgramFiles(x86)} 'Microsoft SDKs\Azure\CLI2\wbin')
-) -Attempts @(
-    @{
-        Description = 'the official installer'
-        # The Azure CLI publishes no ARM64 build, so an ARM64 machine gets the
-        # x64 one and runs it emulated.
-        Action      = { Install-Msi -Uri 'https://aka.ms/installazurecliwindowsx64' -Name 'azure-cli.msi' }
-    },
-    @{
-        Description = 'Chocolatey'
-        Action      = {
-            if (-not (Test-Tool -Candidates @('choco'))) { return $false }
-            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'azure-cli', '-y', '--no-progress') `
-                -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
-        }
-    }
-)
-
-Install-Tool -Name 'the GitHub CLI' -Candidates @('gh') -PathCandidates @(
-    (Join-Path $env:ProgramFiles 'GitHub CLI')
-) -Attempts @(
-    @{
-        Description = 'the latest published installer'
-        Action      = {
-            try {
-                $release = Invoke-RestMethod -Uri 'https://api.github.com/repos/cli/cli/releases/latest' `
-                    -UseBasicParsing -Headers @{ 'User-Agent' = 'mxc-setup' } -TimeoutSec 120
-            } catch {
-                Write-Host "  $($_.Exception.Message.Trim())"
-                return $false
-            }
-            # Assets are named by Go's architecture, not Windows'.
-            $suffix = if ($Architecture -eq 'arm64') { 'windows_arm64.msi' } else { 'windows_amd64.msi' }
-            $asset = $release.assets |
-                Where-Object { $_.name -like "*$suffix" } |
-                Select-Object -First 1
-            if (-not $asset) {
-                Write-Host "  no $suffix asset in release $($release.tag_name)."
-                return $false
-            }
-            return (Install-Msi -Uri $asset.browser_download_url -Name 'gh.msi')
-        }
-    },
-    @{
-        Description = 'Chocolatey'
-        Action      = {
-            if (-not (Test-Tool -Candidates @('choco'))) { return $false }
-            return (Invoke-Program -FilePath 'choco' -Arguments @('install', 'gh', '-y', '--no-progress') `
-                -RetryCodes @($script:InstallerBusy) -MaxAttempts 3)
-        }
-    }
-)
-
-Install-Tool -Name 'the NuGet CLI' -Candidates @('nuget') -PathCandidates @(
-    (Join-Path $env:ProgramData 'MXC\bin')
-) -Attempts @(
-    @{
-        Description = 'a direct download of the standalone executable'
-        Action      = {
-            # Published as a bare executable rather than a package, so it needs
-            # a directory of its own and an entry on PATH.
-            $target = Join-Path $env:ProgramData 'MXC\bin'
-            if (-not (Test-Path -LiteralPath $target)) {
-                New-Item -ItemType Directory -Path $target -Force -ErrorAction SilentlyContinue | Out-Null
-            }
-            if (-not (Test-Path -LiteralPath $target)) {
-                Write-Host "  could not create $target."
-                return $false
-            }
-            return (Get-Download -Uri 'https://dist.nuget.org/win-x86-commandline/latest/nuget.exe' `
-                    -OutFile (Join-Path $target 'nuget.exe'))
-        }
-    }
-)
-
-# ---------------------------------------------------------------------------
-# Inventory
-# ---------------------------------------------------------------------------
-
-Update-ProcessPath
-
-Write-Step 'Inventory'
-
-# Everything the finished image is expected to provide, whoever put it there.
-# openssl, winapp and winget are the exception: the first two are only published
-# as packaged applications and are installed once the machine is running, and
-# the third cannot be added to an image at all. They are listed so the log shows
-# the whole picture, but they are not counted as missing.
-$deferred = @('openssl', 'winapp', 'winget')
-
-$inventory = @(
-    @{ Name = 'pwsh';    Candidates = @('pwsh') },
-    @{ Name = 'git';     Candidates = @('git') },
-    @{ Name = 'node';    Candidates = @('node') },
-    @{ Name = 'npm';     Candidates = @('npm') },
-    @{ Name = 'npx';     Candidates = @('npx') },
-    @{ Name = 'python';  Candidates = @('python', 'python3') },
-    @{ Name = 'pip';     Candidates = @('pip', 'pip3') },
-    @{ Name = 'dotnet';  Candidates = @('dotnet') },
-    @{ Name = 'az';      Candidates = @('az') },
-    @{ Name = 'gh';      Candidates = @('gh') },
-    @{ Name = 'nuget';   Candidates = @('nuget') },
-    @{ Name = 'scoop';   Candidates = @('scoop') },
-    @{ Name = 'choco';   Candidates = @('choco') },
-    @{ Name = 'openssl'; Candidates = @('openssl') },
-    @{ Name = 'winapp';  Candidates = @('winapp') },
-    @{ Name = 'winget';  Candidates = @('winget'); AllowStoreAlias = $true }
-)
-
-$absent = @()
-foreach ($tool in $inventory) {
-    $resolved = Resolve-Tool -Candidates $tool.Candidates -AllowStoreAlias:([bool]$tool['AllowStoreAlias'])
-    if ($resolved) {
-        Write-Host ("  {0,-10} {1}" -f $tool.Name, $resolved)
-    } elseif ($deferred -contains $tool.Name) {
-        Write-Host ("  {0,-10} absent (installed once the machine is running)" -f $tool.Name)
-    } else {
-        Write-Host ("  {0,-10} ABSENT" -f $tool.Name)
-        $absent += $tool.Name
-    }
-}
-
-# ---------------------------------------------------------------------------
-# Cleanup and summary
-# ---------------------------------------------------------------------------
-
-Write-Step 'Cleaning up'
-if (Test-Path -LiteralPath $script:Scratch) {
-    Remove-Item -LiteralPath $script:Scratch -Recurse -Force -ErrorAction SilentlyContinue
-}
-Write-Ok 'removed the temporary download directory.'
-
-Write-Step 'Summary'
-if ($absent.Count -gt 0) {
-    Write-Host "Absent after installation: $($absent -join ', ')"
-} else {
-    Write-Host 'Every expected program is present.'
-}
-
-if ($script:Notes.Count -gt 0) {
-    Write-Host ''
-    Write-Host 'Notes:'
-    foreach ($entry in $script:Notes) { Write-Host "  - $entry" }
-}
-
-if ($script:Failures.Count -gt 0) {
-    Write-Host ''
-    Write-Host 'Failures:'
-    foreach ($entry in $script:Failures) { Write-Host "  - $entry" }
-} else {
-    Write-Host ''
-    Write-Host 'No failures.'
-}
-
-Write-Host ''
-Write-Host 'A new session is needed for the PATH changes to be visible.'
-Write-Host 'Setup finished.'
-
-exit 0
diff --git a/scripts/ci/Setup.sh b/scripts/ci/Setup.sh
deleted file mode 100755
index a35d6c00c..000000000
--- a/scripts/ci/Setup.sh
+++ /dev/null
@@ -1,496 +0,0 @@
-#!/usr/bin/env bash
-#
-# Installation script for MXC Linux machines (Debian / Ubuntu).
-#
-# Installs the sandboxing runtimes, kernel prerequisites and workload
-# interpreters an MXC machine is expected to provide, and persists the kernel
-# settings so they survive a reboot.
-#
-# Every step is best effort. Nothing here aborts the run and the script always
-# exits 0; what the machine actually ended up with is printed in the summary at
-# the end.
-
-set -o pipefail
-
-# Guarantee the exit status regardless of how the script leaves.
-trap 'exit 0' EXIT
-
-export DEBIAN_FRONTEND=noninteractive
-
-failures=()
-notes=()
-
-step() {
-    echo ""
-    echo "=== $* ==="
-}
-
-ok() {
-    echo "OK: $*"
-}
-
-note() {
-    echo "NOTE: $*"
-    notes+=("$*")
-}
-
-fail() {
-    echo "FAILED: $*"
-    failures+=("$*")
-}
-
-# Image builders run as root without sudo installed; interactive machines have
-# sudo and an unprivileged user. Resolve once and use the same wrapper for
-# every privileged call.
-if [ "$(id -u)" -eq 0 ]; then
-    priv() { "$@"; }
-elif command -v sudo >/dev/null 2>&1; then
-    priv() { sudo -n "$@"; }
-else
-    priv() { return 1; }
-fi
-
-have() {
-    command -v "$1" >/dev/null 2>&1
-}
-
-# ---------------------------------------------------------------------------
-# Distribution identity
-# ---------------------------------------------------------------------------
-
-distro_id=""
-distro_version=""
-distro_codename=""
-if [ -r /etc/os-release ]; then
-    # shellcheck disable=SC1091
-    . /etc/os-release
-    distro_id="${ID:-}"
-    distro_version="${VERSION_ID:-}"
-    distro_codename="${VERSION_CODENAME:-}"
-fi
-if [ -z "$distro_codename" ] && have lsb_release; then
-    distro_codename="$(lsb_release -cs 2>/dev/null)"
-fi
-
-step "Machine"
-echo "Distribution : ${distro_id:-unknown} ${distro_version:-} (${distro_codename:-unknown codename})"
-echo "Kernel       : $(uname -r 2>/dev/null)"
-echo "Architecture : $(uname -m 2>/dev/null)"
-
-if ! have apt-get; then
-    fail "apt-get is not available; this script targets Debian and Ubuntu machines."
-    step "Summary"
-    echo "Nothing was installed."
-    exit 0
-fi
-
-if ! priv true 2>/dev/null; then
-    fail "cannot obtain root privileges; no packages can be installed."
-    step "Summary"
-    echo "Nothing was installed."
-    exit 0
-fi
-
-# ---------------------------------------------------------------------------
-# Package helpers
-# ---------------------------------------------------------------------------
-
-apt_refresh() {
-    # A broken third-party feed makes the whole refresh non-zero; the installs
-    # that follow still decide success against whatever indexes did update.
-    if priv apt-get update; then
-        return 0
-    fi
-    note "apt-get update reported repository errors; continuing with the available package indexes."
-    return 0
-}
-
-# install <package>...
-install() {
-    priv env DEBIAN_FRONTEND=noninteractive \
-        apt-get install -y --no-install-recommends "$@"
-}
-
-# install_group <description> <package>...
-# Installs as one transaction, then retries individually so a single package
-# that is unavailable on this release cannot cost the machine the rest.
-install_group() {
-    description="$1"
-    shift
-    [ "$#" -eq 0 ] && return 0
-
-    if install "$@"; then
-        ok "$description: $*"
-        return 0
-    fi
-
-    note "combined install for $description failed; retrying each package on its own."
-    for package in "$@"; do
-        if install "$package"; then
-            ok "$description: $package"
-        else
-            fail "could not install '$package' ($description)."
-        fi
-    done
-}
-
-# available <package>
-available() {
-    apt-cache show "$1" >/dev/null 2>&1
-}
-
-step "Refreshing package indexes"
-apt_refresh
-
-step "Installing feed prerequisites"
-install_group "feed prerequisites" ca-certificates curl gnupg apt-transport-https
-
-# ---------------------------------------------------------------------------
-# Vendor package feeds
-# ---------------------------------------------------------------------------
-
-# Carries PowerShell on this family. It does not carry the Azure CLI, which is
-# published separately below.
-add_microsoft_prod_feed() {
-    if [ -f /etc/apt/sources.list.d/microsoft-prod.list ] ||
-        [ -f /etc/apt/sources.list.d/microsoft-prod.sources ]; then
-        ok "the Microsoft package feed is already configured."
-        return 0
-    fi
-
-    if [ -z "$distro_id" ] || [ -z "$distro_version" ]; then
-        fail "could not read the distribution; skipping the Microsoft package feed."
-        return 1
-    fi
-
-    package="/tmp/packages-microsoft-prod.deb"
-    if ! curl -fsSL \
-        "https://packages.microsoft.com/config/${distro_id}/${distro_version}/packages-microsoft-prod.deb" \
-        -o "$package"; then
-        fail "no Microsoft package feed is published for ${distro_id} ${distro_version}."
-        return 1
-    fi
-
-    if ! priv dpkg -i "$package"; then
-        rm -f "$package"
-        fail "could not install the Microsoft package feed."
-        return 1
-    fi
-
-    rm -f "$package"
-    apt_refresh
-    ok "added the Microsoft package feed."
-}
-
-# The Azure CLI has a repository of its own, keyed by distribution codename
-# rather than version. It lags new releases, so the codename this machine
-# reports may not be published yet.
-add_azure_cli_feed() {
-    if [ -f /etc/apt/sources.list.d/azure-cli.list ] ||
-        [ -f /etc/apt/sources.list.d/azure-cli.sources ]; then
-        ok "the Azure CLI package feed is already configured."
-        return 0
-    fi
-
-    if [ -z "$distro_codename" ]; then
-        fail "could not read the distribution codename; skipping the Azure CLI feed."
-        return 1
-    fi
-
-    candidates="$distro_codename"
-    case "$distro_id" in
-        debian) [ "$distro_codename" = "bookworm" ] || candidates="$candidates bookworm" ;;
-        ubuntu) [ "$distro_codename" = "noble" ] || candidates="$candidates noble" ;;
-    esac
-
-    # Probe before writing. An unpublished suite in a source list makes every
-    # later refresh fail, which would cost this machine the packages that were
-    # otherwise going to install.
-    suite=""
-    for candidate in $candidates; do
-        if curl -fsL --head -o /dev/null \
-            "https://packages.microsoft.com/repos/azure-cli/dists/${candidate}/Release"; then
-            suite="$candidate"
-            break
-        fi
-        echo "The Azure CLI feed publishes no '$candidate' suite."
-    done
-
-    if [ -z "$suite" ]; then
-        fail "the Azure CLI feed publishes no suite usable on '${distro_codename}'."
-        return 1
-    fi
-    if [ "$suite" != "$distro_codename" ]; then
-        note "the Azure CLI feed has no '${distro_codename}' suite; using '${suite}'."
-    fi
-
-    keyring="/etc/apt/keyrings/microsoft.gpg"
-    if ! priv install -d -m 0755 /etc/apt/keyrings; then
-        fail "could not create the apt keyring directory."
-        return 1
-    fi
-    if ! curl -fsSL https://packages.microsoft.com/keys/microsoft.asc |
-        gpg --dearmor | priv dd of="$keyring" status=none; then
-        fail "could not install the Microsoft signing key."
-        return 1
-    fi
-    priv chmod go+r "$keyring"
-
-    if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=${keyring}] https://packages.microsoft.com/repos/azure-cli/ ${suite} main" |
-        priv tee /etc/apt/sources.list.d/azure-cli.list >/dev/null; then
-        fail "could not write the Azure CLI package source."
-        return 1
-    fi
-
-    apt_refresh
-    ok "added the Azure CLI package feed (${suite})."
-}
-
-add_github_cli_feed() {
-    if [ -f /etc/apt/sources.list.d/github-cli.list ]; then
-        ok "the GitHub CLI package feed is already configured."
-        return 0
-    fi
-
-    keyring="/usr/share/keyrings/githubcli-archive-keyring.gpg"
-    if ! curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg |
-        priv dd of="$keyring" status=none; then
-        fail "could not download the GitHub CLI signing key."
-        return 1
-    fi
-    priv chmod go+r "$keyring"
-
-    if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=${keyring}] https://cli.github.com/packages stable main" |
-        priv tee /etc/apt/sources.list.d/github-cli.list >/dev/null; then
-        fail "could not write the GitHub CLI package source."
-        return 1
-    fi
-
-    apt_refresh
-    ok "added the GitHub CLI package feed."
-}
-
-step "Adding vendor package feeds"
-add_microsoft_prod_feed
-add_azure_cli_feed
-add_github_cli_feed
-
-# ---------------------------------------------------------------------------
-# Sandboxing runtimes
-# ---------------------------------------------------------------------------
-
-step "Installing unprivileged sandboxing prerequisites"
-install_group "unprivileged sandboxing" \
-    bubblewrap slirp4netns util-linux iproute2 iptables
-
-step "Installing container prerequisites"
-container_packages="lxc dnsmasq-base bridge-utils iptables"
-# Debian folded the tools into the lxc package; Ubuntu still ships them apart.
-if available lxc-utils; then
-    container_packages="$container_packages lxc-utils"
-fi
-if available lxc-templates; then
-    container_packages="$container_packages lxc-templates"
-fi
-# shellcheck disable=SC2086
-install_group "containers" $container_packages
-
-# ---------------------------------------------------------------------------
-# Workload interpreters
-# ---------------------------------------------------------------------------
-
-step "Installing workload interpreters"
-install_group "interpreters" \
-    git openssl nodejs npm python3 python3-pip
-
-# Named separately so an unavailable release does not take the rest with it.
-install_group "the .NET SDK" dotnet-sdk-8.0
-install_group "PowerShell" powershell
-install_group "the Azure CLI" azure-cli
-install_group "the GitHub CLI" gh
-
-# ---------------------------------------------------------------------------
-# Kernel configuration
-# ---------------------------------------------------------------------------
-
-# Written to disk rather than only applied live, so the machine comes back the
-# same way after a reboot.
-persist_module() {
-    module="$1"
-    if echo "$module" | priv tee "/etc/modules-load.d/mxc-${module}.conf" >/dev/null; then
-        ok "${module} will load at boot."
-    else
-        fail "could not configure ${module} to load at boot."
-    fi
-    priv modprobe "$module" 2>/dev/null ||
-        note "could not load ${module} now; it may be built in or unavailable until reboot."
-}
-
-step "Configuring kernel modules"
-# Connection-state matching in the packet filter needs conntrack present.
-persist_module nf_conntrack
-# Bridged traffic is only visible to the packet filter with this loaded.
-persist_module br_netfilter
-
-step "Configuring kernel settings"
-sysctl_file="/etc/sysctl.d/99-mxc.conf"
-sysctl_lines=""
-
-# Unprivileged user namespaces are what unprivileged sandboxing is built on.
-# Recent releases restrict them by default.
-if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then
-    sysctl_lines="${sysctl_lines}kernel.apparmor_restrict_unprivileged_userns = 0
-"
-fi
-if [ -e /proc/sys/kernel/unprivileged_userns_clone ]; then
-    sysctl_lines="${sysctl_lines}kernel.unprivileged_userns_clone = 1
-"
-fi
-# Bridged traffic must traverse the packet filter for container network policy
-# to apply to it.
-sysctl_lines="${sysctl_lines}net.bridge.bridge-nf-call-iptables = 1
-net.bridge.bridge-nf-call-ip6tables = 1
-net.ipv4.ip_forward = 1
-"
-
-if printf '%s' "$sysctl_lines" | priv tee "$sysctl_file" >/dev/null; then
-    ok "wrote ${sysctl_file}."
-else
-    fail "could not write ${sysctl_file}."
-fi
-
-if priv sysctl --system >/dev/null 2>&1; then
-    ok "applied the kernel settings."
-else
-    note "could not apply the kernel settings now; they will take effect after a reboot."
-fi
-
-# ---------------------------------------------------------------------------
-# Container bridge
-# ---------------------------------------------------------------------------
-
-step "Configuring the container bridge"
-# Debian ships the bridge disabled; Ubuntu enables it.
-if [ -f /etc/default/lxc-net ]; then
-    if priv sed -i 's/^\s*#\?\s*USE_LXC_BRIDGE\s*=.*/USE_LXC_BRIDGE="true"/' /etc/default/lxc-net &&
-        grep -q 'USE_LXC_BRIDGE="true"' /etc/default/lxc-net; then
-        ok "enabled the container bridge in /etc/default/lxc-net."
-    else
-        note "could not confirm the bridge setting in /etc/default/lxc-net."
-    fi
-else
-    note "/etc/default/lxc-net is absent; leaving the bridge at its packaged default."
-fi
-
-if have systemctl; then
-    if priv systemctl enable lxc-net >/dev/null 2>&1; then
-        ok "the container bridge will start at boot."
-    else
-        note "could not enable the container bridge service."
-    fi
-    if priv systemctl restart lxc-net >/dev/null 2>&1; then
-        ok "started the container bridge."
-    else
-        note "could not start the container bridge; it should come up on the next boot."
-    fi
-else
-    note "no systemctl on this machine; skipping the bridge service."
-fi
-
-# Package installs do not always activate the confinement profiles the
-# container tooling relies on.
-if have apparmor_parser && [ -d /etc/apparmor.d ]; then
-    if priv apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null; then
-        ok "reloaded the container confinement profiles."
-    else
-        note "could not reload the container confinement profiles."
-    fi
-fi
-
-# ---------------------------------------------------------------------------
-# Inventory
-# ---------------------------------------------------------------------------
-
-step "Inventory"
-
-# name|candidates tried in order
-inventory="bwrap|bwrap
-slirp4netns|slirp4netns
-unshare|unshare
-nsenter|nsenter
-ip|ip
-iptables|iptables
-ip6tables|ip6tables
-lxc-start|lxc-start
-git|git
-openssl|openssl
-node|node,nodejs
-npm|npm
-npx|npx
-python|python3,python
-pip|pip3,pip
-dotnet|dotnet
-pwsh|pwsh
-az|az
-gh|gh"
-
-absent=""
-while IFS='|' read -r name candidates; do
-    [ -z "$name" ] && continue
-    resolved=""
-    old_ifs="$IFS"
-    IFS=','
-    for candidate in $candidates; do
-        if resolved="$(command -v "$candidate" 2>/dev/null)"; then
-            break
-        fi
-        resolved=""
-    done
-    IFS="$old_ifs"
-
-    if [ -n "$resolved" ]; then
-        printf '  %-14s %s\n' "$name" "$resolved"
-    else
-        printf '  %-14s ABSENT\n' "$name"
-        absent="${absent:+$absent }$name"
-    fi
-done <<EOF
-$inventory
-EOF
-
-# ---------------------------------------------------------------------------
-# Cleanup and summary
-# ---------------------------------------------------------------------------
-
-step "Cleaning up"
-priv apt-get clean >/dev/null 2>&1 && ok "cleared the package cache."
-
-step "Summary"
-if [ -n "$absent" ]; then
-    echo "Absent after installation: $absent"
-else
-    echo "Every expected program is present."
-fi
-
-if [ "${#notes[@]}" -gt 0 ]; then
-    echo ""
-    echo "Notes:"
-    for entry in "${notes[@]}"; do
-        echo "  - $entry"
-    done
-fi
-
-if [ "${#failures[@]}" -gt 0 ]; then
-    echo ""
-    echo "Failures:"
-    for entry in "${failures[@]}"; do
-        echo "  - $entry"
-    done
-else
-    echo ""
-    echo "No failures."
-fi
-
-echo ""
-echo "Setup finished."
-exit 0
diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh
index c180307f6..2f91bd80f 100644
--- a/scripts/ci/prepare-linux-host.sh
+++ b/scripts/ci/prepare-linux-host.sh
@@ -46,8 +46,7 @@ resolve_package_manager() {
 
 # install_packages <package>...
 # Installs from the feeds the host already has configured, and returns the
-# package manager's own status so each caller decides what a failure means: a
-# missing backend prerequisite is fatal, a missing workload interpreter is not.
+# package manager's own status so each caller decides what a failure means.
 install_packages() {
     case "$package_manager" in
         apt-get)
@@ -66,18 +65,6 @@ install_packages() {
     esac
 }
 
-# Returns 0 when any of the comma-separated candidates is on PATH.
-have_command() {
-    local candidate
-    local IFS=','
-    for candidate in $1; do
-        if command -v "$candidate" >/dev/null 2>&1; then
-            return 0
-        fi
-    done
-    return 1
-}
-
 # Red Hat ships no third-party content, so epel-release is not in RHEL's own
 # repos; the documented install is the release RPM straight from Fedora. 
 install_epel() {
@@ -255,281 +242,11 @@ enable_bridge_netfilter() {
     done
 }
 
-# PowerShell, the Azure CLI, and the GitHub CLI are in no distribution's own
-# repositories, so each comes from its vendor's feed. A feed is added at most
-# once and a failure is remembered, so the second tool wanting a broken feed
-# does not retry it.
-microsoft_feed_state=""
-add_microsoft_feed() {
-    case "$microsoft_feed_state" in
-        added) return 0 ;;
-        failed) return 1 ;;
-    esac
-    microsoft_feed_state="failed"
-
-    local id="" version_id=""
-    if [[ -r /etc/os-release ]]; then
-        # shellcheck disable=SC1091
-        . /etc/os-release
-        id="${ID:-}"
-        version_id="${VERSION_ID:-}"
-    fi
-    if [[ -z "$id" || -z "$version_id" ]]; then
-        echo "WARNING: could not read the distribution from /etc/os-release; skipping the Microsoft package feed." >&2
-        return 1
-    fi
-
-    if [[ "$package_manager" == "apt-get" ]]; then
-        local package="/tmp/packages-microsoft-prod.deb"
-        if ! curl -fsSL \
-            "https://packages.microsoft.com/config/${id}/${version_id}/packages-microsoft-prod.deb" \
-            -o "$package"; then
-            echo "WARNING: no Microsoft package feed published for ${id} ${version_id}." >&2
-            return 1
-        fi
-        if ! sudo dpkg -i "$package"; then
-            rm -f "$package"
-            echo "WARNING: could not install the Microsoft package feed." >&2
-            return 1
-        fi
-        rm -f "$package"
-        apt_update
-    else
-        # The RPM feed is keyed on the major version alone.
-        if ! install_packages \
-            "https://packages.microsoft.com/config/rhel/${version_id%%.*}/packages-microsoft-prod.rpm"; then
-            echo "WARNING: could not install the Microsoft package feed." >&2
-            return 1
-        fi
-    fi
-
-    microsoft_feed_state="added"
-}
-
-# On apt the Azure CLI is not in packages-microsoft-prod. It is published to a
-# repository of its own, keyed by distribution codename rather than version, so
-# adding the prod feed and then asking for azure-cli resolves nothing and fails
-# with "E: Unable to locate package azure-cli". The RPM side needs none of this:
-# the prod feed carries azure-cli directly, which is why only Debian-family
-# hosts were affected.
-azure_cli_feed_state=""
-add_azure_cli_apt_feed() {
-    case "$azure_cli_feed_state" in
-        added) return 0 ;;
-        failed) return 1 ;;
-    esac
-    azure_cli_feed_state="failed"
-
-    local id="" codename=""
-    if [[ -r /etc/os-release ]]; then
-        # shellcheck disable=SC1091
-        . /etc/os-release
-        id="${ID:-}"
-        codename="${VERSION_CODENAME:-}"
-    fi
-    if [[ -z "$codename" ]] && command -v lsb_release >/dev/null 2>&1; then
-        codename="$(lsb_release -cs 2>/dev/null || true)"
-    fi
-    if [[ -z "$codename" ]]; then
-        echo "WARNING: could not read the distribution codename; skipping the Azure CLI package feed." >&2
-        return 1
-    fi
-
-    # The vendor publishes a suite per codename and lags new releases, so the
-    # host's own codename may not exist yet. An unpublished suite must never
-    # reach a source list: apt then fails every later refresh against it, which
-    # would cost this host the packages that were going to install fine. Probe
-    # first, and fall back to the newest suite the vendor does publish for the
-    # family. The package vendors its own Python interpreter, so it does not
-    # bind to the release it was built against.
-    local suites=("$codename")
-    case "$id" in
-        debian) [[ "$codename" == "bookworm" ]] || suites+=(bookworm) ;;
-        ubuntu) [[ "$codename" == "noble" ]] || suites+=(noble) ;;
-    esac
-
-    local suite="" candidate
-    for candidate in "${suites[@]}"; do
-        if curl -fsL --head -o /dev/null \
-            "https://packages.microsoft.com/repos/azure-cli/dists/$candidate/Release"; then
-            suite="$candidate"
-            break
-        fi
-        echo "The Azure CLI feed publishes no '$candidate' suite." >&2
-    done
-
-    if [[ -z "$suite" ]]; then
-        echo "WARNING: the Azure CLI feed publishes no suite usable on ${id:-this distribution} '$codename'." >&2
-        return 1
-    fi
-    if [[ "$suite" != "$codename" ]]; then
-        echo "WARNING: the Azure CLI feed has no '$codename' suite; using '$suite' instead." >&2
-    fi
-
-    local keyring="/etc/apt/keyrings/microsoft.gpg"
-    if ! sudo install -d -m 0755 /etc/apt/keyrings; then
-        echo "WARNING: could not create the apt keyring directory." >&2
-        return 1
-    fi
-    if ! curl -fsSL https://packages.microsoft.com/keys/microsoft.asc |
-        gpg --dearmor | sudo dd of="$keyring" status=none; then
-        echo "WARNING: could not install the Microsoft signing key." >&2
-        return 1
-    fi
-    if ! sudo chmod go+r "$keyring"; then
-        echo "WARNING: could not make the Microsoft signing key readable." >&2
-        return 1
-    fi
-    if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=$keyring] https://packages.microsoft.com/repos/azure-cli/ $suite main" |
-        sudo tee /etc/apt/sources.list.d/azure-cli.list >/dev/null; then
-        echo "WARNING: could not write the Azure CLI package source." >&2
-        return 1
-    fi
-    apt_update
-
-    azure_cli_feed_state="added"
-}
-
-add_github_feed() {
-    if [[ "$package_manager" == "apt-get" ]]; then
-        local keyring="/usr/share/keyrings/githubcli-archive-keyring.gpg"
-        if ! curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg |
-            sudo dd of="$keyring" status=none; then
-            echo "WARNING: could not download the GitHub CLI signing key." >&2
-            return 1
-        fi
-        if ! sudo chmod go+r "$keyring"; then
-            echo "WARNING: could not make the GitHub CLI signing key readable." >&2
-            return 1
-        fi
-        if ! echo "deb [arch=$(dpkg --print-architecture) signed-by=$keyring] https://cli.github.com/packages stable main" |
-            sudo tee /etc/apt/sources.list.d/github-cli.list >/dev/null; then
-            echo "WARNING: could not write the GitHub CLI package source." >&2
-            return 1
-        fi
-        apt_update
-        return 0
-    fi
-
-    # config-manager is a separate package on some releases and is what adds the
-    # feed, so it is requested before it is used.
-    install_packages "dnf-command(config-manager)" >/dev/null 2>&1 || true
-    if ! sudo "$package_manager" config-manager \
-        --add-repo https://cli.github.com/packages/rpm/gh-cli.repo; then
-        echo "WARNING: could not add the GitHub CLI package feed." >&2
-        return 1
-    fi
-}
-
-# Installs the workload interpreters the image did not already provide.
-#
-# Host preparation verifies rather than installs wherever it can, because a
-# missing backend prerequisite means the image is wrong for the job. The
-# workload interpreters are the exception: they are ordinary developer tools the
-# package manager can supply in one transaction, which is cheaper than losing a
-# suite's coverage to a tool the image happened not to bake.
-#
-# Every step here is best effort and none of them fails the job. What the host
-# actually ended up with is reported by assert_workload_interpreters below.
-install_workload_interpreters() {
-    if ! resolve_package_manager; then
-        echo "WARNING: no supported package manager on this host; skipping workload interpreter installation." >&2
-        return 0
-    fi
-
-    # name|command candidates|apt package(s)|rpm package(s)
-    # npx has no package of its own; it arrives with npm. pwsh, az, and gh need
-    # a vendor feed and are handled separately below.
-    local packaged=(
-        "git|git|git|git"
-        "openssl|openssl|openssl|openssl"
-        "node|node|nodejs|nodejs"
-        "npm|npm|npm|npm"
-        "python|python3,python|python3|python3"
-        "pip|pip3,pip|python3-pip|python3-pip"
-        "dotnet|dotnet|dotnet-sdk-8.0|dotnet-sdk-8.0"
-    )
-
-    local entry name candidates apt_packages rpm_packages
-    local wanted=()
-    for entry in "${packaged[@]}"; do
-        IFS='|' read -r name candidates apt_packages rpm_packages <<<"$entry"
-        if have_command "$candidates"; then
-            continue
-        fi
-        # Deliberately unquoted: an entry may name more than one package.
-        if [[ "$package_manager" == "apt-get" ]]; then
-            wanted+=($apt_packages)
-        else
-            wanted+=($rpm_packages)
-        fi
-    done
-
-    local need_pwsh="false" need_az="false" need_gh="false"
-    have_command pwsh || need_pwsh="true"
-    have_command az || need_az="true"
-    have_command gh || need_gh="true"
-
-    if [[ ${#wanted[@]} -eq 0 && "$need_pwsh" == "false" &&
-        "$need_az" == "false" && "$need_gh" == "false" ]]; then
-        echo "All workload interpreters are already present; nothing to install."
-        return 0
-    fi
-
-    if [[ "$package_manager" == "apt-get" ]]; then
-        apt_update
-    fi
-
-    if [[ "$need_pwsh" == "true" || "$need_az" == "true" || "$need_gh" == "true" ]]; then
-        # Every feed is fetched over HTTPS and verified against a signing key,
-        # so these have to be present before any of them can be added.
-        install_packages ca-certificates curl gnupg ||
-            echo "WARNING: could not install the prerequisites for adding vendor package feeds." >&2
-    fi
-
-    if [[ "$need_pwsh" == "true" ]] && add_microsoft_feed; then
-        wanted+=(powershell)
-    fi
-
-    # Two different feeds: the prod feed carries azure-cli on the RPM side, but
-    # on apt it lives in the Azure CLI's own repository.
-    if [[ "$need_az" == "true" ]]; then
-        if [[ "$package_manager" == "apt-get" ]]; then
-            if add_azure_cli_apt_feed; then
-                wanted+=(azure-cli)
-            fi
-        elif add_microsoft_feed; then
-            wanted+=(azure-cli)
-        fi
-    fi
-
-    if [[ "$need_gh" == "true" ]] && add_github_feed; then
-        wanted+=(gh)
-    fi
-
-    if [[ ${#wanted[@]} -eq 0 ]]; then
-        return 0
-    fi
-
-    echo "Installing workload interpreters: ${wanted[*]}"
-    if install_packages "${wanted[@]}"; then
-        return 0
-    fi
-
-    # A single unavailable package fails the whole transaction, so retry them
-    # one at a time rather than leaving the host with none of them.
-    echo "WARNING: the combined install failed; retrying each package on its own." >&2
-    local package
-    for package in "${wanted[@]}"; do
-        install_packages "$package" ||
-            echo "WARNING: could not install package '$package'." >&2
-    done
-}
-
-# Verifies the interpreters test suites drive inside the sandbox. This runs
-# after install_workload_interpreters and reports what the host ended up with,
-# so a tool that could not be installed stays visible instead of silently
-# absent.
+# Verifies the interpreters test suites drive inside the sandbox and reports
+# what the image actually provides, so a tool the image was built without stays
+# visible instead of silently absent. These are baked into the image rather
+# than installed here: unlike a backend's prerequisites, they are ordinary
+# developer tools that every job expects to find already in place.
 #
 # The check is suite-agnostic: it describes what a validation host is expected
 # to provide, not what any one suite consumes, so a future suite that shells out
@@ -584,7 +301,6 @@ assert_workload_interpreters() {
 chmod +x "$binary_directory/lxc-exec"
 
 # Run for every backend: this is host inventory, not a backend prerequisite.
-install_workload_interpreters
 assert_workload_interpreters
 
 case "$backend" in
diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json
index aa6448abf..d2f00a454 100644
--- a/scripts/ci/validation-test-matrix.json
+++ b/scripts/ci/validation-test-matrix.json
@@ -389,14 +389,6 @@
       }
     ],
     "weekly": [],
-    "enabled": [
-      {
-        "os": "windows-25h2",
-        "backends": [
-          "process-t3",
-          "wslc"
-        ]
-      }
-    ]
+    "enabled": []
   }
 }

From 4293730355af9bcb8eedfa9c2656151290b1a20f Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 2 Sep 2026 13:38:57 -0700
Subject: [PATCH 39/44] Testing nightly plan before PR

---
 .github/workflows/Validation.Infrastructure.Tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
index 87c8b937b..83dd18499 100644
--- a/.github/workflows/Validation.Infrastructure.Tests.yml
+++ b/.github/workflows/Validation.Infrastructure.Tests.yml
@@ -33,4 +33,4 @@ jobs:
     needs: [windows, linux, macos]
     uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
     with:
-      plan: enabled
\ No newline at end of file
+      plan: nightly
\ No newline at end of file

From b8410c6ea8ae6ee93b1bdb661ec9be26d248c263 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 2 Sep 2026 14:18:07 -0700
Subject: [PATCH 40/44] remove infrastructure testing yml

---
 .../Validation.Infrastructure.Tests.yml       | 36 -------------------
 1 file changed, 36 deletions(-)
 delete mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml

diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml
deleted file mode 100644
index 83dd18499..000000000
--- a/.github/workflows/Validation.Infrastructure.Tests.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-name: Validation Infrastructure Testing
-
-on:
-  push:
-    branches:
-      - user/emichlin/validation-infra-3
-
-concurrency:
-  group: validation-infra-pr-tests-${{ github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  actions: read
-  contents: read
-
-jobs:
-  dependency-feed-check:
-    uses: ./.github/workflows/Dependency.Feed.Check.Job.yml
-
-  windows:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.Windows.Job.yml
-
-  linux:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.Linux.Job.yml
-
-  macos:
-    needs: dependency-feed-check
-    uses: ./.github/workflows/Build.MacOS.Job.yml
-
-  test:
-    needs: [windows, linux, macos]
-    uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml
-    with:
-      plan: nightly
\ No newline at end of file

From 8cd7686d912cb9b84d1f366bf2c5118218c5e7e9 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Wed, 2 Sep 2026 15:22:27 -0700
Subject: [PATCH 41/44] fix minor bugs

---
 scripts/ci/prepare-windows-host.ps1 | 21 +++++++++++++++------
 tests/scripts/T3-Workloads.ps1      |  3 ++-
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index c62ed8ca4..9aff3be74 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -286,15 +286,24 @@ function Repair-Winget {
     }
 }
 
-# Re-reads PATH from the registry so a directory an installer just published is
-# visible to this process, which otherwise keeps the PATH it started with.
+# Appends the registry PATH entries this process has not picked up yet, so a
+# directory an installer just published becomes visible to a process that
+# otherwise keeps the PATH it started with.
 function Update-ProcessPath {
-    $parts = foreach ($scope in 'Machine', 'User') {
+    $seen = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
+    foreach ($entry in $env:Path -split ';') {
+        if ($entry) { [void]$seen.Add($entry.TrimEnd('\')) }
+    }
+
+    foreach ($scope in 'Machine', 'User') {
         $value = [Environment]::GetEnvironmentVariable('Path', $scope)
-        if ($value) { $value -split ';' }
+        if (-not $value) { continue }
+        foreach ($entry in $value -split ';') {
+            if ($entry -and $seen.Add($entry.TrimEnd('\'))) {
+                $env:Path = "$env:Path;$entry"
+            }
+        }
     }
-    $seen = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
-    $env:Path = (($parts | Where-Object { $_ -and $seen.Add($_) }) -join ';')
 }
 
 # Resolves a command the way Assert-WorkloadInterpreters does, so a tool this
diff --git a/tests/scripts/T3-Workloads.ps1 b/tests/scripts/T3-Workloads.ps1
index 888ae7d03..2e9a614d2 100644
--- a/tests/scripts/T3-Workloads.ps1
+++ b/tests/scripts/T3-Workloads.ps1
@@ -858,7 +858,8 @@ catch {
     Write-Host ''
     Write-Host "ABORT: $_" -ForegroundColor Red
     Write-Host $_.ScriptStackTrace -ForegroundColor DarkRed
-    exit 2
+    # ABORT = FAIL test.
+    Record-Workload -Id 'ABORT' -Name 'harness aborted' -Status 'fail' -ExitCode 2 -Detail "$_"
 }
 finally {
     Section 'Summary'

From ef8780b1061a725e52ae631ebdf6d8a0850e6507 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:53:08 -0700
Subject: [PATCH 42/44] Fixed duplicated docs

---
 docs/ci-validation-infrastructure.md | 29 ++++------------------------
 1 file changed, 4 insertions(+), 25 deletions(-)

diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md
index 22593e22b..9687fadc6 100644
--- a/docs/ci-validation-infrastructure.md
+++ b/docs/ci-validation-infrastructure.md
@@ -181,13 +181,12 @@ because Seatbelt has no wired suite.
 
 ### `backendDelayedStart`
 
-Optional, and **currently empty** — no backend is staggered today, so every job
-starts as soon as its runner is ready. The section staggers the start of jobs
+Optional. This section staggers the start of jobs
 for a named backend instead of letting them all begin at once:
 
 ```json
 "backendDelayedStart": [
-  { "backend": "wslc", "seconds": 300 }
+  { "backend": "wslc", "seconds": 30 }
 ]
 ```
 
@@ -198,29 +197,9 @@ answer with rate limiting and stalled downloads.
 
 `seconds` is the gap between consecutive jobs of that backend, counted per
 backend and following the resolved job order. With the example entry above,
-four WSLC jobs would start at 0, 300, 600, and 900 seconds.
+four WSLC jobs would start at 0, 30, 60, and 90 seconds.
 
-The resolver puts the offset on every matrix entry as
-`startup_delay_seconds` — `0` where no stagger applies, which is every entry
-while the section is empty — and the job sleeps that long before its first
-network step. Job timeout is a flat 180 minutes, with plenty of room for any
-wait you'd reasonably configure.
-
-Leave the section out (or empty) and every job starts as soon as its runner is
-ready. A backend id that no plan schedules is accepted; it just never applies.
-
-Do keep in mind that the runner is held while it sleeps — Actions can't defer
-allocating a matrix job, so the wait has to happen inside it. Use no more than
-the contention calls for. This spreads simultaneous load and nothing else; a
-single download that stalls on its own is unaffected.
-
-Leave the section out (or empty) and every job starts as soon as its runner is
-ready. A backend id that no plan schedules is accepted; it just never applies.
-
-Do keep in mind that the runner is held while it sleeps — Actions can't defer
-allocating a matrix job, so the wait has to happen inside it. Use no more than
-the contention calls for. This spreads simultaneous load and nothing else; a
-single download that stalls on its own is unaffected.
+Avoid long delays since idle runners might be harvested.
 
 ## Backend status
 

From 1d0041927315947be3ca30e50518455963392a38 Mon Sep 17 00:00:00 2001
From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 8 Sep 2026 11:45:16 -0700
Subject: [PATCH 43/44] condensed copilot instructions and addressed PR
 feedback

---
 .github/copilot-instructions.md     | 122 +++++-----------------------
 scripts/ci/prepare-windows-host.ps1 |   6 ++
 2 files changed, 28 insertions(+), 100 deletions(-)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 50d91877b..f3830d874 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -52,107 +52,29 @@ parallel, then to the lint / versioning / SDK jobs.
 
 **Validation (E2E) test infrastructure.** Fully documented in
 [`docs/ci-validation-infrastructure.md`](../docs/ci-validation-infrastructure.md)
-(matrix contents, job names, per-backend coverage and status, and the runbook
-for adding/removing an OS, backend, or plan). Backend E2E tests run from those
-same build artifacts — never from a fresh build — so artifact production and
-consumption stay in one workflow run:
-
-- `.github/workflows/Validation.Tests.Scheduled.yml` — scheduled entry point.
-  The `nightly` plan runs Mon–Sat; Sunday runs `nightly` *and* `weekly`.
-  `workflow_dispatch` takes a `plan` input to run one on demand.
+— read it before changing any of the pieces below. Backend E2E tests run from
+the build artifacts, never from a fresh build, so an entry point must call the
+three `Build.*.Job.yml` workflows before calling the matrix job.
+
+- `.github/workflows/Validation.Tests.Scheduled.yml` — scheduled entry point
+  (`nightly` Mon–Sat, `nightly` + `weekly` on Sunday); `workflow_dispatch`
+  takes a `plan` input.
 - `.github/workflows/Validation.Tests.Matrix.Job.yml` — workflow-call-only,
-  takes a `plan` input. Its `resolve` job expands the plan into per-family
-  matrices, then the `windows` / `linux` / `macos` jobs each download the
-  artifact, prepare the host, and run the backend suite.
-
-An entry point must build the artifacts (call the three `Build.*.Job.yml`
-workflows) before calling the matrix job.
-
-**The matrix is declarative:**
-
-- `scripts/ci/validation-test-matrix.json` is the catalog: `platforms` (each
-  with per-architecture target/artifact/1ES pool and the backends that platform
-  supports), `triggers` (which OS/backend pairs each plan runs), and the
-  optional `backendDelayedStart` (per-backend job-start stagger, in seconds;
-  currently empty — nothing is staggered).
-  The `triggers` keys *are* the plan list — the resolver reads them at run time,
-  so adding a plan needs no script change.
-- `scripts/ci/resolve-validation-test-matrix.mjs` validates that catalog and
-  expands a plan (currently `pr`, `nightly`, `weekly`, `enabled`) into GitHub
-  Actions matrices. It rejects an invalid catalog before any specialized test
-  runner is allocated, so add a backend to a trigger only where the platform
-  declares it.
-- A non-macOS platform architecture with an empty `pool` is never scheduled,
-  which is how a catalog entry stays declared but dormant. macOS entries use a
-  GitHub-hosted `runner` instead of a 1ES `pool`.
-
-**Host preparation** happens in the matrix job before the tests, keyed by the
-matrix `backend` id: `scripts/ci/prepare-windows-host.ps1`,
-`scripts/ci/prepare-linux-host.sh`, and `scripts/ci/prepare-macos-host.sh`. A
-backend with no prerequisites is an explicit no-op, so the step runs
-unconditionally for every entry. Independent of the backend id, all three also
-verify the host's workload interpreters and CLIs (`pwsh`, `git`, `node`, `npm`,
-`npx`, `python`, `pip`, `dotnet`, `az`, `gh`, `openssl`, plus `nuget`, `winapp`,
-`winget`, `scoop`, and `choco` on Windows only, and `brew` on macOS only) —
-Windows in `Assert-WorkloadInterpreters`, Linux and macOS in a deliberately
-duplicated bash-3.2-compatible `assert_workload_interpreters` function so each
-platform's list can diverge. Every pool runs an image provisioned ahead of time,
-so the whole list is normally present before a job starts. The provisioning
-scripts are **not in this repository** — they live in the
-`validation-provision-artifacts` branch of the ADO repo as
-`ubuntu-debian-provision.sh` / `rhel-provision.sh` (Linux) and
-`windows-provision.ps1` (Windows). No job installs a workload interpreter on any
-platform: all three scripts verify and report only, and a missing one warns
-rather than failing (except `pwsh` on Windows). A backend's own prerequisites
-are separate and are still installed per job — `install_bubblewrap` /
-`install_lxc` in `prepare-linux-host.sh` reach the package manager through
-`resolve_package_manager` and `install_packages`, so a new distribution family
-is one arm in `install_packages` rather than a branch in every installer. The
-Windows provisioning script installs
-no packaged application and never uses winget, because neither is available
-during image provisioning; it takes only `-Architecture` (`x64`/`arm64`, which
-selects the `gh` MSI and the .NET SDK bundle — the Azure CLI has no ARM64 build
-and is used emulated) and `-ScoopRoot`. It launches every installer through
-`Start-Process` rather
-than the call operator, because `msiexec` is a GUI-subsystem program for which
-the call operator leaves `$LASTEXITCODE` unset — which previously read as
-success and turned a failed install into a silent one. Windows Installer
-serializes machine-wide installs behind the `Global\_MSIExecute` mutex and
-returns 1618 to whoever arrives while another provisioning artifact holds it, so
-every MSI-touching call waits for that mutex to clear before each attempt and
-retries a 1618 up to three times. Its scope is the .NET SDK, `choco`, `scoop`,
-`az`, `gh` and `nuget`. The .NET SDK comes from the arch-matched
-`aka.ms/dotnet/LTS/dotnet-sdk-win-<arch>.exe` bundle rather than
-`dotnet-install.ps1`, because that script's only PATH write is to `$env:path` —
-it persists nothing, so an SDK it installs is on disk but resolves from nowhere
-in any later process. The remaining runtimes (`node`, `python`, `pwsh`, `git`)
-arrive from separate image artifacts, which it only inventories.
-Windows therefore provisions
-two entries at job time, also best-effort and also ahead of the inventory.
-`Repair-Winget`
-re-registers the App Installer package (`Add-AppxPackage -RegisterByFamilyName`)
-when `winget` resolves on `PATH` but fails to run, the symptom of a package the
-image shipped but never registered for the account the job runs as. It decides
-by invoking `winget --version`, not by resolving the command, since a resolvable
-alias is the broken case. `Install-PackagedTooling` then installs `winapp` and
-`openssl`, the two interpreters that cannot be baked into an image at all
-because both ship only as packaged applications; `winapp` is requested as
-`--installer-type zip` so the portable build lands on `PATH` instead of behind a
-`WindowsApps` alias the inventory's store-alias filter would reject, and
-`openssl`'s `bin` directory is appended to `PATH` afterwards because its
-installer publishes none.
-
-**Test dispatch** goes through `scripts/ci/run_backend_validation_tests.ps1`
-(Windows) and `scripts/ci/run_backend_validation_tests.sh` (Linux/macOS), which map
-the matrix `backend` id to the repository's existing backend suite. Ids that
-share a suite get their own case (`process-t1` and `process-t3` both run
-`WinProcessContainer-Tests.ps1`, which derives the tier it expects from the
-host's own `--probe`; `process-t3` additionally runs `T3-Workloads.ps1` and
-reports both suites' exit codes together, so one failing suite never hides the
-other). A backend with no wired suite fails loudly rather than
-reporting a false success. The Windows dispatcher points `TEMP` at
-`$RUNNER_TEMP` before running a suite, so anything a test writes to the temp
-directory is picked up by the job's log upload without per-file CI wiring.
+  takes a `plan`. Its `resolve` job expands the plan into per-family matrices;
+  the `windows` / `linux` / `macos` jobs then download the artifact, prepare
+  the host, and run the backend suite.
+- `scripts/ci/validation-test-matrix.json` — the declarative catalog
+  (`platforms`, `triggers`, `backendDelayedStart`). Its `triggers` keys *are*
+  the plan list. `scripts/ci/resolve-validation-test-matrix.mjs` validates the
+  catalog and expands a plan, so a backend may only be triggered where its
+  platform declares it.
+- `scripts/ci/prepare-{windows,linux,macos}-host.{ps1,sh}` — per-backend host
+  prep, plus a verify-only inventory of the workload interpreters. Backend
+  prerequisites are installed per job; workload interpreters never are (they
+  come from image provisioning scripts that live outside this repository).
+- `scripts/ci/run_backend_validation_tests.{ps1,sh}` — map a matrix `backend`
+  id to the repository's existing backend suite. An unwired id fails loudly
+  rather than reporting a false success.
 
 ### Individual components
 
diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1
index 9aff3be74..491e861f4 100644
--- a/scripts/ci/prepare-windows-host.ps1
+++ b/scripts/ci/prepare-windows-host.ps1
@@ -35,6 +35,8 @@ param(
 Set-StrictMode -Version Latest
 $ErrorActionPreference = 'Stop'
 
+$script:WingetPath = $null
+
 function Exit-WithError {
     param([Parameter(Mandatory)][string]$Message)
 
@@ -366,11 +368,13 @@ function Install-PackagedTooling {
     $wanted = $packages | Where-Object { -not (Resolve-Interpreter -Candidates $_.Candidates) }
     if (-not $wanted) {
         Write-Host "Packaged workload tooling is already present."
+        $global:LASTEXITCODE = 0
         return
     }
 
     if (-not $script:WingetPath) {
         Write-Host "::warning::winget is unavailable, so $(($wanted.Name) -join ' and ') cannot be installed."
+        $global:LASTEXITCODE = 0
         return
     }
 
@@ -411,6 +415,8 @@ function Install-PackagedTooling {
             Write-Host "::warning::$($package.Name) installed but still does not resolve on PATH."
         }
     }
+
+    $global:LASTEXITCODE = 0
 }
 
 function Initialize-MicroVmHost {

From 3b8701fcd7133510565c8c9d531743d5a696e690 Mon Sep 17 00:00:00 2001
From: Elliot <31219104+theelliotm@users.noreply.github.com>
Date: Tue, 8 Sep 2026 11:52:27 -0700
Subject: [PATCH 44/44] Update backend preparation and validation instructions

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 .github/copilot-instructions.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index f3830d874..50a1e90c5 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -69,9 +69,10 @@ three `Build.*.Job.yml` workflows before calling the matrix job.
   catalog and expands a plan, so a backend may only be triggered where its
   platform declares it.
 - `scripts/ci/prepare-{windows,linux,macos}-host.{ps1,sh}` — per-backend host
-  prep, plus a verify-only inventory of the workload interpreters. Backend
-  prerequisites are installed per job; workload interpreters never are (they
-  come from image provisioning scripts that live outside this repository).
+  prep, plus an inventory of the workload interpreters. Backend prerequisites

+  are installed per job; most workload interpreters come from image provisioning

+  scripts outside this repository, while Windows prep installs packaged `winapp`

+  and OpenSSL per job.

 - `scripts/ci/run_backend_validation_tests.{ps1,sh}` — map a matrix `backend`
   id to the repository's existing backend suite. An unwired id fails loudly
   rather than reporting a false success.