diff --git a/README.md b/README.md
index f884f4d70..ea2174587 100644
--- a/README.md
+++ b/README.md
@@ -306,6 +306,7 @@ The following sets of tools are available (toolsets marked with ✓ in the Defau
| ossm | Most common tools for managing OSSM, check the [OSSM documentation](https://github.com/openshift/openshift-mcp-server/blob/main/docs/OSSM.md) for more details. | |
| ovn-kubernetes | OVN-Kubernetes CNI network troubleshooting tools | |
| tekton | Tekton pipeline management tools for Pipelines, PipelineRuns, Tasks, TaskRuns, and troubleshooting. | |
+| tnf | Two-Node Fencing (TNF) cluster diagnostics | |
@@ -1733,6 +1734,20 @@ Example output:
+
+
+tnf
+
+- **tnf_check_fencing_config** - Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. Validates cluster topology, critical operator health (etcd, machine-api, baremetal), Machine/Node/BareMetalHost correlation, BMC addresses and credentials, pacemaker/STONITH fencing status via Kubernetes resources. Returns a diagnostic summary identifying configuration issues that could prevent fencing from functioning correctly.
+ - `namespace` (`string`) - Namespace containing BareMetalHost resources (e.g. 'openshift-machine-api'). If omitted, searches all namespaces.
+
+- **tnf_check_stonith_status** - Check STONITH and pacemaker fencing status on a Two-Node Fencing (TNF) cluster. Creates a temporary privileged debug pod on a control-plane node to run pcs diagnostic commands. Returns pacemaker cluster state, STONITH device configuration, quorum status, and recent fencing history. The debug pod is automatically cleaned up after execution.
+ - `namespace` (`string`) - Namespace to create the temporary debug pod in (optional, defaults to 'default').
+ - `node` (`string`) - Name of the node to run diagnostics on. If omitted, auto-detects the first control-plane node.
+ - `timeout_seconds` (`integer`) - Maximum time in seconds to wait for the diagnostic commands to complete (optional, defaults to 120).
+
+
+
@@ -1846,6 +1861,16 @@ Example output:
+
+
+tnf
+
+- **tnf-troubleshoot** - Generate a step-by-step troubleshooting guide for diagnosing Two-Node Fencing (TNF) cluster issues including STONITH configuration, quorum status, BMC health, and split-brain risk assessment
+ - `node` (`string`) - Node to run STONITH diagnostics on (auto-detects if omitted)
+ - `namespace` (`string`) - Namespace for BareMetalHost resources (default: openshift-machine-api)
+
+
+
@@ -1877,6 +1902,15 @@ Example output:
- MIME Type: `text/plain`
+
+
+tnf
+
+- **tnf-fencing-domain-knowledge** - TNF two-node fencing domain knowledge: quorum rules, split-brain risk matrix, and recovery procedures
+ - URI: `tnf://domain-knowledge/fencing`
+ - MIME Type: `text/markdown`
+
+
diff --git a/docs/README.md b/docs/README.md
index 893e9ed0f..495570243 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -31,6 +31,7 @@ Choose the guide that matches your needs:
- **[NetObserv](NETOBSERV.md)** - Network observability flows, metrics, and alerts (Helm on OpenShift)
- **[KubeVirt](kubevirt.md)** - KubeVirt virtual machine management tools
- **[Tekton](tekton.md)** - Tekton PipelineRun, TaskRun, and troubleshooting tools
+- **[TNF](tnf.md)** - Two-Node Fencing diagnostics (pacemaker, STONITH, BMC health)
## Feature Specifications
diff --git a/docs/configuration.md b/docs/configuration.md
index 757585b71..f84f6eb9f 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -369,6 +369,7 @@ Toolsets group related tools together. Enable only the toolsets you need to redu
| ossm | Most common tools for managing OSSM, check the [OSSM documentation](https://github.com/openshift/openshift-mcp-server/blob/main/docs/OSSM.md) for more details. | |
| ovn-kubernetes | OVN-Kubernetes CNI network troubleshooting tools | |
| tekton | Tekton pipeline management tools for Pipelines, PipelineRuns, Tasks, TaskRuns, and troubleshooting. | |
+| tnf | Two-Node Fencing (TNF) cluster diagnostics | |
@@ -406,6 +407,15 @@ toolsets = ["core", "config", "helm", "kubevirt"]
- MIME Type: `text/plain`
+
+
+tnf
+
+- **tnf-fencing-domain-knowledge** - TNF two-node fencing domain knowledge: quorum rules, split-brain risk matrix, and recovery procedures
+ - URI: `tnf://domain-knowledge/fencing`
+ - MIME Type: `text/markdown`
+
+
diff --git a/docs/tnf.md b/docs/tnf.md
new file mode 100644
index 000000000..f92cf5b2d
--- /dev/null
+++ b/docs/tnf.md
@@ -0,0 +1,114 @@
+# TNF (Two-Node Fencing) Support
+
+The `tnf` toolset extends the OpenShift MCP server with diagnostics for Two-Node Fencing clusters. These are 2-node bare metal OpenShift clusters that use pacemaker and STONITH for fencing to prevent split-brain scenarios.
+
+## Tools
+
+### tnf_check_fencing_config
+
+Validates fencing configuration and readiness for a TNF cluster. Checks cluster topology (platform, node count, TNF profile), critical operator health (etcd, machine-api, baremetal), Machine/Node/BareMetalHost correlation, BMC addresses and credential secrets, FenceAgentsRemediation templates and active remediations, and NodeHealthCheck resources. Returns a diagnostic summary identifying configuration issues that could prevent fencing from functioning correctly.
+
+**Arguments:**
+- `namespace` (optional): Namespace containing BareMetalHost resources (default: searches all namespaces)
+
+### tnf_check_stonith_status
+
+Creates a temporary privileged debug pod on a control-plane node to run `pcs` diagnostic commands. Returns pacemaker cluster state, STONITH device configuration, quorum status, and recent fencing history. The debug pod is automatically cleaned up after execution.
+
+**Arguments:**
+- `node` (optional): Name of the node to run diagnostics on (default: auto-detects the first control-plane node)
+- `namespace` (optional): Namespace to create the temporary debug pod in (default: `default`)
+- `timeout_seconds` (optional): Maximum time in seconds to wait for the diagnostic commands to complete (default: 120)
+
+## Prompt
+
+### tnf-troubleshoot
+
+Runs a full TNF fencing diagnostic workflow that collects cluster topology, node health, critical operator status, BareMetalHost/BMC health, pacemaker/STONITH status, and remediation operator status into a single structured report. Guides the LLM through analysis using the domain knowledge resource to assess split-brain risk and recommend actions.
+
+**Arguments:**
+- `node` (optional): Preferred node to run STONITH diagnostics from
+
+**Invoke in Claude Code:**
+```text
+/mcp__kubernetes-mcp-server__tnf-troubleshoot
+```
+
+## Resource
+
+### tnf://domain-knowledge/fencing
+
+Static reference material covering:
+- How two-node fencing works (corosync, pacemaker, STONITH)
+- Two-node quorum rules and `wait_for_all` behavior
+- Split-brain risk assessment matrix
+- Common issues and recovery procedures (STONITH disabled, BMC unreachable, node won't rejoin, quorum lost, fence race)
+
+## Enable the TNF Toolset
+
+### Option 1: Command Line
+
+```bash
+kubernetes-mcp-server --toolsets core,config,tnf
+```
+
+### Option 2: Configuration File
+
+```toml
+toolsets = ["core", "config", "tnf"]
+```
+
+### Option 3: MCP Client Configuration
+
+```json
+{
+ "mcpServers": {
+ "kubernetes": {
+ "command": "kubernetes-mcp-server",
+ "args": ["--toolsets", "core,config,tnf"]
+ }
+ }
+}
+```
+
+## Prerequisites
+
+TNF support requires:
+
+1. **Two-node bare metal OpenShift cluster** with pacemaker/STONITH fencing configured
+2. **BareMetalHost CRDs** installed (standard on bare metal clusters via the baremetal-operator)
+3. **Proper RBAC** for reading Nodes, BareMetalHosts, Secrets, and creating debug pods
+
+### Verify TNF Setup
+
+```bash
+# Check for 2-node bare metal cluster
+oc get nodes
+oc get infrastructure cluster -o jsonpath='{.status.platform}'
+
+# Check BareMetalHosts exist
+oc get baremetalhosts -A
+
+# Check pacemaker is running
+oc debug node/ -- chroot /host pcs status
+```
+
+## What It Diagnoses
+
+| Area | What's Checked |
+|------|---------------|
+| Cluster topology | Platform, node count, TNF profile detection |
+| Node health | Kubernetes Ready status, conditions |
+| BareMetalHost | Provisioning state, power status, BMC address |
+| BMC credentials | Secret existence, username/password keys present |
+| Pacemaker | Cluster name, online/offline/standby nodes |
+| STONITH devices | Configuration, agent type, target node, started status |
+| Quorum | Quorate status, vote count, two-node flags |
+| Fencing history | Recent fencing events, success/failure |
+| Remediation operators | FenceAgentsRemediation and NodeHealthCheck CRD status |
+
+## Limitations
+
+- **API must be reachable**: All tools depend on the Kubernetes API. When the API is unreachable (e.g. during a fencing event that takes down the API server), the tools detect the failure and return an out-of-band recovery guide with BMC access and manual diagnostic procedures instead of raw cluster data. The `tnf-troubleshoot` prompt integrates this fallback automatically.
+- **Read-only diagnostics**: The tools do not trigger, stop, or modify fencing operations.
+- **Debug pod privileges**: The STONITH check requires creating a privileged debug pod with host access (same as `oc debug node/`).
diff --git a/evals/core-eval-testing/acp-anthropic/eval-tnf.yaml b/evals/core-eval-testing/acp-anthropic/eval-tnf.yaml
new file mode 120000
index 000000000..b03603404
--- /dev/null
+++ b/evals/core-eval-testing/acp-anthropic/eval-tnf.yaml
@@ -0,0 +1 @@
+../builtin-openai/eval-tnf.yaml
\ No newline at end of file
diff --git a/evals/core-eval-testing/acp-google/eval-tnf.yaml b/evals/core-eval-testing/acp-google/eval-tnf.yaml
new file mode 120000
index 000000000..b03603404
--- /dev/null
+++ b/evals/core-eval-testing/acp-google/eval-tnf.yaml
@@ -0,0 +1 @@
+../builtin-openai/eval-tnf.yaml
\ No newline at end of file
diff --git a/evals/core-eval-testing/builtin-anthropic/eval-tnf.yaml b/evals/core-eval-testing/builtin-anthropic/eval-tnf.yaml
new file mode 120000
index 000000000..b03603404
--- /dev/null
+++ b/evals/core-eval-testing/builtin-anthropic/eval-tnf.yaml
@@ -0,0 +1 @@
+../builtin-openai/eval-tnf.yaml
\ No newline at end of file
diff --git a/evals/core-eval-testing/builtin-google/eval-tnf.yaml b/evals/core-eval-testing/builtin-google/eval-tnf.yaml
new file mode 120000
index 000000000..b03603404
--- /dev/null
+++ b/evals/core-eval-testing/builtin-google/eval-tnf.yaml
@@ -0,0 +1 @@
+../builtin-openai/eval-tnf.yaml
\ No newline at end of file
diff --git a/evals/core-eval-testing/builtin-openai/eval-all.yaml b/evals/core-eval-testing/builtin-openai/eval-all.yaml
index df40b0d26..aedd32938 100644
--- a/evals/core-eval-testing/builtin-openai/eval-all.yaml
+++ b/evals/core-eval-testing/builtin-openai/eval-all.yaml
@@ -83,3 +83,13 @@ config:
toolPattern: "netobserv_.*"
minToolCalls: 1
maxToolCalls: 10
+ # TNF (Two-Node Fencing) tasks - requires 2-node bare metal cluster
+ - glob: ../../tasks/tnf/*/*.yaml
+ labelSelector:
+ suite: tnf
+ assertions:
+ toolsUsed:
+ - server: kubernetes
+ toolPattern: "tnf_.*"
+ minToolCalls: 1
+ maxToolCalls: 10
diff --git a/evals/core-eval-testing/builtin-openai/eval-tnf.yaml b/evals/core-eval-testing/builtin-openai/eval-tnf.yaml
new file mode 100644
index 000000000..1ef14ef1c
--- /dev/null
+++ b/evals/core-eval-testing/builtin-openai/eval-tnf.yaml
@@ -0,0 +1,43 @@
+kind: Eval
+metadata:
+ name: "tnf-e2e"
+config:
+ agent:
+ type: "file"
+ path: agent.yaml
+ mcpConfigFile: ../../mcp-config.yaml
+ extensions:
+ kubernetes:
+ package: https://github.com/mcpchecker/kubernetes-extension@v0.0.4
+ llmJudge:
+ ref:
+ type: file
+ path: agent.yaml
+ taskSets:
+ - glob: ../../tasks/*/*/*.yaml
+ labelSelector:
+ suite: core
+ assertions:
+ toolsUsed:
+ - server: kubernetes
+ toolPattern: ".*"
+ minToolCalls: 1
+ maxToolCalls: 20
+ - glob: ../../tasks/*/*/*.yaml
+ labelSelector:
+ suite: config
+ assertions:
+ toolsUsed:
+ - server: kubernetes
+ toolPattern: "configuration_.*"
+ minToolCalls: 1
+ maxToolCalls: 10
+ - glob: ../../tasks/tnf/*/*.yaml
+ labelSelector:
+ suite: tnf
+ assertions:
+ toolsUsed:
+ - server: kubernetes
+ toolPattern: "tnf_.*"
+ minToolCalls: 1
+ maxToolCalls: 10
diff --git a/evals/tasks/tnf/check-fencing-config/task.yaml b/evals/tasks/tnf/check-fencing-config/task.yaml
new file mode 100644
index 000000000..fc3f11ef8
--- /dev/null
+++ b/evals/tasks/tnf/check-fencing-config/task.yaml
@@ -0,0 +1,15 @@
+kind: Task
+metadata:
+ labels:
+ suite: tnf
+ name: check-fencing-config
+ difficulty: easy
+steps:
+ prompt:
+ inline: Check the fencing configuration for this Two-Node Fencing cluster. Verify that BareMetalHost resources exist, BMC credentials are valid, and the cluster topology matches a TNF profile.
+ verify:
+ contains: "BMC"
+ assertions:
+ toolsUsed:
+ - server: kubernetes
+ toolPattern: "tnf_check_fencing_config"
diff --git a/evals/tasks/tnf/check-stonith-status/task.yaml b/evals/tasks/tnf/check-stonith-status/task.yaml
new file mode 100644
index 000000000..d37501caf
--- /dev/null
+++ b/evals/tasks/tnf/check-stonith-status/task.yaml
@@ -0,0 +1,15 @@
+kind: Task
+metadata:
+ labels:
+ suite: tnf
+ name: check-stonith-status
+ difficulty: medium
+steps:
+ prompt:
+ inline: Check the pacemaker and STONITH status on this cluster. Report which STONITH devices are configured, whether they are started, and the current quorum status.
+ verify:
+ contains: "STONITH"
+ assertions:
+ toolsUsed:
+ - server: kubernetes
+ toolPattern: "tnf_check_stonith_status"
diff --git a/evals/tasks/tnf/diagnose-fencing-health/task.yaml b/evals/tasks/tnf/diagnose-fencing-health/task.yaml
new file mode 100644
index 000000000..b532390c1
--- /dev/null
+++ b/evals/tasks/tnf/diagnose-fencing-health/task.yaml
@@ -0,0 +1,15 @@
+kind: Task
+metadata:
+ labels:
+ suite: tnf
+ name: diagnose-fencing-health
+ difficulty: hard
+steps:
+ prompt:
+ inline: Perform a full fencing health assessment for this Two-Node Fencing cluster. Check the fencing configuration, STONITH status, and analyze whether there is any split-brain risk. Provide a summary with an overall health status and any recommended actions.
+ verify:
+ contains: "fencing"
+ assertions:
+ toolsUsed:
+ - server: kubernetes
+ toolPattern: "tnf_.*"
diff --git a/pkg/mcp/modules.go b/pkg/mcp/modules.go
index 0c5ba3c6c..89e63f733 100644
--- a/pkg/mcp/modules.go
+++ b/pkg/mcp/modules.go
@@ -9,4 +9,5 @@ import (
_ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/kubevirt"
_ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/netobserv"
_ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tekton"
+ _ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf"
)
diff --git a/pkg/mcp/testdata/toolsets-tnf-tools.json b/pkg/mcp/testdata/toolsets-tnf-tools.json
new file mode 100644
index 000000000..af7012ef9
--- /dev/null
+++ b/pkg/mcp/testdata/toolsets-tnf-tools.json
@@ -0,0 +1,53 @@
+[
+ {
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": false,
+ "readOnlyHint": true,
+ "title": "TNF: Check Fencing Config"
+ },
+ "description": "Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. Validates cluster topology, critical operator health (etcd, machine-api, baremetal), Machine/Node/BareMetalHost correlation, BMC addresses and credentials, pacemaker/STONITH fencing status via Kubernetes resources. Returns a diagnostic summary identifying configuration issues that could prevent fencing from functioning correctly.",
+ "inputSchema": {
+ "properties": {
+ "namespace": {
+ "description": "Namespace containing BareMetalHost resources (e.g. 'openshift-machine-api'). If omitted, searches all namespaces.",
+ "type": "string"
+ }
+ },
+ "type": "object"
+ },
+ "name": "tnf_check_fencing_config",
+ "title": "TNF: Check Fencing Config"
+ },
+ {
+ "annotations": {
+ "destructiveHint": false,
+ "idempotentHint": true,
+ "openWorldHint": false,
+ "readOnlyHint": true,
+ "title": "TNF: Check STONITH Status"
+ },
+ "description": "Check STONITH and pacemaker fencing status on a Two-Node Fencing (TNF) cluster. Creates a temporary privileged debug pod on a control-plane node to run pcs diagnostic commands. Returns pacemaker cluster state, STONITH device configuration, quorum status, and recent fencing history. The debug pod is automatically cleaned up after execution.",
+ "inputSchema": {
+ "properties": {
+ "namespace": {
+ "description": "Namespace to create the temporary debug pod in (optional, defaults to 'default').",
+ "type": "string"
+ },
+ "node": {
+ "description": "Name of the node to run diagnostics on. If omitted, auto-detects the first control-plane node.",
+ "type": "string"
+ },
+ "timeout_seconds": {
+ "description": "Maximum time in seconds to wait for the diagnostic commands to complete (optional, defaults to 120).",
+ "minimum": 1,
+ "type": "integer"
+ }
+ },
+ "type": "object"
+ },
+ "name": "tnf_check_stonith_status",
+ "title": "TNF: Check STONITH Status"
+ }
+]
diff --git a/pkg/mcp/toolsets_test.go b/pkg/mcp/toolsets_test.go
index 6d9f50f05..abf85b629 100644
--- a/pkg/mcp/toolsets_test.go
+++ b/pkg/mcp/toolsets_test.go
@@ -25,6 +25,7 @@ import (
"github.com/containers/kubernetes-mcp-server/pkg/toolsets/openshift"
"github.com/containers/kubernetes-mcp-server/pkg/toolsets/ovnkubernetes"
"github.com/containers/kubernetes-mcp-server/pkg/toolsets/tekton"
+ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf"
"github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/stretchr/testify/suite"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -262,6 +263,7 @@ func (s *ToolsetsSuite) TestGranularToolsetsTools() {
&tekton.Toolset{},
&clusterDiagnosticsToolset.Toolset{},
&cniDiagnosticsToolset.Toolset{},
+ &tnf.Toolset{},
}
for _, testCase := range testCases {
s.Run("Toolset "+testCase.GetName(), func() {
diff --git a/pkg/toolsets/tnf/fencing/api_check.go b/pkg/toolsets/tnf/fencing/api_check.go
new file mode 100644
index 000000000..2ec11f190
--- /dev/null
+++ b/pkg/toolsets/tnf/fencing/api_check.go
@@ -0,0 +1,254 @@
+package fencing
+
+import (
+ "errors"
+ "net"
+ "strings"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+)
+
+// IsAPIUnreachable probes the Kubernetes API server via the discovery
+// client. It returns true only when the API server cannot be reached
+// at the network level (connection refused, no route, timeout). API-level
+// errors such as Forbidden or Unauthorized indicate the server IS
+// reachable and return false.
+func IsAPIUnreachable(client api.KubernetesClient) bool {
+ if client == nil {
+ return false
+ }
+ _, err := client.DiscoveryClient().ServerVersion()
+ if err == nil {
+ return false
+ }
+ return isConnectionError(err)
+}
+
+func isConnectionError(err error) bool {
+ var netErr net.Error
+ if errors.As(err, &netErr) {
+ return true
+ }
+ var opErr *net.OpError
+ if errors.As(err, &opErr) {
+ return true
+ }
+ msg := err.Error()
+ return strings.Contains(msg, "connection refused") ||
+ strings.Contains(msg, "no route to host") ||
+ strings.Contains(msg, "no such host") ||
+ strings.Contains(msg, "i/o timeout") ||
+ strings.Contains(msg, "dial tcp") ||
+ strings.Contains(msg, "context deadline exceeded") ||
+ strings.Contains(msg, "connection reset by peer")
+}
+
+// APIUnreachableGuide returns a structured troubleshooting guide for when
+// the Kubernetes API server is unreachable. It covers three scenarios:
+// failed install, upgrade/reboot, and crash/degraded.
+func APIUnreachableGuide() string {
+ return `# TNF Diagnostic: Kubernetes API Unreachable
+
+The Kubernetes API server is not responding. This prevents all cluster-level
+diagnostics. Use out-of-band access (BMC, SSH) to determine what happened.
+
+---
+
+## Determine Your Scenario
+
+| Clue | Likely Scenario |
+|------|----------------|
+| Install was recently started, cluster never came up | **Scenario 1: Install failure** |
+| Upgrade or MachineConfig change was in progress | **Scenario 2: Upgrade/reboot** |
+| Cluster was running normally, then stopped | **Scenario 3: Crash/degraded** |
+
+---
+
+## Common First Steps (All Scenarios)
+
+### 1. Check Node Power State via BMC
+
+Use IPMI or Redfish to verify nodes are powered on:
+
+` + "```" + `
+# IPMI
+ipmitool -I lanplus -H -U -P power status
+
+# Redfish
+curl -k -u : https:///redfish/v1/Systems/1 | jq '.PowerState'
+` + "```" + `
+
+### 2. SSH to Nodes
+
+If nodes booted far enough, SSH access may be available:
+
+` + "```" + `
+ssh core@
+` + "```" + `
+
+### 3. Check API VIP Assignment
+
+From any node, verify the API VIP is assigned to a network interface:
+
+` + "```" + `
+ip addr show | grep
+` + "```" + `
+
+If the VIP is not assigned, the API server or keepalived is not running.
+
+### 4. Check DNS Resolution
+
+` + "```" + `
+dig api..
+` + "```" + `
+
+---
+
+## Scenario 1: Failed or In-Progress Install
+
+The cluster has never fully bootstrapped. The assisted-installer agent
+or bootstrap process may still be running or may have failed.
+
+### Check Agent-Based Installer Status
+
+` + "```" + `
+# Agent service logs (ABI installs)
+journalctl -u agent --no-pager -n 100
+
+# Assisted-installer local API (if agent is running)
+curl -s http://localhost:8090/api/assisted-install/v2/clusters | jq '.[].status'
+` + "```" + `
+
+### Check Bootstrap Progression
+
+` + "```" + `
+# Has bootstrap completed?
+ls -la /opt/openshift/.bootkube.done
+
+# Are static pod manifests present?
+ls /etc/kubernetes/manifests/
+
+# Is kubelet running?
+systemctl status kubelet
+
+# etcd status
+sudo crictl ps --name etcd
+sudo crictl logs $(sudo crictl ps --name etcd -q) 2>&1 | tail -20
+` + "```" + `
+
+### Common Install Failures
+
+- **DNS misconfiguration** — api/api-int/*.apps must resolve correctly
+- **Pull secret invalid** — check /root/.docker/config.json or /var/lib/kubelet/config.json
+- **BMC unreachable during install** — needed for host provisioning
+- **Certificate issues** — clock skew between nodes causes TLS failures
+- **Disk/storage** — insufficient space on /var or /sysroot
+
+---
+
+## Scenario 2: Upgrade or Reboot in Progress
+
+A node rebooted for an OS update (MCO) or configuration change. The API
+should return within 5-15 minutes on a two-node cluster.
+
+### Check OS Update Status
+
+` + "```" + `
+# Current and pending OS deployments
+rpm-ostree status
+
+# Was a reboot triggered by MCO?
+journalctl -u machine-config-daemon --no-pager -n 50
+` + "```" + `
+
+### Check Kubelet Recovery
+
+` + "```" + `
+systemctl status kubelet
+journalctl -u kubelet --no-pager -n 50
+` + "```" + `
+
+### Two-Node Etcd Considerations
+
+In a 2-node cluster, if one node reboots, etcd loses quorum until it
+returns. The API will be unavailable until both etcd members are healthy:
+
+` + "```" + `
+sudo crictl ps --name etcd
+sudo crictl exec $(sudo crictl ps --name etcd -q) etcdctl endpoint health \
+ --cluster --cacert /etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \
+ --cert /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.crt \
+ --key /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.key
+` + "```" + `
+
+### If the Upgrade Appears Stuck
+
+- Check if the node completed its reboot: ` + "`uptime`" + `
+- Check if the other node is still running and healthy
+- Check pacemaker: has fencing been triggered during the upgrade?
+
+---
+
+## Scenario 3: Crash or Degraded State
+
+The cluster was running and stopped unexpectedly. A node may have crashed,
+or a critical service may have failed.
+
+### Check What Happened
+
+` + "```" + `
+# Recent kubelet activity
+journalctl -u kubelet --since "30 min ago" --no-pager
+
+# Recent system events
+journalctl --since "30 min ago" --priority err --no-pager
+
+# Are control-plane containers running?
+sudo crictl ps --name kube-apiserver
+sudo crictl ps --name etcd
+sudo crictl ps --name kube-controller-manager
+` + "```" + `
+
+### Check Pacemaker and Fencing
+
+On a TNF cluster, the surviving node may have fenced the failed node:
+
+` + "```" + `
+pcs status
+pcs stonith history
+` + "```" + `
+
+If fencing occurred:
+- Identify which node was fenced and which survived
+- Check BMC power state of the fenced node
+- The fenced node may need manual power-on and cluster rejoin
+
+### Etcd Recovery
+
+` + "```" + `
+# Check etcd member list and health
+sudo crictl exec $(sudo crictl ps --name etcd -q) etcdctl member list \
+ --cacert /etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \
+ --cert /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.crt \
+ --key /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-master-0.key
+` + "```" + `
+
+### If Both Nodes Are Down
+
+This is a critical situation. Use BMC to:
+1. Check power state of both nodes
+2. Power on nodes via IPMI/Redfish
+3. Access BMC virtual console to observe boot process
+4. Once nodes boot, follow Scenario 1 or 2 steps depending on whether
+ the cluster was previously installed
+
+---
+
+## Next Steps
+
+Once the API server is reachable again, re-run the TNF diagnostic tools:
+- ` + "`tnf_check_fencing_config`" + ` — validate fencing configuration
+- ` + "`tnf_check_stonith_status`" + ` — check pacemaker and STONITH state
+- ` + "`tnf-troubleshoot`" + ` — full troubleshooting guide with live data
+`
+}
diff --git a/pkg/toolsets/tnf/fencing/api_check_test.go b/pkg/toolsets/tnf/fencing/api_check_test.go
new file mode 100644
index 000000000..2ae19d90d
--- /dev/null
+++ b/pkg/toolsets/tnf/fencing/api_check_test.go
@@ -0,0 +1,106 @@
+package fencing
+
+import (
+ "errors"
+ "fmt"
+ "net"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIsConnectionError(t *testing.T) {
+ tests := []struct {
+ name string
+ err error
+ expected bool
+ }{
+ {
+ name: "connection refused",
+ err: fmt.Errorf("dial tcp 192.168.111.5:6443: connect: connection refused"),
+ expected: true,
+ },
+ {
+ name: "no route to host",
+ err: fmt.Errorf("dial tcp 192.168.111.5:6443: connect: no route to host"),
+ expected: true,
+ },
+ {
+ name: "no such host",
+ err: fmt.Errorf("dial tcp: lookup api.cluster.example.com: no such host"),
+ expected: true,
+ },
+ {
+ name: "i/o timeout",
+ err: fmt.Errorf("dial tcp 192.168.111.5:6443: i/o timeout"),
+ expected: true,
+ },
+ {
+ name: "net.OpError",
+ err: &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")},
+ expected: true,
+ },
+ {
+ name: "wrapped dial error",
+ err: fmt.Errorf("get server version: %w", fmt.Errorf("dial tcp 10.0.0.1:6443: connect: connection refused")),
+ expected: true,
+ },
+ {
+ name: "context deadline exceeded",
+ err: fmt.Errorf("Get \"https://127.0.0.1:6443/version?timeout=5s\": context deadline exceeded"),
+ expected: true,
+ },
+ {
+ name: "connection reset by peer",
+ err: fmt.Errorf("read tcp 127.0.0.1:47428->127.0.0.1:6443: read: connection reset by peer"),
+ expected: true,
+ },
+ {
+ name: "k8s wrapPreviousError style",
+ err: fmt.Errorf("Get \"https://127.0.0.1:16443/version?timeout=5s\": context deadline exceeded - error from a previous attempt: read tcp 127.0.0.1:47428->127.0.0.1:16443: read: connection reset by peer"),
+ expected: true,
+ },
+ {
+ name: "forbidden (API reachable)",
+ err: fmt.Errorf("the server has asked for the client to provide credentials"),
+ expected: false,
+ },
+ {
+ name: "not found (API reachable)",
+ err: fmt.Errorf("the server could not find the requested resource"),
+ expected: false,
+ },
+ {
+ name: "generic error",
+ err: errors.New("something went wrong"),
+ expected: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := isConnectionError(tt.err)
+ assert.Equal(t, tt.expected, result)
+ })
+ }
+}
+
+func TestIsAPIUnreachableNilClient(t *testing.T) {
+ assert.False(t, IsAPIUnreachable(nil))
+}
+
+func TestAPIUnreachableGuide(t *testing.T) {
+ guide := APIUnreachableGuide()
+
+ assert.Contains(t, guide, "API Unreachable")
+ assert.Contains(t, guide, "Scenario 1")
+ assert.Contains(t, guide, "Scenario 2")
+ assert.Contains(t, guide, "Scenario 3")
+ assert.Contains(t, guide, "BMC")
+ assert.Contains(t, guide, "ssh core@")
+ assert.Contains(t, guide, "journalctl -u agent")
+ assert.Contains(t, guide, "rpm-ostree status")
+ assert.Contains(t, guide, "pcs status")
+ assert.Contains(t, guide, "etcdctl")
+ assert.Contains(t, guide, "tnf_check_fencing_config")
+}
diff --git a/pkg/toolsets/tnf/fencing/fencing.go b/pkg/toolsets/tnf/fencing/fencing.go
new file mode 100644
index 000000000..9da5626e6
--- /dev/null
+++ b/pkg/toolsets/tnf/fencing/fencing.go
@@ -0,0 +1,709 @@
+package fencing
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "github.com/containers/kubernetes-mcp-server/pkg/klogutil"
+ "strings"
+
+ "github.com/google/jsonschema-go/jsonschema"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ apimeta "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/client-go/dynamic"
+ corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+ "k8s.io/utils/ptr"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+)
+
+var (
+ BareMetalHostGVR = schema.GroupVersionResource{
+ Group: "metal3.io", Version: "v1alpha1", Resource: "baremetalhosts",
+ }
+ ClusterOperatorGVR = schema.GroupVersionResource{
+ Group: "config.openshift.io", Version: "v1", Resource: "clusteroperators",
+ }
+ InfrastructureGVR = schema.GroupVersionResource{
+ Group: "config.openshift.io", Version: "v1", Resource: "infrastructures",
+ }
+ MachineGVR = schema.GroupVersionResource{
+ Group: "machine.openshift.io", Version: "v1beta1", Resource: "machines",
+ }
+ FenceAgentsRemediationGVR = schema.GroupVersionResource{
+ Group: "fence-agents-remediation.medik8s.io", Version: "v1alpha1",
+ Resource: "fenceagentsremediations",
+ }
+ FenceAgentsRemediationTemplateGVR = schema.GroupVersionResource{
+ Group: "fence-agents-remediation.medik8s.io", Version: "v1alpha1",
+ Resource: "fenceagentsremediationtemplates",
+ }
+ NodeHealthCheckGVR = schema.GroupVersionResource{
+ Group: "remediation.medik8s.io", Version: "v1alpha1",
+ Resource: "nodehealthchecks",
+ }
+)
+
+func InitFencing() []api.ServerTool {
+ return []api.ServerTool{
+ {
+ Tool: api.Tool{
+ Name: "tnf_check_fencing_config",
+ Description: "Check fencing configuration and readiness for a Two-Node Fencing (TNF) cluster. " +
+ "Validates cluster topology, critical operator health (etcd, machine-api, baremetal), " +
+ "Machine/Node/BareMetalHost correlation, BMC addresses and credentials, " +
+ "pacemaker/STONITH fencing status via Kubernetes resources. " +
+ "Returns a diagnostic summary identifying configuration issues that could " +
+ "prevent fencing from functioning correctly.",
+ InputSchema: &jsonschema.Schema{
+ Type: "object",
+ Properties: map[string]*jsonschema.Schema{
+ "namespace": {
+ Type: "string",
+ Description: "Namespace containing BareMetalHost resources (e.g. 'openshift-machine-api'). If omitted, searches all namespaces.",
+ },
+ },
+ },
+ Annotations: api.ToolAnnotations{
+ Title: "TNF: Check Fencing Config",
+ ReadOnlyHint: ptr.To(true),
+ DestructiveHint: ptr.To(false),
+ IdempotentHint: ptr.To(true),
+ OpenWorldHint: ptr.To(false),
+ },
+ },
+ Handler: checkFencingHealth,
+ },
+ }
+}
+
+func checkFencingHealth(params api.ToolHandlerParams) (*api.ToolCallResult, error) {
+ if IsAPIUnreachable(params.KubernetesClient) {
+ return api.NewToolCallResult(APIUnreachableGuide(), nil), nil
+ }
+
+ p := api.WrapParams(params)
+ namespace := p.OptionalString("namespace", "")
+ if err := p.Err(); err != nil {
+ return api.NewToolCallResult("", fmt.Errorf("failed to check fencing health: %w", err)), nil
+ }
+
+ dynamicClient := params.DynamicClient()
+ coreClient := params.CoreV1()
+
+ var report strings.Builder
+ var issues []string
+
+ report.WriteString("# TNF Fencing Health Check\n\n")
+
+ isTNF, topoIssues := checkInfrastructureTopology(params.Context, dynamicClient, coreClient, &report)
+ issues = append(issues, topoIssues...)
+
+ coIssues := checkClusterOperatorHealth(params.Context, dynamicClient, &report)
+ issues = append(issues, coIssues...)
+
+ bmhToNode, corrIssues := correlateMachinesWithHosts(params.Context, dynamicClient, &report)
+ issues = append(issues, corrIssues...)
+
+ hosts, err := listBareMetalHosts(params.Context, dynamicClient, namespace)
+ if err != nil {
+ return api.NewToolCallResult("", fmt.Errorf("failed to list BareMetalHost resources: %w", err)), nil
+ }
+
+ if len(hosts) == 0 {
+ msg := "No BareMetalHost resources found"
+ if namespace != "" {
+ msg += fmt.Sprintf(" in namespace %q", namespace)
+ }
+ report.WriteString(msg + ". This cluster may not be a bare metal deployment.\n\n")
+ issues = append(issues, msg)
+ } else {
+ fmt.Fprintf(&report, "BareMetalHosts found: %d\n\n", len(hosts))
+ for _, host := range hosts {
+ hostIssues := writeHostSection(params.Context, coreClient, &report, host, bmhToNode)
+ issues = append(issues, hostIssues...)
+ }
+ }
+
+ if !isTNF {
+ farIssues := checkFenceAgentsRemediation(params.Context, dynamicClient, &report)
+ issues = append(issues, farIssues...)
+
+ nhcIssues := checkNodeHealthChecks(params.Context, dynamicClient, &report)
+ issues = append(issues, nhcIssues...)
+ }
+
+ report.WriteString("## Summary\n\n")
+ if len(issues) == 0 {
+ report.WriteString("No fencing health issues detected.\n")
+ } else {
+ fmt.Fprintf(&report, "Found %d issue(s):\n\n", len(issues))
+ for i, issue := range issues {
+ fmt.Fprintf(&report, "%d. %s\n", i+1, issue)
+ }
+ }
+
+ return api.NewToolCallResult(report.String(), nil), nil
+}
+
+func checkInfrastructureTopology(ctx context.Context, client dynamic.Interface, coreClient corev1.CoreV1Interface, report *strings.Builder) (bool, []string) {
+ var issues []string
+ isTNF := false
+ report.WriteString("## Cluster Topology\n\n")
+
+ infra, err := client.Resource(InfrastructureGVR).Get(ctx, "cluster", metav1.GetOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not get Infrastructure CR", "error", err)
+ if isCRDNotInstalled(err) {
+ report.WriteString("- Infrastructure CR: not available (may not be an OpenShift cluster)\n\n")
+ } else {
+ fmt.Fprintf(report, "- Infrastructure CR: **error** — %v\n\n", err)
+ issues = append(issues, fmt.Sprintf("failed to get Infrastructure CR: %v", err))
+ }
+ return isTNF, issues
+ }
+
+ platform, _, err := unstructured.NestedString(infra.Object, "status", "platform")
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("malformed Infrastructure CR field", "field", "status.platform", "error", err)
+ }
+ infraTopology, _, err := unstructured.NestedString(infra.Object, "status", "infrastructureTopology")
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("malformed Infrastructure CR field", "field", "status.infrastructureTopology", "error", err)
+ }
+ cpTopology, _, err := unstructured.NestedString(infra.Object, "status", "controlPlaneTopology")
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("malformed Infrastructure CR field", "field", "status.controlPlaneTopology", "error", err)
+ }
+
+ fmt.Fprintf(report, "- Platform: %s\n", ValueOrNA(platform))
+ fmt.Fprintf(report, "- Infrastructure Topology: %s\n", ValueOrNA(infraTopology))
+ fmt.Fprintf(report, "- Control Plane Topology: %s\n", ValueOrNA(cpTopology))
+
+ nodes, err := coreClient.Nodes().List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list nodes for topology check", "error", err)
+ } else {
+ cpCount := 0
+ for _, node := range nodes.Items {
+ if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
+ cpCount++
+ } else if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
+ cpCount++
+ }
+ }
+ fmt.Fprintf(report, "- Total nodes: %d\n", len(nodes.Items))
+ fmt.Fprintf(report, "- Control-plane nodes: %d\n", cpCount)
+
+ isTNF = strings.EqualFold(platform, "BareMetal") && cpCount == 2
+ if isTNF {
+ report.WriteString("- TNF Profile: **Yes** (2-node bare metal)\n")
+ } else {
+ report.WriteString("- TNF Profile: No\n")
+ if !strings.EqualFold(platform, "BareMetal") {
+ issues = append(issues, fmt.Sprintf("platform is %q, expected BareMetal for TNF", platform))
+ }
+ if cpCount != 2 {
+ issues = append(issues, fmt.Sprintf("found %d control-plane nodes, expected 2 for TNF", cpCount))
+ }
+ }
+ }
+
+ report.WriteString("\n")
+ return isTNF, issues
+}
+
+func checkClusterOperatorHealth(ctx context.Context, client dynamic.Interface, report *strings.Builder) []string {
+ var issues []string
+ report.WriteString("## Cluster Operator Health\n\n")
+
+ tnfOperators := map[string]bool{
+ "etcd": true,
+ "machine-api": true,
+ "baremetal": true,
+ }
+
+ list, err := client.Resource(ClusterOperatorGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list ClusterOperators", "error", err)
+ if isCRDNotInstalled(err) {
+ report.WriteString("- ClusterOperators: not available (may not be an OpenShift cluster)\n\n")
+ } else {
+ fmt.Fprintf(report, "- ClusterOperators: **error** — %v\n\n", err)
+ issues = append(issues, fmt.Sprintf("failed to list ClusterOperators: %v", err))
+ }
+ return issues
+ }
+
+ foundOps := make(map[string]bool)
+
+ for _, item := range list.Items {
+ name := item.GetName()
+ if !tnfOperators[name] {
+ continue
+ }
+ foundOps[name] = true
+
+ conditions, _, _ := unstructured.NestedSlice(item.Object, "status", "conditions")
+ available := "Unknown"
+ degraded := "Unknown"
+ progressing := "Unknown"
+ var opIssues []string
+
+ for _, cond := range conditions {
+ condMap, ok := cond.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ condType, _ := condMap["type"].(string)
+ condStatus, _ := condMap["status"].(string)
+ message, _ := condMap["message"].(string)
+
+ switch condType {
+ case "Available":
+ available = condStatus
+ if condStatus != "True" {
+ opIssues = append(opIssues, fmt.Sprintf("not available: %s", message))
+ }
+ case "Degraded":
+ degraded = condStatus
+ if condStatus == "True" {
+ opIssues = append(opIssues, fmt.Sprintf("degraded: %s", message))
+ }
+ case "Progressing":
+ progressing = condStatus
+ }
+ }
+
+ fmt.Fprintf(report, "- %s: Available=%s, Degraded=%s, Progressing=%s\n",
+ name, available, degraded, progressing)
+ for _, oi := range opIssues {
+ fmt.Fprintf(report, " - **Issue**: %s\n", oi)
+ issues = append(issues, fmt.Sprintf("operator %s: %s", name, oi))
+ }
+ }
+
+ for op := range tnfOperators {
+ if !foundOps[op] {
+ fmt.Fprintf(report, "- %s: **not found**\n", op)
+ issues = append(issues, fmt.Sprintf("operator %s not found", op))
+ }
+ }
+
+ report.WriteString("\n")
+ return issues
+}
+
+// correlateMachinesWithHosts writes a Machine/Node/BMH correlation table and
+// returns a mapping from "namespace/bmhName" to the Node name resolved via
+// Machine.status.nodeRef.name, along with any issues found.
+func correlateMachinesWithHosts(ctx context.Context, client dynamic.Interface, report *strings.Builder) (map[string]string, []string) {
+ bmhToNode := make(map[string]string)
+ var issues []string
+ report.WriteString("## Machine/Node/BMH Correlation\n\n")
+
+ list, err := client.Resource(MachineGVR).Namespace("openshift-machine-api").List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list Machines", "error", err)
+ if isCRDNotInstalled(err) {
+ report.WriteString("- Machines: not available (Machine API not installed)\n\n")
+ } else {
+ fmt.Fprintf(report, "- Machines: **error** — %v\n\n", err)
+ issues = append(issues, fmt.Sprintf("failed to list Machines: %v", err))
+ }
+ return bmhToNode, issues
+ }
+
+ if len(list.Items) == 0 {
+ report.WriteString("No Machine resources found in openshift-machine-api.\n\n")
+ return bmhToNode, issues
+ }
+
+ report.WriteString("| Machine | Node | BMH | Phase |\n")
+ report.WriteString("|---------|------|-----|-------|\n")
+
+ for _, machine := range list.Items {
+ machineName := machine.GetName()
+ annotations := machine.GetAnnotations()
+
+ nodeRef, _, _ := unstructured.NestedString(machine.Object, "status", "nodeRef", "name")
+ phase, _, _ := unstructured.NestedString(machine.Object, "status", "phase")
+ bmhRef := annotations["metal3.io/BareMetalHost"]
+
+ nodeDisplay := ValueOrNA(nodeRef)
+ bmhDisplay := ValueOrNA(bmhRef)
+
+ fmt.Fprintf(report, "| %s | %s | %s | %s |\n",
+ machineName, nodeDisplay, bmhDisplay, ValueOrNA(phase))
+
+ if nodeRef != "" && bmhRef != "" {
+ bmhToNode[bmhRef] = nodeRef
+ }
+
+ if nodeRef == "" {
+ issues = append(issues, fmt.Sprintf("machine %s has no nodeRef (not yet provisioned or failed)", machineName))
+ }
+ if bmhRef == "" {
+ issues = append(issues, fmt.Sprintf("machine %s has no metal3.io/BareMetalHost annotation", machineName))
+ }
+ }
+
+ report.WriteString("\n")
+ return bmhToNode, issues
+}
+
+func writeHostSection(ctx context.Context, coreClient corev1.CoreV1Interface, report *strings.Builder, host unstructured.Unstructured, bmhToNode map[string]string) []string {
+ var issues []string
+ hostName := host.GetName()
+ hostNamespace := host.GetNamespace()
+
+ fmt.Fprintf(report, "## Host: %s/%s\n\n", hostNamespace, hostName)
+
+ provisioningState, _, _ := unstructured.NestedString(host.Object, "status", "provisioning", "state")
+ operationalStatus, _, _ := unstructured.NestedString(host.Object, "status", "operationalStatus")
+ poweredOn, _, _ := unstructured.NestedBool(host.Object, "status", "poweredOn")
+ errorMessage, _, _ := unstructured.NestedString(host.Object, "status", "errorMessage")
+ errorType, _, _ := unstructured.NestedString(host.Object, "status", "errorType")
+ bmcAddress, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "address")
+ credentialsName, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "credentialsName")
+ online, _, _ := unstructured.NestedBool(host.Object, "spec", "online")
+ goodCredsName, _, _ := unstructured.NestedString(host.Object, "status", "goodCredentials", "credentials", "name")
+
+ consumerKind, _, _ := unstructured.NestedString(host.Object, "spec", "consumerRef", "kind")
+ consumerName, _, _ := unstructured.NestedString(host.Object, "spec", "consumerRef", "name")
+
+ manufacturer, _, _ := unstructured.NestedString(host.Object, "status", "hardware", "systemVendor", "manufacturer")
+ productName, _, _ := unstructured.NestedString(host.Object, "status", "hardware", "systemVendor", "productName")
+
+ isUnmanaged := provisioningState == "" || provisioningState == "unmanaged"
+
+ if isUnmanaged && bmcAddress == "" {
+ report.WriteString("- State: **Unmanaged** (not controlled by baremetal-operator)\n")
+ report.WriteString("- BMC/credentials: not applicable for unmanaged hosts\n")
+ if operationalStatus != "" {
+ fmt.Fprintf(report, "- Operational Status: %s\n", operationalStatus)
+ }
+ if consumerKind != "" {
+ fmt.Fprintf(report, "- Consumer: %s/%s\n", consumerKind, consumerName)
+ }
+ } else {
+ fmt.Fprintf(report, "- Provisioning State: %s\n", ValueOrNA(provisioningState))
+ fmt.Fprintf(report, "- Operational Status: %s\n", ValueOrNA(operationalStatus))
+ fmt.Fprintf(report, "- Online (desired): %t\n", online)
+ fmt.Fprintf(report, "- Powered On (actual): %t\n", poweredOn)
+ fmt.Fprintf(report, "- BMC Address: %s\n", ValueOrNA(bmcAddress))
+ fmt.Fprintf(report, "- Credentials Secret: %s\n", ValueOrNA(credentialsName))
+
+ if goodCredsName != "" {
+ fmt.Fprintf(report, "- Good Credentials: %s (verified by BMO)\n", goodCredsName)
+ }
+
+ if consumerKind != "" {
+ fmt.Fprintf(report, "- Consumer: %s/%s\n", consumerKind, consumerName)
+ }
+
+ if manufacturer != "" || productName != "" {
+ fmt.Fprintf(report, "- Hardware: %s %s\n", manufacturer, productName)
+ }
+
+ if errorMessage != "" {
+ fmt.Fprintf(report, "- **Error**: [%s] %s\n", errorType, errorMessage)
+ issues = append(issues, fmt.Sprintf("%s/%s: BMH error — [%s] %s", hostNamespace, hostName, errorType, errorMessage))
+ }
+
+ if bmcAddress == "" {
+ issues = append(issues, fmt.Sprintf("%s/%s: no BMC address configured", hostNamespace, hostName))
+ }
+
+ if credentialsName != "" {
+ credIssues := checkCredentials(ctx, coreClient, hostNamespace, credentialsName, hostName)
+ issues = append(issues, credIssues...)
+ if len(credIssues) == 0 {
+ report.WriteString("- Credentials: present and valid\n")
+ } else {
+ for _, ci := range credIssues {
+ fmt.Fprintf(report, "- **Credentials Issue**: %s\n", ci)
+ }
+ }
+ } else if !isUnmanaged {
+ issues = append(issues, fmt.Sprintf("%s/%s: no BMC credentials secret configured", hostNamespace, hostName))
+ }
+ }
+
+ nodeName := hostName
+ bmhKey := fmt.Sprintf("%s/%s", hostNamespace, hostName)
+ if resolved, ok := bmhToNode[bmhKey]; ok {
+ nodeName = resolved
+ }
+
+ nodeIssues := checkNodeHealth(ctx, coreClient, nodeName)
+ issues = append(issues, nodeIssues...)
+ for _, ni := range nodeIssues {
+ fmt.Fprintf(report, "- **Node Issue**: %s\n", ni)
+ }
+ if len(nodeIssues) == 0 {
+ report.WriteString("- Node: healthy\n")
+ }
+
+ report.WriteString("\n")
+ return issues
+}
+
+func checkFenceAgentsRemediation(ctx context.Context, client dynamic.Interface, report *strings.Builder) []string {
+ var issues []string
+ report.WriteString("## Fencing Remediation\n\n")
+
+ report.WriteString("### Templates\n\n")
+ templates, err := client.Resource(FenceAgentsRemediationTemplateGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list FenceAgentsRemediationTemplates", "error", err)
+ if isCRDNotInstalled(err) {
+ report.WriteString("FenceAgentsRemediationTemplate CRD not installed. Cluster may use traditional pacemaker/STONITH fencing instead.\n\n")
+ } else {
+ fmt.Fprintf(report, "FenceAgentsRemediationTemplates: **error** — %v\n\n", err)
+ issues = append(issues, fmt.Sprintf("failed to list FenceAgentsRemediationTemplates: %v", err))
+ }
+ } else if len(templates.Items) == 0 {
+ report.WriteString("No FenceAgentsRemediationTemplates found. Cluster may use traditional pacemaker/STONITH fencing instead.\n\n")
+ } else {
+ for _, tmpl := range templates.Items {
+ name := tmpl.GetName()
+ ns := tmpl.GetNamespace()
+ agent, _, _ := unstructured.NestedString(tmpl.Object, "spec", "template", "spec", "agent")
+ sharedParams, _, _ := unstructured.NestedMap(tmpl.Object, "spec", "template", "spec", "sharedparameters")
+ retryCount, _, _ := unstructured.NestedInt64(tmpl.Object, "spec", "template", "spec", "retrycount")
+
+ location := name
+ if ns != "" {
+ location = fmt.Sprintf("%s/%s", ns, name)
+ }
+ fmt.Fprintf(report, "- **%s**: agent=%s", location, ValueOrNA(agent))
+ if retryCount > 0 {
+ fmt.Fprintf(report, ", retryCount=%d", retryCount)
+ }
+ if len(sharedParams) > 0 {
+ var paramKeys []string
+ for k := range sharedParams {
+ paramKeys = append(paramKeys, k)
+ }
+ fmt.Fprintf(report, ", sharedParams={%s}", strings.Join(paramKeys, ", "))
+ }
+ report.WriteString("\n")
+ }
+ }
+
+ report.WriteString("\n### Active Remediations\n\n")
+ remediations, err := client.Resource(FenceAgentsRemediationGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list FenceAgentsRemediations", "error", err)
+ if isCRDNotInstalled(err) {
+ report.WriteString("FenceAgentsRemediation CRD not installed.\n\n")
+ } else {
+ fmt.Fprintf(report, "Active Remediations: **error** — %v\n\n", err)
+ issues = append(issues, fmt.Sprintf("failed to list FenceAgentsRemediations: %v", err))
+ }
+ return issues
+ }
+
+ if len(remediations.Items) == 0 {
+ report.WriteString("No active fencing remediations.\n\n")
+ } else {
+ for _, rem := range remediations.Items {
+ remName := rem.GetName()
+ remNs := rem.GetNamespace()
+ agent, _, _ := unstructured.NestedString(rem.Object, "spec", "agent")
+ conditions, _, _ := unstructured.NestedSlice(rem.Object, "status", "conditions")
+
+ fmt.Fprintf(report, "- **%s/%s**: agent=%s\n", remNs, remName, ValueOrNA(agent))
+ for _, cond := range conditions {
+ condMap, ok := cond.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ condType, _ := condMap["type"].(string)
+ condStatus, _ := condMap["status"].(string)
+ message, _ := condMap["message"].(string)
+ fmt.Fprintf(report, " - %s=%s: %s\n", condType, condStatus, message)
+ }
+ issues = append(issues, fmt.Sprintf("active fencing remediation in progress: %s/%s", remNs, remName))
+ }
+ }
+
+ report.WriteString("\n")
+ return issues
+}
+
+func checkNodeHealthChecks(ctx context.Context, client dynamic.Interface, report *strings.Builder) []string {
+ var issues []string
+ report.WriteString("## Node Health Checks\n\n")
+
+ list, err := client.Resource(NodeHealthCheckGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list NodeHealthChecks", "error", err)
+ if isCRDNotInstalled(err) {
+ report.WriteString("NodeHealthCheck CRD not installed. Cluster may use traditional pacemaker-based health monitoring.\n\n")
+ } else {
+ fmt.Fprintf(report, "NodeHealthChecks: **error** — %v\n\n", err)
+ issues = append(issues, fmt.Sprintf("failed to list NodeHealthChecks: %v", err))
+ }
+ return issues
+ }
+
+ if len(list.Items) == 0 {
+ report.WriteString("No NodeHealthCheck resources found. Cluster may use traditional pacemaker-based health monitoring.\n\n")
+ return issues
+ }
+
+ for _, nhc := range list.Items {
+ nhcName := nhc.GetName()
+ phase, _, _ := unstructured.NestedString(nhc.Object, "status", "phase")
+ minHealthy, _, _ := unstructured.NestedString(nhc.Object, "spec", "minHealthy")
+
+ selectorLabels, _, _ := unstructured.NestedStringMap(nhc.Object, "spec", "selector", "matchLabels")
+
+ var selectorStr string
+ if len(selectorLabels) > 0 {
+ var parts []string
+ for k, v := range selectorLabels {
+ if v == "" {
+ parts = append(parts, k)
+ } else {
+ parts = append(parts, fmt.Sprintf("%s=%s", k, v))
+ }
+ }
+ selectorStr = strings.Join(parts, ", ")
+ } else {
+ selectorStr = "(all nodes)"
+ }
+
+ fmt.Fprintf(report, "- **%s**: selector={%s}\n", nhcName, selectorStr)
+ fmt.Fprintf(report, " - Phase: %s\n", ValueOrNA(phase))
+ if minHealthy != "" {
+ fmt.Fprintf(report, " - Min Healthy: %s\n", minHealthy)
+ }
+
+ escalating, found, _ := unstructured.NestedSlice(nhc.Object, "spec", "escalatingRemediations")
+ if found && len(escalating) > 0 {
+ report.WriteString(" - Escalating Remediations:\n")
+ for i, rem := range escalating {
+ remMap, ok := rem.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ remRef, _ := remMap["remediationTemplate"].(map[string]interface{})
+ kind, _ := remRef["kind"].(string)
+ name, _ := remRef["name"].(string)
+ ns, _ := remRef["namespace"].(string)
+ timeout, _ := remMap["timeout"].(string)
+
+ location := name
+ if ns != "" {
+ location = fmt.Sprintf("%s/%s", ns, name)
+ }
+ fmt.Fprintf(report, " %d. %s/%s", i+1, kind, location)
+ if timeout != "" {
+ fmt.Fprintf(report, " (timeout=%s)", timeout)
+ }
+ report.WriteString("\n")
+ }
+ }
+
+ unhealthyConditions, found, _ := unstructured.NestedSlice(nhc.Object, "spec", "unhealthyConditions")
+ if found && len(unhealthyConditions) > 0 {
+ report.WriteString(" - Unhealthy Conditions:\n")
+ for _, uc := range unhealthyConditions {
+ ucMap, ok := uc.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ ucType, _ := ucMap["type"].(string)
+ ucStatus, _ := ucMap["status"].(string)
+ ucDuration, _ := ucMap["duration"].(string)
+ fmt.Fprintf(report, " - %s=%s (duration=%s)\n", ucType, ucStatus, ucDuration)
+ }
+ }
+ }
+
+ report.WriteString("\n")
+ return issues
+}
+
+func listBareMetalHosts(ctx context.Context, client dynamic.Interface, namespace string) ([]unstructured.Unstructured, error) {
+ var list *unstructured.UnstructuredList
+ var err error
+
+ if namespace != "" {
+ list, err = client.Resource(BareMetalHostGVR).Namespace(namespace).List(ctx, metav1.ListOptions{})
+ } else {
+ list, err = client.Resource(BareMetalHostGVR).List(ctx, metav1.ListOptions{})
+ }
+
+ if err != nil {
+ return nil, err
+ }
+ return list.Items, nil
+}
+
+func checkCredentials(ctx context.Context, client corev1.CoreV1Interface, namespace, secretName, hostName string) []string {
+ var issues []string
+
+ secret, err := client.Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{})
+ if err != nil {
+ issues = append(issues, fmt.Sprintf("%s/%s: credentials secret %q not found — %v", namespace, hostName, secretName, err))
+ return issues
+ }
+
+ if v, ok := secret.Data["username"]; !ok || len(v) == 0 {
+ issues = append(issues, fmt.Sprintf("%s/%s: credentials secret %q missing or empty 'username' key", namespace, hostName, secretName))
+ }
+ if v, ok := secret.Data["password"]; !ok || len(v) == 0 {
+ issues = append(issues, fmt.Sprintf("%s/%s: credentials secret %q missing or empty 'password' key", namespace, hostName, secretName))
+ }
+
+ return issues
+}
+
+func checkNodeHealth(ctx context.Context, client corev1.CoreV1Interface, hostName string) []string {
+ var issues []string
+
+ node, err := client.Nodes().Get(ctx, hostName, metav1.GetOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not find Node matching BareMetalHost", "host", hostName, "error", err)
+ issues = append(issues, fmt.Sprintf("%s: no matching Node resource found", hostName))
+ return issues
+ }
+
+ for _, condition := range node.Status.Conditions {
+ switch condition.Type {
+ case "Ready":
+ if condition.Status != "True" {
+ issues = append(issues, fmt.Sprintf("node %s: NotReady — %s", hostName, condition.Message))
+ }
+ case "MemoryPressure", "DiskPressure", "PIDPressure":
+ if condition.Status == "True" {
+ issues = append(issues, fmt.Sprintf("node %s: %s — %s", hostName, condition.Type, condition.Message))
+ }
+ }
+ }
+
+ return issues
+}
+
+// ValueOrNA returns s if non-empty, otherwise "N/A".
+func ValueOrNA(s string) string {
+ if s == "" {
+ return "N/A"
+ }
+ return s
+}
+
+func isCRDNotInstalled(err error) bool {
+ if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
+ return true
+ }
+ var ve *api.ValidationError
+ return errors.As(err, &ve) && ve.Code == api.ErrorCodeResourceNotFound
+}
diff --git a/pkg/toolsets/tnf/fencing/fencing_test.go b/pkg/toolsets/tnf/fencing/fencing_test.go
new file mode 100644
index 000000000..a39f496f8
--- /dev/null
+++ b/pkg/toolsets/tnf/fencing/fencing_test.go
@@ -0,0 +1,36 @@
+package fencing
+
+import (
+ "testing"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+ "github.com/stretchr/testify/suite"
+)
+
+type FencingHandlerSuite struct {
+ suite.Suite
+}
+
+type staticRequest struct {
+ args map[string]any
+}
+
+func (s staticRequest) GetArguments() map[string]any {
+ return s.args
+}
+
+func (s *FencingHandlerSuite) TestInvalidNamespaceType() {
+ params := api.ToolHandlerParams{
+ ToolCallRequest: staticRequest{args: map[string]any{
+ "namespace": 12345,
+ }},
+ }
+ result, err := checkFencingHealth(params)
+ s.Require().NoError(err)
+ s.Require().NotNil(result.Error)
+ s.Contains(result.Error.Error(), "failed to check fencing health")
+}
+
+func TestFencingHandler(t *testing.T) {
+ suite.Run(t, new(FencingHandlerSuite))
+}
diff --git a/pkg/toolsets/tnf/fencing/stonith.go b/pkg/toolsets/tnf/fencing/stonith.go
new file mode 100644
index 000000000..94a792641
--- /dev/null
+++ b/pkg/toolsets/tnf/fencing/stonith.go
@@ -0,0 +1,463 @@
+package fencing
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "github.com/containers/kubernetes-mcp-server/pkg/klogutil"
+ "math"
+ "strings"
+ "time"
+
+ "github.com/google/jsonschema-go/jsonschema"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+ "k8s.io/utils/ptr"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+ "github.com/containers/kubernetes-mcp-server/pkg/cluster-diagnostics/nodesdebug"
+)
+
+const defaultSTONITHTimeout = 120 * time.Second
+
+var stonithDiagnosticScript = strings.Join([]string{
+ "echo '===PCS_STATUS==='",
+ "pcs status 2>&1",
+ "echo '===PCS_STONITH_CONFIG==='",
+ "pcs stonith config 2>&1",
+ "echo '===PCS_STONITH_STATUS==='",
+ "pcs stonith status 2>&1",
+ "echo '===PCS_STONITH_HISTORY==='",
+ "pcs stonith history 2>&1",
+ "echo '===PCS_QUORUM_STATUS==='",
+ "corosync-quorumtool 2>&1",
+ "echo '===PCS_PROPERTY==='",
+ "{ pcs property config 2>&1 || pcs property list 2>&1; }",
+ "echo '===END==='",
+}, "; ")
+
+func InitSTONITH() []api.ServerTool {
+ return []api.ServerTool{
+ {
+ Tool: api.Tool{
+ Name: "tnf_check_stonith_status",
+ Description: "Check STONITH and pacemaker fencing status on a Two-Node Fencing (TNF) cluster. " +
+ "Creates a temporary privileged debug pod on a control-plane node to run pcs diagnostic " +
+ "commands. Returns pacemaker cluster state, STONITH device configuration, quorum status, " +
+ "and recent fencing history. The debug pod is automatically cleaned up after execution.",
+ InputSchema: &jsonschema.Schema{
+ Type: "object",
+ Properties: map[string]*jsonschema.Schema{
+ "node": {
+ Type: "string",
+ Description: "Name of the node to run diagnostics on. If omitted, auto-detects the first control-plane node.",
+ },
+ "namespace": {
+ Type: "string",
+ Description: "Namespace to create the temporary debug pod in (optional, defaults to 'default').",
+ },
+ "timeout_seconds": {
+ Type: "integer",
+ Description: "Maximum time in seconds to wait for the diagnostic commands to complete (optional, defaults to 120).",
+ Minimum: ptr.To(float64(1)),
+ },
+ },
+ },
+ Annotations: api.ToolAnnotations{
+ Title: "TNF: Check STONITH Status",
+ ReadOnlyHint: ptr.To(true),
+ DestructiveHint: ptr.To(false),
+ IdempotentHint: ptr.To(true),
+ OpenWorldHint: ptr.To(false),
+ },
+ },
+ Handler: checkSTONITHStatus,
+ },
+ }
+}
+
+func checkSTONITHStatus(params api.ToolHandlerParams) (*api.ToolCallResult, error) {
+ if IsAPIUnreachable(params.KubernetesClient) {
+ return api.NewToolCallResult(APIUnreachableGuide(), nil), nil
+ }
+
+ p := api.WrapParams(params)
+ nodeName := p.OptionalString("node", "")
+ namespace := p.OptionalString("namespace", "")
+ if err := p.Err(); err != nil {
+ return api.NewToolCallResult("", fmt.Errorf("failed to check STONITH status: %w", err)), nil
+ }
+
+ timeout := defaultSTONITHTimeout
+ if timeoutRaw, exists := params.GetArguments()["timeout_seconds"]; exists && timeoutRaw != nil {
+ t, err := parseTimeout(timeoutRaw)
+ if err != nil {
+ return api.NewToolCallResult("", fmt.Errorf("failed to check STONITH status: %w", err)), nil
+ }
+ timeout = t
+ }
+
+ report, issues, err := RunSTONITHDiagnostics(params.Context, params.KubernetesClient, params.CoreV1(), nodeName, namespace, timeout)
+ if err != nil {
+ return api.NewToolCallResult(report, err), nil
+ }
+
+ var reportBuilder strings.Builder
+ reportBuilder.WriteString(report)
+
+ reportBuilder.WriteString("\n## Summary\n\n")
+ if len(issues) == 0 {
+ reportBuilder.WriteString("No STONITH issues detected.\n")
+ } else {
+ fmt.Fprintf(&reportBuilder, "Found %d issue(s):\n\n", len(issues))
+ for i, issue := range issues {
+ fmt.Fprintf(&reportBuilder, "%d. %s\n", i+1, issue)
+ }
+ }
+
+ return api.NewToolCallResult(reportBuilder.String(), nil), nil
+}
+
+// RunSTONITHDiagnostics runs pcs diagnostics via a debug pod and returns a
+// structured report and list of issues. If nodeName is empty, auto-detects
+// the first control-plane node.
+func RunSTONITHDiagnostics(ctx context.Context, k8sClient api.KubernetesClient, coreClient corev1client.CoreV1Interface, nodeName, namespace string, timeout time.Duration) (string, []string, error) {
+ if nodeName == "" {
+ detected, err := DetectControlPlaneNode(ctx, coreClient)
+ if err != nil {
+ return "", nil, fmt.Errorf("failed to auto-detect control-plane node: %w", err)
+ }
+ nodeName = detected
+ }
+
+ client := nodesdebug.NewNodeDebug(k8sClient)
+ command := []string{"chroot", "/host", "sh", "-c", stonithDiagnosticScript}
+
+ stdout, stderr, execErr := client.NodesDebugExec(
+ ctx, namespace, nodeName, "", command, "", "", timeout,
+ )
+
+ if execErr != nil {
+ output := stderr
+ if stdout != "" {
+ output = fmt.Sprintf("Partial output:\n%s\n\nError: %v", stdout, execErr)
+ }
+ return output, nil, execErr
+ }
+
+ report, issues := buildSTONITHReport(nodeName, stdout)
+
+ if stderr != "" {
+ klogutil.FromContext(ctx).V(1).Info("STONITH diagnostic stderr", "node", nodeName, "stderr", stderr)
+ }
+
+ return report, issues, nil
+}
+
+// DetectControlPlaneNode finds the first control-plane node by label.
+// Checks both node-role.kubernetes.io/control-plane and the legacy
+// node-role.kubernetes.io/master label.
+func DetectControlPlaneNode(ctx context.Context, client corev1client.CoreV1Interface) (string, error) {
+ for _, label := range []string{"node-role.kubernetes.io/control-plane", "node-role.kubernetes.io/master"} {
+ nodes, err := client.Nodes().List(ctx, metav1.ListOptions{
+ LabelSelector: label,
+ })
+ if err != nil {
+ return "", fmt.Errorf("failed to list control-plane nodes: %w", err)
+ }
+ if len(nodes.Items) > 0 {
+ return nodes.Items[0].Name, nil
+ }
+ }
+ return "", errors.New("no control-plane nodes found")
+}
+
+func buildSTONITHReport(nodeName, rawOutput string) (string, []string) {
+ sections := parseSections(rawOutput)
+ var report strings.Builder
+ var issues []string
+
+ fmt.Fprintf(&report, "# TNF STONITH Status (from node: %s)\n\n", nodeName)
+
+ pcsIssues := writePCSStatusSection(&report, sections["PCS_STATUS"])
+ issues = append(issues, pcsIssues...)
+
+ propIssues := writePropertySection(&report, sections["PCS_PROPERTY"])
+ issues = append(issues, propIssues...)
+
+ deviceIssues := writeStonithDeviceSection(&report, sections["PCS_STONITH_CONFIG"], sections["PCS_STONITH_STATUS"])
+ issues = append(issues, deviceIssues...)
+
+ quorumIssues := writeQuorumSection(&report, sections["PCS_QUORUM_STATUS"])
+ issues = append(issues, quorumIssues...)
+
+ writeHistorySection(&report, sections["PCS_STONITH_HISTORY"])
+
+ return report.String(), issues
+}
+
+func parseSections(raw string) map[string]string {
+ sections := make(map[string]string)
+ markers := []string{"PCS_STATUS", "PCS_STONITH_CONFIG", "PCS_STONITH_STATUS",
+ "PCS_STONITH_HISTORY", "PCS_QUORUM_STATUS", "PCS_PROPERTY", "END"}
+
+ for i, marker := range markers {
+ start := strings.Index(raw, "==="+marker+"===")
+ if start == -1 {
+ continue
+ }
+ contentStart := start + len("==="+marker+"===")
+ contentEnd := len(raw)
+ for j := i + 1; j < len(markers); j++ {
+ nextStart := strings.Index(raw, "==="+markers[j]+"===")
+ if nextStart != -1 {
+ contentEnd = nextStart
+ break
+ }
+ }
+ if contentStart < contentEnd {
+ sections[marker] = strings.TrimSpace(raw[contentStart:contentEnd])
+ }
+ }
+
+ return sections
+}
+
+func writePCSStatusSection(report *strings.Builder, section string) []string {
+ var issues []string
+ report.WriteString("## Pacemaker Cluster\n\n")
+
+ if section == "" {
+ report.WriteString("- pcs status: not available\n\n")
+ issues = append(issues, "pcs status command returned no output")
+ return issues
+ }
+
+ var onlineNodes, offlineNodes, standbyNodes []string
+
+ for _, line := range strings.Split(section, "\n") {
+ trimmed := strings.TrimSpace(line)
+
+ if strings.HasPrefix(trimmed, "Cluster name:") {
+ fmt.Fprintf(report, "- %s\n", trimmed)
+ }
+
+ if strings.Contains(trimmed, "Online:") && strings.Contains(trimmed, "[") {
+ nodes := extractBracketedList(trimmed)
+ onlineNodes = nodes
+ if len(nodes) > 0 {
+ fmt.Fprintf(report, "- Nodes Online: %s\n", strings.Join(nodes, ", "))
+ }
+ }
+ if strings.Contains(trimmed, "OFFLINE:") && strings.Contains(trimmed, "[") {
+ nodes := extractBracketedList(trimmed)
+ offlineNodes = nodes
+ if len(nodes) > 0 {
+ fmt.Fprintf(report, "- Nodes Offline: %s\n", strings.Join(nodes, ", "))
+ issues = append(issues, fmt.Sprintf("nodes offline: %s", strings.Join(nodes, ", ")))
+ }
+ }
+ if strings.Contains(trimmed, "Standby:") && strings.Contains(trimmed, "[") {
+ nodes := extractBracketedList(trimmed)
+ standbyNodes = append(standbyNodes, nodes...)
+ }
+ if strings.HasPrefix(trimmed, "* Node") && strings.Contains(trimmed, "standby") {
+ parts := strings.Fields(trimmed)
+ if len(parts) >= 3 {
+ name := strings.TrimSuffix(parts[2], ":")
+ standbyNodes = append(standbyNodes, name)
+ }
+ }
+
+ if strings.Contains(trimmed, "STONITH") && strings.Contains(trimmed, "enabled") {
+ fmt.Fprintf(report, "- %s\n", trimmed)
+ }
+
+ if strings.Contains(trimmed, "Failed") && strings.Contains(trimmed, "Resource Actions") {
+ fmt.Fprintf(report, "- **%s**\n", trimmed)
+ }
+ }
+
+ if len(standbyNodes) > 0 {
+ fmt.Fprintf(report, "- Nodes Standby: %s\n", strings.Join(standbyNodes, ", "))
+ issues = append(issues, fmt.Sprintf("nodes in standby: %s", strings.Join(standbyNodes, ", ")))
+ }
+
+ if len(onlineNodes) == 0 && len(offlineNodes) == 0 && len(standbyNodes) == 0 {
+ report.WriteString("- (could not parse node status from pcs output)\n")
+ }
+
+ report.WriteString("\n")
+
+ if strings.Contains(section, "Failed Resource Actions") || strings.Contains(section, "Failed Actions") {
+ failedSection := extractAfter(section, "Failed")
+ if failedSection != "" && !strings.Contains(failedSection, "0 resource") {
+ issues = append(issues, "pcs reports failed resource actions")
+ }
+ }
+
+ return issues
+}
+
+func writePropertySection(report *strings.Builder, section string) []string {
+ var issues []string
+
+ if section == "" {
+ return issues
+ }
+
+ for _, line := range strings.Split(section, "\n") {
+ trimmed := strings.TrimSpace(line)
+ lower := strings.ToLower(trimmed)
+
+ if strings.HasPrefix(lower, "stonith-enabled:") {
+ val := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[1])
+ if strings.EqualFold(val, "false") {
+ issues = append(issues, "STONITH is disabled (stonith-enabled: false)")
+ }
+ }
+ if strings.HasPrefix(lower, "no-quorum-policy:") {
+ val := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[1])
+ fmt.Fprintf(report, "- No-Quorum Policy: %s\n", val)
+ }
+ }
+
+ return issues
+}
+
+func writeStonithDeviceSection(report *strings.Builder, configSection, statusSection string) []string {
+ var issues []string
+ report.WriteString("## STONITH Devices\n\n")
+
+ if configSection == "" {
+ report.WriteString("No STONITH device configuration found.\n\n")
+ issues = append(issues, "no STONITH devices configured")
+ return issues
+ }
+
+ if strings.Contains(configSection, "NO stonith devices") || strings.Contains(configSection, "No stonith") {
+ report.WriteString("No STONITH devices configured.\n\n")
+ issues = append(issues, "no STONITH devices configured")
+ return issues
+ }
+
+ report.WriteString("### Configuration\n\n")
+ report.WriteString("```\n")
+ report.WriteString(configSection)
+ report.WriteString("\n```\n\n")
+
+ if statusSection != "" {
+ report.WriteString("### Status\n\n")
+ for _, line := range strings.Split(statusSection, "\n") {
+ trimmed := strings.TrimSpace(line)
+ if trimmed == "" {
+ continue
+ }
+ fmt.Fprintf(report, "- %s\n", trimmed)
+
+ lower := strings.ToLower(trimmed)
+ if strings.Contains(lower, "stopped") {
+ issues = append(issues, fmt.Sprintf("STONITH device stopped: %s", trimmed))
+ }
+ if strings.Contains(lower, "failed") {
+ issues = append(issues, fmt.Sprintf("STONITH device failed: %s", trimmed))
+ }
+ }
+ report.WriteString("\n")
+ }
+
+ return issues
+}
+
+func writeQuorumSection(report *strings.Builder, section string) []string {
+ var issues []string
+ report.WriteString("## Quorum\n\n")
+
+ if section == "" {
+ report.WriteString("- corosync-quorumtool: not available\n\n")
+ return issues
+ }
+
+ for _, line := range strings.Split(section, "\n") {
+ trimmed := strings.TrimSpace(line)
+ lower := strings.ToLower(trimmed)
+
+ if strings.HasPrefix(lower, "quorate:") {
+ fmt.Fprintf(report, "- %s\n", trimmed)
+ val := strings.TrimSpace(strings.SplitN(trimmed, ":", 2)[1])
+ if strings.EqualFold(val, "No") {
+ issues = append(issues, "cluster is NOT quorate")
+ }
+ }
+ if strings.HasPrefix(lower, "expected votes:") || strings.HasPrefix(lower, "total votes:") {
+ fmt.Fprintf(report, "- %s\n", trimmed)
+ }
+ if strings.HasPrefix(lower, "flags:") || strings.Contains(lower, "two_node") || strings.Contains(lower, "two node") {
+ fmt.Fprintf(report, "- %s\n", trimmed)
+ }
+ }
+
+ report.WriteString("\n")
+ return issues
+}
+
+func writeHistorySection(report *strings.Builder, section string) {
+ report.WriteString("## Fencing History\n\n")
+
+ if section == "" || strings.Contains(section, "No fencing") || strings.Contains(section, "no fencing") {
+ report.WriteString("No fencing events recorded.\n\n")
+ return
+ }
+
+ for _, line := range strings.Split(section, "\n") {
+ trimmed := strings.TrimSpace(line)
+ if trimmed == "" {
+ continue
+ }
+ fmt.Fprintf(report, "- %s\n", trimmed)
+ }
+ report.WriteString("\n")
+}
+
+func extractBracketedList(line string) []string {
+ start := strings.Index(line, "[")
+ end := strings.Index(line, "]")
+ if start == -1 || end == -1 || end <= start {
+ return nil
+ }
+ inner := strings.TrimSpace(line[start+1 : end])
+ if inner == "" {
+ return nil
+ }
+ return strings.Fields(inner)
+}
+
+func extractAfter(text, marker string) string {
+ idx := strings.Index(text, marker)
+ if idx == -1 {
+ return ""
+ }
+ return text[idx:]
+}
+
+func parseTimeout(raw interface{}) (time.Duration, error) {
+ switch v := raw.(type) {
+ case float64:
+ if v < 1 || v != math.Trunc(v) {
+ return 0, errors.New("timeout_seconds must be an integer >= 1")
+ }
+ return time.Duration(int64(v)) * time.Second, nil
+ case int:
+ if v < 1 {
+ return 0, errors.New("timeout_seconds must be >= 1")
+ }
+ return time.Duration(v) * time.Second, nil
+ case int64:
+ if v < 1 {
+ return 0, errors.New("timeout_seconds must be >= 1")
+ }
+ return time.Duration(v) * time.Second, nil
+ default:
+ return 0, errors.New("timeout_seconds must be a numeric value")
+ }
+}
diff --git a/pkg/toolsets/tnf/fencing/stonith_test.go b/pkg/toolsets/tnf/fencing/stonith_test.go
new file mode 100644
index 000000000..d549e8fd2
--- /dev/null
+++ b/pkg/toolsets/tnf/fencing/stonith_test.go
@@ -0,0 +1,180 @@
+package fencing
+
+import (
+ "testing"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+ "github.com/stretchr/testify/suite"
+)
+
+type STONITHHandlerSuite struct {
+ suite.Suite
+}
+
+func (s *STONITHHandlerSuite) TestInvalidNodeType() {
+ params := api.ToolHandlerParams{
+ ToolCallRequest: staticRequest{args: map[string]any{
+ "node": 12345,
+ }},
+ }
+ result, err := checkSTONITHStatus(params)
+ s.Require().NoError(err)
+ s.Require().NotNil(result.Error)
+ s.Contains(result.Error.Error(), "failed to check STONITH status")
+}
+
+func (s *STONITHHandlerSuite) TestInvalidTimeoutType() {
+ params := api.ToolHandlerParams{
+ ToolCallRequest: staticRequest{args: map[string]any{
+ "timeout_seconds": "not-a-number",
+ }},
+ }
+ result, err := checkSTONITHStatus(params)
+ s.Require().NoError(err)
+ s.Require().NotNil(result.Error)
+ s.Contains(result.Error.Error(), "timeout_seconds must be a numeric value")
+}
+
+func (s *STONITHHandlerSuite) TestInvalidTimeoutValue() {
+ params := api.ToolHandlerParams{
+ ToolCallRequest: staticRequest{args: map[string]any{
+ "timeout_seconds": float64(0),
+ }},
+ }
+ result, err := checkSTONITHStatus(params)
+ s.Require().NoError(err)
+ s.Require().NotNil(result.Error)
+ s.Contains(result.Error.Error(), "timeout_seconds must be an integer >= 1")
+}
+
+func (s *STONITHHandlerSuite) TestParseSectionsBasic() {
+ raw := `===PCS_STATUS===
+Cluster name: ostest
+Online: [ master-0 master-1 ]
+===PCS_STONITH_CONFIG===
+ Resource: fence_master_0 (class=stonith type=fence_ipmilan)
+ Attributes: fence_master_0-instance_attributes
+ ipaddr=192.168.1.10
+ lanplus=1
+===PCS_STONITH_STATUS===
+ * fence_master_0 (stonith:fence_ipmilan): Started master-1
+===PCS_STONITH_HISTORY===
+No fencing actions recorded.
+===PCS_QUORUM_STATUS===
+Quorate: Yes
+Expected votes: 2
+===PCS_PROPERTY===
+stonith-enabled: true
+no-quorum-policy: stop
+===END===`
+
+ sections := parseSections(raw)
+ s.Contains(sections["PCS_STATUS"], "Cluster name: ostest")
+ s.Contains(sections["PCS_STONITH_CONFIG"], "fence_master_0")
+ s.Contains(sections["PCS_STONITH_STATUS"], "Started master-1")
+ s.Contains(sections["PCS_QUORUM_STATUS"], "Quorate")
+ s.Contains(sections["PCS_PROPERTY"], "stonith-enabled: true")
+}
+
+func (s *STONITHHandlerSuite) TestBuildReportHealthy() {
+ raw := `===PCS_STATUS===
+Cluster name: ostest
+Online: [ master-0 master-1 ]
+OFFLINE: [ ]
+===PCS_STONITH_CONFIG===
+ Resource: fence_master_0 (class=stonith type=fence_ipmilan)
+===PCS_STONITH_STATUS===
+ * fence_master_0 (stonith:fence_ipmilan): Started master-1
+ * fence_master_1 (stonith:fence_ipmilan): Started master-0
+===PCS_STONITH_HISTORY===
+No fencing actions recorded.
+===PCS_QUORUM_STATUS===
+Quorate: Yes
+Expected votes: 2
+===PCS_PROPERTY===
+stonith-enabled: true
+no-quorum-policy: stop
+===END===`
+
+ report, issues := buildSTONITHReport("master-0", raw)
+ s.Contains(report, "TNF STONITH Status")
+ s.Contains(report, "master-0, master-1")
+ s.Contains(report, "fence_master_0")
+ s.Contains(report, "Quorate")
+ s.Empty(issues)
+}
+
+func (s *STONITHHandlerSuite) TestBuildReportStonithDisabled() {
+ raw := `===PCS_STATUS===
+Cluster name: ostest
+Online: [ master-0 master-1 ]
+===PCS_STONITH_CONFIG===
+ Resource: fence_master_0 (class=stonith type=fence_ipmilan)
+===PCS_STONITH_STATUS===
+ * fence_master_0 (stonith:fence_ipmilan): Started master-1
+===PCS_STONITH_HISTORY===
+===PCS_QUORUM_STATUS===
+Quorate: Yes
+===PCS_PROPERTY===
+stonith-enabled: false
+no-quorum-policy: stop
+===END===`
+
+ _, issues := buildSTONITHReport("master-0", raw)
+ s.Contains(issues, "STONITH is disabled (stonith-enabled: false)")
+}
+
+func (s *STONITHHandlerSuite) TestBuildReportNodeOffline() {
+ raw := `===PCS_STATUS===
+Cluster name: ostest
+Online: [ master-0 ]
+OFFLINE: [ master-1 ]
+===PCS_STONITH_CONFIG===
+ Resource: fence_master_0 (class=stonith type=fence_ipmilan)
+===PCS_STONITH_STATUS===
+ * fence_master_0 (stonith:fence_ipmilan): Stopped
+===PCS_STONITH_HISTORY===
+===PCS_QUORUM_STATUS===
+Quorate: No
+===PCS_PROPERTY===
+stonith-enabled: true
+===END===`
+
+ _, issues := buildSTONITHReport("master-0", raw)
+ s.Contains(issues, "nodes offline: master-1")
+ s.Contains(issues, "STONITH device stopped: * fence_master_0\t(stonith:fence_ipmilan):\t Stopped")
+ s.Contains(issues, "cluster is NOT quorate")
+}
+
+func (s *STONITHHandlerSuite) TestBuildReportNoDevices() {
+ raw := `===PCS_STATUS===
+Cluster name: ostest
+Online: [ master-0 master-1 ]
+===PCS_STONITH_CONFIG===
+NO stonith devices configured
+===PCS_STONITH_STATUS===
+===PCS_STONITH_HISTORY===
+===PCS_QUORUM_STATUS===
+Quorate: Yes
+===PCS_PROPERTY===
+stonith-enabled: true
+===END===`
+
+ _, issues := buildSTONITHReport("master-0", raw)
+ s.Contains(issues, "no STONITH devices configured")
+}
+
+func (s *STONITHHandlerSuite) TestExtractBracketedList() {
+ nodes := extractBracketedList("Online: [ master-0 master-1 ]")
+ s.Equal([]string{"master-0", "master-1"}, nodes)
+
+ empty := extractBracketedList("OFFLINE: [ ]")
+ s.Nil(empty)
+
+ none := extractBracketedList("no brackets here")
+ s.Nil(none)
+}
+
+func TestSTONITHHandler(t *testing.T) {
+ suite.Run(t, new(STONITHHandlerSuite))
+}
diff --git a/pkg/toolsets/tnf/mcp_resources.go b/pkg/toolsets/tnf/mcp_resources.go
new file mode 100644
index 000000000..9244fc65a
--- /dev/null
+++ b/pkg/toolsets/tnf/mcp_resources.go
@@ -0,0 +1,100 @@
+package tnf
+
+import (
+ "context"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+)
+
+func initMCPResources() []api.ServerResource {
+ return []api.ServerResource{
+ {
+ Resource: api.Resource{
+ URI: "tnf://domain-knowledge/fencing",
+ Name: "tnf-fencing-domain-knowledge",
+ Description: "TNF two-node fencing domain knowledge: quorum rules, split-brain risk matrix, and recovery procedures",
+ MIMEType: "text/markdown",
+ },
+ Handler: resourceFencingDomainKnowledge,
+ },
+ }
+}
+
+func resourceFencingDomainKnowledge(_ context.Context) (*api.ResourceContent, error) {
+ return &api.ResourceContent{Text: fencingDomainKnowledge}, nil
+}
+
+const fencingDomainKnowledge = `# TNF Fencing Domain Knowledge
+
+## How Two-Node Fencing Works
+
+TNF clusters use **pacemaker** and **corosync** for high availability:
+
+1. **Corosync** provides cluster communication and membership between the 2 nodes
+2. **Pacemaker** manages cluster resources (services, virtual IPs) and decides placement
+3. **STONITH** (Shoot The Other Node In The Head) is the fencing mechanism:
+ - Each node has a fence device (IPMI via fence_ipmilan or Redfish via fence_redfish) that targets the OTHER node
+ - When a node fails or loses communication, the surviving node uses the fence device to power-cycle the failed node via its BMC
+ - This prevents **split-brain**: a condition where both nodes think they own shared resources, causing data corruption
+
+**Two-node quorum** is special:
+- Normal quorum requires a majority (>50%) of votes. With 2 nodes, losing 1 node means only 50% — no majority
+- Corosync uses the two_node: 1 and wait_for_all: 1 settings to handle this
+- wait_for_all: Both nodes must be present at initial startup before quorum is granted
+- Once running, if one node fails, the remaining node retains quorum (provided it successfully fences the other)
+- The no-quorum-policy property controls what happens when quorum is lost (usually "stop" — stop all resources)
+
+## Split-Brain Risk Assessment
+
+Analyze collected data for these risk conditions:
+
+| Condition | Risk Level | Action Required |
+|-----------|-----------|-----------------|
+| STONITH disabled | **CRITICAL** | Re-enable immediately — cluster has no fencing protection |
+| Fence device stopped/failed | **HIGH** | Fencing will fail on demand — check BMC connectivity |
+| BMC credentials missing | **HIGH** | Fence agent cannot authenticate to BMC |
+| BMC address not configured | **HIGH** | Fence agent has no target to fence |
+| One node offline (pacemaker) | **MEDIUM** | Check if fencing occurred — verify other node has quorum |
+| Quorum lost | **HIGH** | Resources may be stopped depending on no-quorum-policy |
+| Both nodes online, healthy | **LOW** | Normal operation |
+
+## Common Issues and Recovery Procedures
+
+### 1. STONITH Disabled
+
+- **Symptom:** stonith-enabled: false in pcs property list
+- **Risk:** No fencing protection — split-brain can occur
+- **Recovery:** Verify fence devices are configured and working, then run: pcs property set stonith-enabled=true
+- **Verify:** pcs stonith status shows devices Started
+
+### 2. BMC Unreachable / Credentials Invalid
+
+- **Symptom:** Fence device fails to operate; BMH shows credential errors
+- **Risk:** Fencing will fail when needed — node cannot be power-cycled
+- **Recovery:** Check BMC network connectivity, update the credential Secret referenced by the BareMetalHost
+- **Verify:** fence_ -a -l -p -o status (from the node)
+
+### 3. Node Won't Rejoin After Fencing
+
+- **Symptom:** Node was fenced but pacemaker shows it OFFLINE
+- **Recovery:** On the fenced node after it boots: check pcs status, clear stale fencing state with pcs stonith history cleanup , then pcs cluster start if needed
+- **Check:** BMC power state via fence_ -o status, verify node booted successfully
+
+### 4. Quorum Lost (Single Node Running)
+
+- **Symptom:** Quorate: No in corosync-quorumtool; resources may be stopped
+- **Context:** With no-quorum-policy: stop, pacemaker stops all resources when quorum is lost
+- **Recovery:** If the other node is truly down and fenced, the remaining node can be given quorum with pcs quorum unblock or by using --force flags (use with caution)
+
+### 5. Fence Device Misconfigured
+
+- **Symptom:** Wrong agent type, incorrect BMC IP, or wrong parameters
+- **Recovery:** pcs stonith update = to correct parameters
+- **Verify:** pcs stonith config to review, fence_ -o status to test
+
+### 6. Fence Race (Both Nodes Fencing Each Other)
+
+- **Symptom:** Both nodes tried to fence each other simultaneously
+- **Diagnosis:** Check pcs stonith history on both nodes — one should have won
+- **Recovery:** The node that lost the fence race should have been power-cycled. If both are up, check which one has quorum and resources
+`
diff --git a/pkg/toolsets/tnf/mcp_resources_test.go b/pkg/toolsets/tnf/mcp_resources_test.go
new file mode 100644
index 000000000..348ec7d09
--- /dev/null
+++ b/pkg/toolsets/tnf/mcp_resources_test.go
@@ -0,0 +1,37 @@
+package tnf
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestInitMCPResources(t *testing.T) {
+ resources := initMCPResources()
+ require.Len(t, resources, 1)
+
+ r := resources[0]
+ assert.Equal(t, "tnf://domain-knowledge/fencing", r.Resource.URI)
+ assert.Equal(t, "tnf-fencing-domain-knowledge", r.Resource.Name)
+ assert.Equal(t, "text/markdown", r.Resource.MIMEType)
+ assert.NotNil(t, r.Handler)
+}
+
+func TestFencingDomainKnowledgeResource(t *testing.T) {
+ resources := initMCPResources()
+ require.Len(t, resources, 1)
+
+ content, err := resources[0].Handler(context.Background())
+ require.NoError(t, err)
+ require.NotNil(t, content)
+ assert.NotEmpty(t, content.Text)
+
+ assert.Contains(t, content.Text, "Split-Brain Risk Assessment")
+ assert.Contains(t, content.Text, "STONITH")
+ assert.Contains(t, content.Text, "quorum")
+ assert.Contains(t, content.Text, "fence_ipmilan")
+ assert.Contains(t, content.Text, "Recovery")
+ assert.Contains(t, content.Text, "pcs property set stonith-enabled=true")
+}
diff --git a/pkg/toolsets/tnf/toolset.go b/pkg/toolsets/tnf/toolset.go
new file mode 100644
index 000000000..5fcecc42c
--- /dev/null
+++ b/pkg/toolsets/tnf/toolset.go
@@ -0,0 +1,44 @@
+package tnf
+
+import (
+ "slices"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+ "github.com/containers/kubernetes-mcp-server/pkg/toolsets"
+ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf/fencing"
+)
+
+type Toolset struct{}
+
+var _ api.Toolset = (*Toolset)(nil)
+
+func (t *Toolset) GetName() string {
+ return "tnf"
+}
+
+func (t *Toolset) GetDescription() string {
+ return "Two-Node Fencing (TNF) cluster diagnostics"
+}
+
+func (t *Toolset) GetTools(_ api.FilteringProvider) []api.ServerTool {
+ return slices.Concat(
+ fencing.InitFencing(),
+ fencing.InitSTONITH(),
+ )
+}
+
+func (t *Toolset) GetPrompts() []api.ServerPrompt {
+ return initTNFTroubleshoot()
+}
+
+func (t *Toolset) GetResources() []api.ServerResource {
+ return initMCPResources()
+}
+
+func (t *Toolset) GetResourceTemplates() []api.ServerResourceTemplate {
+ return nil
+}
+
+func init() {
+ toolsets.Register(&Toolset{})
+}
diff --git a/pkg/toolsets/tnf/troubleshoot.go b/pkg/toolsets/tnf/troubleshoot.go
new file mode 100644
index 000000000..ec0709994
--- /dev/null
+++ b/pkg/toolsets/tnf/troubleshoot.go
@@ -0,0 +1,560 @@
+package tnf
+
+import (
+ "context"
+ "fmt"
+ "github.com/containers/kubernetes-mcp-server/pkg/klogutil"
+ "strings"
+ "time"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/client-go/dynamic"
+ corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf/fencing"
+)
+
+const defaultTroubleshootTimeout = 120 * time.Second
+
+func initTNFTroubleshoot() []api.ServerPrompt {
+ return []api.ServerPrompt{
+ {
+ Prompt: api.Prompt{
+ Name: "tnf-troubleshoot",
+ Title: "TNF Fencing Troubleshoot",
+ Description: "Generate a step-by-step troubleshooting guide for diagnosing " +
+ "Two-Node Fencing (TNF) cluster issues including STONITH configuration, " +
+ "quorum status, BMC health, and split-brain risk assessment",
+ Arguments: []api.PromptArgument{
+ {
+ Name: "node",
+ Description: "Node to run STONITH diagnostics on (auto-detects if omitted)",
+ Required: false,
+ },
+ {
+ Name: "namespace",
+ Description: "Namespace for BareMetalHost resources (default: openshift-machine-api)",
+ Required: false,
+ },
+ },
+ },
+ Handler: tnfTroubleshootHandler,
+ },
+ }
+}
+
+func tnfTroubleshootHandler(params api.PromptHandlerParams) (*api.PromptCallResult, error) {
+ if fencing.IsAPIUnreachable(params.KubernetesClient) {
+ guide := fencing.APIUnreachableGuide()
+ return api.NewPromptCallResult(
+ "Kubernetes API is unreachable — returning out-of-band troubleshooting guide",
+ []api.PromptMessage{
+ {
+ Role: "user",
+ Content: api.PromptContent{
+ Type: "text",
+ Text: guide,
+ },
+ },
+ {
+ Role: "assistant",
+ Content: api.PromptContent{
+ Type: "text",
+ Text: "The Kubernetes API is unreachable. I'll walk through the out-of-band troubleshooting guide to help determine whether this is a failed install, an upgrade/reboot, or a crash scenario.",
+ },
+ },
+ },
+ nil,
+ ), nil
+ }
+
+ args := params.GetArguments()
+ nodeName := args["node"]
+ namespace := args["namespace"]
+ if namespace == "" {
+ namespace = "openshift-machine-api"
+ }
+
+ ctx := params.Context
+
+ dynamicClient := params.DynamicClient()
+ coreClient := params.CoreV1()
+
+ topologyData, isTNF := fetchClusterTopology(ctx, dynamicClient, coreClient)
+ nodeHealthData := fetchNodeHealth(ctx, coreClient)
+ operatorData := fetchOperatorHealth(ctx, dynamicClient)
+ bmhData := fetchBMHStatus(ctx, dynamicClient, coreClient, namespace)
+
+ stonithData := fetchSTONITHData(ctx, params.KubernetesClient, coreClient, nodeName)
+
+ var remediationData string
+ if isTNF {
+ remediationData = "TNF clusters use Pacemaker/STONITH for fencing — see STONITH data above.\n"
+ } else {
+ remediationData = fetchRemediationStatus(ctx, dynamicClient)
+ }
+
+ guideText := fmt.Sprintf(`# TNF Fencing Troubleshooting Guide
+
+Use this guide to diagnose issues with Two-Node Fencing (TNF) clusters. All relevant
+cluster data has been collected below.
+
+---
+
+## Step 1: Cluster Topology
+
+Verify this is a TNF cluster (2-node bare metal with pacemaker fencing).
+- Expect: Platform=BareMetal, 2 control-plane nodes, TNF Profile=Yes
+- If not TNF: this guide may not apply
+
+%s
+
+---
+
+## Step 2: Node Health
+
+Both nodes must be Ready for the cluster to function. A NotReady node may trigger fencing.
+- Both nodes Ready = healthy
+- One node NotReady = potential fencing trigger (check pacemaker)
+- Both nodes NotReady = critical — cluster is down
+
+%s
+
+---
+
+## Step 3: Critical Operators
+
+These operators are essential for TNF cluster health:
+- **etcd**: Stores all cluster state; losing quorum is fatal for 2-node clusters
+- **machine-api**: Manages BareMetalHost lifecycle
+- **baremetal**: Manages BMC communication and provisioning
+
+%s
+
+---
+
+## Step 4: BareMetalHost & BMC Health
+
+BareMetalHost resources represent the physical nodes. BMC credentials must be valid
+for STONITH fencing to work — the fence agent uses BMC to power-cycle nodes.
+- Check: BMH provisioning state, BMC address configured, credentials secret exists
+- Missing or invalid BMC credentials = fencing will fail when needed
+
+%s
+
+---
+
+## Step 5: Pacemaker & STONITH Status
+
+This section shows the live pacemaker cluster state gathered from the node via a
+debug pod. Key things to check:
+- **STONITH enabled**: Must be true for production. Disabling STONITH is a split-brain risk.
+- **Nodes online**: Both nodes should be in the pacemaker cluster
+- **Fence devices**: Each node needs a working fence device targeting the OTHER node
+- **Quorum**: Cluster must be quorate. Two-node mode uses special quorum rules.
+
+%s
+
+---
+
+## Step 6: Remediation Operators
+
+%s
+
+---
+
+## TNF Domain Knowledge
+
+For detailed domain knowledge about two-node fencing mechanics, split-brain
+risk assessment, and common recovery procedures, read the MCP resource:
+
+ tnf://domain-knowledge/fencing
+
+Reference this resource to interpret the data collected above and assess
+split-brain risk.
+
+---
+
+## Troubleshooting Analysis
+
+Based on the data collected above (and the domain knowledge resource), analyze:
+
+1. **Is this a TNF cluster?** Check topology — 2-node bare metal with pacemaker
+2. **Are both nodes healthy?** Check node Ready status and pacemaker online list
+3. **Is STONITH properly configured?** Devices exist, started, targeting correct nodes
+4. **Are BMC credentials valid?** Secrets exist with username/password keys
+5. **Is the cluster quorate?** Check corosync quorum status
+6. **Any active issues?** Failed resources, active remediations, fencing events
+7. **Split-brain risk level?** Use the risk matrix from the domain knowledge resource
+
+---
+
+## Report Findings
+
+After analysis, report:
+- **Status:** Healthy / At Risk / Degraded / Critical
+- **Split-Brain Risk:** None / Low / Medium / High / Critical
+- **Issues Found:** List of specific problems
+- **Root Cause:** Description of the primary issue (or "None found")
+- **Recommended Actions:** Specific steps to resolve issues
+- **Verification:** How to confirm the fix worked
+`, topologyData, nodeHealthData, operatorData, bmhData, stonithData, remediationData)
+
+ return api.NewPromptCallResult(
+ "TNF fencing troubleshooting guide generated",
+ []api.PromptMessage{
+ {
+ Role: "user",
+ Content: api.PromptContent{
+ Type: "text",
+ Text: guideText,
+ },
+ },
+ {
+ Role: "assistant",
+ Content: api.PromptContent{
+ Type: "text",
+ Text: "I'll analyze the collected TNF cluster data to assess fencing health, identify split-brain risks, and provide specific remediation guidance.",
+ },
+ },
+ },
+ nil,
+ ), nil
+}
+
+func fetchClusterTopology(ctx context.Context, dynamicClient dynamic.Interface, coreClient corev1client.CoreV1Interface) (string, bool) {
+ var result strings.Builder
+ result.WriteString("### Cluster Topology\n\n")
+
+ infra, err := dynamicClient.Resource(fencing.InfrastructureGVR).Get(ctx, "cluster", metav1.GetOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not get Infrastructure CR", "error", err)
+ result.WriteString("*Infrastructure CR not available*\n")
+ return result.String(), false
+ }
+
+ platform, _, _ := unstructured.NestedString(infra.Object, "status", "platform")
+ infraTopology, _, _ := unstructured.NestedString(infra.Object, "status", "infrastructureTopology")
+ cpTopology, _, _ := unstructured.NestedString(infra.Object, "status", "controlPlaneTopology")
+
+ fmt.Fprintf(&result, "- **Platform:** %s\n", fencing.ValueOrNA(platform))
+ fmt.Fprintf(&result, "- **Infrastructure Topology:** %s\n", fencing.ValueOrNA(infraTopology))
+ fmt.Fprintf(&result, "- **Control Plane Topology:** %s\n", fencing.ValueOrNA(cpTopology))
+
+ nodes, err := coreClient.Nodes().List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list nodes", "error", err)
+ return result.String(), false
+ }
+
+ cpCount := 0
+ for _, node := range nodes.Items {
+ if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
+ cpCount++
+ } else if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
+ cpCount++
+ }
+ }
+ fmt.Fprintf(&result, "- **Total Nodes:** %d\n", len(nodes.Items))
+ fmt.Fprintf(&result, "- **Control-Plane Nodes:** %d\n", cpCount)
+
+ isTNF := strings.EqualFold(platform, "BareMetal") && cpCount == 2
+ if isTNF {
+ result.WriteString("- **TNF Profile:** Yes (2-node bare metal)\n")
+ } else {
+ result.WriteString("- **TNF Profile:** No\n")
+ }
+
+ return result.String(), isTNF
+}
+
+func fetchNodeHealth(ctx context.Context, coreClient corev1client.CoreV1Interface) string {
+ var result strings.Builder
+ result.WriteString("### Node Status\n\n")
+
+ nodes, err := coreClient.Nodes().List(ctx, metav1.ListOptions{})
+ if err != nil {
+ fmt.Fprintf(&result, "*Error listing nodes: %v*\n", err)
+ return result.String()
+ }
+
+ if len(nodes.Items) == 0 {
+ result.WriteString("*No nodes found*\n")
+ return result.String()
+ }
+
+ result.WriteString("| Node | Ready | Roles | Conditions |\n")
+ result.WriteString("|------|-------|-------|------------|\n")
+
+ for _, node := range nodes.Items {
+ ready := "Unknown"
+ var conditions []string
+
+ for _, cond := range node.Status.Conditions {
+ switch cond.Type {
+ case "Ready":
+ if cond.Status == "True" {
+ ready = "Yes"
+ } else {
+ ready = "**No**"
+ }
+ case "MemoryPressure", "DiskPressure", "PIDPressure":
+ if cond.Status == "True" {
+ conditions = append(conditions, string(cond.Type))
+ }
+ }
+ }
+
+ var roles []string
+ for label := range node.Labels {
+ if strings.HasPrefix(label, "node-role.kubernetes.io/") {
+ role := strings.TrimPrefix(label, "node-role.kubernetes.io/")
+ roles = append(roles, role)
+ }
+ }
+
+ condStr := "None"
+ if len(conditions) > 0 {
+ condStr = strings.Join(conditions, ", ")
+ }
+
+ fmt.Fprintf(&result, "| %s | %s | %s | %s |\n",
+ node.Name, ready, strings.Join(roles, ", "), condStr)
+ }
+
+ return result.String()
+}
+
+func fetchOperatorHealth(ctx context.Context, dynamicClient dynamic.Interface) string {
+ var result strings.Builder
+ result.WriteString("### Cluster Operators\n\n")
+
+ tnfOperators := []string{"etcd", "machine-api", "baremetal"}
+
+ list, err := dynamicClient.Resource(fencing.ClusterOperatorGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list ClusterOperators", "error", err)
+ result.WriteString("*ClusterOperators not available (may not be an OpenShift cluster)*\n")
+ return result.String()
+ }
+
+ result.WriteString("| Operator | Available | Degraded | Progressing |\n")
+ result.WriteString("|----------|-----------|----------|-------------|\n")
+
+ found := make(map[string]bool)
+ for _, item := range list.Items {
+ name := item.GetName()
+ isRelevant := false
+ for _, op := range tnfOperators {
+ if name == op {
+ isRelevant = true
+ break
+ }
+ }
+ if !isRelevant {
+ continue
+ }
+ found[name] = true
+
+ conditions, _, _ := unstructured.NestedSlice(item.Object, "status", "conditions")
+ available, degraded, progressing := "Unknown", "Unknown", "Unknown"
+
+ for _, cond := range conditions {
+ condMap, ok := cond.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ condType, _ := condMap["type"].(string)
+ condStatus, _ := condMap["status"].(string)
+
+ switch condType {
+ case "Available":
+ available = condStatus
+ case "Degraded":
+ degraded = condStatus
+ case "Progressing":
+ progressing = condStatus
+ }
+ }
+
+ fmt.Fprintf(&result, "| %s | %s | %s | %s |\n",
+ name, available, degraded, progressing)
+ }
+
+ for _, op := range tnfOperators {
+ if !found[op] {
+ fmt.Fprintf(&result, "| %s | **NOT FOUND** | - | - |\n", op)
+ }
+ }
+
+ return result.String()
+}
+
+func fetchBMHStatus(ctx context.Context, dynamicClient dynamic.Interface, coreClient corev1client.CoreV1Interface, namespace string) string {
+ var result strings.Builder
+ result.WriteString("### BareMetalHost Status\n\n")
+
+ var list *unstructured.UnstructuredList
+ var err error
+ if namespace != "" {
+ list, err = dynamicClient.Resource(fencing.BareMetalHostGVR).Namespace(namespace).List(ctx, metav1.ListOptions{})
+ } else {
+ list, err = dynamicClient.Resource(fencing.BareMetalHostGVR).List(ctx, metav1.ListOptions{})
+ }
+ if err != nil {
+ fmt.Fprintf(&result, "*Error listing BareMetalHosts: %v*\n", err)
+ return result.String()
+ }
+
+ if len(list.Items) == 0 {
+ result.WriteString("*No BareMetalHost resources found — this may not be a bare metal cluster*\n")
+ return result.String()
+ }
+
+ fmt.Fprintf(&result, "Found %d BareMetalHost(s):\n\n", len(list.Items))
+
+ for _, host := range list.Items {
+ hostName := host.GetName()
+ hostNS := host.GetNamespace()
+
+ provState, _, _ := unstructured.NestedString(host.Object, "status", "provisioning", "state")
+ opStatus, _, _ := unstructured.NestedString(host.Object, "status", "operationalStatus")
+ poweredOn, _, _ := unstructured.NestedBool(host.Object, "status", "poweredOn")
+ errorMsg, _, _ := unstructured.NestedString(host.Object, "status", "errorMessage")
+ bmcAddr, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "address")
+ credName, _, _ := unstructured.NestedString(host.Object, "spec", "bmc", "credentialsName")
+
+ fmt.Fprintf(&result, "#### %s/%s\n\n", hostNS, hostName)
+
+ isUnmanaged := provState == "" || provState == "unmanaged"
+ if isUnmanaged && bmcAddr == "" {
+ result.WriteString("- State: Unmanaged (not controlled by baremetal-operator)\n")
+ } else {
+ fmt.Fprintf(&result, "- Provisioning State: %s\n", fencing.ValueOrNA(provState))
+ fmt.Fprintf(&result, "- Operational Status: %s\n", fencing.ValueOrNA(opStatus))
+ fmt.Fprintf(&result, "- Powered On: %t\n", poweredOn)
+ fmt.Fprintf(&result, "- BMC Address: %s\n", fencing.ValueOrNA(bmcAddr))
+ fmt.Fprintf(&result, "- Credentials Secret: %s\n", fencing.ValueOrNA(credName))
+
+ if errorMsg != "" {
+ fmt.Fprintf(&result, "- **Error:** %s\n", errorMsg)
+ }
+
+ if credName != "" {
+ credStatus := checkCredentialSecret(ctx, coreClient, hostNS, credName)
+ fmt.Fprintf(&result, "- Credentials Check: %s\n", credStatus)
+ } else if !isUnmanaged {
+ result.WriteString("- Credentials Check: **No credentials secret configured**\n")
+ }
+ }
+ result.WriteString("\n")
+ }
+
+ return result.String()
+}
+
+func checkCredentialSecret(ctx context.Context, coreClient corev1client.CoreV1Interface, namespace, secretName string) string {
+ secret, err := coreClient.Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{})
+ if err != nil {
+ return fmt.Sprintf("**Secret %q not found** — %v", secretName, err)
+ }
+
+ userVal, hasUser := secret.Data["username"]
+ passVal, hasPass := secret.Data["password"]
+ userValid := hasUser && len(userVal) > 0
+ passValid := hasPass && len(passVal) > 0
+
+ if !userValid && !passValid {
+ return fmt.Sprintf("**Secret %q missing or empty 'username' and 'password'**", secretName)
+ }
+ if !userValid {
+ return fmt.Sprintf("**Secret %q missing or empty 'username'**", secretName)
+ }
+ if !passValid {
+ return fmt.Sprintf("**Secret %q missing or empty 'password'**", secretName)
+ }
+ return "Valid (username and password present)"
+}
+
+func fetchSTONITHData(ctx context.Context, k8sClient api.KubernetesClient, coreClient corev1client.CoreV1Interface, nodeName string) string {
+ report, issues, err := fencing.RunSTONITHDiagnostics(ctx, k8sClient, coreClient, nodeName, "", defaultTroubleshootTimeout)
+ if err != nil {
+ return fmt.Sprintf("### STONITH Diagnostics\n\n*Error running STONITH diagnostics: %v*\n\n"+
+ "This may indicate:\n- No control-plane nodes found\n- Debug pod could not be created\n"+
+ "- pcs/corosync not installed (not a pacemaker-based cluster)\n", err)
+ }
+
+ if len(issues) > 0 {
+ var issueSection strings.Builder
+ issueSection.WriteString("\n### Detected Issues\n\n")
+ for i, issue := range issues {
+ fmt.Fprintf(&issueSection, "%d. **%s**\n", i+1, issue)
+ }
+ return report + issueSection.String()
+ }
+
+ return report
+}
+
+func fetchRemediationStatus(ctx context.Context, dynamicClient dynamic.Interface) string {
+ var result strings.Builder
+ result.WriteString("### Remediation Operators\n\n")
+
+ // FenceAgentsRemediation Templates
+ result.WriteString("#### FenceAgentsRemediation Templates\n\n")
+ templates, err := dynamicClient.Resource(fencing.FenceAgentsRemediationTemplateGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list FAR templates", "error", err)
+ result.WriteString("*FenceAgentsRemediationTemplate CRD not installed — cluster may use traditional pacemaker/STONITH fencing*\n\n")
+ } else if len(templates.Items) == 0 {
+ result.WriteString("*No FenceAgentsRemediationTemplates configured*\n\n")
+ } else {
+ for _, tmpl := range templates.Items {
+ name := tmpl.GetName()
+ ns := tmpl.GetNamespace()
+ agent, _, _ := unstructured.NestedString(tmpl.Object, "spec", "template", "spec", "agent")
+ location := name
+ if ns != "" {
+ location = fmt.Sprintf("%s/%s", ns, name)
+ }
+ fmt.Fprintf(&result, "- %s: agent=%s\n", location, fencing.ValueOrNA(agent))
+ }
+ result.WriteString("\n")
+ }
+
+ // Active remediations
+ result.WriteString("#### Active Remediations\n\n")
+ remediations, err := dynamicClient.Resource(fencing.FenceAgentsRemediationGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list FAR remediations", "error", err)
+ result.WriteString("*Could not check active remediations*\n\n")
+ } else if len(remediations.Items) == 0 {
+ result.WriteString("*No active fencing remediations*\n\n")
+ } else {
+ for _, rem := range remediations.Items {
+ agent, _, _ := unstructured.NestedString(rem.Object, "spec", "agent")
+ fmt.Fprintf(&result, "- **%s/%s**: agent=%s\n", rem.GetNamespace(), rem.GetName(), fencing.ValueOrNA(agent))
+ }
+ result.WriteString("\n")
+ }
+
+ // NodeHealthChecks
+ result.WriteString("#### NodeHealthCheck\n\n")
+ nhcList, err := dynamicClient.Resource(fencing.NodeHealthCheckGVR).List(ctx, metav1.ListOptions{})
+ if err != nil {
+ klogutil.FromContext(ctx).V(1).Info("could not list NHC", "error", err)
+ result.WriteString("*NodeHealthCheck CRD not installed*\n\n")
+ } else if len(nhcList.Items) == 0 {
+ result.WriteString("*No NodeHealthCheck resources configured*\n\n")
+ } else {
+ for _, nhc := range nhcList.Items {
+ phase, _, _ := unstructured.NestedString(nhc.Object, "status", "phase")
+ fmt.Fprintf(&result, "- %s: phase=%s\n", nhc.GetName(), fencing.ValueOrNA(phase))
+ }
+ result.WriteString("\n")
+ }
+
+ return result.String()
+}
diff --git a/pkg/toolsets/tnf/troubleshoot_test.go b/pkg/toolsets/tnf/troubleshoot_test.go
new file mode 100644
index 000000000..8c6eeb884
--- /dev/null
+++ b/pkg/toolsets/tnf/troubleshoot_test.go
@@ -0,0 +1,373 @@
+package tnf
+
+import (
+ "context"
+ "testing"
+
+ "github.com/containers/kubernetes-mcp-server/pkg/api"
+ "github.com/containers/kubernetes-mcp-server/pkg/toolsets/tnf/fencing"
+ "github.com/stretchr/testify/suite"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/client-go/dynamic/fake"
+ corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+)
+
+type mockPromptCallRequest struct {
+ args map[string]string
+}
+
+func (m *mockPromptCallRequest) GetArguments() map[string]string {
+ return m.args
+}
+
+var _ api.PromptCallRequest = (*mockPromptCallRequest)(nil)
+
+type fakeCoreV1 struct {
+ corev1client.CoreV1Interface
+ nodes []*corev1.Node
+ secrets []*corev1.Secret
+}
+
+func (f *fakeCoreV1) Nodes() corev1client.NodeInterface {
+ return &fakeNodes{nodes: f.nodes}
+}
+
+func (f *fakeCoreV1) Secrets(namespace string) corev1client.SecretInterface {
+ return &fakeSecrets{namespace: namespace, secrets: f.secrets}
+}
+
+type fakeNodes struct {
+ corev1client.NodeInterface
+ nodes []*corev1.Node
+}
+
+func (f *fakeNodes) List(_ context.Context, _ metav1.ListOptions) (*corev1.NodeList, error) {
+ items := make([]corev1.Node, len(f.nodes))
+ for i, n := range f.nodes {
+ items[i] = *n
+ }
+ return &corev1.NodeList{Items: items}, nil
+}
+
+type fakeSecrets struct {
+ corev1client.SecretInterface
+ namespace string
+ secrets []*corev1.Secret
+}
+
+func (f *fakeSecrets) Get(_ context.Context, name string, _ metav1.GetOptions) (*corev1.Secret, error) {
+ for _, s := range f.secrets {
+ if s.Name == name && s.Namespace == f.namespace {
+ return s, nil
+ }
+ }
+ return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "secrets"}, name)
+}
+
+func newFakeCoreV1(objects ...runtime.Object) corev1client.CoreV1Interface {
+ f := &fakeCoreV1{}
+ for _, obj := range objects {
+ switch o := obj.(type) {
+ case *corev1.Node:
+ f.nodes = append(f.nodes, o)
+ case *corev1.Secret:
+ f.secrets = append(f.secrets, o)
+ }
+ }
+ return f
+}
+
+type TNFTroubleshootSuite struct {
+ suite.Suite
+}
+
+func (s *TNFTroubleshootSuite) TestPromptRegistration() {
+ prompts := initTNFTroubleshoot()
+ s.Require().Len(prompts, 1)
+ s.Equal("tnf-troubleshoot", prompts[0].Prompt.Name)
+ s.Equal("TNF Fencing Troubleshoot", prompts[0].Prompt.Title)
+ s.Len(prompts[0].Prompt.Arguments, 2)
+ s.Equal("node", prompts[0].Prompt.Arguments[0].Name)
+ s.False(prompts[0].Prompt.Arguments[0].Required)
+ s.Equal("namespace", prompts[0].Prompt.Arguments[1].Name)
+ s.False(prompts[0].Prompt.Arguments[1].Required)
+ s.NotNil(prompts[0].Handler)
+}
+
+func (s *TNFTroubleshootSuite) TestToolsetGetPrompts() {
+ toolset := &Toolset{}
+ prompts := toolset.GetPrompts()
+ s.Require().Len(prompts, 1)
+ s.Equal("tnf-troubleshoot", prompts[0].Prompt.Name)
+}
+
+func (s *TNFTroubleshootSuite) TestFetchClusterTopology() {
+ ctx := context.Background()
+ gvrToListKind := map[schema.GroupVersionResource]string{
+ fencing.InfrastructureGVR: "InfrastructureList",
+ }
+
+ s.Run("detects platform from infrastructure CR", func() {
+ infra := &unstructured.Unstructured{}
+ infra.SetUnstructuredContent(map[string]interface{}{
+ "apiVersion": "config.openshift.io/v1",
+ "kind": "Infrastructure",
+ "metadata": map[string]interface{}{
+ "name": "cluster",
+ },
+ "status": map[string]interface{}{
+ "platform": "BareMetal",
+ "infrastructureTopology": "HighlyAvailable",
+ "controlPlaneTopology": "HighlyAvailable",
+ },
+ })
+
+ node1 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "master-0",
+ Labels: map[string]string{"node-role.kubernetes.io/master": ""},
+ },
+ }
+ node2 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "master-1",
+ Labels: map[string]string{"node-role.kubernetes.io/master": ""},
+ },
+ }
+
+ dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, infra)
+ coreClient := newFakeCoreV1(node1, node2)
+
+ result, isTNF := fetchClusterTopology(ctx, dynamicClient, coreClient)
+ s.Contains(result, "BareMetal")
+ s.Contains(result, "TNF Profile:** Yes")
+ s.Contains(result, "Total Nodes:** 2")
+ s.True(isTNF)
+ })
+
+ s.Run("handles missing infrastructure CR", func() {
+ dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind)
+ coreClient := newFakeCoreV1()
+
+ result, isTNF := fetchClusterTopology(ctx, dynamicClient, coreClient)
+ s.Contains(result, "not available")
+ s.False(isTNF)
+ })
+}
+
+func (s *TNFTroubleshootSuite) TestFetchNodeHealth() {
+ ctx := context.Background()
+
+ s.Run("reports node status table", func() {
+ node := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "master-0",
+ Labels: map[string]string{
+ "node-role.kubernetes.io/master": "",
+ "node-role.kubernetes.io/control-plane": "",
+ },
+ },
+ Status: corev1.NodeStatus{
+ Conditions: []corev1.NodeCondition{
+ {Type: "Ready", Status: "True"},
+ },
+ },
+ }
+
+ coreClient := newFakeCoreV1(node)
+ result := fetchNodeHealth(ctx, coreClient)
+ s.Contains(result, "master-0")
+ s.Contains(result, "Yes")
+ s.Contains(result, "master")
+ })
+
+ s.Run("reports NotReady node", func() {
+ node := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "master-0",
+ Labels: map[string]string{"node-role.kubernetes.io/master": ""},
+ },
+ Status: corev1.NodeStatus{
+ Conditions: []corev1.NodeCondition{
+ {Type: "Ready", Status: "False"},
+ },
+ },
+ }
+
+ coreClient := newFakeCoreV1(node)
+ result := fetchNodeHealth(ctx, coreClient)
+ s.Contains(result, "**No**")
+ })
+}
+
+func (s *TNFTroubleshootSuite) TestFetchOperatorHealth() {
+ ctx := context.Background()
+ gvrToListKind := map[schema.GroupVersionResource]string{
+ fencing.ClusterOperatorGVR: "ClusterOperatorList",
+ }
+
+ s.Run("reports operator status", func() {
+ etcdOp := &unstructured.Unstructured{}
+ etcdOp.SetUnstructuredContent(map[string]interface{}{
+ "apiVersion": "config.openshift.io/v1",
+ "kind": "ClusterOperator",
+ "metadata": map[string]interface{}{
+ "name": "etcd",
+ },
+ "status": map[string]interface{}{
+ "conditions": []interface{}{
+ map[string]interface{}{
+ "type": "Available",
+ "status": "True",
+ },
+ map[string]interface{}{
+ "type": "Degraded",
+ "status": "False",
+ },
+ },
+ },
+ })
+
+ dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, etcdOp)
+ result := fetchOperatorHealth(ctx, dynamicClient)
+ s.Contains(result, "etcd")
+ s.Contains(result, "True")
+ s.Contains(result, "machine-api")
+ s.Contains(result, "NOT FOUND")
+ })
+}
+
+func (s *TNFTroubleshootSuite) TestFetchBMHStatus() {
+ ctx := context.Background()
+ gvrToListKind := map[schema.GroupVersionResource]string{
+ fencing.BareMetalHostGVR: "BareMetalHostList",
+ }
+
+ s.Run("reports BMH details", func() {
+ bmh := &unstructured.Unstructured{}
+ bmh.SetUnstructuredContent(map[string]interface{}{
+ "apiVersion": "metal3.io/v1alpha1",
+ "kind": "BareMetalHost",
+ "metadata": map[string]interface{}{
+ "name": "master-0",
+ "namespace": "openshift-machine-api",
+ },
+ "spec": map[string]interface{}{
+ "bmc": map[string]interface{}{
+ "address": "redfish://192.168.1.10/redfish/v1/Systems/1",
+ "credentialsName": "master-0-bmc-secret",
+ },
+ "online": true,
+ },
+ "status": map[string]interface{}{
+ "provisioning": map[string]interface{}{
+ "state": "externally provisioned",
+ },
+ "operationalStatus": "OK",
+ "poweredOn": true,
+ },
+ })
+
+ secret := &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "master-0-bmc-secret",
+ Namespace: "openshift-machine-api",
+ },
+ Data: map[string][]byte{
+ "username": []byte("admin"),
+ "password": []byte("secret"),
+ },
+ }
+
+ dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, bmh)
+ coreClient := newFakeCoreV1(secret)
+
+ result := fetchBMHStatus(ctx, dynamicClient, coreClient, "openshift-machine-api")
+ s.Contains(result, "master-0")
+ s.Contains(result, "redfish://192.168.1.10")
+ s.Contains(result, "externally provisioned")
+ s.Contains(result, "Valid")
+ })
+
+ s.Run("handles no BMH resources", func() {
+ dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind)
+ coreClient := newFakeCoreV1()
+
+ result := fetchBMHStatus(ctx, dynamicClient, coreClient, "openshift-machine-api")
+ s.Contains(result, "No BareMetalHost resources found")
+ })
+}
+
+func (s *TNFTroubleshootSuite) TestFetchRemediationStatus() {
+ ctx := context.Background()
+ gvrToListKind := map[schema.GroupVersionResource]string{
+ fencing.FenceAgentsRemediationTemplateGVR: "FenceAgentsRemediationTemplateList",
+ fencing.FenceAgentsRemediationGVR: "FenceAgentsRemediationList",
+ fencing.NodeHealthCheckGVR: "NodeHealthCheckList",
+ }
+
+ s.Run("reports empty state", func() {
+ dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind)
+ result := fetchRemediationStatus(ctx, dynamicClient)
+ s.Contains(result, "No FenceAgentsRemediationTemplates configured")
+ s.Contains(result, "No active fencing remediations")
+ s.Contains(result, "No NodeHealthCheck resources configured")
+ })
+}
+
+func (s *TNFTroubleshootSuite) TestCheckCredentialSecret() {
+ ctx := context.Background()
+
+ s.Run("valid credentials", func() {
+ secret := &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "bmc-secret",
+ Namespace: "ns",
+ },
+ Data: map[string][]byte{
+ "username": []byte("admin"),
+ "password": []byte("pass"),
+ },
+ }
+
+ coreClient := newFakeCoreV1(secret)
+ result := checkCredentialSecret(ctx, coreClient, "ns", "bmc-secret")
+ s.Contains(result, "Valid")
+ })
+
+ s.Run("missing username", func() {
+ secret := &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "bmc-secret",
+ Namespace: "ns",
+ },
+ Data: map[string][]byte{
+ "password": []byte("pass"),
+ },
+ }
+
+ coreClient := newFakeCoreV1(secret)
+ result := checkCredentialSecret(ctx, coreClient, "ns", "bmc-secret")
+ s.Contains(result, "missing or empty 'username'")
+ })
+
+ s.Run("missing secret", func() {
+ coreClient := newFakeCoreV1()
+ result := checkCredentialSecret(ctx, coreClient, "ns", "nonexistent")
+ s.Contains(result, "not found")
+ })
+}
+
+func (s *TNFTroubleshootSuite) TestValueOrNA() {
+ s.Equal("N/A", fencing.ValueOrNA(""))
+ s.Equal("test", fencing.ValueOrNA("test"))
+}
+
+func TestTNFTroubleshoot(t *testing.T) {
+ suite.Run(t, new(TNFTroubleshootSuite))
+}