diff --git a/build/openshift/e2e.mk b/build/openshift/e2e.mk index 827a574d2..a339c38b1 100644 --- a/build/openshift/e2e.mk +++ b/build/openshift/e2e.mk @@ -10,4 +10,4 @@ e2e-ci-test: ## Run e2e tests against a CI-provisioned cluster HELM_PATH=$(HELM) \ MCP_SERVER_IMAGE=$(MCP_SERVER_IMAGE) \ CHART_PATH=$(shell pwd)/charts/kubernetes-mcp-server \ - go test -tags e2e -v -count=1 -timeout 20m ./test/e2e/ $(E2E_ARGS) + go test -tags e2e -v -count=1 -timeout 30m ./test/e2e/ $(E2E_ARGS) diff --git a/test/e2e/keycloak_test.go b/test/e2e/keycloak_test.go index 80a9c123a..32b31fdc9 100644 --- a/test/e2e/keycloak_test.go +++ b/test/e2e/keycloak_test.go @@ -29,8 +29,8 @@ func TestKeycloakOIDC(t *testing.T) { s := keycloakTS.get(ctx) d := discoverOIDC(t, s.localURL, keycloakRealm) - require.Contains(t, d.Issuer, "keycloak.keycloak.svc", - "issuer must use internal service DNS to match API server --oidc-issuer-url") + require.Contains(t, d.Issuer, "keycloak", + "issuer must reference the Keycloak server") require.Contains(t, d.Issuer, keycloakRealm) require.NotEmpty(t, d.TokenEndpoint) require.NotEmpty(t, d.AuthorizationEndpoint) diff --git a/test/e2e/oauth_flows_test.go b/test/e2e/oauth_flows_test.go index b56a56509..d542ff60b 100644 --- a/test/e2e/oauth_flows_test.go +++ b/test/e2e/oauth_flows_test.go @@ -85,7 +85,7 @@ func TestOAuthOIDCFlows(t *testing.T) { // verifies the signature against Keycloak's JWKS — a different path // than A4 (unparseable) and A5 (offline audience mismatch). badlySigned := mintJWT(t, jwt.Claims{ - Issuer: "https://keycloak.keycloak.svc:8443/realms/openshift", + Issuer: keycloakIssuerURL(), Subject: "e2e-throwaway", Audience: jwt.Audience{"mcp-server"}, Expiry: jwt.NewNumericDate(time.Now().Add(time.Hour)), @@ -101,7 +101,7 @@ func TestOAuthOIDCFlows(t *testing.T) { // A4 (unparseable). The expiry is set well beyond go-jose's default // 1-minute leeway. expired := mintJWT(t, jwt.Claims{ - Issuer: "https://keycloak.keycloak.svc:8443/realms/openshift", + Issuer: keycloakIssuerURL(), Subject: "e2e-throwaway", Audience: jwt.Audience{"mcp-server"}, Expiry: jwt.NewNumericDate(time.Now().Add(-time.Hour)), @@ -118,7 +118,7 @@ func TestOAuthOIDCFlows(t *testing.T) { // E1: openid-configuration is proxied from Keycloak. oidcCfg, _ := requireWellKnown(t, base, "/.well-known/openid-configuration") issuer, _ := oidcCfg["issuer"].(string) - require.Contains(t, issuer, "keycloak.keycloak.svc", "issuer = %q", issuer) + require.Contains(t, issuer, "keycloak", "issuer = %q", issuer) require.NotEmpty(t, oidcCfg["token_endpoint"], "openid-configuration token_endpoint") // E2: oauth-protected-resource (RFC 9728) metadata, plus E8 CORS header. @@ -264,7 +264,7 @@ func TestOAuthForwardedIdentity(t *testing.T) { // forwarded user token it runs as the cluster-admin user. s := oauthIdentityTS.get(ctx) dep := deployServer(ctx, t, cfg, "oauth-open-fwd", - withConfig("require_oauth = false"), + withConfig("require_oauth = false\ndenied_resources = []"), withValues(viewClusterRoleBindingValues()), ) @@ -291,7 +291,7 @@ func TestOAuthForwardedIdentity(t *testing.T) { // authority. The forwarded user token acts as the cluster-admin user. s := oauthIdentityTS.get(ctx) dep := deployServer(ctx, t, cfg, "oauth-passthrough-fwd", - withConfig("require_oauth = true\nskip_jwt_verification = true"), + withConfig("require_oauth = true\nskip_jwt_verification = true\ndenied_resources = []"), withValues(viewClusterRoleBindingValues()), ) @@ -473,7 +473,7 @@ func TestOAuthSTSAssertion(t *testing.T) { oauth_audience = "mcp-server-jwt" oauth_scopes = ["openid", "mcp-server-jwt"] validate_token = false - authorization_url = "https://keycloak.keycloak.svc:8443/realms/openshift" + authorization_url = "%s" sts_client_id = "mcp-server-jwt" sts_audience = "openshift" sts_scopes = ["mcp:openshift"] @@ -482,7 +482,8 @@ func TestOAuthSTSAssertion(t *testing.T) { sts_client_cert_file = %q sts_client_key_file = %q certificate_authority = "%s/ca.crt" - `, stsAssertionCertPath, stsAssertionKeyPath, caMountPath) + denied_resources = [] + `, keycloakIssuerURL(), stsAssertionCertPath, stsAssertionKeyPath, caMountPath) dep := deployServer(ctx, t, cfg, "oauth-sts-assertion", withConfig(assertionConfig), diff --git a/test/e2e/oauth_test.go b/test/e2e/oauth_test.go index 99daf94ab..8d6e47143 100644 --- a/test/e2e/oauth_test.go +++ b/test/e2e/oauth_test.go @@ -31,12 +31,35 @@ import ( const ( keycloakRealm = "openshift" + // keycloakDefaultBaseURL is the internal cluster URL for Keycloak when no + // override is set. On Minikube this is the only URL; on OCP the env var + // KEYCLOAK_ISSUER_URL overrides it to the Route URL so the kube-apiserver + // (which runs on host networking) can reach Keycloak. + keycloakDefaultBaseURL = "https://keycloak.keycloak.svc:8443" + // caSecretName is the secret holding the CA cert the MCP server trusts when // talking to Keycloak; caMountPath is where the chart mounts it in the pod. caSecretName = "keycloak-ca" caMountPath = "/etc/keycloak-ca" ) +// keycloakIssuerURL returns the Keycloak issuer URL (base + realm) used in +// OIDC configuration and token issuer assertions. On OCP CI this is overridden +// via KEYCLOAK_ISSUER_URL to the Route URL. +func keycloakIssuerURL() string { + base := envOrDefault("KEYCLOAK_ISSUER_URL", keycloakDefaultBaseURL+"/realms/"+keycloakRealm) + return base +} + +// keycloakBaseURL returns just the Keycloak base URL (without /realms/...). +func keycloakBaseURL() string { + issuer := keycloakIssuerURL() + if idx := strings.Index(issuer, "/realms/"); idx != -1 { + return issuer[:idx] + } + return envOrDefault("KEYCLOAK_BASE_URL", keycloakDefaultBaseURL) +} + // oidcDiscovery is the subset of the OIDC discovery document the tests use. type oidcDiscovery struct { Issuer string `json:"issuer"` @@ -245,17 +268,33 @@ func mcpViewerToken(t *testing.T, keycloakURL string, scopes ...string) string { }) } -// copyKeycloakCASecret copies the cert-manager self-signed CA into the test -// namespace as the caSecretName secret so the MCP server pod can mount it and -// trust Keycloak's TLS. Intended as a deployServer preInstall hook. +// copyKeycloakCASecret copies the CA certificate that the MCP server needs to +// trust Keycloak's TLS into the test namespace. On Minikube this is the +// cert-manager self-signed CA (from cert-manager/selfsigned-ca-secret). On OCP +// CI the env var KEYCLOAK_CA_SECRET can override the source to a pre-created +// secret (e.g. the OpenShift ingress CA). The format is "namespace/name". func copyKeycloakCASecret(ctx context.Context, t *testing.T, clientset kubernetes.Interface, namespace string) { t.Helper() - caSecret, err := clientset.CoreV1().Secrets("cert-manager").Get(ctx, "selfsigned-ca-secret", metav1.GetOptions{}) - require.NoError(t, err, "get cert-manager CA secret") - _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ + var caCert []byte + if src := os.Getenv("KEYCLOAK_CA_SECRET"); src != "" { + parts := strings.SplitN(src, "/", 2) + require.Len(t, parts, 2, "KEYCLOAK_CA_SECRET must be namespace/name, got %q", src) + secret, err := clientset.CoreV1().Secrets(parts[0]).Get(ctx, parts[1], metav1.GetOptions{}) + require.NoError(t, err, "get CA secret %s", src) + for _, v := range secret.Data { + caCert = v + break + } + } else { + caSecret, err := clientset.CoreV1().Secrets("cert-manager").Get(ctx, "selfsigned-ca-secret", metav1.GetOptions{}) + require.NoError(t, err, "get cert-manager CA secret") + caCert = caSecret.Data["ca.crt"] + } + + _, err := clientset.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: caSecretName}, - Data: map[string][]byte{"ca.crt": caSecret.Data["ca.crt"]}, + Data: map[string][]byte{"ca.crt": caCert}, }, metav1.CreateOptions{}) require.NoError(t, err, "create CA secret in test namespace") } @@ -443,13 +482,14 @@ func oidcServerConfig(oauthScopes []string) string { oauth_audience = "mcp-server" oauth_scopes = %s validate_token = false - authorization_url = "https://keycloak.keycloak.svc:8443/realms/openshift" + authorization_url = "%s" sts_client_id = "mcp-server" sts_client_secret = "mcp-server-dev-secret" sts_audience = "openshift" sts_scopes = ["mcp:openshift"] certificate_authority = "%s/ca.crt" - `, scopes, caMountPath) + denied_resources = [] + `, scopes, keycloakIssuerURL(), caMountPath) } // tomlStringArray renders a Go string slice as a TOML array literal. diff --git a/test/openshift/e2e-commands.sh b/test/openshift/e2e-commands.sh index bb22f72a4..cc24181c2 100755 --- a/test/openshift/e2e-commands.sh +++ b/test/openshift/e2e-commands.sh @@ -17,5 +17,13 @@ unset KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT export MCP_SERVER_IMAGE="${IMAGE_OPENSHIFT_MCP_SERVER}" +bash test/openshift/keycloak-setup.sh + +# Source Keycloak env vars if setup produced them (skipped on older clusters) +if [[ -f /tmp/keycloak-env.sh ]]; then + # shellcheck disable=SC1091 + source /tmp/keycloak-env.sh +fi + make e2e-ci-setup make e2e-ci-test diff --git a/test/openshift/keycloak-rbac.yaml b/test/openshift/keycloak-rbac.yaml new file mode 100644 index 000000000..53b9fd6e7 --- /dev/null +++ b/test/openshift/keycloak-rbac.yaml @@ -0,0 +1,29 @@ +# RBAC bindings for Keycloak OIDC users on OpenShift. +# +# On OCP the Authentication CR uses prefixPolicy: NoPrefix, so usernames +# are bare (e.g. "mcp") rather than "#mcp" as on Minikube. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: oidc-mcp-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: mcp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: oidc-mcp-viewers-view +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: mcp-viewers diff --git a/test/openshift/keycloak-setup.sh b/test/openshift/keycloak-setup.sh new file mode 100755 index 000000000..f02a518b2 --- /dev/null +++ b/test/openshift/keycloak-setup.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# +# Installs cert-manager, deploys Keycloak with the project's realm, and +# configures the OCP API server for OIDC authentication. +# +# Designed to run inside a CI test pod with oc in PATH and KUBECONFIG +# pointing at an IPI-provisioned OCP cluster. +# +# The kube-apiserver on OCP uses hostNetwork and cannot resolve cluster DNS +# names like keycloak.keycloak.svc. This script exposes Keycloak via an +# OpenShift Route and uses the ingress CA for TLS trust. +# +# Outputs: +# /tmp/keycloak-env.sh — env vars for the test runner (source this file) +set -euo pipefail + +CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.16.2}" +KEYCLOAK_ENV_FILE="/tmp/keycloak-env.sh" + +echo "=== Keycloak CI Setup ===" + +# --- Version gate -------------------------------------------------------- +CLUSTER_VERSION=$(oc get clusterversion version -o jsonpath='{.status.desired.version}') +OCP_MAJOR=$(echo "$CLUSTER_VERSION" | cut -d. -f1) +OCP_MINOR=$(echo "$CLUSTER_VERSION" | cut -d. -f2) +if [[ "$OCP_MAJOR" -eq 4 && "$OCP_MINOR" -lt 20 ]]; then + echo "ExternalOIDC requires OCP 4.20+, got $CLUSTER_VERSION — skipping Keycloak setup" + exit 0 +fi +echo "Cluster version: $CLUSTER_VERSION (ExternalOIDC supported)" + +# --- cert-manager -------------------------------------------------------- +echo "Installing cert-manager ${CERT_MANAGER_VERSION}..." +oc apply -f "https://github.com/cert-manager/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml" +echo "Waiting for cert-manager deployments..." +oc wait --namespace cert-manager --for=condition=available deployment/cert-manager --timeout=120s +oc wait --namespace cert-manager --for=condition=available deployment/cert-manager-cainjector --timeout=120s +oc wait --namespace cert-manager --for=condition=available deployment/cert-manager-webhook --timeout=120s +echo "cert-manager ready" + +# --- Self-signed CA issuer chain ----------------------------------------- +echo "Creating self-signed CA issuer chain..." +oc apply -f dev/config/cert-manager/selfsigned-issuer.yaml +oc wait --for=condition=ready certificate/selfsigned-ca -n cert-manager --timeout=60s +echo "CA issuer chain ready" + +# --- STS assertion keypair ----------------------------------------------- +echo "Generating STS assertion keypair..." +dev/config/keycloak/gen-sts-assertion-keypair.sh + +# --- Keycloak deployment ------------------------------------------------- +echo "Rendering realm import with STS assertion cert..." +STS_CERT_DER=$(grep -v -- '-----' test/e2e/testdata/generated/sts-assertion.crt | tr -d '\n') +sed "s|@@STS_ASSERTION_CERT_DER@@|${STS_CERT_DER}|" \ + dev/config/keycloak/realm-import.yaml > /tmp/realm-import.rendered.yaml + +echo "Deploying Keycloak..." +oc create namespace keycloak --dry-run=client -o yaml | oc apply -f - +oc apply -f /tmp/realm-import.rendered.yaml +oc apply -f dev/config/keycloak/deployment.yaml + +echo "Waiting for Keycloak TLS certificate..." +oc wait --for=condition=ready certificate/keycloak-tls -n keycloak --timeout=120s + +# --- Expose Keycloak via Route ------------------------------------------- +# Use a re-encrypt Route: the router terminates client TLS (signed by the +# ingress CA), then re-encrypts to Keycloak's cert-manager TLS on port 8443. +echo "Extracting cert-manager CA for Route backend verification..." +oc get secret selfsigned-ca-secret -n cert-manager \ + -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/cert-manager-ca.crt + +echo "Creating re-encrypt Route for Keycloak..." +oc create route reencrypt keycloak --service=keycloak --port=8443 \ + --dest-ca-cert=/tmp/cert-manager-ca.crt -n keycloak + +KEYCLOAK_ROUTE_HOST=$(oc get route keycloak -n keycloak -o jsonpath='{.spec.host}') +KEYCLOAK_URL="https://${KEYCLOAK_ROUTE_HOST}" +echo "Keycloak Route: ${KEYCLOAK_URL}" + +# Update KC_HOSTNAME so tokens carry the Route URL as issuer. This triggers +# a deployment rollout that also picks up the realm import. +echo "Setting KC_HOSTNAME to Route URL..." +oc set env deployment/keycloak KC_HOSTNAME="${KEYCLOAK_URL}" -n keycloak + +echo "Waiting for Keycloak to be ready..." +oc rollout status deployment/keycloak -n keycloak --timeout=300s +echo "Keycloak ready" + +# --- Verify Keycloak via Route ------------------------------------------- +echo "Extracting OpenShift ingress CA..." +oc get configmap default-ingress-cert -n openshift-config-managed \ + -o jsonpath='{.data.ca-bundle\.crt}' > /tmp/ingress-ca.crt + +DISCOVERY_URL="${KEYCLOAK_URL}/realms/openshift/.well-known/openid-configuration" +echo "Verifying Keycloak OIDC discovery via Route..." +for i in $(seq 1 12); do + if curl -sSf --cacert /tmp/ingress-ca.crt "${DISCOVERY_URL}" > /dev/null 2>&1; then + echo "OIDC discovery: OK" + break + fi + if [[ $i -eq 12 ]]; then + echo "ERROR: Keycloak OIDC discovery not reachable at ${DISCOVERY_URL}" + curl -v --cacert /tmp/ingress-ca.crt "${DISCOVERY_URL}" 2>&1 || true + oc get route keycloak -n keycloak -o yaml + oc get pods -n keycloak + exit 1 + fi + echo "Waiting for Route to become ready... (${i}/12)" + sleep 5 +done + +# --- OCP API server OIDC configuration ----------------------------------- +# The kube-apiserver connects to Keycloak via the Route, so it needs the +# OpenShift ingress CA to trust the Route's TLS certificate. +echo "Creating CA ConfigMap in openshift-config..." +oc create configmap keycloak-oidc-ca \ + --from-file=ca-bundle.crt=/tmp/ingress-ca.crt \ + -n openshift-config \ + --dry-run=client -o yaml | oc apply -f - + +echo "Patching Authentication CR for OIDC..." +oc patch authentication.config/cluster --type=merge -p "{ + \"spec\": { + \"type\": \"OIDC\", + \"webhookTokenAuthenticator\": null, + \"oidcProviders\": [{ + \"name\": \"keycloak\", + \"issuer\": { + \"issuerURL\": \"${KEYCLOAK_URL}/realms/openshift\", + \"audiences\": [\"openshift\"], + \"issuerCertificateAuthority\": {\"name\": \"keycloak-oidc-ca\"} + }, + \"claimMappings\": { + \"username\": {\"claim\": \"preferred_username\", \"prefixPolicy\": \"NoPrefix\"}, + \"groups\": {\"claim\": \"groups\", \"prefix\": \"\"} + }, + \"oidcClients\": [] + }] + } +}" + +echo "Waiting for kube-apiserver to start rolling out..." +if ! oc wait co/kube-apiserver --for=condition=Progressing --timeout=120s; then + echo "API server did not start progressing — checking status" + oc get co/kube-apiserver + exit 1 +fi + +echo "Waiting for kube-apiserver rollout to complete (this may take several minutes)..." +if ! oc wait co/kube-apiserver --for=condition=Progressing=false --timeout=900s; then + echo "API server rollout timed out" + oc get co/kube-apiserver + oc get po -n openshift-kube-apiserver -L revision -l apiserver + exit 1 +fi + +echo "Verifying cluster operators are healthy..." +if oc get co kube-apiserver authentication --no-headers | grep -v "True *False *False"; then + echo "WARNING: some cluster operators not in expected state" + oc get co kube-apiserver authentication +fi +echo "API server OIDC configuration complete" + +# --- RBAC for OIDC users ------------------------------------------------- +echo "Applying OIDC RBAC bindings..." +oc apply -f test/openshift/keycloak-rbac.yaml + +# --- Test environment variables ------------------------------------------ +# The MCP server pods connect to Keycloak via the Route URL and need the +# ingress CA to verify its TLS certificate. +echo "Creating ingress CA secret for tests..." +oc create secret generic keycloak-ingress-ca \ + --from-file=ca.crt=/tmp/ingress-ca.crt \ + -n keycloak \ + --dry-run=client -o yaml | oc apply -f - + +cat > "${KEYCLOAK_ENV_FILE}" <