diff --git a/go/core/pkg/sandboxbackend/substrate/agent_lifecycle.go b/go/core/pkg/sandboxbackend/substrate/agent_lifecycle.go index 520d7387d4..d62407db5b 100644 --- a/go/core/pkg/sandboxbackend/substrate/agent_lifecycle.go +++ b/go/core/pkg/sandboxbackend/substrate/agent_lifecycle.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -64,10 +65,11 @@ func (p *Lifecycle) buildSandboxAgentActorTemplate( wpKey types.NamespacedName, podTemplate corev1.PodTemplateSpec, ) (*atev1alpha1.ActorTemplate, error) { - kagentContainer := findKagentContainer(podTemplate.Spec.Containers) - if kagentContainer == nil { + kagentIdx := findKagentContainerIndex(podTemplate.Spec.Containers) + if kagentIdx < 0 { return nil, fmt.Errorf("pod template is missing the kagent container") } + kagentContainer := &podTemplate.Spec.Containers[kagentIdx] image, err := pinImageRef(kagentContainer.Image) if err != nil { return nil, err @@ -77,11 +79,19 @@ func (p *Lifecycle) buildSandboxAgentActorTemplate( if err != nil { return nil, err } + // Extra containers (byo.deployment.extraContainers / declarative deployment.extraContainers) + // run alongside the agent in the same gVisor sandbox — shared loopback, separate rootfs — so + // sidecars keep working on substrate. They are appended AFTER the agent container: + // applyDurableDirSessionStore mounts /data on Containers[0], which must stay the agent. + extraContainers, err := buildSubstrateExtraContainers(podTemplate.Spec.Containers, kagentIdx) + if err != nil { + return nil, err + } spec := atev1alpha1.ActorTemplateSpec{ PauseImage: p.Defaults.PauseImage, SandboxClass: atev1alpha1.SandboxClassGvisor, - Containers: []atev1alpha1.Container{{ + Containers: append([]atev1alpha1.Container{{ Name: defaultKagentContainer, Image: image, Command: command, @@ -98,7 +108,7 @@ func (p *Lifecycle) buildSandboxAgentActorTemplate( Port: substrateKagentListenPort, }, }, - }}, + }}, extraContainers...), WorkerSelector: workerSelectorForPool(wpKey), SnapshotsConfig: atev1alpha1.SnapshotsConfig{ Location: sandboxAgentSnapshotsLocation(sa), @@ -176,16 +186,122 @@ func applyDurableDirSessionStore(spec *atev1alpha1.ActorTemplateSpec) { spec.SnapshotsConfig.OnCommit = atev1alpha1.SnapshotScopeData } -func findKagentContainer(containers []corev1.Container) *corev1.Container { +// findKagentContainerIndex returns the index of the agent's main container: the one named +// "kagent", falling back to the first container when none carries the reserved name (mirrors +// the old findKagentContainer pointer semantics). +func findKagentContainerIndex(containers []corev1.Container) int { for i := range containers { if containers[i].Name == defaultKagentContainer { - return &containers[i] + return i } } if len(containers) > 0 { - return &containers[0] + return 0 + } + return -1 +} + +// maxActorTemplateContainers mirrors the ActorTemplate CRD's spec.containers MaxItems=10. +const maxActorTemplateContainers = 10 + +// buildSubstrateExtraContainers converts the pod template's non-kagent containers (BYO +// byo.deployment.extraContainers, or a declarative agent's deployment.extraContainers) into +// ActorTemplate containers, so sidecars keep running alongside the agent inside the same +// gVisor sandbox: shared loopback network, separate rootfs and mount namespaces. +// +// The ActorTemplate container contract is a strict subset of the Kubernetes one: +// - the command must be fully explicit (verbatim OCI Process.Args, no image-entrypoint +// fallback — the same rule ValidateSubstrateSandboxAgentSpec enforces for the BYO cmd); +// - the image must be digest-pinned (pinImageRef); +// - env supports literals and secretKeyRef only; envFrom, configMapKeyRef and any other +// valueFrom are dropped (sanitizeActorTemplateEnvVar); +// - an HTTP readiness probe maps to Readyz so actor readiness keeps gating on the sidecar; +// TCP/exec probes and lifecycle hooks have no substrate equivalent and are ignored. +// +// Volumes are NOT mapped: ActorTemplate volumes only support durableDir (one per template, +// already consumed by the agent's /data session store), so secret/configMap/emptyDir mounts +// would silently misconfigure a sidecar — an extra container carrying volumeMounts is +// rejected instead. +func buildSubstrateExtraContainers(containers []corev1.Container, kagentIdx int) ([]atev1alpha1.Container, error) { + var out []atev1alpha1.Container + for i := range containers { + if i == kagentIdx { + continue + } + c := &containers[i] + if c.Name == "" { + return nil, fmt.Errorf("extra container %d has no name", i) + } + if len(c.VolumeMounts) > 0 { + return nil, fmt.Errorf("extra container %q mounts volumes, which substrate ActorTemplates do not support (durableDir is reserved for the agent's /data session store)", c.Name) + } + image, err := pinImageRef(c.Image) + if err != nil { + return nil, fmt.Errorf("extra container %q: %w", c.Name, err) + } + if len(c.Command) == 0 { + return nil, fmt.Errorf("extra container %q on substrate must set command (substrate does not fall back to the image entrypoint)", c.Name) + } + ec := atev1alpha1.Container{ + Name: c.Name, + Image: image, + Command: append(append([]string{}, c.Command...), c.Args...), + Env: actorTemplateEnvFromPodEnv(c.Env), + } + if get := probeHTTPGet(c.ReadinessProbe); get != nil { + readyz, err := substrateReadyzFromProbe(c, get) + if err != nil { + return nil, fmt.Errorf("extra container %q: %w", c.Name, err) + } + ec.Readyz = readyz + } + out = append(out, ec) + } + if 1+len(out) > maxActorTemplateContainers { + return nil, fmt.Errorf("substrate ActorTemplates support at most %d containers (the agent plus %d extra)", maxActorTemplateContainers, maxActorTemplateContainers-1) + } + return out, nil +} + +// probeHTTPGet returns the HTTP handler of a Kubernetes readiness probe, or nil when the +// probe is absent or is a TCP/exec probe (which have no substrate equivalent). +func probeHTTPGet(p *corev1.Probe) *corev1.HTTPGetAction { + if p == nil { + return nil + } + return p.HTTPGet +} + +// substrateReadyzFromProbe maps an HTTP readiness probe onto an ActorTemplate ContainerReadyz. +// ActorTemplate probe ports are plain integers: a named port must resolve against the extra +// container's own containerPorts (the translator only registers the agent container's "http" +// port). An empty path takes the substrate default, /readyz. +func substrateReadyzFromProbe(c *corev1.Container, get *corev1.HTTPGetAction) (*atev1alpha1.ContainerReadyz, error) { + var port int32 + switch get.Port.Type { + case intstr.Int: + port = get.Port.IntVal + case intstr.String: + for i := range c.Ports { + if c.Ports[i].Name == get.Port.StrVal { + port = c.Ports[i].ContainerPort + break + } + } + if port == 0 { + return nil, fmt.Errorf("readiness probe names port %q which the container does not declare", get.Port.StrVal) + } + } + if port < 1 || port > 65535 { + return nil, fmt.Errorf("readiness probe port %q is not a valid port", get.Port.String()) + } + path := get.Path + if path == "" { + path = "/readyz" } - return nil + return &atev1alpha1.ContainerReadyz{ + HTTPGet: &atev1alpha1.HTTPGetAction{Path: path, Port: port}, + }, nil } // buildSubstrateKagentContainerCommand returns the ActorTemplate command and the prepended diff --git a/go/core/pkg/sandboxbackend/substrate/agent_lifecycle_test.go b/go/core/pkg/sandboxbackend/substrate/agent_lifecycle_test.go index 3d5bf3c9a8..d1aaaa1253 100644 --- a/go/core/pkg/sandboxbackend/substrate/agent_lifecycle_test.go +++ b/go/core/pkg/sandboxbackend/substrate/agent_lifecycle_test.go @@ -1,6 +1,8 @@ package substrate import ( + "fmt" + "k8s.io/apimachinery/pkg/util/intstr" "testing" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -326,3 +328,138 @@ func TestBuildSandboxAgentActorTemplateDurableDirSessions(t *testing.T) { }) } } + +// byoSandboxAgentWithSidecar builds a BYO SandboxAgent whose translated pod template carries a +// credential-brokering sidecar next to the kagent container, the shape byo.deployment +// .extraContainers produces. +func byoSandboxAgentWithSidecar(extra corev1.Container) (*v1alpha2.SandboxAgent, corev1.PodTemplateSpec) { + const pinnedImage = "registry.example/kagent-dev/kagent/app@sha256:1111111111111111111111111111111111111111111111111111111111111111111" + cmd := "/serve" + sa := &v1alpha2.SandboxAgent{ + ObjectMeta: metav1.ObjectMeta{Name: "byo-sidecar", Namespace: "kagent"}, + Spec: v1alpha2.SandboxAgentSpec{ + AgentSpec: v1alpha2.AgentSpec{Type: v1alpha2.AgentType_BYO, BYO: &v1alpha2.BYOAgentSpec{ + Deployment: &v1alpha2.ByoDeploymentSpec{Image: pinnedImage, Cmd: &cmd}, + }}, + }, + } + podTemplate := corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: defaultKagentContainer, Image: pinnedImage, Command: []string{"/serve"}, Env: []corev1.EnvVar{{Name: "ADDR", Value: "0.0.0.0:80"}}}, + extra, + }}} + return sa, podTemplate +} + +func TestBuildSandboxAgentActorTemplateExtraContainers(t *testing.T) { + t.Parallel() + const sidecarImage = "registry.example/videoamp/central@sha256:2222222222222222222222222222222222222222222222222222222222222222" + wpKey := types.NamespacedName{Namespace: "kagent", Name: "kagent-default"} + + sidecar := corev1.Container{ + Name: "agent-vault", + Image: sidecarImage, + Command: []string{"/sidecar"}, + Env: []corev1.EnvVar{ + {Name: "AGENT_VAULT_LISTEN_ADDR", Value: "127.0.0.1:14322"}, + {Name: "AGENT_VAULT_SECRET_ANTHROPIC_AUTH_TOKEN", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "vault-secrets"}, Key: "anthropic-token"}, + }}, + {Name: "DROPPED_CONFIGMAP", ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "cm"}, Key: "k"}, + }}, + }, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt32(8080)}}}, + } + + t.Run("renders alongside the agent", func(t *testing.T) { + t.Parallel() + p := newTestLifecycle(t) + sa, podTemplate := byoSandboxAgentWithSidecar(sidecar) + tmpl, err := p.buildSandboxAgentActorTemplate(sa, wpKey, podTemplate) + require.NoError(t, err) + require.Len(t, tmpl.Spec.Containers, 2) + + agent := tmpl.Spec.Containers[0] + require.Equal(t, defaultKagentContainer, agent.Name, "the agent stays container 0: /data is mounted there") + require.Equal(t, durableDataMount, agent.VolumeMounts[0].MountPath) + require.Equal(t, "/.well-known/agent-card.json", agent.Readyz.HTTPGet.Path) + + sc := tmpl.Spec.Containers[1] + require.Equal(t, "agent-vault", sc.Name) + require.Equal(t, sidecarImage, sc.Image, "extra images must stay digest-pinned") + require.Equal(t, []string{"/sidecar"}, sc.Command) + require.NotNil(t, sc.Readyz, "an HTTP readiness probe maps to readyz so actor readiness gates on the sidecar") + require.Equal(t, "/readyz", sc.Readyz.HTTPGet.Path) + require.Equal(t, int32(8080), sc.Readyz.HTTPGet.Port) + names := actorEnvNames(sc.Env) + require.True(t, names["AGENT_VAULT_LISTEN_ADDR"]) + require.True(t, names["AGENT_VAULT_SECRET_ANTHROPIC_AUTH_TOKEN"], "secretKeyRef env survives (substrate resolves it server-side)") + require.False(t, names["DROPPED_CONFIGMAP"], "configMapKeyRef is not expressible in an ActorTemplate and is dropped") + require.Empty(t, sc.VolumeMounts, "extra containers get no volume mounts: durableDir belongs to the agent's /data") + + // The sidecar changes the shape hash, so it fans out blue-green like any spec change. + p2 := newTestLifecycle(t) + sa2, bare := byoSandboxAgentWithSidecar(corev1.Container{}) + bare.Spec.Containers = bare.Spec.Containers[:1] + tmpl2, err := p2.buildSandboxAgentActorTemplate(sa2, wpKey, bare) + require.NoError(t, err) + require.NotEqual(t, tmpl.Name, tmpl2.Name, "extra containers must change the ActorTemplate name (shape hash)") + }) + + t.Run("named readiness port resolves against the container ports", func(t *testing.T) { + t.Parallel() + p := newTestLifecycle(t) + named := sidecar + named.Ports = []corev1.ContainerPort{{Name: "health", ContainerPort: 8081}} + named.ReadinessProbe.HTTPGet.Port = intstr.FromString("health") + sa, podTemplate := byoSandboxAgentWithSidecar(named) + tmpl, err := p.buildSandboxAgentActorTemplate(sa, wpKey, podTemplate) + require.NoError(t, err) + require.Equal(t, int32(8081), tmpl.Spec.Containers[1].Readyz.HTTPGet.Port) + }) + + for _, tc := range []struct { + name string + extra corev1.Container + }{ + { + name: "unpinned image is rejected", + extra: corev1.Container{Name: "agent-vault", Image: "registry.example/videoamp/central:latest", Command: []string{"/sidecar"}}, + }, + { + name: "missing command is rejected", + extra: corev1.Container{Name: "agent-vault", Image: sidecarImage}, + }, + { + name: "volume mounts are rejected", + extra: corev1.Container{Name: "agent-vault", Image: sidecarImage, Command: []string{"/sidecar"}, + VolumeMounts: []corev1.VolumeMount{{Name: "secrets", MountPath: "/etc/agent-vault/secrets"}}}, + }, + { + name: "unresolvable named readiness port is rejected", + extra: corev1.Container{Name: "agent-vault", Image: sidecarImage, Command: []string{"/sidecar"}, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromString("nope")}}}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + p := newTestLifecycle(t) + sa, podTemplate := byoSandboxAgentWithSidecar(tc.extra) + _, err := p.buildSandboxAgentActorTemplate(sa, wpKey, podTemplate) + require.Error(t, err) + }) + } + + t.Run("more than 10 containers is rejected", func(t *testing.T) { + t.Parallel() + p := newTestLifecycle(t) + sa, podTemplate := byoSandboxAgentWithSidecar(sidecar) + for i := 0; i < maxActorTemplateContainers; i++ { + podTemplate.Spec.Containers = append(podTemplate.Spec.Containers, corev1.Container{ + Name: fmt.Sprintf("extra-%d", i), Image: sidecarImage, Command: []string{"/bin/true"}, + }) + } + _, err := p.buildSandboxAgentActorTemplate(sa, wpKey, podTemplate) + require.Error(t, err) + }) +}