diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 050b0499..3be77a9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -364,6 +364,9 @@ jobs: - name: Test release image digest resolution run: make test-release-image-digests + - name: Test release install rendering + run: make test-release-install + tokenize-cgo: name: Tokenizer cgo Build Tag (${{ matrix.build-tag }}) runs-on: ubuntu-latest @@ -488,7 +491,7 @@ jobs: - name: Verify image SBOM outputs run: | set -euo pipefail - for component in controller server subscriber; do + for component in controller server subscriber cleanup; do sbom="$RUNNER_TEMP/sbom-image-smoke/inference-cache-${component}-${IMAGE_TAG}.spdx.json" test -s "$sbom" jq -e '.spdxVersion and ((.packages | type) == "array") and ((.packages | length) > 0)' "$sbom" >/dev/null diff --git a/.github/workflows/release-sbom.yml b/.github/workflows/release-sbom.yml index 88e282d6..1a734389 100644 --- a/.github/workflows/release-sbom.yml +++ b/.github/workflows/release-sbom.yml @@ -28,6 +28,7 @@ env: CONTROLLER_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-controller SERVER_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-server SUBSCRIBER_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-subscriber + CLEANUP_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-shm-cleanup jobs: secure-release: @@ -132,6 +133,7 @@ jobs: build_image controller "$CONTROLLER_IMAGE_REPO" build_image server "$SERVER_IMAGE_REPO" build_image subscriber "$SUBSCRIBER_IMAGE_REPO" + build_image cleanup "$CLEANUP_IMAGE_REPO" - name: Resolve published release image digests id: published-image-digests @@ -144,14 +146,31 @@ jobs: BUILT_CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }} BUILT_SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }} BUILT_SUBSCRIBER_DIGEST: ${{ steps.built-image-digests.outputs.subscriber }} + BUILT_CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }} PUBLISHED_CONTROLLER_DIGEST: ${{ steps.published-image-digests.outputs.controller }} PUBLISHED_SERVER_DIGEST: ${{ steps.published-image-digests.outputs.server }} PUBLISHED_SUBSCRIBER_DIGEST: ${{ steps.published-image-digests.outputs.subscriber }} + PUBLISHED_CLEANUP_DIGEST: ${{ steps.published-image-digests.outputs.cleanup }} run: | set -euo pipefail test "$BUILT_CONTROLLER_DIGEST" = "$PUBLISHED_CONTROLLER_DIGEST" test "$BUILT_SERVER_DIGEST" = "$PUBLISHED_SERVER_DIGEST" test "$BUILT_SUBSCRIBER_DIGEST" = "$PUBLISHED_SUBSCRIBER_DIGEST" + test "$BUILT_CLEANUP_DIGEST" = "$PUBLISHED_CLEANUP_DIGEST" + + - name: Render digest-pinned install manifest + env: + CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }} + SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }} + CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }} + run: | + make -C release-source kustomize + CONTROLLER_IMAGE="${CONTROLLER_IMAGE_REPO}@${CONTROLLER_DIGEST}" \ + SERVER_IMAGE="${SERVER_IMAGE_REPO}@${SERVER_DIGEST}" \ + CLEANUP_IMAGE="${CLEANUP_IMAGE_REPO}@${CLEANUP_DIGEST}" \ + OUTPUT_FILE="$GITHUB_WORKSPACE/dist/install/inference-cache-${RELEASE_TAG}.yaml" \ + KUSTOMIZE_CMD="$GITHUB_WORKSPACE/release-source/bin/kustomize" \ + release-source/hack/render-release-install.sh - name: Generate published image SBOMs run: make -f "$GITHUB_WORKSPACE/Makefile" -C "$GITHUB_WORKSPACE/release-source" sbom-registry-images TAG="${RELEASE_TAG}" SBOM_DIR="$GITHUB_WORKSPACE/dist/sbom" @@ -166,11 +185,13 @@ jobs: CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }} SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }} SUBSCRIBER_DIGEST: ${{ steps.built-image-digests.outputs.subscriber }} + CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }} run: | set -euo pipefail cosign sign --yes "${CONTROLLER_IMAGE_REPO}@${CONTROLLER_DIGEST}" cosign sign --yes "${SERVER_IMAGE_REPO}@${SERVER_DIGEST}" cosign sign --yes "${SUBSCRIBER_IMAGE_REPO}@${SUBSCRIBER_DIGEST}" + cosign sign --yes "${CLEANUP_IMAGE_REPO}@${CLEANUP_DIGEST}" - name: Generate controller SLSA provenance id: attest-controller @@ -196,17 +217,27 @@ jobs: subject-digest: ${{ steps.built-image-digests.outputs.subscriber }} push-to-registry: true + - name: Generate NodeLocal cleanup SLSA provenance + id: attest-cleanup + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.CLEANUP_IMAGE_REPO }} + subject-digest: ${{ steps.built-image-digests.outputs.cleanup }} + push-to-registry: true + - name: Collect provenance bundles env: CONTROLLER_BUNDLE: ${{ steps.attest-controller.outputs.bundle-path }} SERVER_BUNDLE: ${{ steps.attest-server.outputs.bundle-path }} SUBSCRIBER_BUNDLE: ${{ steps.attest-subscriber.outputs.bundle-path }} + CLEANUP_BUNDLE: ${{ steps.attest-cleanup.outputs.bundle-path }} run: | set -euo pipefail mkdir -p dist/provenance cp -- "$CONTROLLER_BUNDLE" "dist/provenance/inference-cache-controller-${RELEASE_TAG}.provenance.sigstore.json" cp -- "$SERVER_BUNDLE" "dist/provenance/inference-cache-server-${RELEASE_TAG}.provenance.sigstore.json" cp -- "$SUBSCRIBER_BUNDLE" "dist/provenance/inference-cache-subscriber-${RELEASE_TAG}.provenance.sigstore.json" + cp -- "$CLEANUP_BUNDLE" "dist/provenance/inference-cache-cleanup-${RELEASE_TAG}.provenance.sigstore.json" - name: Verify image signatures and SLSA provenance env: @@ -215,12 +246,14 @@ jobs: CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }} SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }} SUBSCRIBER_DIGEST: ${{ steps.built-image-digests.outputs.subscriber }} + CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }} run: | set -euo pipefail for ref in \ "${CONTROLLER_IMAGE_REPO}@${CONTROLLER_DIGEST}" \ "${SERVER_IMAGE_REPO}@${SERVER_DIGEST}" \ - "${SUBSCRIBER_IMAGE_REPO}@${SUBSCRIBER_DIGEST}"; do + "${SUBSCRIBER_IMAGE_REPO}@${SUBSCRIBER_DIGEST}" \ + "${CLEANUP_IMAGE_REPO}@${CLEANUP_DIGEST}"; do cosign verify \ --certificate-identity "$COSIGN_IDENTITY" \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ @@ -240,6 +273,7 @@ jobs: with: name: release-supply-chain path: | + dist/install/*.yaml dist/sbom/*.spdx.json dist/provenance/*.sigstore.json if-no-files-found: error @@ -262,4 +296,4 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ env.RELEASE_TAG }} - run: gh release upload "${RELEASE_TAG}" dist/sbom/*.spdx.json dist/provenance/*.sigstore.json --clobber + run: gh release upload "${RELEASE_TAG}" dist/install/*.yaml dist/sbom/*.spdx.json dist/provenance/*.sigstore.json --clobber diff --git a/Makefile b/Makefile index 97b4ba73..cfb34224 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,14 @@ TAG ?= $(shell git describe --tags --dirty --always 2>/dev/null || echo dev) CONTROLLER_IMAGE_REPO ?= $(REGISTRY)/inference-cache-controller SERVER_IMAGE_REPO ?= $(REGISTRY)/inference-cache-server SUBSCRIBER_IMAGE_REPO ?= $(REGISTRY)/inference-cache-subscriber +CLEANUP_IMAGE_REPO ?= $(REGISTRY)/inference-cache-shm-cleanup IMG ?= $(CONTROLLER_IMAGE_REPO):$(TAG) SERVER_IMG ?= $(SERVER_IMAGE_REPO):$(TAG) SUBSCRIBER_IMG ?= $(SUBSCRIBER_IMAGE_REPO):$(TAG) +CLEANUP_IMG ?= $(CLEANUP_IMAGE_REPO):$(TAG) +INSTALL_MANIFEST ?= dist/install/inference-cache.yaml +INSTALL_KUSTOMIZATION ?= default +KUBECTL ?= kubectl DOCKER_BUILD_CMD ?= docker KIND ?= $(shell command -v kind 2>/dev/null || echo $(LOCAL_KIND)) KIND_CLUSTER ?= inference-cache @@ -265,10 +270,11 @@ proto-lint: buf ## Lint the gRPC contract with buf (lint-only; codegen stays on $(BUF) lint .PHONY: build -build: ## Build controller, server, kvevent-subscriber, and inferencecache binaries. +build: ## Build controller, server, kvevent-subscriber, NodeLocal cleanup, and inferencecache binaries. $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/controller ./cmd/controller $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/server ./cmd/server $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/kvevent-subscriber ./cmd/kvevent-subscriber + $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/node-local-shm-cleanup ./cmd/node-local-shm-cleanup $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/inferencecache ./cmd/inferencecache .PHONY: test @@ -370,7 +376,7 @@ verify-ranker-calibration: ## Verify the checked-in ranker calibration output ma $(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT) -check .PHONY: image-build -image-build: controller-image server-image subscriber-image ## Build controller, server, and kvevent-subscriber images. +image-build: controller-image server-image subscriber-image cleanup-image ## Build all release images. .PHONY: controller-image controller-image: ## Build the controller container image. @@ -384,6 +390,10 @@ server-image: ## Build the server container image. subscriber-image: ## Build the kvevent-subscriber container image (sidecar auto-attached to engine pods). $(DOCKER_BUILD_CMD) build -f dockerfiles/Dockerfile --target subscriber -t "$(SUBSCRIBER_IMG)" . +.PHONY: cleanup-image +cleanup-image: ## Build the NodeLocal SHM cleanup helper image. + $(DOCKER_BUILD_CMD) build -f dockerfiles/Dockerfile --target cleanup -t "$(CLEANUP_IMG)" . + .PHONY: verify-minimal-base verify-minimal-base: ## Verify every shipped runtime stage uses the approved Distroless non-root base. @MINIMAL_IMAGE_DOCKERFILE="$(MINIMAL_IMAGE_DOCKERFILE)" \ @@ -399,7 +409,7 @@ verify-minimal-images: verify-minimal-base ## Inspect built images for non-root @DOCKER="$(DOCKER_BUILD_CMD)" \ MINIMAL_IMAGE_DOCKERFILE="$(MINIMAL_IMAGE_DOCKERFILE)" \ MINIMAL_RUNTIME_BASE="$(MINIMAL_RUNTIME_BASE)" \ - IMG="$(IMG)" SERVER_IMG="$(SERVER_IMG)" SUBSCRIBER_IMG="$(SUBSCRIBER_IMG)" \ + IMG="$(IMG)" SERVER_IMG="$(SERVER_IMG)" SUBSCRIBER_IMG="$(SUBSCRIBER_IMG)" CLEANUP_IMG="$(CLEANUP_IMG)" \ bash hack/verify-minimal-images.sh .PHONY: syft-check @@ -425,7 +435,7 @@ sbom-release: syft-check ## Generate a source/release SBOM for the checked-out t -o "spdx-json=$(SBOM_DIR)/inference-cache-$(SBOM_TAG).spdx.json" .PHONY: sbom-images -sbom-images: syft-check ## Generate SBOMs for controller, server, and kvevent-subscriber images. +sbom-images: syft-check ## Generate SBOMs for all release images. @if [ "$(SBOM_IMAGE_SOURCE)" = "docker" ] && [ "$(SBOM_IMAGE_BUILD)" = "1" ]; then \ $(MAKE) -f "$(MAKEFILE_SELF)" image-build; \ fi @@ -433,6 +443,7 @@ sbom-images: syft-check ## Generate SBOMs for controller, server, and kvevent-su "$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-controller-$(SBOM_TAG).spdx.json" "$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(SERVER_IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-server-$(SBOM_TAG).spdx.json" "$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(SUBSCRIBER_IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-subscriber-$(SBOM_TAG).spdx.json" + "$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(CLEANUP_IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-cleanup-$(SBOM_TAG).spdx.json" .PHONY: sbom-registry-images sbom-registry-images: syft-check ## Generate SBOMs for published release images by immutable registry digest. @@ -441,7 +452,8 @@ sbom-registry-images: syft-check ## Generate SBOMs for published release images for image in \ "controller|$(CONTROLLER_IMAGE_REPO)" \ "server|$(SERVER_IMAGE_REPO)" \ - "subscriber|$(SUBSCRIBER_IMAGE_REPO)"; do \ + "subscriber|$(SUBSCRIBER_IMAGE_REPO)" \ + "cleanup|$(CLEANUP_IMAGE_REPO)"; do \ component="$${image%%|*}"; \ repo="$${image#*|}"; \ ref="$${repo}:$(TAG)"; \ @@ -529,6 +541,20 @@ sbom-registry-images: syft-check ## Generate SBOMs for published release images test-release-image-digests: ## Test fail-closed release image digest resolution with a fake registry client. @bash hack/resolve-release-image-digests_test.sh +.PHONY: test-release-install +test-release-install: kustomize ## Test digest-pinned release install rendering. + @KUSTOMIZE_CMD="$(LOCAL_KUSTOMIZE)" bash hack/render-release-install_test.sh + +.PHONY: render-install +render-install: kustomize ## Render an install manifest; IMG, SERVER_IMG, and CLEANUP_IMG must be digest-pinned. + @CONTROLLER_IMAGE="$(IMG)" SERVER_IMAGE="$(SERVER_IMG)" CLEANUP_IMAGE="$(CLEANUP_IMG)" \ + OUTPUT_FILE="$(INSTALL_MANIFEST)" KUSTOMIZATION_PATH="$(INSTALL_KUSTOMIZATION)" \ + KUSTOMIZE_CMD="$(LOCAL_KUSTOMIZE)" hack/render-release-install.sh + +.PHONY: deploy +deploy: render-install ## Apply the digest-pinned install manifest to the current Kubernetes context. + $(KUBECTL) apply -f "$(INSTALL_MANIFEST)" + .PHONY: dev-cluster dev-cluster: kind ## Create a local kind cluster for development. @if $(KIND) get clusters 2>/dev/null | grep -qx "$(KIND_CLUSTER)"; then \ diff --git a/README.md b/README.md index d5a7948f..5071a9f8 100644 --- a/README.md +++ b/README.md @@ -195,8 +195,14 @@ The default Kustomize overlay brings up both control-plane components: [docs/design/cachebackend-api.md#functional-probe-gate](docs/design/cachebackend-api.md#functional-probe-gate); it is intentionally not visible on a default install with no engine workload. +Use the digest-pinned manifest attached to the selected GitHub release. The +checked-in `config/default` is a rendering template, not a directly installable +bundle; see [container image operations](docs/operations/container-images.md) +for the equivalent source-tree `make deploy` command. + ```bash -kubectl apply -k config/default +RELEASE_TAG=vX.Y.Z +kubectl apply -f "inference-cache-${RELEASE_TAG}.yaml" kubectl -n inference-cache-system wait --for=condition=Available deployment --all --timeout=180s kubectl get cacheindex cluster-default -o yaml ``` diff --git a/api/v1alpha1/cachebackend_types.go b/api/v1alpha1/cachebackend_types.go index 2acb87d6..eb566891 100644 --- a/api/v1alpha1/cachebackend_types.go +++ b/api/v1alpha1/cachebackend_types.go @@ -192,9 +192,10 @@ type LMCachePodLocalServerSpec struct { // +kubebuilder:validation:Maximum=65535 Port int32 `json:"port"` - // L1Capacity is the server's host-memory cache capacity. The renderer sizes - // /dev/shm to this value plus 1Gi; container memory requests and limits must - // each cover that complete budget. + // L1Capacity is the server's eagerly allocated private pinned host-memory + // cache capacity. The PodLocal renderer also keeps a conservative /dev/shm + // IPC budget of this value plus 1Gi; container memory requests and limits + // must each cover the same complete budget. // +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero" L1Capacity resource.Quantity `json:"l1Capacity"` @@ -204,7 +205,7 @@ type LMCachePodLocalServerSpec struct { // Resources are applied to the injected MP server container. Admission // requires a positive CPU request and requires both the memory request and - // memory limit to cover l1Capacity plus 1Gi of /dev/shm headroom. + // memory limit to cover l1Capacity plus 1Gi of server headroom. Resources corev1.ResourceRequirements `json:"resources"` } @@ -233,8 +234,8 @@ type LMCacheNodeLocalServerSpec struct { // +kubebuilder:validation:Maximum=65535 HTTPPort int32 `json:"httpPort"` - // L1Capacity is one shared host-memory budget per active engine node, not per - // selected engine Pod. + // L1Capacity is one eagerly allocated private pinned host-memory budget per + // active engine node, not per selected engine Pod. // +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero" L1Capacity resource.Quantity `json:"l1Capacity"` @@ -263,6 +264,8 @@ type LMCacheNodeLocalSchedulingSpec struct { // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // ImagePullSecrets are used by controller-managed Server and cleanup Pods. + // The cleanup Pod does not inherit registry credentials from an Engine Pod. // +optional ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` diff --git a/api/v1alpha1/cachebackend_types_test.go b/api/v1alpha1/cachebackend_types_test.go index 70606466..d75f628f 100644 --- a/api/v1alpha1/cachebackend_types_test.go +++ b/api/v1alpha1/cachebackend_types_test.go @@ -74,7 +74,8 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { requireEnum(t, mustProperty(t, specSchema, "runtime"), []string{"VLLM", "SGLang"}) lmCacheSchema := mustProperty(t, specSchema, "lmCache") requireNoProperty(t, lmCacheSchema, "multiprocess") - requireEnum(t, mustProperty(t, lmCacheSchema, "topology"), []string{"PodLocal", "NodeLocal"}) + topologySchema := mustProperty(t, lmCacheSchema, "topology") + requireEnum(t, topologySchema, []string{"PodLocal", "NodeLocal"}) podLocalSchema := mustProperty(t, lmCacheSchema, "podLocal") requireRequired(t, podLocalSchema, "server") podLocalServerSchema := mustProperty(t, podLocalSchema, "server") @@ -86,6 +87,7 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { requireMinimum(t, mustProperty(t, podLocalServerSchema, "maxWorkers"), 1) nodeLocalSchema := mustProperty(t, lmCacheSchema, "nodeLocal") requireRequired(t, nodeLocalSchema, "server") + requireNoProperty(t, nodeLocalSchema, "cleanupImage") requireRequired(t, nodeLocalSchema, "idleRetentionSeconds") idleRetentionSchema := mustProperty(t, nodeLocalSchema, "idleRetentionSeconds") requireMinimum(t, idleRetentionSchema, 0) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 2a682bc0..a910d734 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -7,7 +7,10 @@ package main import ( "crypto/tls" "flag" + "fmt" "os" + "regexp" + "strings" "time" "go.uber.org/zap/zapcore" @@ -31,6 +34,10 @@ import ( const leaderLockName = "inference-cache-controller-leader-lock" +var sha256ImagePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[a-f0-9]{64}$`) + +const zeroSHA256Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" + var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") @@ -53,6 +60,7 @@ type options struct { cacheIndexRefreshEvery time.Duration policyPushEvery time.Duration subscriberImage string + nodeLocalCleanupImage string policyServerGRPCAddress string zapOpts zap.Options } @@ -69,6 +77,7 @@ func defaultOptions() options { cacheIndexRefreshEvery: controller.DefaultRefreshInterval, policyPushEvery: controller.DefaultPolicyPushInterval, subscriberImage: "", + nodeLocalCleanupImage: "", policyServerGRPCAddress: "inference-cache-server.inference-cache-system.svc.cluster.local:9090", zapOpts: zap.Options{ TimeEncoder: zapcore.RFC3339TimeEncoder, @@ -89,6 +98,7 @@ func parseOptions() options { flag.DurationVar(&opts.cacheIndexRefreshEvery, "cacheindex-refresh-interval", opts.cacheIndexRefreshEvery, "How often to refresh the CacheIndex status from the server snapshot.") flag.DurationVar(&opts.policyPushEvery, "cachepolicy-push-interval", opts.policyPushEvery, "How often to re-push the full CachePolicy snapshot to the server (self-healing on server restart).") flag.StringVar(&opts.subscriberImage, "kvevent-subscriber-image", opts.subscriberImage, "Image reference the pod-mutating webhook uses for the kvevent-subscriber sidecar it auto-attaches to managed-LMCache engine pods (vLLM and SGLang). Empty (default) disables auto-attach — the engine pod wiring still happens but no subscriber container is appended. Pin to a digest in production.") + flag.StringVar(&opts.nodeLocalCleanupImage, "node-local-shm-cleanup-image", opts.nodeLocalCleanupImage, "Digest-pinned inference-cache helper image used to reclaim NodeLocal LMCache SHM pools. Required.") flag.StringVar(&opts.policyServerGRPCAddress, "policy-server-grpc-address", opts.policyServerGRPCAddress, "host:port the kvevent-subscriber sidecar dials to ReportCacheState. Defaults to the in-cluster Service DNS in the inference-cache-system namespace.") opts.zapOpts.BindFlags(flag.CommandLine) flag.Parse() @@ -98,6 +108,10 @@ func parseOptions() options { func main() { opts := parseOptions() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts.zapOpts))) + if err := validateOptions(opts); err != nil { + setupLog.Error(err, "invalid controller configuration") + os.Exit(1) + } setupLog.Info("initializing", "gitVersion", version.GitVersion, "gitCommit", version.GitCommit) tlsOpts := []func(*tls.Config){} @@ -147,14 +161,15 @@ func main() { probeClient := &controller.ProbeClient{ProbeURL: opts.serverProbeURL} if err := (&controller.CacheBackendReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: ctrl.Log.WithName("controllers").WithName("CacheBackend"), - Recorder: mgr.GetEventRecorder("cachebackend-controller"), - APIReader: mgr.GetAPIReader(), - Registry: adapterRegistry, - BackendRegistry: adapterRegistries.Storage, - ProbeClient: probeClient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: ctrl.Log.WithName("controllers").WithName("CacheBackend"), + Recorder: mgr.GetEventRecorder("cachebackend-controller"), + APIReader: mgr.GetAPIReader(), + Registry: adapterRegistry, + BackendRegistry: adapterRegistries.Storage, + ProbeClient: probeClient, + NodeLocalShmCleanupImage: opts.nodeLocalCleanupImage, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CacheBackend") os.Exit(1) @@ -241,3 +256,10 @@ func main() { os.Exit(1) } } + +func validateOptions(opts options) error { + if !sha256ImagePattern.MatchString(opts.nodeLocalCleanupImage) || strings.HasSuffix(opts.nodeLocalCleanupImage, zeroSHA256Digest) { + return fmt.Errorf("--node-local-shm-cleanup-image must be a digest-pinned image reference") + } + return nil +} diff --git a/cmd/controller/main_test.go b/cmd/controller/main_test.go new file mode 100644 index 00000000..095fe560 --- /dev/null +++ b/cmd/controller/main_test.go @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import "testing" + +func TestValidateOptionsRequiresDigestPinnedCleanupImage(t *testing.T) { + valid := defaultOptions() + valid.nodeLocalCleanupImage = "registry.example/inference-cache-shm-cleanup@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + if err := validateOptions(valid); err != nil { + t.Fatalf("valid options: %v", err) + } + + for _, image := range []string{ + "", + "registry.example/inference-cache-shm-cleanup:latest", + "registry.example/bad@@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:0000000000000000000000000000000000000000000000000000000000000000", + } { + opts := defaultOptions() + opts.nodeLocalCleanupImage = image + if err := validateOptions(opts); err == nil { + t.Fatalf("cleanup image %q was accepted without a sha256 digest", image) + } + } +} diff --git a/cmd/node-local-shm-cleanup/main.go b/cmd/node-local-shm-cleanup/main.go new file mode 100644 index 00000000..5dafec18 --- /dev/null +++ b/cmd/node-local-shm-cleanup/main.go @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" +) + +func emptyDirectory(root string) error { + entries, err := os.ReadDir(root) + if err != nil { + return err + } + for _, entry := range entries { + child := filepath.Join(root, entry.Name()) + if err := os.RemoveAll(child); err != nil { + return fmt.Errorf("remove %s: %w", child, err) + } + } + return nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: node-local-shm-cleanup DIRECTORY") + os.Exit(2) + } + if err := emptyDirectory(os.Args[1]); err != nil { + fmt.Fprintf(os.Stderr, "empty %s: %v\n", os.Args[1], err) + os.Exit(1) + } +} diff --git a/cmd/node-local-shm-cleanup/main_test.go b/cmd/node-local-shm-cleanup/main_test.go new file mode 100644 index 00000000..d05c39b3 --- /dev/null +++ b/cmd/node-local-shm-cleanup/main_test.go @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestEmptyDirectoryPreservesRootAndDoesNotFollowSymlinks(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + outsideFile := filepath.Join(outside, "keep") + if err := os.WriteFile(outsideFile, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, "nested"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "nested", "ipc"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "outside")); err != nil { + t.Fatal(err) + } + + if err := emptyDirectory(root); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatalf("root directory was removed: %v", err) + } + if len(entries) != 0 { + t.Fatalf("root still contains %d entries", len(entries)) + } + if _, err := os.Stat(outsideFile); err != nil { + t.Fatalf("cleanup followed symlink outside root: %v", err) + } +} diff --git a/config/crd/bases/inferencecache.io_cachebackends.yaml b/config/crd/bases/inferencecache.io_cachebackends.yaml index 1585b340..61627cfa 100644 --- a/config/crd/bases/inferencecache.io_cachebackends.yaml +++ b/config/crd/bases/inferencecache.io_cachebackends.yaml @@ -493,6 +493,9 @@ spec: It cannot select nodes; inference-engine scheduling remains authoritative. properties: imagePullSecrets: + description: |- + ImagePullSecrets are used by controller-managed Server and cleanup Pods. + The cleanup Pod does not inherit registry credentials from an Engine Pod. items: description: |- LocalObjectReference contains enough information to let you locate the @@ -831,8 +834,8 @@ spec: - type: integer - type: string description: |- - L1Capacity is one shared host-memory budget per active engine node, not per - selected engine Pod. + L1Capacity is one eagerly allocated private pinned host-memory budget per + active engine node, not per selected engine Pod. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true x-kubernetes-validations: @@ -948,9 +951,10 @@ spec: - type: integer - type: string description: |- - L1Capacity is the server's host-memory cache capacity. The renderer sizes - /dev/shm to this value plus 1Gi; container memory requests and limits must - each cover that complete budget. + L1Capacity is the server's eagerly allocated private pinned host-memory + cache capacity. The PodLocal renderer also keeps a conservative /dev/shm + IPC budget of this value plus 1Gi; container memory requests and limits + must each cover the same complete budget. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true x-kubernetes-validations: @@ -973,7 +977,7 @@ spec: description: |- Resources are applied to the injected MP server container. Admission requires a positive CPU request and requires both the memory request and - memory limit to cover l1Capacity plus 1Gi of /dev/shm headroom. + memory limit to cover l1Capacity plus 1Gi of server headroom. properties: claims: description: |- diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 6ea18663..783f4101 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -25,6 +25,10 @@ spec: containers: - args: - --leader-elect + # Source-tree rendering placeholder. The controller rejects it at + # startup; supported install tooling replaces it with the matching + # cleanup image digest. + - --node-local-shm-cleanup-image=ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:0000000000000000000000000000000000000000000000000000000000000000 command: - /controller image: controller:latest diff --git a/config/samples/README.md b/config/samples/README.md index 8f178378..762a18b1 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -68,6 +68,8 @@ they go `Ready` as soon as admission accepts the endpoint. See the [quickstart](../../docs/quickstart.md). NodeLocal focused samples are not five-minute recipes. Before applying one, +configure the controller's `--node-local-shm-cleanup-image` with the matching +digest from the inference-cache release, then reserve its MP and HTTP host ports on every node where the inference system may place a selected engine, and ensure all selected engine Pods belong to one mutually trusted tenant domain. CacheBackend does not select nodes or rewrite diff --git a/config/samples/cachebackend-sglang-nodelocal-host-only.yaml b/config/samples/cachebackend-sglang-nodelocal-host-only.yaml index 9ff21f9f..74a245d6 100644 --- a/config/samples/cachebackend-sglang-nodelocal-host-only.yaml +++ b/config/samples/cachebackend-sglang-nodelocal-host-only.yaml @@ -9,7 +9,8 @@ # intentionally disjoint from the sibling vLLM sample if both backends have # engines on the same node. Set scheduling.runtimeClassName when the engine's # runtime does not provide the server's required NVIDIA visibility. The server -# and engines mount only this backend UID's host SHM directory as `/dev/shm`. +# and engines mount only this backend UID's host IPC directory as `/dev/shm`; +# L1 itself is private pinned host memory, not a POSIX SHM file. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 5c49e38f..b794d179 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -22,6 +22,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ RUN --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ go build -ldflags="-s -w" -o /out/kvevent-subscriber ./cmd/kvevent-subscriber +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ + go build -ldflags="-s -w" -o /out/node-local-shm-cleanup ./cmd/node-local-shm-cleanup FROM gcr.io/distroless/static-debian13:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 AS controller COPY --from=builder /out/controller /controller @@ -37,3 +40,8 @@ FROM gcr.io/distroless/static-debian13:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae COPY --from=builder /out/kvevent-subscriber /kvevent-subscriber USER 65532:65532 ENTRYPOINT ["/kvevent-subscriber"] + +FROM gcr.io/distroless/static-debian13:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 AS cleanup +COPY --from=builder /out/node-local-shm-cleanup /node-local-shm-cleanup +USER 65532:65532 +ENTRYPOINT ["/node-local-shm-cleanup"] diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index 717df19c..df26f4e0 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -217,7 +217,9 @@ For the demo / debugging-flow walkthrough, on a local kind cluster: ```sh make dev-cluster -kubectl apply -k config/default # install CRDs + controller + server +# Install CRDs + controller + server from the selected release manifest first. +RELEASE_TAG=vX.Y.Z +kubectl apply -f "inference-cache-${RELEASE_TAG}.yaml" kubectl apply -f config/samples/cache_v1alpha1_cachebackend.yaml # Validate configuration without the live server (works pre-port-forward): diff --git a/docs/concepts/cachebackend-engine-binding.md b/docs/concepts/cachebackend-engine-binding.md index fc06259e..a16cd6b0 100644 --- a/docs/concepts/cachebackend-engine-binding.md +++ b/docs/concepts/cachebackend-engine-binding.md @@ -30,13 +30,14 @@ server uses exact node-name affinity, so Kubernetes still evaluates taints, resources, and declared host-port conflicts. The Downward API supplies the engine's `status.hostIP`; no ClusterIP participates. The init gate blocks normal engine startup until `/config` and `/healthcheck` verify the same -name/UID/generation and live server configuration. Each CacheBackend UID also -derives an explicit `lmcache_l1_pool_inferencecache_` POSIX SHM name; the -gate verifies both the declared and effective live name before starting the -engine. The server and engines mount only +name/UID/generation and live server configuration. The server explicitly +selects `lmcache_driven`, disables lazy allocation, and passes an empty +`shm_name`; L1 is therefore eagerly allocated as private pinned host memory, +not as a named POSIX SHM object. The server and engines still mount only `/dev/shm/inference-cache/` from the host as their container -`/dev/shm`, so normally behaving co-located pools do not see one another's SHM -objects. This remains ownership isolation rather than cryptographic +`/dev/shm`, so normally behaving co-located pools do not see one another's +CUDA/PyTorch auxiliary IPC objects. This remains ownership isolation rather +than cryptographic authentication: host root, privileged Pods, or processes mounting the parent directory can bypass it, so the host-network/server pool still requires one trusted tenant domain. @@ -146,7 +147,7 @@ A fuller paired sample is | A matching Pod has no injection annotation | Admission failed open because of an invalid/colliding Pod shape or an unavailable managed Redis endpoint. | Read webhook logs and Pod Events, fix the reported shape, then recreate the Pod. | | Engine crashes after successful injection | The runtime-owned image lacks a compatible LMCache client/API, or another engine startup requirement failed. | Inspect engine logs and use a compatible pinned image; CacheBackend does not replace it. | | Multiple CacheBackends could match one Pod | A cache-domain value was reused by concurrent creates. CacheBackend admission normally rejects the duplicate; the Pod webhook also denies an ambiguous live match rather than choosing a backend. | Give every CacheBackend a unique namespace-scoped `inferencecache.io/cache-domain` value and put that value on only the intended engine Pod templates. | -| NodeLocal engine stays in `lmcache-node-local-gate` | Its on-demand same-node server is not healthy, the host ports conflict, the effective UID-scoped SHM pool is unavailable/mismatched, or live config belongs to another backend. | Inspect the server Pod args, `/config`, scheduler events, and `status.connector.enginePodCoverage`; fix ports, host `/dev/shm` capacity, resources, or runtime configuration and recreate or reschedule. | +| NodeLocal engine stays in `lmcache-node-local-gate` | Its on-demand same-node server is not healthy, the host ports conflict, the effective CUDA/non-lazy/private-L1 profile is mismatched, or live config belongs to another backend. | Inspect the server Pod args, `/config`, scheduler events, and `status.connector.enginePodCoverage`; fix ports, pinned host-memory capacity, resources, or runtime configuration and recreate or reschedule. | | Pod was relabeled after creation | Admission is CREATE-only. | Recreate the Pod. | | Pod intentionally needs no cache injection | No explicit opt-out was set. | Put `inferencecache.io/skip-inject: "true"` on the Pod template and recreate it. | diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 892da039..6b960c90 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -247,14 +247,20 @@ vLLM image/version, KV reuse, TP determinism, and failure recovery remain Phase 4 runtime gates. Canonical examples are the three PodLocal profiles plus `config/samples/cachebackend-vllm-nodelocal-host-only.yaml`. -For both typed vLLM and SGLang PodLocal adapters, `l1Capacity` is the usable L1 -target, not the complete container budget. The common renderer creates a -memory-backed `/dev/shm` with `sizeLimit: l1Capacity + 1Gi`; admission requires -both the MP-server memory request and memory limit to be at least that value. If -the engine already mounts `/dev/shm`, the adapter reuses it only when it is a -memory-backed `emptyDir` with a `sizeLimit` at least as large as that budget. -This keeps scheduling/cgroup accounting aligned with the tmpfs and leaves room -for LMCache metadata and shared-memory allocator overhead. +For both typed vLLM and SGLang adapters, the supported server profile is +CUDA-only and explicit: `--supported-transfer-mode lmcache_driven`, +`--no-l1-use-lazy`, and `--shm-name ""`. LMCache therefore allocates the full +`l1Capacity` eagerly as private CUDA-pinned host memory during server startup; +allocation failure prevents health from becoming ready. It does not create a +POSIX SHM-backed `/dev/shm/lmcache_l1_pool_*` L1 arena. Admission still requires +the MP-server memory request and limit to cover `l1Capacity + 1Gi` so scheduling +and the container cgroup account for L1 plus server overhead. + +PodLocal continues to provide a memory-backed `/dev/shm` with +`sizeLimit: l1Capacity + 1Gi` as a conservative budget for CUDA/PyTorch IPC +objects. If the engine already mounts `/dev/shm`, the adapter reuses it only +when it is a memory-backed `emptyDir` with at least that `sizeLimit`. This tmpfs +is an IPC surface; it is not where the supported LMCache L1 KV cache resides. For NodeLocal, `l1Capacity` is instead one shared per-node budget. CacheBackend creation alone creates no server and never changes engine placement. After an @@ -265,30 +271,42 @@ backend's `/dev/shm/inference-cache/` host directory as their container `/dev/shm`; they do not mount the whole node SHM namespace. Exact node-name affinity sends the server through the normal scheduler on the engine's node; `status.hostIP` prevents ClusterIP or cross-node CUDA IPC. -`maxGPUWorkers` must cover all selected engine instances on one node. Every -server receives the controller-derived -`lmcache_l1_pool_inferencecache_` through `--shm-name`; the -engine gate verifies both the declared MP value and the effective L1 -memory-manager value before startup. Different CacheBackend UIDs therefore do -not accidentally unlink or rebind the same POSIX SHM object. The server sets +`maxGPUWorkers` must cover all selected engine instances on one node. The +engine gate verifies the declared `lmcache_driven` transfer mode and the +effective non-lazy, empty-`shm_name` L1 configuration before startup. Different +CacheBackend UIDs do not share a POSIX SHM-backed L1 object because this profile +creates none; the UID directory remains to separate observed CUDA, PyTorch, and +semaphore auxiliary IPC objects. The server sets `NVIDIA_VISIBLE_DEVICES=all` but requests no -allocatable GPU. It inherits the source engine's runtime class, tolerations, -image-pull secrets, priority class, and scheduler unless optional +allocatable GPU. The server inherits the source engine's runtime class, +tolerations, image-pull secrets, priority class, and scheduler unless optional `nodeLocal.scheduling` server overrides are supplied. The FastAPI/MP listeners are unauthenticated and host networking bypasses NetworkPolicy, so this topology requires one trusted tenant domain per pool plus node firewall controls. CacheBackend name/UID/generation verification detects wrong ownership but is not cryptographic authentication. The UID-scoped mount prevents normal pool -processes from seeing another pool through their container `/dev/shm`, but does -not isolate host root, privileged Pods, or processes that independently mount -the parent host directory. Co-located pools therefore remain limited to one -trusted node domain. After the last +processes from seeing another pool's auxiliary IPC directory through their +container `/dev/shm`, but does not isolate host root, privileged Pods, or +processes that independently mount the parent host directory. Co-located pools +therefore remain limited to one trusted node domain. After the last selected engine leaves a node, `nodeLocal.idleRetentionSeconds` keeps the server and shared L1 warm for the configured window (300 seconds by default); new demand on that node reuses the same Pod. Set it to zero for immediate deletion. A retained Pod continues to reserve its declared host ports, so another NodeLocal backend using the same pair remains in the normal Kubernetes host-port conflict path until expiry. +Before an idle server or deleting CacheBackend releases the pool, a gated +one-shot Pod runs the controller-wide, digest-pinned +`--node-local-shm-cleanup-image` and clears only +that backend UID directory after its consumers are gone. This small helper is +independent of LMCache and uses only `nodeLocal.scheduling.imagePullSecrets`; +it never copies registry credentials from an Engine Pod. Consumer checks are +cluster-wide because the path is node-global. A managed Engine arriving after +cleanup starts cannot use the directory: its main container remains behind the +same-node startup gate, and Server recreation stays blocked until cleanup +succeeds. Failed helpers are retried at most three times to bound API churn. +Unmanaged processes that race an exact hostPath mount remain outside +the trusted-node boundary described above. NodeLocal ports are explicit rather than dynamically allocated. Engine Pods are immutable and receive their endpoint during admission, before the @@ -338,10 +356,12 @@ With a RESP binding it offloads to Redis; without a binding it runs host-only. `NVIDIA_VISIBLE_DEVICES=all` lets the GPU-less sidecar use CUDA-IPC with no device-plugin allocation, an `exec` startup-probe on the loopback ZMQ port gates the engine's -start, and a shared `emptyDir` carries the config file. For `/dev/shm` (the L1 tier) -it reuses the engine's own volume when the engine already mounts one (a duplicate -mountPath is an invalid Pod), else adds a sized `emptyDir{medium: Memory}` — see the -reserved-names note below for the reuse/reject rules. On the engine container (name +start, and a shared `emptyDir` carries the config file. For `/dev/shm`, it reuses +the engine's own volume when the engine already mounts one (a duplicate +mountPath is an invalid Pod), else adds a sized `emptyDir{medium: Memory}` for +CUDA/PyTorch auxiliary IPC objects. L1 KV bytes are private pinned memory, not +files in that tmpfs. See the reserved-names note below for the reuse/reject +rules. On the engine container (name `sglang`) it injects: - `--enable-lmcache` — SGLang's boolean flag (an argparse `store_true`) that activates its LMCache connector. This replaces vLLM's `--kv-transfer-config` JSON. @@ -373,11 +393,11 @@ The old lm:// `LMCACHE_REMOTE_URL` / serde / chunk-size / local-CPU env is | `lmCache.topology` | required | `PodLocal`, `NodeLocal` | Chooses native-sidecar or engine-demanded per-node server placement. | | `lmCache.podLocal.server.image` | required | digest-pinned reference | Independently owned LMCache server image; never copied from or into the engine image. | | `lmCache.podLocal.server.port` | required | `1`–`65535` | Loopback MP port. | -| `lmCache.podLocal.server.l1Capacity` | required | positive quantity | Usable L1; `/dev/shm` and memory resources must cover this plus 1Gi. | +| `lmCache.podLocal.server.l1Capacity` | required | positive quantity | Eager private pinned L1; memory resources and the conservative IPC tmpfs budget cover this plus 1Gi. | | `lmCache.podLocal.server.maxWorkers` | required | `>=1` | Server worker bound. | | `lmCache.podLocal.server.resources` | required | validated K8s resources | Positive CPU request and sufficient memory request/limit. | | `lmCache.nodeLocal.server.{image,port,httpPort}` | required | digest plus distinct ports | One server image/config and real node-bound listeners for the pool. | -| `lmCache.nodeLocal.server.l1Capacity` | required | positive quantity | Shared L1 budget per active engine node; memory request/limit cover it plus 1Gi. | +| `lmCache.nodeLocal.server.l1Capacity` | required | positive quantity | Eager private pinned L1 budget per active engine node; memory request/limit cover it plus 1Gi. | | `lmCache.nodeLocal.server.{maxGPUWorkers,maxCPUWorkers}` | required | `>=1` | Shared per-server worker bounds. | | `lmCache.nodeLocal.idleRetentionSeconds` | `300` | `0`–`86400` | Warm retention after the last selected engine leaves a node; `0` deletes immediately. | | `lmCache.nodeLocal.scheduling` | optional | server operational overrides | May override tolerations, image-pull secrets, ServiceAccount, Pod security context, priority/scheduler, runtime class, and termination grace on server Pods. It does not expose node selection or mutate engine placement. | diff --git a/docs/design/grpc-tls.md b/docs/design/grpc-tls.md index 6faa5ea4..cd4b3fa7 100644 --- a/docs/design/grpc-tls.md +++ b/docs/design/grpc-tls.md @@ -51,7 +51,7 @@ The admission webhook (B3) uses a **namespaced `Issuer`** (`selfsigned-issuer`), - **Server flags** (`cmd/server`): `--tls-cert-file`, `--tls-key-file`. Both set → TLS; both empty → plaintext; **exactly one set → the server refuses to start** (the both-or-neither rule lives in `server.LoadGRPCTLSCredentials`, so it's the single, unit-tested source of truth). - **Posture observability**: `inferencecache_server_grpc_tls_enabled` (gauge, 0/1) so operators can confirm the wire posture from Prometheus; the startup log line also carries a `grpc_tls` boolean field. - **Default install** (`config/default`): **plaintext** — no cert, no `--tls-*` args. (This is what the C6 canary and any real engine deployment apply, so the plaintext `kvevent-subscriber` keeps ingesting.) -- **Opt-in TLS** — `config/overlays/server-tls` = `config/default` + the `config/server/tls/` kustomize component. The component ships the `Issuer` + `Certificate` (Secret `inference-cache-server-tls`) and patches the server Deployment to mount the Secret read-only at `/var/run/secrets/tls/` and pass the two `--tls-*` args. Service FQDN DNS names (`inference-cache-server.inference-cache-system.svc[.cluster.local]`) are written directly into the Certificate. Enable with `kubectl apply -k config/overlays/server-tls` (cert-manager required); the install smoke exercises exactly this overlay. +- **Opt-in TLS** — `config/overlays/server-tls` = `config/default` + the `config/server/tls/` kustomize component. The component ships the `Issuer` + `Certificate` (Secret `inference-cache-server-tls`) and patches the server Deployment to mount the Secret read-only at `/var/run/secrets/tls/` and pass the two `--tls-*` args. Service FQDN DNS names (`inference-cache-server.inference-cache-system.svc[.cluster.local]`) are written directly into the Certificate. Render it with `make deploy INSTALL_KUSTOMIZATION=overlays/server-tls` plus the three digest-pinned image variables documented in [`container-images.md`](../operations/container-images.md) (cert-manager required). - **kind / dev**: the default is plaintext (no cert-manager dependency for a plaintext loop). To exercise TLS locally, apply the `config/overlays/server-tls` overlay (needs cert-manager). ## kubelet probe compatibility diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 767ac2fa..808a73ff 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -97,14 +97,14 @@ code lands. | D3 | `LMCacheServer` is removed from the canonical `remoteStorage.provider` set. | `lm://` is a legacy IP remote connector and is absent from the MP L3 adapter catalog. | | D4 | PodLocal is the first production candidate and migration target. | It has the smallest scheduling and ownership surface and builds on the existing SGLang proof. | | D5 | NodeLocal means one controller-owned server Pod per active engine node per `CacheBackend`; the inference system remains the placement authority. This replaces the earlier DaemonSet/server-first decision on 2026-08-12. | A DaemonSet requires a node set before engines are scheduled and therefore inverted ownership by forcing engines onto cache-selected nodes. Engine-demanded Pods preserve arbitrary inference-system scheduling while still allowing same-node sharing. | -| D6 | A generic Deployment behind a load-balanced Service is not a valid CUDA MP topology. | CUDA IPC and shared memory require the engine to reach the MP server on its own node. | +| D6 | A generic Deployment behind a load-balanced Service is not a valid CUDA MP topology. | CUDA IPC requires the engine to reach the MP server on its own node; a load balancer may select a server that cannot open that engine's GPU handle. | | D7 | Connector endpoints are not published in the generic `status.endpoint`. | PodLocal uses loopback; NodeLocal is node-dependent. Only remote L3 has a globally meaningful provider endpoint. | | D8 | Unsupported combinations are rejected at admission. | An accepted but inert cache field commonly produces silent zero-hit behavior. | | D9 | Fail-open is rendered into runtime-native behavior and tested. | A custom environment variable without a known consumer is not an enforceable serving contract. | | D10 | Component lifecycle ownership is capability-specific. | The Pod-local MP process is kubelet-owned while remote L3 is independently managed; connector re-registration after an MP-process restart is a post-migration enhancement, not an MVP contract. | | D11 | Each supported vLLM integration explicitly identifies its MP connector implementation; the initial reference baseline uses the LMCache-shipped connector. | With vLLM 0.20 or newer, `LMCacheMPConnector` without a module path selects vLLM's built-in implementation. The initial adapter uses `kv_connector_module_path: lmcache.integration.vllm.lmcache_mp_connector` so the tested client tracks the pinned LMCache server protocol; a future adapter revision may validate a different implementation explicitly. | | D12 | CacheBackend never owns or rewrites the inference engine image. Engine images in validation matrices are reproducible fixtures only; CacheBackend digest-pins only cache components it injects or manages. | The inference system owns its runtime lifecycle. The selected adapter renders its engine-specific connector contract, while normal engine initialization is the authoritative compatibility check; tested images are neither an admission allowlist nor a mutation default. | -| D13 | Selecting `NodeLocal` explicitly opts the backend into one host-networked MP server per active engine node. Server and selected engine Pods mount only `/dev/shm/inference-cache/` from the host as container `/dev/shm`; engine Pods themselves remain off host networking and host IPC. | LMCache 0.5.3 requires node-visible networking and shared host memory for cross-Pod CUDA IPC. A UID-scoped bind mount retains that path without exposing the entire node SHM namespace to normally behaving pool processes. Keeping engine placement and networking under the inference system reduces coupling, while the topology choice and documented trust domain make the remaining host access explicit. | +| D13 | Selecting `NodeLocal` explicitly opts the backend into one host-networked MP server per active engine node. Server and selected engine Pods mount only `/dev/shm/inference-cache/` from the host as container `/dev/shm`; engine Pods themselves remain off host networking and host IPC. | LMCache 0.5.3 CUDA IPC requires node-visible networking and GPU visibility. The UID-scoped mount is retained for observed CUDA/PyTorch auxiliary IPC objects without exposing the entire node SHM namespace to normally behaving pool processes; L1 KV bytes are private pinned host memory. Keeping engine placement and networking under the inference system reduces coupling, while the topology choice and documented trust domain make the remaining host access explicit. | ## Migration baseline (before Phase 1) @@ -878,8 +878,8 @@ skipped because `xxhash` was unavailable; `make ci` still passed. ## Phase 8 — NodeLocal shared MP servers -- **Status:** Complete (2026-08-12), including focused live-node and GPU - validation of the UID-scoped POSIX SHM remediation. +- **Status:** Implementation revised (2026-08-22); focused GPU revalidation of + the explicit non-lazy/private-pinned-L1 profile is pending. - **Depends on:** Phases 3–4; does not block Phase 7 ### Objective @@ -898,7 +898,9 @@ sharing remain out of scope. The final contract is: -- **API:** `nodeLocal.server` explicitly requires a digest-pinned image, +- **API:** `nodeLocal.server` explicitly requires a digest-pinned LMCache image; + the controller-wide `--node-local-shm-cleanup-image` supplies the separate + digest-pinned cleanup helper, distinct MP and FastAPI host ports, per-node L1 capacity, GPU/CPU worker limits, and resources covering `l1Capacity + 1Gi`. The FastAPI listener also serves `/metrics`; no third metrics port is created. `nodeLocal.scheduling` @@ -916,33 +918,40 @@ The final contract is: deletion. No Deployment, ReplicaSet, or DaemonSet owns these Pods. - **Host boundary:** Server Pods use `hostNetwork`, `ClusterFirstWithHostNet`, the selected NVIDIA runtime without reserving allocatable GPUs, and a - restrictive container security context. Servers and selected engines mount - only the backend's `/dev/shm/inference-cache/` host directory as their - container `/dev/shm`; engines remain off host networking and host IPC. + restrictive container security context. The controller-created server and + injected startup gate disable privilege escalation, drop all Linux + capabilities, and use the runtime-default seccomp profile; the server also + disables host IPC and service-account token automount. Servers and selected + engines mount only the backend's `/dev/shm/inference-cache/` host + directory as their container `/dev/shm` for CUDA/PyTorch auxiliary IPC + objects. The injector does not enable or otherwise govern engine-owned + Pod/container settings such as host networking, host IPC, PID namespace, + privileged mode, capabilities, or seccomp policy; those remain + inference-system responsibilities. L1 itself is private pinned host memory. - **Endpoint and gate:** Engines derive the same-node address from Downward API `status.hostIP`. A blocking init gate requires healthy `/healthcheck` plus an exact `/config` match for namespace/name/UID/generation, ports, and chunk size - before the engine starts. It also verifies both the declared MP `shm_name` - and the effective L1 memory-manager `shm_name`, so an unsafe pool or LMCache - shared-memory fallback cannot silently admit the engine. SGLang writes its + before the engine starts. It also verifies `supported_transfer_mode` is + `lmcache_driven`, lazy L1 allocation is disabled, and both declared and + effective `shm_name` are empty. SGLang writes its engine-specific client YAML; vLLM retains its connector JSON. No Service or ClusterIP participates in CUDA MP traffic. - **Ownership and isolation:** One CacheBackend name/UID/runtime and its sole namespace-unique `inferencecache.io/cache-domain` value own one server pool. CREATE and UPDATE reject non-canonical or duplicate ownership; Pod admission - denies concurrent ambiguity. Every server receives the full UID-derived - `lmcache_l1_pool_inferencecache_` name through `--shm-name`; the name and - UID host directory are stable across same-UID generation/server replacement - and distinct after CacheBackend delete/recreate. Disjoint port pairs prevent - network bind conflicts, while the UID-scoped name and mount prevent normal - co-located pools from accidentally unlinking/rebinding or seeing each - other's POSIX SHM objects. Idle-retained servers continue reserving their - ports and SHM budget until expiry. UID matching and mount scoping are - routing/ownership identities, not authentication: host root, privileged - Pods, and processes mounting the parent host directory remain outside this - isolation boundary. Co-located pools must therefore remain inside one - mutually trusted node domain; host firewall controls are required, and - NetworkPolicy does not isolate host-network listeners. + denies concurrent ambiguity. Every server explicitly selects + `lmcache_driven`, non-lazy allocation, and empty `shm_name`, so no named + POSIX SHM-backed L1 pool exists to collide. The UID host directory is stable + across same-UID generation/server replacement and distinct after + CacheBackend delete/recreate. Disjoint port pairs prevent network bind + conflicts, while the UID-scoped mount separates normally created auxiliary + IPC files. Idle-retained servers continue reserving their ports, pinned L1, + and IPC directory until expiry. UID matching and mount scoping are + routing/ownership identities, not authentication. The supported contract + assumes that co-located workloads belong to one mutually trusted node + domain; controlling unrelated or privileged Pods, host root, namespace + security policy, node separation, and host firewall policy belongs to the + inference/cluster platform and is not enforced by this controller. - **Runtime consistency:** A pool cannot mix vLLM and SGLang. CacheBackend supplies one server image, chunk size, port tuple, generation, and runtime for the pool. The inference-system owner remains responsible for engine @@ -952,15 +961,16 @@ The final contract is: domain requires a separate CacheBackend. - **Status:** `desiredServers` is the distinct active scheduled-engine node count. `readyServers` counts current-generation, name/UID-verified Ready - servers carrying the expected UID-scoped SHM annotation and exact - `--shm-name` argument on those nodes. An engine is covered only by exactly one - healthy current server on its own node; unscheduled, stale-generation, - SHM-mismatched, ambiguous, or serverless engines are uncovered. Connector + servers carrying the exact CUDA/non-lazy/empty-`shm_name` runtime profile and + UID-scoped host IPC directory on those nodes. An engine is covered only by + exactly one healthy current server on its own node; unscheduled, + stale-generation, runtime-profile-mismatched, ambiguous, or serverless + engines are uncovered. Connector readiness requires all desired servers and all matched engines to be Ready and covered. -- **Capacity:** `l1Capacity` is one shared budget per active node, not per engine - Pod. `maxGPUWorkers` must cover the maximum engine instances expected on one - node. +- **Capacity:** `l1Capacity` is one eagerly allocated private pinned-memory + budget per active node, not per engine Pod. `maxGPUWorkers` must cover the + maximum engine instances expected on one node. ### Deliverables @@ -977,9 +987,9 @@ The final contract is: CREATE and UPDATE; deny ambiguous Pod injection and cross-backend sharing. - [x] Gate engine startup on the healthy same-node server's exact name/UID/generation/port/chunk-size identity. -- [x] Derive one full UID-scoped POSIX SHM name per CacheBackend, pass it - explicitly to every NodeLocal server, verify declared and effective live - configuration, and replace or un-cover servers missing that identity. +- [x] Explicitly select `lmcache_driven`, non-lazy allocation, and empty + `shm_name` on every server; verify declared and effective live + configuration, and replace or un-cover servers missing that profile. - [x] Mount only `/dev/shm/inference-cache/` as `/dev/shm` in each NodeLocal server and selected engine; replace or un-cover a server whose declared hostPath does not match its owner UID. @@ -1005,6 +1015,12 @@ on-demand server-Pod implementation only. Local and repository validation completed on 2026-08-12 PDT: +The CPU collision and UID-named rows below are retained as historical evidence +for the former `auto`/engine-driven POSIX-SHM profile. They do not validate the +current CUDA-only profile. The 2026-08-22 implementation now pins +`lmcache_driven`, non-lazy allocation, and empty `shm_name`; repository +validation and focused GPU requalification are tracked separately. + | Check | Result | |---|---| | Generated API artifacts | `make generate manifests` passed after the final engine-first/idle-retention API change; deepcopy, served CRD, Pod create/patch/delete RBAC, and webhook manifests are synchronized. No DaemonSet RBAC remains. | @@ -1012,9 +1028,10 @@ Local and repository validation completed on 2026-08-12 PDT: | Samples | `make verify-samples` passed: 27 admitted, one pre-existing explicit skip, zero failures. Both engine-first NodeLocal samples passed real admission. | | Coverage | `make cover-check` passed. | | CI | The baseline was amended as `628194e` with a matching `Signed-off-by`; `make verify-dco` and the complete `make ci` target passed. The optional Python golden-vector regeneration explicitly skipped because `xxhash` is unavailable. | +| Explicit CUDA allocation profile (2026-08-22) | `make generate manifests`, `git diff --check`, gate-script Python syntax parsing, `bash -n` for the reference smoke, `go test ./...`, `make verify-samples` (27 passed, one explicit skip), `make cover-check` (90.7%), and complete `make ci` passed. Tests require `lmcache_driven`, non-lazy allocation, and empty `shm_name` in rendered server args, the live-config gate, status/lifecycle identity, and vLLM connector JSON. Focused GPU revalidation remains pending and is not implied by these repository results. | | Fresh install | Dedicated Kubernetes 1.32 kind clusters passed the engine-first CRD/controller/webhook installation, real duplicate cache-domain rejection, zero speculative server Pods, placement-preserving NodeLocal engine admission, scheduler-selected engine node followed by one exact-node-affinity direct server Pod, host boundary/host ports, no MP Service, PodLocal admission, current samples, doctor, idempotent re-apply, served idle-retention default/bounds, Pod patch RBAC, idle marking, and same-UID reuse. VPN-safe temporary self-signed webhook TLS replaced only the cert-manager download; both clusters were deleted. The later UID-scoped `--shm-name` delta updated this smoke with exact annotation/argument checks but was not rerun end-to-end because no kind node image/cluster was cached and the standard cert-manager URL remained unavailable through the VPN. That delta instead passed real envtest admission plus the SJC current-controller live tests recorded below. | | Legacy production search | Production Go/manifests contain no `LMCacheConnectorV1`, `LMCACHE_REMOTE_URL`, `LMCACHE_REMOTE_SERDE`, `ProtocolLMCache`, `lm://`, or LMCacheServer provider path. Remaining LMCacheServer matches are sample comments explicitly describing its removal. | -| Confirmed SHM collision root cause | A focused SJC dev test placed two independent LMCache 0.5.3 standalone Pods on node `10.0.103.182`, with disjoint ports and instance IDs but no `--shm-name`. Both ran as PID 1: A created `/dev/shm/lmcache_l1_pool_1` at inode `14092`; B unlinked that name and recreated inode `14097`; A retained a mapping to deleted inode `14092`. The dedicated namespace was deleted and no control-plane object changed. | +| Confirmed historical SHM collision root cause | A focused SJC CPU dev test placed two independent LMCache 0.5.3 standalone Pods on node `10.0.103.182`, with disjoint ports and instance IDs but no `--shm-name`. Both ran as PID 1: A created `/dev/shm/lmcache_l1_pool_1` at inode `14092`; B unlinked that name and recreated inode `14097`; A retained a mapping to deleted inode `14092`. This proves the collision in the former CPU/engine-driven profile, not in the current CUDA-only profile. The dedicated namespace was deleted and no control-plane object changed. | | UID-scoped SHM implementation | `git diff --check`, gate-script Python syntax parsing, `go test ./...`, `make verify-samples` (27 passed, one explicit skip), `make cover-check`, and complete `make ci` passed. Tests cover deterministic full-UID naming, distinct UIDs, unsafe/oversized UID rejection, exact server args/annotation, declared and effective startup-gate checks, status exclusion, automatic replacement of an existing server missing the managed SHM identity, and PodLocal regression. Fresh-install could not run locally because no kind node image/cluster is cached and the standard cert-manager bootstrap requires the known-unavailable GitHub path; real envtest API-server admission did run. | | UID-directory mount hardening | Local validation passed `git diff --check`, `go test ./...`, `make verify-samples` (27 passed, one explicit skip), `make cover-check` at 90.0%, complete `make ci`, and a fresh Kubernetes 1.32 kind install smoke. Tests cover stable and distinct full-UID host paths, exact `DirectoryOrCreate` mounts in server and engine Pods, rejection of the whole host `/dev/shm` and another backend's directory, status exclusion, automatic stale-server replacement, PodLocal regression, current samples, and idempotent re-apply. The live kind node physically created `/dev/shm/inference-cache/` as `root:root 0755`; focused SJC GPU evidence for the pinned root engine/server identities is recorded below. The temporary kind cluster was deleted. Arbitrary non-root runtime compatibility remains outside this validation. | | Focused live SHM remediation | On SJC Kubernetes 1.31.1, two raw LMCache 0.5.3 servers with distinct explicit UID-style names ran together on CPU node `10.0.103.182`: A remained at inode `14166`, while B used inode `14176` and then `14181` after replacement. A's mapping remained named and unchanged throughout. The current controller then created two independent NodeLocal pools on the same node with real CacheBackend UIDs `61a98028-653a-4cfb-83ef-2dc3a9321b50` and `47205c02-6d7d-45aa-bf40-0e1882346309`: their effective names and inodes were respectively `14196` and `14200`; replacing only B moved it to `14207` while A stayed `14196`; deleting and recreating B's engine demand inside idle retention reused B's same server Pod UID and inode `14207`. Both pools reported server/engine coverage `1/1/1/1`. All CPU test resources were deleted. | @@ -1067,13 +1084,16 @@ mechanism and was not supplied by CacheBackend. before expiry and removes it only after expiry (or immediately at zero). - [x] Before/after MP `/metrics` snapshots on the representative common server data path, in addition to the separate vLLM and SGLang functional runs. -- [x] Two CacheBackends with disjoint ports retain distinct UID-scoped SHM - names and inodes on one live node through server startup/replacement and - idle-retention reuse. +- [x] Historical CPU-profile test: two CacheBackends with disjoint ports retain + distinct UID-scoped SHM names and inodes on one live node through server + startup/replacement and idle-retention reuse. - [x] Two GPU engines independently pass store → engine GPU/local KV clear → L1 - retrieve and worker restart/reconnect; exact effective SHM identities, + retrieve and worker restart/reconnect; the former exact effective SHM identities, disjoint worker registrations, and an identical-prompt cold miss prove that neither pool retrieved or registered the other pool's object/worker. +- [ ] Re-run NodeLocal vLLM and SGLang store/reset-or-flush/retrieve with the + explicit `lmcache_driven` + non-lazy + empty-`shm_name` profile; verify + startup fails before health when the full pinned allocation is unavailable. ### Exit criteria @@ -1086,9 +1106,10 @@ mechanism and was not supplied by CacheBackend. - [x] Cross-`CacheBackend` sharing remains rejected by the name+UID demand filter. - [x] Required vLLM and SGLang functional matrix evidence is supplemented by before/after metrics from their common standalone MP-server data path. -- [x] UID-scoped SHM object-name isolation passes focused live-node and GPU - validation; missing or mismatched name identity never counts as Ready or - covered and never admits an engine. +- [x] The current implementation no longer creates or treats a UID-scoped + POSIX SHM L1 name as identity; status and the startup gate instead require + the explicit CUDA/non-lazy/empty-`shm_name` profile. +- [ ] Focused GPU validation passes for that revised allocation profile. - [x] UID-directory mounts pass focused SJC GPU validation for one pool, two same-node engines, and two co-located CacheBackends; the pinned root engine/server identities successfully use the kubelet-created @@ -1143,8 +1164,9 @@ The migration is complete only when all of the following are true: including LMCache access through the kubelet-created directory and isolation between two co-located CacheBackends. -Phase 8 production behavior and its UID-scoped object-name and directory-mount -validation are complete. This remains the final phase of the migration; the +Phase 8 remains the final phase of the migration. Its engine-first topology and +UID-directory evidence are complete; the revised explicit CUDA allocation +profile remains pending focused GPU revalidation. The future capability profiles below are independent backlog items rather than additional phases. @@ -1157,35 +1179,33 @@ A profile enters the supported matrix only after its API contract, GPU correctness, failure-recovery, security, and operability gates pass against immutable artifacts. The numbered items below are the future-work backlog. -### 1. NodeLocal hostile-process isolation and aggregate SHM capacity - -Phase 8 owns the accidental-collision fix: every NodeLocal pool uses a -deterministic full-UID `--shm-name` and mounts only its full-UID host directory; -startup/status verify that exact identity. This future item covers stronger -security and capacity guarantees that mount scoping cannot provide. - -- [ ] Define the hostile-process boundary. A UID-directory mount does not stop - host root, a privileged Pod, or another process that independently mounts - the parent host `/dev/shm` from deliberately opening or unlinking another - pool. Decide whether production support requires distinct Unix - identities, admission-enforced node separation, or a combination. -- [ ] Account for aggregate host `/dev/shm` capacity across co-located pools and - expose actionable admission/Pending/status behavior before publishing a - supported multi-pool capacity envelope. -- [ ] Define ownership-verified reclamation for an idle/deleted pool's UID - directory. The focused GPU run found CUDA, torch, and semaphore files - still present after every engine and server Pod had exited. This SHM - lifetime behavior pre-dates UID directories: under the former whole-host - mount the same classes of objects shared the unowned `/dev/shm` root and - could not be attributed or safely reclaimed. UID scoping makes that - existing lifecycle problem ownership-visible; it adds only the directory - entry itself. Without safe last-user cleanup, the objects and their tmpfs - pages can remain until explicit node cleanup or reboot. Cleanup must prove - that no selected engine or server still uses the directory and must never - traverse or delete another CacheBackend UID. -- [ ] Validate the selected tenant boundary with unauthorized open/unlink tests; - until then, multiple pools on one node are supported only inside one - mutually trusted node domain. +### 1. NodeLocal shared `/dev/shm` and UID-directory reclamation + +- [x] Use `lmcache_driven`, non-lazy allocation, and empty `shm_name`, keeping + `l1Capacity` in private pinned host memory rather than POSIX SHM. +- [x] Retain the exact `/dev/shm/inference-cache/` hostPath for + PyTorch/CUDA IPC lifetime objects. It is an IPC correctness boundary, not + L1 storage or capacity control. +- [x] Limit inference-cache ownership to its rendered Server/helper containers, + startup gate, and exact LMCache configuration and UID-volume wiring. The + Engine Pod and unrelated workloads retain their existing policy owners. +- [x] Disable privilege escalation, drop all capabilities, and use + `RuntimeDefault` seccomp for managed containers. The NodeLocal Server also + uses `hostIPC: false`, disables service-account token automount, declares + only its MP/HTTP host ports, and is the only renderer enabling + `hostNetwork`. +- [x] Install no namespace security exemption or cluster-wide Pod policy. + Co-located pools remain within one mutually trusted node domain; UID + directories isolate managed pools but are not hostile-process boundaries. +- [x] Use declared memory requests plus non-lazy startup as the capacity + contract. Allocation failure keeps the Server unhealthy and the Engine + gate closed; no aggregate pinned-memory prediction, SHM quota, or separate + capacity API is planned. +- [x] Reclaim an idle/deleted pool with a gated, one-shot cleanup Pod after no + Pod uses its exact UID hostPath. Cleanup mounts only that UID, clears its + contents with a dedicated inference-cache helper image, and blocks Server + recreation until completion; failed helpers have a bounded retry count, + and a finalizer covers CacheBackend deletion. ### 2. MP client/server compatibility signaling diff --git a/docs/design/sglang-lmcache-mp-mode.md b/docs/design/sglang-lmcache-mp-mode.md index d4b8d0b5..f402c223 100644 --- a/docs/design/sglang-lmcache-mp-mode.md +++ b/docs/design/sglang-lmcache-mp-mode.md @@ -143,10 +143,10 @@ engine starts. For **NodeLocal**, the controller creates one direct server Pod per active engine node using exact-node affinity, host networking, declared host ports, -and a UID-scoped `--shm-name`. The engine receives a same-node endpoint derived -from the Downward API and a startup gate that verifies server ownership, -generation, shared-memory identity, and health. No load-balanced MP Service is -created. +and the explicit CUDA profile (`lmcache_driven`, non-lazy, empty `shm_name`). +The engine receives a same-node endpoint derived from the Downward API and a +startup gate that verifies server ownership, generation, effective allocation +profile, and health. No load-balanced MP Service is created. Redis is an optional remote tier for either topology. Managed Redis is a single-replica Deployment and Service; external Redis publishes the declared @@ -252,9 +252,8 @@ serving component), and Phase 2 must *validate* rather than assume it: Upstream's documented MP deploy is a per-node DaemonSet worker (`hostNetwork` + host `/dev/shm` + `hostIPC`) shared by all engines on the node. That is a heavier privilege posture and does not fit the per-CacheBackend, engines-anywhere model: -the shared-memory data path (CUDA-IPC / POSIX `/dev/shm`) requires the engine and -worker to share an IPC namespace + `/dev/shm`, which across separate pods means -`hostIPC` + a host-path `/dev/shm` mount on every engine pod. The same-pod +the CUDA-IPC data path requires the engine and worker to run on the same node +with the required GPU visibility and runtime IPC surfaces. The same-pod sidecar avoids all of that — one worker per engine pod, isolated, no host namespaces — at the cost of not sharing an L1 across co-located engines (they share instead through the L2, which is the cross-node path anyway). The DaemonSet @@ -267,10 +266,12 @@ nodes, but is not what the managed adapter renders. to end.** This section records the answer, because the reasoning that got here is load-bearing for anyone touching the worker's security posture. -The question was which data path the MP worker uses: **CUDA-IPC** (the worker maps -the engine's GPU KV directly — needs GPU visibility) or **POSIX `/dev/shm`** -(`non_gpu` transfer via `--shm-name` — no GPU needed). The answer is **CUDA-IPC**, -and the worker therefore **does** need to see the engine's GPU. +The question was which data path the MP worker uses: **CUDA-IPC** (the worker +maps the engine's GPU KV directly — needs GPU visibility) or the engine-driven +CPU/POSIX-SHM path. The supported answer is **CUDA-IPC**. The server and vLLM +client now select `lmcache_driven` explicitly, so `--shm-name ""` disables a +POSIX SHM-backed L1 instead of naming one; the worker still needs to see the +engine's GPU. But the anticipated consequence — "a separate sidecar cannot get the engine's GPU, so fall back to one container" — **does not hold**, which is why the clean design @@ -384,9 +385,10 @@ data plane), different resolution because the data planes differ: gives no cross-node sharing). These are different valid MP sub-configs, and the exact surface moves between LMCache versions — Phase 2 re-confirms the config wire against the version actually pinned in `VERSIONS.md`, not the quickstart. -- **`/dev/shm` sizing** — the L1 lives in `/dev/shm`; too small (default 64 MB) - silently falls back to slow pickle serialization. The shared `emptyDir` must be - `medium: Memory` and sized ≥ the L1. +- **`/dev/shm` sizing** — L1 KV bytes live in eager private pinned host memory; + `/dev/shm` carries CUDA/PyTorch auxiliary IPC objects. The shared `emptyDir` + remains `medium: Memory` with the conservative typed L1 + 1Gi budget so the + IPC path does not fall back to slow serialization. - **L2 durability/HA** — a single managed Redis is a simple default, not an HA store. Future provider-specific work may add a real managed Redis Cluster; separately, a typed `mooncake_store` L2 adapter can provide Mooncake without diff --git a/docs/operations/container-images.md b/docs/operations/container-images.md index dffe6962..ebba76d2 100644 --- a/docs/operations/container-images.md +++ b/docs/operations/container-images.md @@ -1,8 +1,8 @@ # Container Image Hardening -The repository ships three Go binaries as container images: the controller, -policy server, and KV-event subscriber. They share one multi-stage Dockerfile -at `dockerfiles/Dockerfile`. +The repository ships four Go binaries as container images: the controller, +policy server, KV-event subscriber, and NodeLocal SHM cleanup helper. They share +one multi-stage Dockerfile at `dockerfiles/Dockerfile`. ## Runtime Base @@ -11,14 +11,38 @@ the multi-architecture index digest `sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6`. The explicit Debian version and immutable digest prevent upstream aliases or tags from silently changing the runtime. The `static` variant is sufficient -because all three binaries are built with `CGO_ENABLED=0`; it contains neither +because all four binaries are built with `CGO_ENABLED=0`; it contains neither a shell nor a package manager. Each target also declares `USER 65532:65532` -and a vector-form entrypoint. +and a vector-form entrypoint. The cleanup image is still non-root by default; +its controller-rendered Pod explicitly runs the helper as UID 0 with no Linux +capabilities so it can remove IPC files created by different container users. The `golang` image appears only in the builder stage. It is not present in any shipped runtime image. BusyBox images created by reference-stack smoke scripts are short-lived test fixtures loaded into kind; they are not release targets. +The cleanup image is a platform implementation detail, configured once on the +controller with `--node-local-shm-cleanup-image=@sha256:`. +Use the cleanup image published by the same inference-cache release; individual +CacheBackends do not select or override it. + +Each GitHub release attaches `inference-cache-.yaml`, rendered from +`config/default` with the matching controller, server, and cleanup image +digests. Prefer that artifact for release installs; the checked-in manager YAML +contains an all-zero cleanup digest only as a source-tree rendering placeholder, +and the controller deliberately rejects that value. A source checkout can render +and apply the same contract with: + +```bash +make deploy \ + IMG="${CONTROLLER_IMAGE}" \ + SERVER_IMG="${SERVER_IMAGE}" \ + CLEANUP_IMG="${CLEANUP_IMAGE}" +``` + +All three variables must contain full `repository@sha256:digest` references. +Set `INSTALL_KUSTOMIZATION=overlays/server-tls` when rendering the TLS overlay. + ## Verification The source-only policy check is available without Docker: @@ -35,7 +59,7 @@ make image-build make verify-minimal-images ``` -`verify-minimal-images` checks all three targets for: +`verify-minimal-images` checks all four targets for: - the approved Distroless runtime stage; - the numeric non-root user and group `65532:65532`; @@ -50,10 +74,10 @@ while this gate enforces the runtime-image shape. ## Release Signatures and Provenance The `Release Supply Chain` workflow checks out the release tag and -unconditionally builds and publishes the controller, server, and subscriber -images from that source. It captures each immutable digest directly from -Buildx, verifies the published tag resolves to the same digest, and only then -signs and attests that digest. The workflow uses keyless Cosign, generates +unconditionally builds and publishes the controller, server, subscriber, and +NodeLocal cleanup images from that source. It captures each immutable digest +directly from Buildx, verifies the published tag resolves to the same digest, +and only then signs and attests that digest. The workflow uses keyless Cosign, generates signed SLSA v1 build provenance, pushes the provenance to GHCR, and verifies both the signature and provenance before attaching release artifacts. The Sigstore provenance bundles are also attached to the corresponding GitHub @@ -83,9 +107,10 @@ gh attestation verify "oci://${image}@${digest}" \ --predicate-type https://slsa.dev/provenance/v1 ``` -Repeat the same verification for `inference-cache-server` and -`inference-cache-subscriber`. Verification is intentionally digest-based; a -mutable release tag is never accepted as the signed subject. +Repeat the same verification for `inference-cache-server`, +`inference-cache-subscriber`, and `inference-cache-shm-cleanup`. Verification +is intentionally digest-based; a mutable release tag is never accepted as the +signed subject. See the upstream [Distroless project](https://github.com/GoogleContainerTools/distroless) for image contents, supported Debian variants, and signature-verification diff --git a/docs/operations/sbom.md b/docs/operations/sbom.md index 8dd6fc32..c23467d0 100644 --- a/docs/operations/sbom.md +++ b/docs/operations/sbom.md @@ -27,8 +27,9 @@ Override the output directory with `SBOM_DIR=/path/to/out`. make sbom-images TAG=v0.1.0 ``` -By default this builds the controller, server, and kvevent-subscriber images -locally before scanning them. To scan already-built local images, pass +By default this builds the controller, server, kvevent-subscriber, and +NodeLocal cleanup images locally before scanning them. To scan already-built +local images, pass `SBOM_IMAGE_BUILD=0`. Outputs: @@ -37,6 +38,7 @@ Outputs: dist/sbom/inference-cache-controller-v0.1.0.spdx.json dist/sbom/inference-cache-server-v0.1.0.spdx.json dist/sbom/inference-cache-subscriber-v0.1.0.spdx.json +dist/sbom/inference-cache-cleanup-v0.1.0.spdx.json ``` ## Registry Image SBOMs @@ -52,6 +54,7 @@ before scanning: ghcr.io/cachebox-project/inference-cache-controller:v0.1.0 ghcr.io/cachebox-project/inference-cache-server:v0.1.0 ghcr.io/cachebox-project/inference-cache-subscriber:v0.1.0 +ghcr.io/cachebox-project/inference-cache-shm-cleanup:v0.1.0 ``` For manifest-list images, the target discovers the platforms present in the diff --git a/docs/quickstart.md b/docs/quickstart.md index d0eb0dbd..1e51bd29 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -52,6 +52,8 @@ To share one MP L1 across several engine Pods on each GPU node, use the focused NodeLocal samples for [vLLM](../config/samples/cachebackend-vllm-nodelocal-host-only.yaml) or [SGLang](../config/samples/cachebackend-sglang-nodelocal-host-only.yaml). +Configure the controller's `--node-local-shm-cleanup-image` once with the +cleanup helper digest published by the same inference-cache release. NodeLocal is an advanced host-bound topology. The inference system schedules each engine without CacheBackend changing its placement; the controller then creates one shared server Pod on every node that actually has an active diff --git a/docs/reference-stack/GPU-RUNBOOK.md b/docs/reference-stack/GPU-RUNBOOK.md index e470a857..34d58cfe 100644 --- a/docs/reference-stack/GPU-RUNBOOK.md +++ b/docs/reference-stack/GPU-RUNBOOK.md @@ -72,8 +72,8 @@ Rules of thumb: | Resource | Reference (8B) | Why | |---|---|---| -| Host RAM | engine budget + MP L1 + headroom | The reference uses `l1Capacity: 4Gi`; the MP server request/limit and `/dev/shm` must cover L1 plus at least 1Gi. | -| `/dev/shm` | ≥ typed L1 + 1Gi | Shared by the engine and injected MP server; the reference uses 8Gi. | +| Host RAM | engine budget + MP L1 + headroom | The reference uses `l1Capacity: 4Gi`; the MP server request/limit must cover the eagerly allocated pinned L1 plus at least 1Gi. | +| `/dev/shm` | conservative IPC budget ≥ typed L1 + 1Gi | Shared by the engine and injected MP server for CUDA/PyTorch IPC objects; L1 KV bytes themselves are private pinned memory. The reference uses 8Gi. | | Local disk | model size × 1.5 | HF weight cache. The host-only reference does not claim a local-disk LMCache tier. | | Network | 100 Gb+ RDMA for multi-node | Only if you later shard across nodes; single-node TP uses NVLink. | | Driver/runtime | NVIDIA driver + Container Toolkit; `nvidia` default Docker runtime | So local or managed-cluster pods can request `nvidia.com/gpu`. | @@ -96,9 +96,11 @@ kubectl get nodes -o json | jq '.items[].status.allocatable["nvidia.com/gpu"]' kubectl create namespace cache-substrate kubectl -n cache-substrate create secret generic hf-token --from-literal=token="$HF_TOKEN" -# 2. Install inference-cache first, then apply. Replace the deliberately -# non-pullable engine-image placeholder before creating the Deployment. -kubectl apply -k ../../config/default +# 2. Install inference-cache from its digest-pinned release manifest first, +# then apply. Replace the deliberately non-pullable engine-image placeholder +# before creating the Deployment. +RELEASE_TAG=vX.Y.Z +kubectl apply -f "inference-cache-${RELEASE_TAG}.yaml" kubectl apply -f manifests/namespace.yaml -f manifests/deployment.yaml -f manifests/service.yaml kubectl -n cache-substrate rollout status deploy/vllm-lmcache-llama-8b --timeout=20m ``` @@ -127,4 +129,4 @@ like"): subscribe with `scripts/kv_events_subscriber.py` and fire | Loads but low throughput / frequent recompute | KV pool too small | bigger card, raise `gpu_memory_utilization`, or lean on LMCache offload | | `tensor-parallel-size` mismatch / hang at startup | TP ≠ GPU count, or heads not divisible | set TP = `nvidia.com/gpu`; check head count divisibility | | NCCL / loader hang on multi-GPU | small `/dev/shm`, or no NVLink (multi-GPU VM) | raise `/dev/shm`; use a bare-metal NVLink shape for TP | -| Host OOM with LMCache enabled | typed `l1Capacity` + 1Gi headroom exceeds the sidecar/pod memory budget | lower `l1Capacity` consistently or raise the MP-server request/limit and `/dev/shm` size | +| Host OOM with LMCache enabled | typed `l1Capacity` + 1Gi headroom exceeds the server/container memory budget | lower `l1Capacity` consistently or raise the MP-server request/limit; size `/dev/shm` separately for runtime IPC | diff --git a/docs/reference-stack/README.md b/docs/reference-stack/README.md index 163728d2..e37e80ed 100644 --- a/docs/reference-stack/README.md +++ b/docs/reference-stack/README.md @@ -30,11 +30,12 @@ native sidecar contract, so inventing a chart mapping would be unsafe. ## Install the operator -Install `config/default` (or the equivalent published release) before creating -the reference `CacheBackend`: +Install the digest-pinned manifest attached to the selected inference-cache +release before creating the reference `CacheBackend`: ```bash -kubectl apply -k config/default +RELEASE_TAG=vX.Y.Z +kubectl apply -f "inference-cache-${RELEASE_TAG}.yaml" kubectl -n inference-cache-system wait \ --for=condition=Available deployment --all --timeout=180s ``` diff --git a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml index b7f316fd..087aeab3 100644 --- a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml +++ b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml @@ -221,8 +221,9 @@ spec: volumes: - name: cache-home emptyDir: {} - # The MP L1 tier lives here; shared by the engine and injected server. Sized - # to cover the engine's own /dev/shm use plus the 4Gi typed L1 budget. + # Shared by the engine and injected server for a conservative + # CUDA/PyTorch IPC budget. The 4Gi LMCache L1 itself is eagerly allocated + # private pinned host memory by the server. - name: shm emptyDir: medium: Memory diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index 99ce72fb..9950efc1 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -103,6 +103,13 @@ kubectl -n cert-manager wait --for=condition=Available deployment --all --timeou tmpdir="$(mktemp -d)" cp -R config "$tmpdir/config" +# This smoke never starts the cleanup helper. Replace the source placeholder +# with a syntactically valid test-only digest so controller startup exercises +# the same fail-closed option validation as a rendered release install. +sed -i.bak \ + 's|sha256:0000000000000000000000000000000000000000000000000000000000000000|sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb|' \ + "$tmpdir/config/manager/manager.yaml" +rm -f "$tmpdir/config/manager/manager.yaml.bak" ( cd "$tmpdir/config/default" if command -v kustomize >/dev/null 2>&1; then @@ -367,6 +374,7 @@ EOF grep -Fq 'lmcache-mp-server' "$pod_json" || fail "native MP sidecar was not injected" grep -Fq 'LMCacheMPConnector' "$pod_json" || fail "vLLM MP connector was not injected" grep -Fq 'lmcache.mp.host' "$pod_json" || fail "MP loopback host was not injected" +grep -Fq 'lmcache.mp.mp_transfer_mode' "$pod_json" || fail "vLLM MP connector does not pin lmcache-driven transfer" for retired in LMCacheConnectorV1 LMCACHE_REMOTE_URL LMCACHE_REMOTE_SERDE 'lm://'; do if grep -Fq "$retired" "$pod_json"; then fail "admitted Pod contains retired wire: $retired" @@ -391,7 +399,8 @@ spec: command: ["sh", "-c", "sleep 3600"] EOF grep -Fq 'lmcache-node-local-gate' "$node_pod_json" || fail "NodeLocal ownership/health startup gate was not injected" -grep -Fq 'EXPECTED_SHM_NAME' "$node_pod_json" || fail "NodeLocal startup gate does not verify UID-scoped shared memory" +grep -Fq 'supported_transfer_mode' "$node_pod_json" || fail "NodeLocal startup gate does not verify lmcache-driven transfer" +grep -Fq 'use_lazy' "$node_pod_json" || fail "NodeLocal startup gate does not verify eager L1 allocation" grep -Fq 'INFERENCECACHE_NODE_IP' "$node_pod_json" || fail "NodeLocal hostIP Downward API was not injected" grep -Fq 'status.hostIP' "$node_pod_json" || fail "NodeLocal endpoint is not derived from status.hostIP" grep -Fq 'kubernetes.io/os' "$node_pod_json" || fail "inference-owned nodeSelector was not preserved" @@ -429,14 +438,15 @@ node_local_host_ipc="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o j || fail "NodeLocal server does not mount its UID-scoped host SHM directory" [ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.volumes[0].hostPath.type}')" = "DirectoryOrCreate" ] \ || fail "NodeLocal server UID-scoped SHM hostPath is not DirectoryOrCreate" -node_local_shm_name="lmcache_l1_pool_inferencecache_${node_local_backend_uid}" -[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.metadata.annotations.inferencecache\.io/node-local-shm-name}')" = "$node_local_shm_name" ] \ - || fail "NodeLocal server does not carry its UID-scoped shared-memory identity" -node_local_shm_arg="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" \ +node_local_runtime_profile="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" \ -o jsonpath='{range .spec.containers[?(@.name=="lmcache-mp-server")].args[*]}{@}{"\n"}{end}' | \ - awk 'previous == "--shm-name" && value == "" { value = $0 } { previous = $0 } END { print value }')" -[ "$node_local_shm_arg" = "$node_local_shm_name" ] \ - || fail "NodeLocal server does not pass its UID-scoped --shm-name: $node_local_shm_arg" + awk 'previous == "--supported-transfer-mode" && $0 == "lmcache_driven" { transfer = 1 } + $0 == "--no-l1-use-lazy" { eager = 1 } + previous == "--shm-name" { shm_seen = 1; if ($0 == "") shm_empty = 1 } + { previous = $0 } + END { if (transfer && eager && shm_seen && shm_empty) print "ok" }')" +[ "$node_local_runtime_profile" = "ok" ] \ + || fail "NodeLocal server does not declare lmcache_driven + non-lazy + empty shm-name" node_local_ports="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{range .spec.containers[?(@.name=="lmcache-mp-server")].ports[*]}{.containerPort}:{.hostPort}{" "}{end}')" [ "$node_local_ports" = "5556:5556 8081:8081 " ] || fail "NodeLocal host ports were not declared: $node_local_ports" node_affinity_target="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchFields[0].values[0]}')" diff --git a/hack/render-release-install.sh b/hack/render-release-install.sh new file mode 100755 index 00000000..1f771f48 --- /dev/null +++ b/hack/render-release-install.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${CONTROLLER_IMAGE:?CONTROLLER_IMAGE is required}" +: "${SERVER_IMAGE:?SERVER_IMAGE is required}" +: "${CLEANUP_IMAGE:?CLEANUP_IMAGE is required}" +: "${OUTPUT_FILE:?OUTPUT_FILE is required}" + +kustomize_cmd="${KUSTOMIZE_CMD:-kustomize}" +kustomization_path="${KUSTOMIZATION_PATH:-default}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +placeholder='ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:0000000000000000000000000000000000000000000000000000000000000000' +zero_digest='sha256:0000000000000000000000000000000000000000000000000000000000000000' + +for ref in "$CONTROLLER_IMAGE" "$SERVER_IMAGE" "$CLEANUP_IMAGE"; do + if [[ ! "$ref" =~ ^[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}$ || "$ref" == *"@$zero_digest" ]]; then + echo "install image is not digest-pinned: $ref" >&2 + exit 1 + fi +done +if ! command -v "$kustomize_cmd" >/dev/null 2>&1; then + echo "kustomize is unavailable: $kustomize_cmd" >&2 + exit 1 +fi +if [[ "$kustomization_path" == /* || "$kustomization_path" == *..* ]]; then + echo "KUSTOMIZATION_PATH must stay within config: $kustomization_path" >&2 + exit 1 +fi + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT +cp -R "$repo_root/config" "$workdir/config" + +manager="$workdir/config/manager/manager.yaml" +if [[ "$(grep -Fc -- "$placeholder" "$manager")" != 1 ]]; then + echo "cleanup image placeholder is missing or duplicated in $manager" >&2 + exit 1 +fi +sed "s|$placeholder|$CLEANUP_IMAGE|" "$manager" >"$manager.tmp" +mv "$manager.tmp" "$manager" + +render_dir="$workdir/config/$kustomization_path" +if [[ ! -f "$render_dir/kustomization.yaml" ]]; then + echo "kustomization is unavailable: config/$kustomization_path" >&2 + exit 1 +fi +( + cd "$workdir/config/default" + "$kustomize_cmd" edit set image \ + "controller=$CONTROLLER_IMAGE" \ + "server=$SERVER_IMAGE" +) + +mkdir -p "$(dirname "$OUTPUT_FILE")" +"$kustomize_cmd" build "$render_dir" >"$OUTPUT_FILE" + +for ref in "$CONTROLLER_IMAGE" "$SERVER_IMAGE" "$CLEANUP_IMAGE"; do + grep -Fq -- "$ref" "$OUTPUT_FILE" || { + echo "rendered install manifest is missing $ref" >&2 + exit 1 + } +done +if grep -Fq -- "$placeholder" "$OUTPUT_FILE"; then + echo "rendered install manifest retained the cleanup image placeholder" >&2 + exit 1 +fi diff --git a/hack/render-release-install_test.sh b/hack/render-release-install_test.sh new file mode 100755 index 00000000..704afdaf --- /dev/null +++ b/hack/render-release-install_test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT + +controller='ghcr.io/cachebox-project/inference-cache-controller@sha256:1111111111111111111111111111111111111111111111111111111111111111' +server='ghcr.io/cachebox-project/inference-cache-server@sha256:2222222222222222222222222222222222222222222222222222222222222222' +cleanup='ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:3333333333333333333333333333333333333333333333333333333333333333' +output="$workdir/install.yaml" + +CONTROLLER_IMAGE="$controller" \ +SERVER_IMAGE="$server" \ +CLEANUP_IMAGE="$cleanup" \ +OUTPUT_FILE="$output" \ +KUSTOMIZE_CMD="${KUSTOMIZE_CMD:?KUSTOMIZE_CMD is required}" \ + hack/render-release-install.sh + +for ref in "$controller" "$server" "$cleanup"; do + grep -Fq -- "$ref" "$output" +done +if grep -Fq -- 'sha256:0000000000000000000000000000000000000000000000000000000000000000' "$output"; then + echo "rendered install retained an all-zero digest" >&2 + exit 1 +fi +grep -Fq -- 'sha256:0000000000000000000000000000000000000000000000000000000000000000' config/manager/manager.yaml + +CONTROLLER_IMAGE="$controller" SERVER_IMAGE="$server" CLEANUP_IMAGE="$cleanup" \ +OUTPUT_FILE="$workdir/tls.yaml" KUSTOMIZATION_PATH=overlays/server-tls KUSTOMIZE_CMD="$KUSTOMIZE_CMD" \ + hack/render-release-install.sh +grep -Fq -- '--tls-cert-file=/var/run/secrets/tls/tls.crt' "$workdir/tls.yaml" + +if CONTROLLER_IMAGE='controller:latest' SERVER_IMAGE="$server" CLEANUP_IMAGE="$cleanup" \ + OUTPUT_FILE="$workdir/invalid.yaml" KUSTOMIZE_CMD="$KUSTOMIZE_CMD" \ + hack/render-release-install.sh >/dev/null 2>&1; then + echo "expected a mutable controller image to fail" >&2 + exit 1 +fi + +if CONTROLLER_IMAGE="$controller" SERVER_IMAGE="$server" \ + CLEANUP_IMAGE='ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:0000000000000000000000000000000000000000000000000000000000000000' \ + OUTPUT_FILE="$workdir/zero.yaml" KUSTOMIZE_CMD="$KUSTOMIZE_CMD" \ + hack/render-release-install.sh >/dev/null 2>&1; then + echo "expected an all-zero cleanup digest to fail" >&2 + exit 1 +fi + +echo "release install rendering tests passed" diff --git a/hack/resolve-release-image-digests.sh b/hack/resolve-release-image-digests.sh index 6352e0b5..778135c3 100755 --- a/hack/resolve-release-image-digests.sh +++ b/hack/resolve-release-image-digests.sh @@ -17,6 +17,7 @@ docker_cmd="${DOCKER_BUILD_CMD:-docker}" controller_repo="${CONTROLLER_IMAGE_REPO:-ghcr.io/cachebox-project/inference-cache-controller}" server_repo="${SERVER_IMAGE_REPO:-ghcr.io/cachebox-project/inference-cache-server}" subscriber_repo="${SUBSCRIBER_IMAGE_REPO:-ghcr.io/cachebox-project/inference-cache-subscriber}" +cleanup_repo="${CLEANUP_IMAGE_REPO:-ghcr.io/cachebox-project/inference-cache-shm-cleanup}" if ! command -v "$docker_cmd" >/dev/null 2>&1; then echo "registry client is unavailable: $docker_cmd" >&2 @@ -49,3 +50,4 @@ resolve_digest() { resolve_digest controller "$controller_repo" resolve_digest server "$server_repo" resolve_digest subscriber "$subscriber_repo" +resolve_digest cleanup "$cleanup_repo" diff --git a/hack/resolve-release-image-digests_test.sh b/hack/resolve-release-image-digests_test.sh index 29805338..a74e9b63 100755 --- a/hack/resolve-release-image-digests_test.sh +++ b/hack/resolve-release-image-digests_test.sh @@ -41,6 +41,9 @@ case "$ref" in ghcr.io/cachebox-project/inference-cache-subscriber:v1.2.3) digit=3 ;; + ghcr.io/cachebox-project/inference-cache-shm-cleanup:v1.2.3) + digit=4 + ;; *) echo "unexpected image ref: $ref" >&2 exit 2 @@ -62,7 +65,8 @@ actual="$(RELEASE_TAG=v1.2.3 hack/resolve-release-image-digests.sh)" expected="$(printf '%s\n' \ 'controller=sha256:0000000000000000000000000000000000000000000000000000000000000001' \ 'server=sha256:0000000000000000000000000000000000000000000000000000000000000002' \ - 'subscriber=sha256:0000000000000000000000000000000000000000000000000000000000000003')" + 'subscriber=sha256:0000000000000000000000000000000000000000000000000000000000000003' \ + 'cleanup=sha256:0000000000000000000000000000000000000000000000000000000000000004')" if [[ "$actual" != "$expected" ]]; then echo "unexpected digest output" >&2 diff -u <(printf '%s\n' "$expected") <(printf '%s\n' "$actual") >&2 diff --git a/hack/sbom-registry-smoke.sh b/hack/sbom-registry-smoke.sh index 550e95af..78e756a6 100755 --- a/hack/sbom-registry-smoke.sh +++ b/hack/sbom-registry-smoke.sh @@ -42,7 +42,8 @@ if [ "$1" = "buildx" ] && [ "$2" = "imagetools" ] && [ "$3" = "inspect" ]; then case "$ref" in ghcr.io/cachebox-project/inference-cache-controller:ci-*|\ ghcr.io/cachebox-project/inference-cache-server:ci-*|\ - ghcr.io/cachebox-project/inference-cache-subscriber:ci-*) ;; + ghcr.io/cachebox-project/inference-cache-subscriber:ci-*|\ + ghcr.io/cachebox-project/inference-cache-shm-cleanup:ci-*) ;; *) echo "unexpected inspect ref: $ref" >&2; exit 2 ;; esac case "${DOCKER_FAKE_MODE:-missing}" in @@ -106,6 +107,7 @@ if [ "$1" = "buildx" ] && [ "$2" = "build" ]; then ghcr.io/cachebox-project/inference-cache-controller:ci-*) expected_target=controller ;; ghcr.io/cachebox-project/inference-cache-server:ci-*) expected_target=server ;; ghcr.io/cachebox-project/inference-cache-subscriber:ci-*) expected_target=subscriber ;; + ghcr.io/cachebox-project/inference-cache-shm-cleanup:ci-*) expected_target=cleanup ;; *) echo "unexpected build tag: $tag" >&2; exit 2 ;; esac test -n "$metadata" @@ -154,7 +156,8 @@ if [ -z "$platform" ]; then case "$source" in registry:ghcr.io/cachebox-project/inference-cache-controller@sha256:4444444444444444444444444444444444444444444444444444444444444444|\ registry:ghcr.io/cachebox-project/inference-cache-server@sha256:4444444444444444444444444444444444444444444444444444444444444444|\ - registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:4444444444444444444444444444444444444444444444444444444444444444) ;; + registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:4444444444444444444444444444444444444444444444444444444444444444|\ + registry:ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:4444444444444444444444444444444444444444444444444444444444444444) ;; *) echo "unexpected syft source/platform: $source $platform" >&2; exit 2 ;; esac else @@ -165,15 +168,20 @@ else registry:ghcr.io/cachebox-project/inference-cache-server@sha256:1111111111111111111111111111111111111111111111111111111111111111\|linux/arm64|\ registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:1111111111111111111111111111111111111111111111111111111111111111\|linux/amd64|\ registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:1111111111111111111111111111111111111111111111111111111111111111\|linux/arm64|\ + registry:ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:1111111111111111111111111111111111111111111111111111111111111111\|linux/amd64|\ + registry:ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:1111111111111111111111111111111111111111111111111111111111111111\|linux/arm64|\ registry:ghcr.io/cachebox-project/inference-cache-controller@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/amd64|\ registry:ghcr.io/cachebox-project/inference-cache-controller@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/arm64|\ registry:ghcr.io/cachebox-project/inference-cache-server@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/amd64|\ registry:ghcr.io/cachebox-project/inference-cache-server@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/arm64|\ registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/amd64|\ registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/arm64|\ + registry:ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/amd64|\ + registry:ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:2222222222222222222222222222222222222222222222222222222222222222\|linux/arm64|\ registry:ghcr.io/cachebox-project/inference-cache-controller@sha256:3333333333333333333333333333333333333333333333333333333333333333\|linux/amd64|\ registry:ghcr.io/cachebox-project/inference-cache-server@sha256:3333333333333333333333333333333333333333333333333333333333333333\|linux/amd64|\ - registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:3333333333333333333333333333333333333333333333333333333333333333\|linux/amd64) ;; + registry:ghcr.io/cachebox-project/inference-cache-subscriber@sha256:3333333333333333333333333333333333333333333333333333333333333333\|linux/amd64|\ + registry:ghcr.io/cachebox-project/inference-cache-shm-cleanup@sha256:3333333333333333333333333333333333333333333333333333333333333333\|linux/amd64) ;; *) echo "unexpected syft source/platform: $source $platform" >&2; exit 2 ;; esac fi @@ -201,7 +209,7 @@ if PATH="$fakebin:$PATH" DOCKER_FAKE_MODE=denied make sbom-registry-images TAG=" fi PATH="$fakebin:$PATH" DOCKER_FAKE_MODE=existing make sbom-registry-images TAG="$IMAGE_TAG" SBOM_DIR="$outdir/existing" SBOM_IMAGE_CONTEXT=. SBOM_DOCKERFILE=dockerfiles/Dockerfile -for component in controller server subscriber; do +for component in controller server subscriber cleanup; do for platform in linux_amd64 linux_arm64; do sbom="$outdir/existing/inference-cache-${component}-${platform}-${IMAGE_TAG}.spdx.json" test -s "$sbom" @@ -210,7 +218,7 @@ for component in controller server subscriber; do done PATH="$fakebin:$PATH" DOCKER_FAKE_MODE=existing-amd64 make sbom-registry-images TAG="$IMAGE_TAG" SBOM_DIR="$outdir/existing-amd64" SBOM_IMAGE_CONTEXT=. SBOM_DOCKERFILE=dockerfiles/Dockerfile -for component in controller server subscriber; do +for component in controller server subscriber cleanup; do sbom="$outdir/existing-amd64/inference-cache-${component}-linux_amd64-${IMAGE_TAG}.spdx.json" test -s "$sbom" jq -e '.spdxVersion and ((.packages | type) == "array") and ((.packages | length) > 0)' "$sbom" >/dev/null @@ -218,7 +226,7 @@ for component in controller server subscriber; do done PATH="$fakebin:$PATH" DOCKER_FAKE_MODE=existing-single make sbom-registry-images TAG="$IMAGE_TAG" SBOM_DIR="$outdir/existing-single" SBOM_IMAGE_CONTEXT=. SBOM_DOCKERFILE=dockerfiles/Dockerfile -for component in controller server subscriber; do +for component in controller server subscriber cleanup; do sbom="$outdir/existing-single/inference-cache-${component}-${IMAGE_TAG}.spdx.json" test -s "$sbom" jq -e '.spdxVersion and ((.packages | type) == "array") and ((.packages | length) > 0)' "$sbom" >/dev/null @@ -227,7 +235,7 @@ for component in controller server subscriber; do done PATH="$fakebin:$PATH" DOCKER_FAKE_MODE=missing make sbom-registry-images TAG="$IMAGE_TAG" SBOM_DIR="$outdir/missing" SBOM_REGISTRY_PUBLISH_MISSING=1 SBOM_IMAGE_CONTEXT=. SBOM_DOCKERFILE=dockerfiles/Dockerfile -for component in controller server subscriber; do +for component in controller server subscriber cleanup; do for platform in linux_amd64 linux_arm64; do sbom="$outdir/missing/inference-cache-${component}-${platform}-${IMAGE_TAG}.spdx.json" test -s "$sbom" @@ -236,7 +244,7 @@ for component in controller server subscriber; do done PATH="$fakebin:$PATH" DOCKER_FAKE_MODE=missing-notfound make sbom-registry-images TAG="$IMAGE_TAG" SBOM_DIR="$outdir/missing-notfound" SBOM_REGISTRY_PUBLISH_MISSING=1 SBOM_IMAGE_CONTEXT=. SBOM_DOCKERFILE=dockerfiles/Dockerfile -for component in controller server subscriber; do +for component in controller server subscriber cleanup; do for platform in linux_amd64 linux_arm64; do sbom="$outdir/missing-notfound/inference-cache-${component}-${platform}-${IMAGE_TAG}.spdx.json" test -s "$sbom" diff --git a/hack/verify-minimal-images.sh b/hack/verify-minimal-images.sh index ad45e1f0..36d689dc 100755 --- a/hack/verify-minimal-images.sh +++ b/hack/verify-minimal-images.sh @@ -31,12 +31,13 @@ if [ ! -f "$dockerfile" ]; then exit 1 fi -targets=(controller server subscriber) -entrypoints=(/controller /server /kvevent-subscriber) +targets=(controller server subscriber cleanup) +entrypoints=(/controller /server /kvevent-subscriber /node-local-shm-cleanup) images=( "${IMG:-ghcr.io/cachebox-project/inference-cache-controller:dev}" "${SERVER_IMG:-ghcr.io/cachebox-project/inference-cache-server:dev}" "${SUBSCRIBER_IMG:-ghcr.io/cachebox-project/inference-cache-subscriber:dev}" + "${CLEANUP_IMG:-ghcr.io/cachebox-project/inference-cache-shm-cleanup:dev}" ) stage_base() { diff --git a/hack/verify-minimal-images_test.sh b/hack/verify-minimal-images_test.sh index d21f5c43..267ba8f4 100755 --- a/hack/verify-minimal-images_test.sh +++ b/hack/verify-minimal-images_test.sh @@ -18,6 +18,7 @@ FROM golang:1.26.6 AS builder FROM gcr.io/distroless/static-debian13:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 AS controller FROM gcr.io/distroless/static-debian13:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 AS server FROM gcr.io/distroless/static-debian13:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 AS subscriber +FROM gcr.io/distroless/static-debian13:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 AS cleanup EOF expect_failure() { @@ -71,6 +72,7 @@ if [ "$1" = "image" ] && [ "$2" = "inspect" ]; then controller:test) echo '["/controller"]' ;; server:test) echo '["/server"]' ;; subscriber:test) echo '["/kvevent-subscriber"]' ;; + cleanup:test) echo '["/node-local-shm-cleanup"]' ;; *) echo "unexpected image: $image" >&2; exit 2 ;; esac ;; @@ -84,6 +86,7 @@ if [ "$1" = "create" ]; then controller:test) echo cid-controller ;; server:test) echo cid-server ;; subscriber:test) echo cid-subscriber ;; + cleanup:test) echo cid-cleanup ;; *) echo "unexpected create image: $2" >&2; exit 2 ;; esac exit 0 @@ -99,6 +102,7 @@ if [ "$1" = "export" ] && [ "$2" = "-o" ]; then controller) payload="$root/controller" ;; server) payload="$root/server" ;; subscriber) payload="$root/kvevent-subscriber" ;; + cleanup) payload="$root/node-local-shm-cleanup" ;; *) echo "unexpected container: $4" >&2; exit 2 ;; esac if [ "${FAKE_MISSING_PAYLOAD:-0}" != "1" ]; then @@ -158,6 +162,7 @@ run_runtime_check() { IMG=controller:test \ SERVER_IMG=server:test \ SUBSCRIBER_IMG=subscriber:test \ + CLEANUP_IMG=cleanup:test \ "$@" \ "$verifier" "$valid_dockerfile" } @@ -173,7 +178,7 @@ expect_failure missing-payload run_runtime_check FAKE_MISSING_PAYLOAD=1 expect_failure payload-name-as-field run_runtime_check \ FAKE_MISSING_PAYLOAD=1 \ FAKE_EXTRA_PATH_EXECUTABLE=1 \ - "FAKE_EXTRA_PATH=opt/foo controller server kvevent-subscriber" + "FAKE_EXTRA_PATH=opt/foo controller server kvevent-subscriber node-local-shm-cleanup" expect_failure non-executable-payload run_runtime_check FAKE_NON_EXECUTABLE_PAYLOAD=1 expect_failure owner-only-executable-payload run_runtime_check FAKE_OWNER_ONLY_EXECUTABLE_PAYLOAD=1 expect_failure directory-payload run_runtime_check FAKE_DIRECTORY_PAYLOAD=1 @@ -194,6 +199,7 @@ expect_failure docker-unavailable env \ IMG=controller:test \ SERVER_IMG=server:test \ SUBSCRIBER_IMG=subscriber:test \ + CLEANUP_IMG=cleanup:test \ "$verifier" "$valid_dockerfile" echo "All minimal-image verifier tests passed." diff --git a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go index 8fb0ea6d..8e83b4ba 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" "github.com/cachebox-project/inference-cache/internal/enginebinding" @@ -25,13 +26,15 @@ const ( lmCacheNodeLocalGateManagedEnv = "INFERENCECACHE_NODE_LOCAL_GATE" lmCacheNodeLocalGateManagedValue = "true" lmCacheNodeLocalShmVolumeName = "lmcache-node-shm" + lmCacheNodeLocalShmCleanupName = "lmcache-node-shm-cleanup" + lmCacheNodeLocalShmCleanupGate = "inferencecache.io/await-shm-quiescence" + lmCacheNodeLocalShmCleanupPath = "/var/run/inference-cache/shm-pool" lmCacheNodeLocalConfigVolumeName = "lmcache-node-config" lmCacheNodeLocalConfigMountPath = "/var/run/inference-cache/lmcache-node" lmCacheNodeLocalConfigFilePath = lmCacheNodeLocalConfigMountPath + "/client.yaml" lmCacheNodeIPEnv = "INFERENCECACHE_NODE_IP" - lmCacheNodeLocalShmNamePrefix = "lmcache_l1_pool_inferencecache_" lmCacheNodeLocalShmHostRoot = "/dev/shm/inference-cache" - posixShmNameMaxLength = 255 + hostPathComponentMaxLength = 255 ) // RenderLMCacheNodeLocalServerPod renders the engine-neutral server for one @@ -65,10 +68,6 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding } identity := lmCacheNodeLocalInstanceID(cache) - shmName, err := NodeLocalServerShmName(cache) - if err != nil { - return nil, err - } shmHostPath, err := NodeLocalServerShmHostPath(cache) if err != nil { return nil, err @@ -76,7 +75,9 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding args := []string{ "server", "--instance-id", identity, - "--shm-name", shmName, + lmCacheMPTransferModeArg, lmCacheMPTransferModeLMCacheDriven, + lmCacheMPL1NonLazyArg, + lmCacheMPShmNameArg, "", "--host", "$(" + lmCacheNodeIPEnv + ")", "--port", strconv.FormatInt(int64(server.Port), 10), "--http-host", "$(" + lmCacheNodeIPEnv + ")", @@ -118,7 +119,6 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding LivenessProbe: lmCacheMPHTTPProbeForPort(lmCacheMPHTTPPortName, 10, 3), SecurityContext: lmCacheMPServerSecurityContext(nil), } - labels := map[string]string{ "app.kubernetes.io/name": "lmcache-mp-server", "app.kubernetes.io/managed-by": "inference-cache-controller", @@ -131,7 +131,6 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding enginebinding.AnnotationNodeLocalOwnerUID: string(cache.UID), enginebinding.AnnotationNodeLocalGeneration: strconv.FormatInt(cache.Generation, 10), enginebinding.AnnotationNodeLocalTargetNode: nodeName, - enginebinding.AnnotationNodeLocalShmName: shmName, } pathType := corev1.HostPathDirectoryOrCreate noToken := false @@ -199,6 +198,215 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding }, nil } +func nodeLocalShmHelperSecurityContext() *corev1.SecurityContext { + no := false + zero := int64(0) + readOnlyRoot := true + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: &no, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + ReadOnlyRootFilesystem: &readOnlyRoot, + RunAsNonRoot: &no, + RunAsUser: &zero, + RunAsGroup: &zero, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} + +func nodeLocalShmHelperResources() corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("32Mi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("128Mi")}, + } +} + +// NodeLocalCleanupPodName returns the stable name of one backend/node cleanup +// intent. The name deliberately depends on the immutable backend UID rather +// than the reusable CacheBackend name. +func NodeLocalCleanupPodName(uid types.UID, nodeName string) string { + sum := sha256.Sum256([]byte(string(uid) + "\x00" + nodeName)) + return fmt.Sprintf("lmcache-shm-cleanup-%x", sum[:12]) +} + +// NodeLocalCleanupRetryPodName returns a deterministic successor name for one +// failed cleanup Pod. The failed name changes on each retry, so another +// terminal failure produces a distinct name without requiring extra state. +func NodeLocalCleanupRetryPodName(uid types.UID, nodeName, failedName string) string { + sum := sha256.Sum256([]byte(failedName)) + return fmt.Sprintf("%s-retry-%x", NodeLocalCleanupPodName(uid, nodeName), sum[:6]) +} + +// IsNodeLocalCleanupPodName accepts the initial deterministic intent and its +// controller-rendered retry successors. +func IsNodeLocalCleanupPodName(uid types.UID, nodeName, name string) bool { + base := NodeLocalCleanupPodName(uid, nodeName) + if name == base { + return true + } + suffix := strings.TrimPrefix(name, base+"-retry-") + if suffix == name || len(suffix) != 12 { + return false + } + for _, char := range suffix { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} + +// RenderLMCacheNodeLocalCleanupPod renders a gated, one-shot cleanup intent. +// It mounts only the exact UID directory; the controller removes the scheduling +// gate after all prior consumers have disappeared. +func RenderLMCacheNodeLocalCleanupPod(cache *cachev1alpha1.CacheBackend, nodeName, image string, source *corev1.Pod) (*corev1.Pod, error) { + if cache == nil || cache.UID == "" || strings.TrimSpace(image) == "" || strings.TrimSpace(nodeName) == "" || source == nil { + return nil, fmt.Errorf("render LMCache NodeLocal SHM cleanup Pod: backend UID, target node, cleanup image, and scheduling source are required") + } + shmHostPath, err := NodeLocalServerShmHostPath(cache) + if err != nil { + return nil, err + } + image = strings.TrimSpace(image) + pathType := corev1.HostPathDirectoryOrCreate + noToken := false + enableServiceLinks := false + grace := int64(5) + var imagePullSecrets []corev1.LocalObjectReference + if cache.Spec.LMCache != nil && cache.Spec.LMCache.NodeLocal != nil && cache.Spec.LMCache.NodeLocal.Scheduling != nil { + imagePullSecrets = append(imagePullSecrets, cache.Spec.LMCache.NodeLocal.Scheduling.ImagePullSecrets...) + } + cleanup := corev1.Container{ + Name: lmCacheNodeLocalShmCleanupName, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/node-local-shm-cleanup"}, + Args: []string{lmCacheNodeLocalShmCleanupPath}, + Resources: nodeLocalShmHelperResources(), + VolumeMounts: []corev1.VolumeMount{{Name: lmCacheNodeLocalShmVolumeName, MountPath: lmCacheNodeLocalShmCleanupPath}}, + SecurityContext: nodeLocalShmHelperSecurityContext(), + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: NodeLocalCleanupPodName(cache.UID, nodeName), + Namespace: cache.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "lmcache-shm-cleanup", + "app.kubernetes.io/managed-by": "inference-cache-controller", + enginebinding.LabelLMCacheNodeLocalCleanup: "true", + enginebinding.LabelCacheBackendUID: string(cache.UID), + }, + Annotations: map[string]string{ + enginebinding.AnnotationNodeLocalOwner: cache.Namespace + "/" + cache.Name, + enginebinding.AnnotationNodeLocalOwnerUID: string(cache.UID), + enginebinding.AnnotationNodeLocalTargetNode: nodeName, + }, + }, + Spec: corev1.PodSpec{ + HostNetwork: false, + HostPID: false, + HostIPC: false, + AutomountServiceAccountToken: &noToken, + EnableServiceLinks: &enableServiceLinks, + Affinity: exactNodeAffinity(nodeName), + Tolerations: append([]corev1.Toleration(nil), source.Spec.Tolerations...), + ImagePullSecrets: imagePullSecrets, + PriorityClassName: source.Spec.PriorityClassName, + SchedulerName: source.Spec.SchedulerName, + TerminationGracePeriodSeconds: &grace, + RestartPolicy: corev1.RestartPolicyNever, + SchedulingGates: []corev1.PodSchedulingGate{{Name: lmCacheNodeLocalShmCleanupGate}}, + Containers: []corev1.Container{cleanup}, + Volumes: []corev1.Volume{{ + Name: lmCacheNodeLocalShmVolumeName, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: shmHostPath, Type: &pathType, + }}, + }}, + }, + }, nil +} + +// RenderLMCacheNodeLocalCleanupRetryPod clones the already validated cleanup +// contract into a fresh, gated Pod. Re-adding the gate makes every retry prove +// consumer quiescence again before touching the UID directory. +func RenderLMCacheNodeLocalCleanupRetryPod(cache *cachev1alpha1.CacheBackend, failed *corev1.Pod) (*corev1.Pod, error) { + if cache == nil || cache.UID == "" || failed == nil || strings.TrimSpace(failed.Name) == "" { + return nil, fmt.Errorf("render LMCache NodeLocal SHM cleanup retry: backend UID and failed cleanup Pod are required") + } + nodeName := failed.Annotations[enginebinding.AnnotationNodeLocalTargetNode] + if strings.TrimSpace(nodeName) == "" { + return nil, fmt.Errorf("render LMCache NodeLocal SHM cleanup retry: target node is required") + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: NodeLocalCleanupRetryPodName(cache.UID, nodeName, failed.Name), + Namespace: cache.Namespace, + Labels: copyStringMap(failed.Labels), + Annotations: copyStringMap(failed.Annotations), + }, + Spec: func() corev1.PodSpec { + spec := *failed.Spec.DeepCopy() + // A terminal Pod may already be scheduler-bound. Clear NodeName so the + // retry's gate is authoritative before exact-node scheduling happens. + spec.NodeName = "" + spec.SchedulingGates = []corev1.PodSchedulingGate{{Name: lmCacheNodeLocalShmCleanupGate}} + return spec + }(), + }, nil +} + +// IsLMCacheNodeLocalCleanupPod reports whether pod has the exact executable +// contract rendered for one backend/node cleanup. Scheduling-only fields copied +// from the source Pod are intentionally excluded. +func IsLMCacheNodeLocalCleanupPod(pod *corev1.Pod, cache *cachev1alpha1.CacheBackend, nodeName, image string) bool { + if pod == nil || cache == nil || cache.UID == "" || strings.TrimSpace(nodeName) == "" || strings.TrimSpace(image) == "" || + pod.Namespace != cache.Namespace || !metav1.IsControlledBy(pod, cache) || + !IsNodeLocalCleanupPodName(cache.UID, nodeName, pod.Name) || + pod.Labels[enginebinding.LabelLMCacheNodeLocalCleanup] != "true" || + pod.Labels[enginebinding.LabelCacheBackendUID] != string(cache.UID) || + pod.Annotations[enginebinding.AnnotationNodeLocalOwnerUID] != string(cache.UID) || + pod.Annotations[enginebinding.AnnotationNodeLocalTargetNode] != nodeName || + (pod.Spec.NodeName != "" && pod.Spec.NodeName != nodeName) || + !equality.Semantic.DeepEqual(pod.Spec.Affinity, exactNodeAffinity(nodeName)) || + pod.Spec.RestartPolicy != corev1.RestartPolicyNever || len(pod.Spec.InitContainers) != 0 || + len(pod.Spec.EphemeralContainers) != 0 || len(pod.Spec.Containers) != 1 || len(pod.Spec.Volumes) != 1 || + (len(pod.Spec.SchedulingGates) != 0 && !LMCacheNodeLocalCleanupIsGated(pod)) { + return false + } + + volume := pod.Spec.Volumes[0] + wantPath, err := NodeLocalServerShmHostPath(cache) + if err != nil || volume.Name != lmCacheNodeLocalShmVolumeName || volume.HostPath == nil || + volume.HostPath.Path != wantPath || volume.HostPath.Type == nil || *volume.HostPath.Type != corev1.HostPathDirectoryOrCreate { + return false + } + + container := pod.Spec.Containers[0] + return container.Name == lmCacheNodeLocalShmCleanupName && container.Image == strings.TrimSpace(image) && + equality.Semantic.DeepEqual(container.Command, []string{"/node-local-shm-cleanup"}) && + equality.Semantic.DeepEqual(container.Args, []string{lmCacheNodeLocalShmCleanupPath}) && + len(container.VolumeMounts) == 1 && container.VolumeMounts[0].Name == lmCacheNodeLocalShmVolumeName && + container.VolumeMounts[0].MountPath == lmCacheNodeLocalShmCleanupPath && !container.VolumeMounts[0].ReadOnly && + container.VolumeMounts[0].SubPath == "" && container.VolumeMounts[0].SubPathExpr == "" && + container.VolumeMounts[0].MountPropagation == nil +} + +// LMCacheNodeLocalCleanupIsGated reports whether the cleanup Pod still carries +// the controller-authored quiescence gate. +func LMCacheNodeLocalCleanupIsGated(pod *corev1.Pod) bool { + return pod != nil && len(pod.Spec.SchedulingGates) == 1 && pod.Spec.SchedulingGates[0].Name == lmCacheNodeLocalShmCleanupGate +} + +// LMCacheNodeLocalCleanupSucceeded reports success only for the rendered +// helper container on the intended node. +func LMCacheNodeLocalCleanupSucceeded(pod *corev1.Pod, nodeName string) bool { + if pod == nil || pod.Status.Phase != corev1.PodSucceeded || pod.Spec.NodeName != nodeName || len(pod.Status.ContainerStatuses) != 1 { + return false + } + status := pod.Status.ContainerStatuses[0] + return status.Name == lmCacheNodeLocalShmCleanupName && status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 +} + // NodeLocalServerPodName returns the stable object name for one backend/node // pair. It is exported for controller lifecycle reconciliation. func NodeLocalServerPodName(backendName, nodeName string) string { @@ -213,13 +421,9 @@ func NodeLocalServerPodName(backendName, nodeName string) string { return prefix + "-" + suffix } -// NodeLocalServerShmName returns the exact LMCache POSIX shared-memory object -// name owned by one CacheBackend UID. It intentionally excludes generation and -// node identity: replacements of the same backend reclaim their own stale -// object, while Kubernetes-assigned UIDs isolate delete/recreate lifecycles. -func NodeLocalServerShmName(cache *cachev1alpha1.CacheBackend) (string, error) { +func nodeLocalBackendUID(cache *cachev1alpha1.CacheBackend) (string, error) { if cache == nil || cache.UID == "" { - return "", fmt.Errorf("derive LMCache NodeLocal shm name: CacheBackend UID is empty") + return "", fmt.Errorf("derive LMCache NodeLocal host path: CacheBackend UID is empty") } uid := string(cache.UID) for _, char := range uid { @@ -227,24 +431,25 @@ func NodeLocalServerShmName(cache *cachev1alpha1.CacheBackend) (string, error) { (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' { continue } - return "", fmt.Errorf("derive LMCache NodeLocal shm name: CacheBackend UID contains unsafe character %q", char) + return "", fmt.Errorf("derive LMCache NodeLocal host path: CacheBackend UID contains unsafe character %q", char) } - name := lmCacheNodeLocalShmNamePrefix + uid - if len(name) > posixShmNameMaxLength { - return "", fmt.Errorf("derive LMCache NodeLocal shm name: %d-byte name exceeds POSIX limit %d", len(name), posixShmNameMaxLength) + if len(uid) > hostPathComponentMaxLength { + return "", fmt.Errorf("derive LMCache NodeLocal host path: %d-byte UID exceeds path-component limit %d", len(uid), hostPathComponentMaxLength) } - return name, nil + return uid, nil } // NodeLocalServerShmHostPath returns the host tmpfs directory mounted as // /dev/shm by one CacheBackend's NodeLocal servers and engines. Mounting only -// the UID directory keeps normally behaving co-located pools out of each -// other's POSIX SHM namespace while retaining the node-local CUDA IPC path. +// the UID directory keeps CUDA/PyTorch auxiliary IPC objects from normally +// behaving co-located pools out of each other's directory while retaining the +// node-local CUDA IPC path. The LMCache L1 itself is private pinned memory. func NodeLocalServerShmHostPath(cache *cachev1alpha1.CacheBackend) (string, error) { - if _, err := NodeLocalServerShmName(cache); err != nil { + uid, err := nodeLocalBackendUID(cache) + if err != nil { return "", err } - return lmCacheNodeLocalShmHostRoot + "/" + string(cache.UID), nil + return lmCacheNodeLocalShmHostRoot + "/" + uid, nil } func exactNodeAffinity(nodeName string) *corev1.Affinity { @@ -341,10 +546,6 @@ func renderLMCacheNodeLocalEngine(pod *corev1.PodSpec, engineContainerName strin if err := validateLMCacheNodeLocalServerConfig(server, effectiveLMCacheChunkSize(cache.Spec.LMCache)); err != nil { return "", err } - shmName, err := NodeLocalServerShmName(cache) - if err != nil { - return "", err - } shmHostPath, err := NodeLocalServerShmHostPath(cache) if err != nil { return "", err @@ -405,7 +606,7 @@ func renderLMCacheNodeLocalEngine(pod *corev1.PodSpec, engineContainerName strin gateMounts = append(gateMounts, configMount) } - gate := lmCacheNodeLocalGateContainer(cache, shmName, configPath, gateMounts) + gate := lmCacheNodeLocalGateContainer(cache, configPath, gateMounts) work.InitContainers, err = adoptNodeLocalGate(work.InitContainers, gate, owned) if err != nil { return "", err @@ -414,7 +615,7 @@ func renderLMCacheNodeLocalEngine(pod *corev1.PodSpec, engineContainerName strin return configPath, nil } -func lmCacheNodeLocalGateContainer(cache *cachev1alpha1.CacheBackend, shmName, configPath string, mounts []corev1.VolumeMount) corev1.Container { +func lmCacheNodeLocalGateContainer(cache *cachev1alpha1.CacheBackend, configPath string, mounts []corev1.VolumeMount) corev1.Container { server := cache.Spec.LMCache.NodeLocal.Server const gateScript = `import json, os, time, urllib.request ip = os.environ["INFERENCECACHE_NODE_IP"] @@ -422,7 +623,8 @@ host = "[" + ip + "]" if ":" in ip else ip base = "http://%s:%s" % (host, os.environ["EXPECTED_HTTP_PORT"]) expected = { "instance_id": os.environ["EXPECTED_INSTANCE_ID"], - "shm_name": os.environ["EXPECTED_SHM_NAME"], + "supported_transfer_mode": "lmcache_driven", + "shm_name": "", "port": int(os.environ["EXPECTED_MP_PORT"]), "chunk_size": int(os.environ["EXPECTED_CHUNK_SIZE"]), "max_gpu_workers": int(os.environ["EXPECTED_MAX_GPU_WORKERS"]), @@ -438,8 +640,10 @@ while True: if mp.get(key) != value: raise RuntimeError("server config %s=%r, expected %r" % (key, mp.get(key), value)) memory = config.get("storage_manager", {}).get("l1_manager_config", {}).get("memory_config", {}) - if memory.get("shm_name") != expected["shm_name"]: - raise RuntimeError("effective L1 shm_name=%r, expected %r" % (memory.get("shm_name"), expected["shm_name"])) + if memory.get("use_lazy") is not False: + raise RuntimeError("effective L1 use_lazy=%r, expected False" % memory.get("use_lazy")) + if memory.get("shm_name") != "": + raise RuntimeError("effective L1 shm_name=%r, expected empty" % memory.get("shm_name")) if http.get("http_port") != int(os.environ["EXPECTED_HTTP_PORT"]): raise RuntimeError("server HTTP port does not match") with urllib.request.urlopen(base + "/healthcheck", timeout=2) as response: @@ -470,7 +674,6 @@ while True: {Name: lmCacheNodeLocalGateManagedEnv, Value: lmCacheNodeLocalGateManagedValue}, {Name: lmCacheNodeIPEnv, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.hostIP"}}}, {Name: "EXPECTED_INSTANCE_ID", Value: lmCacheNodeLocalInstanceID(cache)}, - {Name: "EXPECTED_SHM_NAME", Value: shmName}, {Name: "EXPECTED_MP_PORT", Value: strconv.FormatInt(int64(server.Port), 10)}, {Name: "EXPECTED_HTTP_PORT", Value: strconv.FormatInt(int64(server.HTTPPort), 10)}, {Name: "EXPECTED_CHUNK_SIZE", Value: strconv.FormatInt(int64(effectiveLMCacheChunkSize(cache.Spec.LMCache)), 10)}, diff --git a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go index 211ede0b..d2806d6e 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go @@ -106,7 +106,6 @@ func TestRenderLMCacheNodeLocalServerPod(t *testing.T) { args := strings.Join(server.Args, " ") for _, want := range []string{ "--instance-id team-a/node-cache@11111111-2222-3333-4444-555555555555#7", - "--shm-name lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555", "--host $(INFERENCECACHE_NODE_IP)", "--http-port 18080", "--max-gpu-workers 4", "--max-cpu-workers 8", } { @@ -114,6 +113,9 @@ func TestRenderLMCacheNodeLocalServerPod(t *testing.T) { t.Fatalf("server args %q missing %q", args, want) } } + if !IsLMCacheMPCUDAServerProfile(server.Args) { + t.Fatalf("server args do not explicitly select the CUDA/private-pinned-L1 profile: %v", server.Args) + } if got := serverPod.Annotations[enginebinding.AnnotationNodeLocalOwnerUID]; got != string(cache.UID) { t.Fatalf("owner UID annotation = %q", got) } @@ -123,8 +125,122 @@ func TestRenderLMCacheNodeLocalServerPod(t *testing.T) { if got := serverPod.Annotations[enginebinding.AnnotationNodeLocalTargetNode]; got != "gpu-node-a" { t.Fatalf("target-node annotation = %q", got) } - if got := serverPod.Annotations[enginebinding.AnnotationNodeLocalShmName]; got != "lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555" { - t.Fatalf("shared-memory annotation = %q", got) + if _, found := serverPod.Annotations["inferencecache.io/node-local-shm-name"]; found { + t.Fatalf("obsolete POSIX SHM identity annotation remains: %v", serverPod.Annotations) + } +} + +func TestRenderLMCacheNodeLocalCleanupPod(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + cache.Spec.LMCache.NodeLocal.Scheduling = &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "cache-pull"}}, + } + source := nodeLocalSourceEngine() + source.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "engine-pull"}} + cleanup, err := RenderLMCacheNodeLocalCleanupPod(cache, "gpu-node-a", testNodeLocalCleanupImage, source) + if err != nil { + t.Fatal(err) + } + if cleanup.Name != NodeLocalCleanupPodName(cache.UID, "gpu-node-a") || cleanup.Spec.NodeName != "" || len(cleanup.Spec.SchedulingGates) != 1 { + t.Fatalf("cleanup identity/initial gate = name:%q node:%q gates:%+v", cleanup.Name, cleanup.Spec.NodeName, cleanup.Spec.SchedulingGates) + } + if cleanup.Spec.HostNetwork || cleanup.Spec.HostPID || cleanup.Spec.HostIPC || cleanup.Spec.AutomountServiceAccountToken == nil || *cleanup.Spec.AutomountServiceAccountToken { + t.Fatalf("cleanup host boundary = %+v", cleanup.Spec) + } + if cleanup.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Fatalf("cleanup restart policy = %q, want Never so controller retries terminal failures", cleanup.Spec.RestartPolicy) + } + if len(cleanup.Spec.Volumes) != 1 || cleanup.Spec.Volumes[0].HostPath == nil || + cleanup.Spec.Volumes[0].HostPath.Path != "/dev/shm/inference-cache/11111111-2222-3333-4444-555555555555" { + t.Fatalf("cleanup volumes = %+v", cleanup.Spec.Volumes) + } + if len(cleanup.Spec.ImagePullSecrets) != 1 || cleanup.Spec.ImagePullSecrets[0].Name != "cache-pull" { + t.Fatalf("cleanup imagePullSecrets = %+v", cleanup.Spec.ImagePullSecrets) + } + container := cleanup.Spec.Containers[0] + if container.Image != testNodeLocalCleanupImage || !reflect.DeepEqual(container.Command, []string{"/node-local-shm-cleanup"}) || + !reflect.DeepEqual(container.Args, []string{lmCacheNodeLocalShmCleanupPath}) { + t.Fatalf("cleanup container = image:%q command:%v args:%v", container.Image, container.Command, container.Args) + } + security := container.SecurityContext + if security == nil || security.AllowPrivilegeEscalation == nil || *security.AllowPrivilegeEscalation || + security.RunAsUser == nil || *security.RunAsUser != 0 || security.ReadOnlyRootFilesystem == nil || !*security.ReadOnlyRootFilesystem { + t.Fatalf("cleanup security context = %+v", security) + } + controller := true + cleanup.OwnerReferences = []metav1.OwnerReference{{UID: cache.UID, Controller: &controller}} + if !IsLMCacheNodeLocalCleanupPod(cleanup, cache, "gpu-node-a", testNodeLocalCleanupImage) { + t.Fatal("rendered cleanup Pod did not satisfy its executable contract") + } + cleanup.Spec.Containers[0].Command = []string{"/bin/true"} + if IsLMCacheNodeLocalCleanupPod(cleanup, cache, "gpu-node-a", testNodeLocalCleanupImage) { + t.Fatal("cleanup Pod with a foreign command satisfied the executable contract") + } +} + +func TestLMCacheNodeLocalCleanupSucceededRequiresHelperStatus(t *testing.T) { + pod := &corev1.Pod{Spec: corev1.PodSpec{NodeName: "gpu-node-a"}, Status: corev1.PodStatus{ + Phase: corev1.PodSucceeded, + ContainerStatuses: []corev1.ContainerStatus{{Name: "foreign", State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}, + }}}, + }} + if LMCacheNodeLocalCleanupSucceeded(pod, "gpu-node-a") { + t.Fatal("foreign successful container was accepted as cleanup success") + } + pod.Status.ContainerStatuses[0].Name = lmCacheNodeLocalShmCleanupName + if !LMCacheNodeLocalCleanupSucceeded(pod, "gpu-node-a") { + t.Fatal("successful cleanup helper status was rejected") + } +} + +func TestRenderLMCacheNodeLocalCleanupRetryPod(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + cleanup, err := RenderLMCacheNodeLocalCleanupPod(cache, "gpu-node-a", testNodeLocalCleanupImage, nodeLocalSourceEngine()) + if err != nil { + t.Fatal(err) + } + cleanup.Spec.SchedulingGates = nil + cleanup.Spec.NodeName = "gpu-node-a" + retry, err := RenderLMCacheNodeLocalCleanupRetryPod(cache, cleanup) + if err != nil { + t.Fatal(err) + } + if retry.Name != NodeLocalCleanupRetryPodName(cache.UID, "gpu-node-a", cleanup.Name) || + !IsNodeLocalCleanupPodName(cache.UID, "gpu-node-a", retry.Name) || retry.Spec.NodeName != "" || len(retry.Spec.SchedulingGates) != 1 { + t.Fatalf("cleanup retry identity/gate = name:%q node:%q gates:%+v", retry.Name, retry.Spec.NodeName, retry.Spec.SchedulingGates) + } + if retry.Spec.Containers[0].Image != cleanup.Spec.Containers[0].Image || retry.Spec.Volumes[0].HostPath.Path != cleanup.Spec.Volumes[0].HostPath.Path { + t.Fatalf("cleanup retry changed image or hostPath: image:%q volumes:%+v", retry.Spec.Containers[0].Image, retry.Spec.Volumes) + } +} + +func TestNodeLocalCleanupRejectsInvalidInputs(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + base := NodeLocalCleanupPodName(cache.UID, "gpu-node-a") + if IsNodeLocalCleanupPodName(cache.UID, "gpu-node-a", base+"-retry-short") { + t.Fatal("short cleanup retry suffix was accepted") + } + if IsNodeLocalCleanupPodName(cache.UID, "gpu-node-a", base+"-retry-zzzzzzzzzzzz") { + t.Fatal("non-hex cleanup retry suffix was accepted") + } + if _, err := RenderLMCacheNodeLocalCleanupPod(nil, "gpu-node-a", testNodeLocalCleanupImage, nodeLocalSourceEngine()); err == nil { + t.Fatal("nil backend was accepted for cleanup rendering") + } + unsafeUID := cache.DeepCopy() + unsafeUID.UID = "unsafe/uid" + if _, err := RenderLMCacheNodeLocalCleanupPod(unsafeUID, "gpu-node-a", testNodeLocalCleanupImage, nodeLocalSourceEngine()); err == nil { + t.Fatal("unsafe backend UID was accepted for cleanup rendering") + } + if _, err := RenderLMCacheNodeLocalCleanupRetryPod(nil, nil); err == nil { + t.Fatal("nil backend and failed Pod were accepted for retry rendering") + } + failed := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "failed"}} + if _, err := RenderLMCacheNodeLocalCleanupRetryPod(cache, failed); err == nil { + t.Fatal("cleanup retry without target node was accepted") + } + if _, err := NodeLocalServerShmHostPath(nil); err == nil { + t.Fatal("nil backend produced a NodeLocal SHM hostPath") } } @@ -191,27 +307,6 @@ func TestNodeLocalServerPodNameIsStableAndBounded(t *testing.T) { } } -func TestNodeLocalServerShmNameIsStableAndUIDScoped(t *testing.T) { - cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) - want := "lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555" - got, err := NodeLocalServerShmName(cache) - if err != nil || got != want { - t.Fatalf("NodeLocalServerShmName = %q, %v; want %q", got, err, want) - } - cache.Name = "renamed" - cache.Namespace = "other" - cache.Generation++ - stable, err := NodeLocalServerShmName(cache) - if err != nil || stable != want { - t.Fatalf("same-UID replacement shm name = %q, %v; want %q", stable, err, want) - } - cache.UID = types.UID("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") - distinct, err := NodeLocalServerShmName(cache) - if err != nil || distinct == want { - t.Fatalf("different-UID shm name = %q, %v; must differ from %q", distinct, err, want) - } -} - func TestNodeLocalServerShmHostPathIsStableAndUIDScoped(t *testing.T) { cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) want := "/dev/shm/inference-cache/11111111-2222-3333-4444-555555555555" @@ -231,14 +326,14 @@ func TestNodeLocalServerShmHostPathIsStableAndUIDScoped(t *testing.T) { } } -func TestNodeLocalServerShmNameRejectsUnsafeUID(t *testing.T) { +func TestNodeLocalServerShmHostPathRejectsUnsafeUID(t *testing.T) { cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) cache.UID = types.UID("unsafe/uid") - if _, err := NodeLocalServerShmName(cache); err == nil || !strings.Contains(err.Error(), "unsafe character") { + if _, err := NodeLocalServerShmHostPath(cache); err == nil || !strings.Contains(err.Error(), "unsafe character") { t.Fatalf("unsafe UID error = %v", err) } - cache.UID = types.UID(strings.Repeat("a", posixShmNameMaxLength)) - if _, err := NodeLocalServerShmName(cache); err == nil || !strings.Contains(err.Error(), "exceeds POSIX limit") { + cache.UID = types.UID(strings.Repeat("a", hostPathComponentMaxLength+1)) + if _, err := NodeLocalServerShmHostPath(cache); err == nil || !strings.Contains(err.Error(), "exceeds path-component limit") { t.Fatalf("oversized UID error = %v", err) } } @@ -271,7 +366,8 @@ func TestVLLMNodeLocalEngineInjection(t *testing.T) { t.Fatalf("engine node IP env = %+v", engine.Env) } joined := strings.Join(engine.Args, " ") - if !strings.Contains(joined, `tcp://$(INFERENCECACHE_NODE_IP)`) || !strings.Contains(joined, `"lmcache.mp.port":"6555"`) { + if !strings.Contains(joined, `tcp://$(INFERENCECACHE_NODE_IP)`) || !strings.Contains(joined, `"lmcache.mp.port":"6555"`) || + !strings.Contains(joined, `"lmcache.mp.mp_transfer_mode":"lmcache_driven"`) { t.Fatalf("vLLM args do not carry node-derived endpoint: %s", joined) } shmMount := mountAtPath(engine.VolumeMounts, "/dev/shm") @@ -291,11 +387,13 @@ func TestVLLMNodeLocalEngineInjection(t *testing.T) { } gate := pod.Spec.InitContainers[0] if !strings.Contains(gate.Args[0], "/config") || !strings.Contains(gate.Args[0], "EXPECTED_INSTANCE_ID") || - !strings.Contains(gate.Args[0], `memory.get("shm_name")`) { + !strings.Contains(gate.Args[0], `"supported_transfer_mode": "lmcache_driven"`) || + !strings.Contains(gate.Args[0], `memory.get("use_lazy") is not False`) || + !strings.Contains(gate.Args[0], `memory.get("shm_name") != ""`) { t.Fatalf("gate script does not verify live server identity: %q", gate.Args[0]) } - if got, found := lookupEnv(gate.Env, "EXPECTED_SHM_NAME"); !found || got != "lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555" { - t.Fatalf("gate EXPECTED_SHM_NAME = %q, found=%v", got, found) + if _, found := lookupEnv(gate.Env, "EXPECTED_SHM_NAME"); found { + t.Fatal("gate still carries obsolete UID-derived POSIX SHM identity") } before := pod.Spec.DeepCopy() @@ -487,6 +585,7 @@ func TestNodeLocalAdaptersValidateTopologyContract(t *testing.T) { sglangCache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeSGLang) sglangPod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ Name: SGLangEngineContainerName, Args: []string{"--page-size", "64"}, + Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{gpuResourceName: resource.MustParse("1")}}, }}}} sglang := sglangLMCacheAdapter{} if !sglang.Supports("sglang", sglangCache) { diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go index 23de7618..3e1f71b6 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go @@ -33,6 +33,12 @@ const ( lmCacheMPShmMountPath = "/dev/shm" lmCacheMPShmHeadroom = "1Gi" + lmCacheMPTransferModeArg = "--supported-transfer-mode" + lmCacheMPTransferModeLMCacheDriven = "lmcache_driven" + lmCacheMPL1NonLazyArg = "--no-l1-use-lazy" + lmCacheMPL1LazyArg = "--l1-use-lazy" + lmCacheMPShmNameArg = "--shm-name" + lmCacheMPServerPortName = "lmcache-mp" lmCacheMPHTTPPortName = "lmcache-http" lmCacheMPHTTPPort = int32(8080) @@ -161,8 +167,9 @@ func lmCacheMPMemoryBudget(l1Capacity resource.Quantity) resource.Quantity { } // checkLMCacheMPShmBudget validates an engine-owned /dev/shm volume without -// mutating it. A larger operator-owned tmpfs is safe to share; an unbounded, -// disk-backed, or undersized volume cannot satisfy the typed L1 contract. +// mutating it. The supported L1 is private pinned host memory, but the engine +// and CUDA runtime still use /dev/shm for auxiliary IPC objects. Keep the +// existing l1Capacity+headroom bound as a conservative PodLocal IPC budget. func checkLMCacheMPShmBudget(volumes []corev1.Volume, mount corev1.VolumeMount, budget resource.Quantity) error { for i := range volumes { if volumes[i].Name != mount.Name { @@ -170,7 +177,7 @@ func checkLMCacheMPShmBudget(volumes []corev1.Volume, mount corev1.VolumeMount, } emptyDir := volumes[i].EmptyDir if emptyDir == nil || emptyDir.Medium != corev1.StorageMediumMemory { - return fmt.Errorf("render LMCache MP server: engine container mounts %q from volume %q, but PodLocal L1 requires a memory-backed emptyDir", lmCacheMPShmMountPath, mount.Name) + return fmt.Errorf("render LMCache MP server: engine container mounts %q from volume %q, but the PodLocal CUDA IPC budget requires a memory-backed emptyDir", lmCacheMPShmMountPath, mount.Name) } if emptyDir.SizeLimit == nil || emptyDir.SizeLimit.Cmp(budget) < 0 { var got string @@ -212,6 +219,9 @@ func lmCacheMPServerContainer(cfg lmCacheMPServerConfig, l2Adapter string, bindi serverArgs := []string{ "server", + lmCacheMPTransferModeArg, lmCacheMPTransferModeLMCacheDriven, + lmCacheMPL1NonLazyArg, + lmCacheMPShmNameArg, "", "--host", "127.0.0.1", "--port", strconv.FormatInt(int64(cfg.Port), 10), "--http-host", "0.0.0.0", @@ -281,6 +291,28 @@ exec "$@"` }, nil } +// IsLMCacheMPCUDAServerProfile reports whether args explicitly select the +// supported CUDA-only MP transfer and L1 allocation contract. An empty +// --shm-name keeps the eagerly allocated pinned L1 private to the server +// process instead of creating a POSIX SHM-backed L1 arena. +func IsLMCacheMPCUDAServerProfile(args []string) bool { + transferModes, transferMalformed := argValues(args, lmCacheMPTransferModeArg) + shmNames, shmMalformed := argValues(args, lmCacheMPShmNameArg) + nonLazyCount := 0 + lazyCount := 0 + for _, arg := range args { + switch arg { + case lmCacheMPL1NonLazyArg: + nonLazyCount++ + case lmCacheMPL1LazyArg: + lazyCount++ + } + } + return !transferMalformed && len(transferModes) == 1 && transferModes[0] == lmCacheMPTransferModeLMCacheDriven && + !shmMalformed && len(shmNames) == 1 && shmNames[0] == "" && + nonLazyCount == 1 && lazyCount == 0 +} + func lmCacheMPHTTPProbe(period, failures int32) *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go index c7d0103a..e68607d5 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go @@ -75,6 +75,9 @@ func TestRenderLMCachePodLocalServerGolden(t *testing.T) { if server.RestartPolicy == nil || *server.RestartPolicy != corev1.ContainerRestartPolicyAlways { t.Fatalf("server is not a native sidecar: %v", server.RestartPolicy) } + if !IsLMCacheMPCUDAServerProfile(server.Args) { + t.Fatalf("server args do not explicitly select the CUDA/private-pinned-L1 profile: %v", server.Args) + } joined := strings.Join(append(server.Command, server.Args...), " ") for _, want := range []string{ "lmcache server", "--host 127.0.0.1", "--port 6500", @@ -110,13 +113,35 @@ func TestRenderLMCachePodLocalServerGolden(t *testing.T) { } shm := findVolume(pod.Volumes, lmCacheMPShmVolumeName) if shm == nil || shm.EmptyDir == nil || shm.EmptyDir.Medium != corev1.StorageMediumMemory || shm.EmptyDir.SizeLimit == nil || shm.EmptyDir.SizeLimit.Cmp(resource.MustParse("5Gi")) != 0 { - t.Fatalf("shared-memory volume = %+v, want bounded 5Gi tmpfs (4Gi L1 + 1Gi headroom)", shm) + t.Fatalf("shared-memory volume = %+v, want conservative bounded 5Gi CUDA IPC tmpfs", shm) } if findVolume(pod.Volumes, lmCacheMPConfigVolumeName) == nil { t.Fatalf("client config volume missing: %+v", pod.Volumes) } } +func TestIsLMCacheMPCUDAServerProfile(t *testing.T) { + valid := []string{"server", lmCacheMPTransferModeArg, lmCacheMPTransferModeLMCacheDriven, lmCacheMPL1NonLazyArg, lmCacheMPShmNameArg, ""} + tests := []struct { + name string + args []string + want bool + }{ + {name: "explicit profile", args: valid, want: true}, + {name: "implicit transfer mode", args: []string{"server", lmCacheMPL1NonLazyArg, lmCacheMPShmNameArg, ""}}, + {name: "lazy", args: []string{"server", lmCacheMPTransferModeArg, lmCacheMPTransferModeLMCacheDriven, lmCacheMPL1LazyArg, lmCacheMPShmNameArg, ""}}, + {name: "named L1 SHM", args: []string{"server", lmCacheMPTransferModeArg, lmCacheMPTransferModeLMCacheDriven, lmCacheMPL1NonLazyArg, lmCacheMPShmNameArg, "pool"}}, + {name: "missing empty SHM override", args: []string{"server", lmCacheMPTransferModeArg, lmCacheMPTransferModeLMCacheDriven, lmCacheMPL1NonLazyArg}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsLMCacheMPCUDAServerProfile(tt.args); got != tt.want { + t.Fatalf("IsLMCacheMPCUDAServerProfile(%v) = %v, want %v", tt.args, got, tt.want) + } + }) + } +} + func TestRenderLMCachePodLocalServerSecretAuthUsesEnvReferences(t *testing.T) { cfg := testLMCacheMPConfig() cfg.Binding = &backendadapter.Binding{ diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_test.go b/internal/adapters/builtin/runtime/sglang_lmcache_test.go index 1b7d4792..9e2fda6a 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache_test.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache_test.go @@ -45,6 +45,14 @@ func typedSGLangBackend() *cachev1alpha1.CacheBackend { } } +func sglangMPValidationPod(args ...string) *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, + Args: args, + Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{gpuResourceName: resource.MustParse("1")}}, + }}}} +} + func observedTypedSGLangBackend(model string) *cachev1alpha1.CacheBackend { backend := typedSGLangBackend() backend.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: model} @@ -91,7 +99,7 @@ func TestSGLangLMCacheInjectsTypedMP(t *testing.T) { func TestSGLangLMCacheValidatesPageSize(t *testing.T) { adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) cache := typedSGLangBackend() - pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Args: []string{"--page-size", "64"}}}}} + pod := sglangMPValidationPod("--page-size", "64") if err := adapter.ValidateMPEnginePod(pod, cache); err != nil { t.Fatalf("ValidateMPEnginePod: %v", err) } @@ -190,10 +198,7 @@ func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, - Args: tc.args, - }}}} + pod := sglangMPValidationPod(tc.args...) err := adapter.ValidateMPEnginePod(pod, typedSGLangBackend()) if tc.wantErr == "" { if err != nil { @@ -210,7 +215,7 @@ func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { func TestSGLangValidateRejectsIncompleteTopology(t *testing.T) { adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) - validPod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Args: []string{"--page-size", "64"}}}}} + validPod := sglangMPValidationPod("--page-size", "64") tests := []struct { name string pod *corev1.Pod diff --git a/internal/adapters/builtin/runtime/test_helpers_test.go b/internal/adapters/builtin/runtime/test_helpers_test.go index b46852b9..08589f63 100644 --- a/internal/adapters/builtin/runtime/test_helpers_test.go +++ b/internal/adapters/builtin/runtime/test_helpers_test.go @@ -9,6 +9,7 @@ import corev1 "k8s.io/api/core/v1" import backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" const testLMCacheServerImage = "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const testNodeLocalCleanupImage = "registry.example/inference-cache-shm-cleanup@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" func findVolume(volumes []corev1.Volume, name string) *corev1.Volume { for i := range volumes { diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go index 1b74458a..b3b0a7d6 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go @@ -192,8 +192,9 @@ type vllmMPKVTransferConfig struct { } type vllmMPConnectorExtraConfig struct { - Host string `json:"lmcache.mp.host"` - Port string `json:"lmcache.mp.port"` + Host string `json:"lmcache.mp.host"` + Port string `json:"lmcache.mp.port"` + TransferMode string `json:"lmcache.mp.mp_transfer_mode"` } func vllmMPKVTransferConfigJSON(role cachev1alpha1.CacheBackendIntegrationRole, port int32) (string, error) { @@ -217,8 +218,9 @@ func vllmMPKVTransferConfigJSONForHost(role cachev1alpha1.CacheBackendIntegratio ConnectorModule: vllmLMCacheMPConnectorModulePath, Role: kvRole, ExtraConfig: vllmMPConnectorExtraConfig{ - Host: host, - Port: strconv.FormatInt(int64(port), 10), + Host: host, + Port: strconv.FormatInt(int64(port), 10), + TransferMode: lmCacheMPTransferModeLMCacheDriven, }, }) if err != nil { diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go index 2b67c6ee..df7e85cf 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go @@ -49,9 +49,10 @@ func newTypedVLLMMPBackend() *cachev1alpha1.CacheBackend { func newVLLMMPEnginePod(args ...string) *corev1.Pod { return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ - Name: EngineContainerName, - Image: "vllm:connector-ready", - Args: args, + Name: EngineContainerName, + Image: "vllm:connector-ready", + Args: args, + Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{gpuResourceName: resource.MustParse("1")}}, }}}} } @@ -240,7 +241,8 @@ func TestVLLMLMCacheMPKVTransferConfigRoles(t *testing.T) { got.ConnectorModule != vllmLMCacheMPConnectorModulePath || got.Role != tc.want || got.ExtraConfig.Host != "tcp://127.0.0.1" || - got.ExtraConfig.Port != "6500" { + got.ExtraConfig.Port != "6500" || + got.ExtraConfig.TransferMode != lmCacheMPTransferModeLMCacheDriven { t.Fatalf("config = %+v", got) } }) diff --git a/internal/controller/cachebackend_lmcache_mp_status.go b/internal/controller/cachebackend_lmcache_mp_status.go index 8ce2cf12..f34d0797 100644 --- a/internal/controller/cachebackend_lmcache_mp_status.go +++ b/internal/controller/cachebackend_lmcache_mp_status.go @@ -220,11 +220,6 @@ func (r *CacheBackendReconciler) refreshLMCacheNodeLocalConnectorStatus(ctx cont wantOwner := backend.Namespace + "/" + backend.Name wantUID := string(backend.UID) wantGeneration := strconv.FormatInt(backend.Generation, 10) - wantShmName, shmNameErr := builtinruntime.NodeLocalServerShmName(backend) - if shmNameErr != nil { - log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: shared-memory identity is invalid", "error", shmNameErr.Error()) - return - } wantShmHostPath, shmPathErr := builtinruntime.NodeLocalServerShmHostPath(backend) if shmPathErr != nil { log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: shared-memory host path is invalid", "error", shmPathErr.Error()) @@ -241,7 +236,7 @@ func (r *CacheBackendReconciler) refreshLMCacheNodeLocalConnectorStatus(ctx cont if annotations[enginebinding.AnnotationNodeLocalOwner] != wantOwner || annotations[enginebinding.AnnotationNodeLocalOwnerUID] != wantUID || annotations[enginebinding.AnnotationNodeLocalGeneration] != wantGeneration || - !nodeLocalServerHasShmIdentity(pod, wantShmName, wantShmHostPath) { + !nodeLocalServerHasRuntimeIdentity(pod, wantShmHostPath) { continue } targetNode := annotations[enginebinding.AnnotationNodeLocalTargetNode] diff --git a/internal/controller/cachebackend_lmcache_mp_status_test.go b/internal/controller/cachebackend_lmcache_mp_status_test.go index bafee175..cad58fa4 100644 --- a/internal/controller/cachebackend_lmcache_mp_status_test.go +++ b/internal/controller/cachebackend_lmcache_mp_status_test.go @@ -23,23 +23,16 @@ import ( "github.com/cachebox-project/inference-cache/internal/enginebinding" ) -func setNodeLocalShmIdentity(t *testing.T, backend *cachev1alpha1.CacheBackend, pod *corev1.Pod) { +func setNodeLocalRuntimeIdentity(t *testing.T, backend *cachev1alpha1.CacheBackend, pod *corev1.Pod) { t.Helper() - name, err := builtinruntime.NodeLocalServerShmName(backend) - if err != nil { - t.Fatal(err) - } - if pod.Annotations == nil { - pod.Annotations = map[string]string{} - } path, err := builtinruntime.NodeLocalServerShmHostPath(backend) if err != nil { t.Fatal(err) } pathType := corev1.HostPathDirectoryOrCreate - pod.Annotations[enginebinding.AnnotationNodeLocalShmName] = name pod.Spec.Containers = []corev1.Container{{ - Name: lmCacheMPServerStatusContainerName, Args: []string{"server", "--shm-name", name}, + Name: lmCacheMPServerStatusContainerName, + Args: []string{"server", "--supported-transfer-mode", "lmcache_driven", "--no-l1-use-lazy", "--shm-name", ""}, VolumeMounts: []corev1.VolumeMount{{Name: "shm", MountPath: "/dev/shm"}}, }} pod.Spec.Volumes = []corev1.Volume{{Name: "shm", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ @@ -217,7 +210,7 @@ func TestRefreshLMCacheNodeLocalConnectorStatusSameNodeCoverage(t *testing.T) { if ready { pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} } - setNodeLocalShmIdentity(t, backend, pod) + setNodeLocalRuntimeIdentity(t, backend, pod) return pod } enginePod := func(name, node string) *corev1.Pod { @@ -330,9 +323,9 @@ func TestRefreshLMCacheNodeLocalConnectorStatusFailureModes(t *testing.T) { wantCondition: reasonNodeLocalWorkerCapacity, wantCoverage: reasonNodeLocalWorkerCapacity, wantReadyServers: 1, }, { - name: "server is missing UID-scoped shared-memory identity", + name: "server uses implicit transfer mode", mutate: func(_ *cachev1alpha1.CacheBackend, servers *[]*corev1.Pod, _ *corev1.Pod) { - delete((*servers)[0].Annotations, enginebinding.AnnotationNodeLocalShmName) + (*servers)[0].Spec.Containers[0].Args = []string{"server", "--no-l1-use-lazy", "--shm-name", ""} }, wantCondition: reasonNodeLocalPoolPending, wantCoverage: reasonMPServersNotReady, wantReadyServers: 0, }, @@ -376,7 +369,7 @@ func TestRefreshLMCacheNodeLocalConnectorStatusFailureModes(t *testing.T) { ContainerStatuses: []corev1.ContainerStatus{{Name: lmCacheMPServerStatusContainerName, Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}, }, } - setNodeLocalShmIdentity(t, backend, server) + setNodeLocalRuntimeIdentity(t, backend, server) servers := []*corev1.Pod{server} engine := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "engine", Namespace: "ns1", Labels: map[string]string{"app": "engine"}, Annotations: map[string]string{ diff --git a/internal/controller/cachebackend_lmcache_nodelocal.go b/internal/controller/cachebackend_lmcache_nodelocal.go index ae8bce16..266b4fdb 100644 --- a/internal/controller/cachebackend_lmcache_nodelocal.go +++ b/internal/controller/cachebackend_lmcache_nodelocal.go @@ -15,6 +15,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -24,6 +25,12 @@ import ( backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" ) +const ( + nodeLocalShmCleanupFinalizer = "inferencecache.io/nodelocal-shm-cleanup" + nodeLocalShmCleanupRetryAnnotation = "inferencecache.io/node-local-shm-cleanup-retry" + nodeLocalShmCleanupMaxRetryAttempts = 3 +) + func isTypedLMCacheNodeLocal(backend *cachev1alpha1.CacheBackend) bool { return backend != nil && backend.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache && @@ -42,16 +49,17 @@ func isTypedLMCacheMP(backend *cachev1alpha1.CacheBackend) bool { // placement. func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context.Context, backend *cachev1alpha1.CacheBackend, binding *backendadapter.Binding) error { if !isTypedLMCacheNodeLocal(backend) { + if controllerutil.ContainsFinalizer(backend, nodeLocalShmCleanupFinalizer) { + if err := r.ensureLMCacheNodeLocalCleanupForConsumers(ctx, backend); err != nil { + return err + } + } return r.cleanupLMCacheNodeLocalServerPods(ctx, backend) } demand, err := r.nodeLocalEngineDemand(ctx, backend) if err != nil { return err } - wantShmName, err := builtinruntime.NodeLocalServerShmName(backend) - if err != nil { - return err - } wantShmHostPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) if err != nil { return err @@ -67,6 +75,10 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context ); err != nil { return fmt.Errorf("list LMCache NodeLocal server Pods for %s/%s: %w", backend.Namespace, backend.Name, err) } + cleanupNodes, err := r.reconcileLMCacheNodeLocalCleanupPods(ctx, backend, demand, true) + if err != nil { + return err + } wantGeneration := strconv.FormatInt(backend.Generation, 10) liveByNode := make(map[string]*corev1.Pod, len(servers.Items)) @@ -75,14 +87,21 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context if !metav1.IsControlledBy(pod, backend) { continue } - targetNode := pod.Annotations[enginebinding.AnnotationNodeLocalTargetNode] + targetNode := nodeLocalServerTargetNode(pod) _, wanted := demand[targetNode] + runtimeCurrent := nodeLocalServerHasRuntimeIdentity(pod, wantShmHostPath) current := pod.Annotations[enginebinding.AnnotationNodeLocalOwnerUID] == string(backend.UID) && pod.Annotations[enginebinding.AnnotationNodeLocalGeneration] == wantGeneration && pod.Name == builtinruntime.NodeLocalServerPodName(backend.Name, targetNode) && - nodeLocalServerHasShmIdentity(pod, wantShmName, wantShmHostPath) + runtimeCurrent if !current { if pod.DeletionTimestamp == nil { + if nodeLocalServerUsesShmHostPath(pod, wantShmHostPath) && (!runtimeCurrent || !wanted) { + if err := r.ensureLMCacheNodeLocalCleanupPod(ctx, backend, pod); err != nil { + return err + } + cleanupNodes[targetNode] = true + } if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("delete stale LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) } @@ -93,6 +112,9 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context retention := time.Duration(backend.Spec.LMCache.NodeLocal.IdleRetentionSeconds) * time.Second if retention <= 0 { if pod.DeletionTimestamp == nil { + if err := r.ensureLMCacheNodeLocalCleanupPod(ctx, backend, pod); err != nil { + return err + } if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("delete idle LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) } @@ -112,6 +134,9 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context continue } if time.Since(idleSince) >= retention && pod.DeletionTimestamp == nil { + if err := r.ensureLMCacheNodeLocalCleanupPod(ctx, backend, pod); err != nil { + return err + } if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("delete expired idle LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) } @@ -142,7 +167,7 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context } sort.Strings(nodes) for _, nodeName := range nodes { - if liveByNode[nodeName] != nil { + if liveByNode[nodeName] != nil || cleanupNodes[nodeName] { continue } desired, err := builtinruntime.RenderLMCacheNodeLocalServerPod(backend, binding, nodeName, demand[nodeName]) @@ -169,8 +194,8 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context return nil } -func nodeLocalServerHasShmIdentity(pod *corev1.Pod, wantName, wantHostPath string) bool { - if pod == nil || wantName == "" || wantHostPath == "" || pod.Annotations[enginebinding.AnnotationNodeLocalShmName] != wantName { +func nodeLocalServerHasRuntimeIdentity(pod *corev1.Pod, wantHostPath string) bool { + if pod == nil || wantHostPath == "" { return false } for i := range pod.Spec.Containers { @@ -201,11 +226,28 @@ func nodeLocalServerHasShmIdentity(pod *corev1.Pod, wantName, wantHostPath strin if !validHostPath { return false } - args := container.Args - for j := 0; j+1 < len(args); j++ { - if args[j] == "--shm-name" && args[j+1] == wantName { - return true - } + return builtinruntime.IsLMCacheMPCUDAServerProfile(container.Args) + } + return false +} + +func nodeLocalServerTargetNode(pod *corev1.Pod) string { + if pod == nil { + return "" + } + if nodeName := pod.Annotations[enginebinding.AnnotationNodeLocalTargetNode]; nodeName != "" { + return nodeName + } + return pod.Spec.NodeName +} + +func nodeLocalServerUsesShmHostPath(pod *corev1.Pod, wantHostPath string) bool { + if pod == nil || wantHostPath == "" { + return false + } + for i := range pod.Spec.Volumes { + if hostPath := pod.Spec.Volumes[i].HostPath; hostPath != nil && hostPath.Path == wantHostPath { + return true } } return false @@ -266,14 +308,329 @@ func (r *CacheBackendReconciler) cleanupLMCacheNodeLocalServerPods(ctx context.C ); err != nil { return fmt.Errorf("list obsolete LMCache NodeLocal server Pods for %s/%s: %w", backend.Namespace, backend.Name, err) } + wantShmHostPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + return err + } for i := range pods.Items { pod := &pods.Items[i] if !metav1.IsControlledBy(pod, backend) || pod.DeletionTimestamp != nil { continue } + if nodeLocalServerUsesShmHostPath(pod, wantShmHostPath) { + if err := r.ensureLMCacheNodeLocalCleanupPod(ctx, backend, pod); err != nil { + return err + } + } if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("delete LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) } } + _, err = r.reconcileLMCacheNodeLocalCleanupPods(ctx, backend, nil, false) + return err +} + +func (r *CacheBackendReconciler) ensureLMCacheNodeLocalCleanupPod(ctx context.Context, backend *cachev1alpha1.CacheBackend, server *corev1.Pod) error { + if backend == nil || server == nil || !metav1.IsControlledBy(server, backend) { + return fmt.Errorf("create LMCache NodeLocal SHM cleanup intent: controlled source server is required") + } + nodeName := nodeLocalServerTargetNode(server) + return r.ensureLMCacheNodeLocalCleanupForNode(ctx, backend, nodeName, server) +} + +func (r *CacheBackendReconciler) ensureLMCacheNodeLocalCleanupForNode(ctx context.Context, backend *cachev1alpha1.CacheBackend, nodeName string, source *corev1.Pod) error { + var cleanups corev1.PodList + if err := r.Client.List(ctx, &cleanups, client.InNamespace(backend.Namespace), client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalCleanup: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }); err != nil { + return fmt.Errorf("list existing LMCache NodeLocal SHM cleanup Pods for %s/%s: %w", backend.Namespace, backend.Name, err) + } + for i := range cleanups.Items { + cleanup := &cleanups.Items[i] + if metav1.IsControlledBy(cleanup, backend) && cleanup.Annotations[enginebinding.AnnotationNodeLocalTargetNode] == nodeName { + if builtinruntime.IsLMCacheNodeLocalCleanupPod(cleanup, backend, nodeName, r.NodeLocalShmCleanupImage) && + builtinruntime.LMCacheNodeLocalCleanupIsGated(cleanup) { + return nil + } + if err := r.Client.Delete(ctx, cleanup); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete invalid LMCache NodeLocal SHM cleanup Pod %s/%s: %w", cleanup.Namespace, cleanup.Name, err) + } + return fmt.Errorf("deleted invalid LMCache NodeLocal SHM cleanup Pod %s/%s", cleanup.Namespace, cleanup.Name) + } + } + + desired, err := builtinruntime.RenderLMCacheNodeLocalCleanupPod(backend, nodeName, r.NodeLocalShmCleanupImage, source) + if err != nil { + return err + } + if err := controllerutil.SetControllerReference(backend, desired, r.Scheme); err != nil { + return fmt.Errorf("own LMCache NodeLocal SHM cleanup Pod %s/%s: %w", desired.Namespace, desired.Name, err) + } + if err := r.Client.Create(ctx, desired); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create LMCache NodeLocal SHM cleanup Pod %s/%s: %w", desired.Namespace, desired.Name, err) + } + var existing corev1.Pod + if getErr := r.Client.Get(ctx, client.ObjectKeyFromObject(desired), &existing); getErr != nil { + return fmt.Errorf("inspect existing LMCache NodeLocal SHM cleanup Pod %s/%s: %w", desired.Namespace, desired.Name, getErr) + } + if !builtinruntime.IsLMCacheNodeLocalCleanupPod(&existing, backend, nodeName, r.NodeLocalShmCleanupImage) || + !builtinruntime.LMCacheNodeLocalCleanupIsGated(&existing) { + return fmt.Errorf("LMCache NodeLocal SHM cleanup Pod name %s/%s is occupied by another object", desired.Namespace, desired.Name) + } + } return nil } + +func (r *CacheBackendReconciler) ensureLMCacheNodeLocalCleanupForConsumers(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { + if backend == nil || backend.UID == "" { + return nil + } + wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + return err + } + reader := client.Reader(r.APIReader) + if reader == nil { + reader = r.Client + } + var pods corev1.PodList + if err := reader.List(ctx, &pods, client.InNamespace(backend.Namespace)); err != nil { + return fmt.Errorf("list NodeLocal SHM consumers for finalization of %s/%s: %w", backend.Namespace, backend.Name, err) + } + wantOwner := backend.Namespace + "/" + backend.Name + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Spec.NodeName == "" || pod.Labels[enginebinding.LabelLMCacheNodeLocalCleanup] == "true" || + pod.Annotations[enginebinding.AnnotationInjectedBy] != wantOwner || + pod.Annotations[enginebinding.AnnotationInjectedByUID] != string(backend.UID) { + continue + } + usesPath := false + for j := range pod.Spec.Volumes { + usesPath = usesPath || (pod.Spec.Volumes[j].HostPath != nil && pod.Spec.Volumes[j].HostPath.Path == wantPath) + } + if usesPath { + if err := r.ensureLMCacheNodeLocalCleanupForNode(ctx, backend, pod.Spec.NodeName, pod); err != nil { + return err + } + } + } + return nil +} + +func (r *CacheBackendReconciler) finalizeLMCacheNodeLocal(ctx context.Context, backend *cachev1alpha1.CacheBackend) (ctrl.Result, error) { + if err := r.cleanupOwnedWorkload(ctx, backend); err != nil { + return ctrl.Result{}, err + } + if err := r.ensureLMCacheNodeLocalCleanupForConsumers(ctx, backend); err != nil { + return ctrl.Result{}, err + } + if err := r.cleanupLMCacheNodeLocalServerPods(ctx, backend); err != nil { + return ctrl.Result{}, err + } + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(ctx, backend, nil, false); err != nil { + return ctrl.Result{}, err + } + + var servers, cleanups corev1.PodList + if err := r.Client.List(ctx, &servers, client.InNamespace(backend.Namespace), client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalServer: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }); err != nil { + return ctrl.Result{}, err + } + if err := r.Client.List(ctx, &cleanups, client.InNamespace(backend.Namespace), client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalCleanup: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }); err != nil { + return ctrl.Result{}, err + } + if len(servers.Items) != 0 || len(cleanups.Items) != 0 { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + consumers, err := r.nodeLocalShmConsumers(ctx, backend, "") + if err != nil { + return ctrl.Result{}, err + } + if len(consumers) != 0 { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + before := backend.DeepCopy() + controllerutil.RemoveFinalizer(backend, nodeLocalShmCleanupFinalizer) + if err := r.Patch(ctx, backend, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("remove NodeLocal SHM cleanup finalizer from %s/%s: %w", backend.Namespace, backend.Name, err) + } + return ctrl.Result{}, nil +} + +// reconcileLMCacheNodeLocalCleanupPods advances existing cleanup intents. A +// gated cleanup is cancelled when normal engine demand returns before it +// starts; once scheduled, it blocks server recreation until it succeeds. +func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalCleanupPods(ctx context.Context, backend *cachev1alpha1.CacheBackend, demand map[string]*corev1.Pod, cancelOnDemand bool) (map[string]bool, error) { + blocked := map[string]bool{} + if backend == nil || backend.UID == "" { + return blocked, nil + } + var pods corev1.PodList + if err := r.Client.List(ctx, &pods, client.InNamespace(backend.Namespace), client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalCleanup: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }); err != nil { + return nil, fmt.Errorf("list LMCache NodeLocal SHM cleanup Pods for %s/%s: %w", backend.Namespace, backend.Name, err) + } + activeByNode := map[string]bool{} + succeededByNode := map[string]bool{} + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, backend) { + continue + } + nodeName := pod.Annotations[enginebinding.AnnotationNodeLocalTargetNode] + if !builtinruntime.IsLMCacheNodeLocalCleanupPod(pod, backend, nodeName, r.NodeLocalShmCleanupImage) { + if pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("delete invalid LMCache NodeLocal SHM cleanup Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + return nil, fmt.Errorf("delete invalid LMCache NodeLocal SHM cleanup Pod %s/%s", pod.Namespace, pod.Name) + } + switch { + case builtinruntime.LMCacheNodeLocalCleanupSucceeded(pod, nodeName): + succeededByNode[nodeName] = true + case pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded: + default: + if pod.DeletionTimestamp == nil { + activeByNode[nodeName] = true + } + } + } + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, backend) { + continue + } + nodeName := pod.Annotations[enginebinding.AnnotationNodeLocalTargetNode] + if builtinruntime.LMCacheNodeLocalCleanupSucceeded(pod, nodeName) { + if pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("delete completed LMCache NodeLocal SHM cleanup Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + continue + } + if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded { + blocked[nodeName] = true + if !succeededByNode[nodeName] { + retryCount, err := strconv.Atoi(pod.Annotations[nodeLocalShmCleanupRetryAnnotation]) + if pod.Annotations[nodeLocalShmCleanupRetryAnnotation] == "" { + retryCount, err = 0, nil + } + if err != nil || retryCount < 0 || retryCount >= nodeLocalShmCleanupMaxRetryAttempts { + continue + } + if !activeByNode[nodeName] { + retry, err := builtinruntime.RenderLMCacheNodeLocalCleanupRetryPod(backend, pod) + if err != nil { + return nil, err + } + retry.Annotations[nodeLocalShmCleanupRetryAnnotation] = strconv.Itoa(retryCount + 1) + if err := controllerutil.SetControllerReference(backend, retry, r.Scheme); err != nil { + return nil, fmt.Errorf("own LMCache NodeLocal SHM cleanup retry Pod %s/%s: %w", retry.Namespace, retry.Name, err) + } + if err := r.Client.Create(ctx, retry); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, fmt.Errorf("create LMCache NodeLocal SHM cleanup retry Pod %s/%s: %w", retry.Namespace, retry.Name, err) + } + var existing corev1.Pod + if getErr := r.Client.Get(ctx, client.ObjectKeyFromObject(retry), &existing); getErr != nil { + return nil, fmt.Errorf("inspect existing LMCache NodeLocal SHM cleanup retry Pod %s/%s: %w", retry.Namespace, retry.Name, getErr) + } + if !metav1.IsControlledBy(&existing, backend) { + return nil, fmt.Errorf("LMCache NodeLocal SHM cleanup retry Pod name %s/%s is occupied by another object", retry.Namespace, retry.Name) + } + } + activeByNode[nodeName] = true + } + } + if pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("delete failed LMCache NodeLocal SHM cleanup Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + continue + } + gated := len(pod.Spec.SchedulingGates) > 0 + if gated && cancelOnDemand && demand[nodeName] != nil { + if pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("cancel LMCache NodeLocal SHM cleanup Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + continue + } + blocked[nodeName] = true + if !gated || pod.DeletionTimestamp != nil { + continue + } + consumers, err := r.nodeLocalShmConsumers(ctx, backend, nodeName) + if err != nil { + return nil, err + } + if len(consumers) != 0 { + continue + } + // A managed Engine admitted after this snapshot cannot use the UID + // directory while cleanup runs: only its main container receives the + // inference-cache-authored SHM mount, and that container remains behind + // the same-node startup gate. This cleanup Pod keeps Server recreation + // blocked until it succeeds, so the gate cannot complete early. Races + // from unmanaged processes mounting the hostPath are outside the trusted + // node boundary, but the cluster-wide snapshot above still waits for any + // such Pod that already exists. + before := pod.DeepCopy() + pod.Spec.SchedulingGates = nil + if err := r.Client.Patch(ctx, pod, client.MergeFrom(before)); err != nil { + return nil, fmt.Errorf("release LMCache NodeLocal SHM cleanup Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + return blocked, nil +} + +func (r *CacheBackendReconciler) nodeLocalShmConsumers(ctx context.Context, backend *cachev1alpha1.CacheBackend, nodeName string) ([]string, error) { + wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + return nil, err + } + reader := client.Reader(r.APIReader) + if reader == nil { + reader = r.Client + } + var pods corev1.PodList + if err := reader.List(ctx, &pods); err != nil { + return nil, fmt.Errorf("list cluster-wide SHM consumers for NodeLocal CacheBackend %s/%s: %w", backend.Namespace, backend.Name, err) + } + consumers := []string{} + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Spec.NodeName == "" || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + if nodeName != "" && pod.Spec.NodeName != nodeName { + continue + } + if pod.Namespace == backend.Namespace && pod.Labels[enginebinding.LabelLMCacheNodeLocalCleanup] == "true" && metav1.IsControlledBy(pod, backend) { + continue + } + for j := range pod.Spec.Volumes { + hostPath := pod.Spec.Volumes[j].HostPath + if hostPath != nil && hostPath.Path == wantPath { + consumers = append(consumers, pod.Namespace+"/"+pod.Name) + break + } + } + } + return consumers, nil +} diff --git a/internal/controller/cachebackend_mp_lifecycle_test.go b/internal/controller/cachebackend_mp_lifecycle_test.go index 07dae13a..8ece292e 100644 --- a/internal/controller/cachebackend_mp_lifecycle_test.go +++ b/internal/controller/cachebackend_mp_lifecycle_test.go @@ -6,6 +6,8 @@ package controller import ( "context" + "errors" + "fmt" "testing" "time" @@ -14,8 +16,11 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" @@ -56,6 +61,388 @@ func nodeLocalEngine(backend *cachev1alpha1.CacheBackend, name, node string) *co } } +func nodeLocalCleanupPod(t *testing.T, backend *cachev1alpha1.CacheBackend) *corev1.Pod { + t.Helper() + pod, err := builtinruntime.RenderLMCacheNodeLocalCleanupPod( + backend, + "node-a", + "registry.example/inference-cache-shm-cleanup@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + nodeLocalEngine(backend, "engine-a", "node-a"), + ) + if err != nil { + t.Fatal(err) + } + if err := controllerutil.SetControllerReference(backend, pod, newScheme(t)); err != nil { + t.Fatal(err) + } + return pod +} + +func TestNodeLocalCleanupHelperGuards(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + reconciler := newReconciler(newScheme(t), backend) + if nodeLocalServerTargetNode(nil) != "" { + t.Fatal("nil server Pod produced a target node") + } + server := &corev1.Pod{Spec: corev1.PodSpec{NodeName: "node-a"}} + if nodeLocalServerTargetNode(server) != "node-a" { + t.Fatal("server target did not fall back to spec.nodeName") + } + if nodeLocalServerUsesShmHostPath(nil, "/dev/shm/inference-cache/x") || nodeLocalServerUsesShmHostPath(server, "") { + t.Fatal("incomplete server identity reported an SHM hostPath") + } + if nodeLocalServerHasRuntimeIdentity(server, "/dev/shm/inference-cache/x") { + t.Fatal("server without the managed container reported a runtime identity") + } + if err := reconciler.cleanupLMCacheNodeLocalServerPods(context.Background(), nil); err != nil { + t.Fatalf("nil backend cleanup: %v", err) + } + if err := reconciler.ensureLMCacheNodeLocalCleanupPod(context.Background(), backend, nil); err == nil { + t.Fatal("nil source server was accepted for cleanup intent") + } + if err := reconciler.ensureLMCacheNodeLocalCleanupForConsumers(context.Background(), nil); err != nil { + t.Fatalf("nil backend consumer intent: %v", err) + } + if _, err := reconciler.reconcileLMCacheNodeLocalCleanupPods(context.Background(), nil, nil, false); err != nil { + t.Fatalf("nil backend cleanup reconcile: %v", err) + } + if _, err := reconciler.nodeLocalShmConsumers(context.Background(), nil, ""); err == nil { + t.Fatal("nil backend produced an SHM consumer snapshot") + } + if err := reconciler.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "", server); err == nil { + t.Fatal("empty cleanup target node was accepted") + } + unsafeUID := backend.DeepCopy() + unsafeUID.UID = "unsafe/uid" + if err := reconciler.ensureLMCacheNodeLocalCleanupForConsumers(context.Background(), unsafeUID); err == nil { + t.Fatal("unsafe backend UID was accepted for consumer cleanup") + } +} + +func TestNodeLocalCleanupPropagatesListErrors(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + listErr := errors.New("list denied") + reconciler := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return listErr + }, + }, backend) + server := nodeLocalEngine(backend, "engine-a", "node-a") + checks := []func() error{ + func() error { return reconciler.cleanupLMCacheNodeLocalServerPods(context.Background(), backend) }, + func() error { + return reconciler.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "node-a", server) + }, + func() error { + return reconciler.ensureLMCacheNodeLocalCleanupForConsumers(context.Background(), backend) + }, + func() error { + _, err := reconciler.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false) + return err + }, + func() error { + _, err := reconciler.nodeLocalShmConsumers(context.Background(), backend, "") + return err + }, + } + for i, check := range checks { + if err := check(); !errors.Is(err, listErr) { + t.Fatalf("list error check %d = %v, want %v", i, err, listErr) + } + } +} + +func TestEnsureNodeLocalCleanupRejectsInvalidOrOccupiedPod(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + source := nodeLocalEngine(backend, "engine-a", "node-a") + + t.Run("invalid existing", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + cleanup.Spec.Containers[0].Command = []string{"/bin/true"} + r := newReconciler(newScheme(t), backend, cleanup) + if err := r.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "node-a", source); err == nil { + t.Fatal("invalid existing cleanup Pod was accepted") + } + }) + + t.Run("invalid existing delete denied", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + cleanup.Spec.Containers[0].Command = []string{"/bin/true"} + deleteErr := errors.New("delete invalid denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return deleteErr + }, + }, backend, cleanup) + if err := r.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "node-a", source); !errors.Is(err, deleteErr) { + t.Fatalf("invalid cleanup delete error = %v, want %v", err, deleteErr) + } + }) + + t.Run("create denied", func(t *testing.T) { + createErr := errors.New("create denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + return createErr + }, + }, backend) + if err := r.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "node-a", source); !errors.Is(err, createErr) { + t.Fatalf("create error = %v, want %v", err, createErr) + } + }) + + t.Run("occupied name", func(t *testing.T) { + foreign := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace, + }, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "foreign"}}}} + r := newReconciler(newScheme(t), backend, foreign) + if err := r.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "node-a", source); err == nil { + t.Fatal("occupied cleanup Pod name was accepted") + } + }) + + t.Run("owner scheme", func(t *testing.T) { + r := newReconciler(newScheme(t), backend) + r.Scheme = runtime.NewScheme() + if err := r.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "node-a", source); err == nil { + t.Fatal("cleanup Pod owner was resolved with an empty scheme") + } + }) + + t.Run("inspect collision denied", func(t *testing.T) { + inspectErr := errors.New("inspect denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Create: func(_ context.Context, _ client.WithWatch, obj client.Object, _ ...client.CreateOption) error { + return apierrors.NewAlreadyExists(corev1.Resource("pods"), obj.GetName()) + }, + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return inspectErr + }, + }, backend) + if err := r.ensureLMCacheNodeLocalCleanupForNode(context.Background(), backend, "node-a", source); !errors.Is(err, inspectErr) { + t.Fatalf("inspect collision error = %v, want %v", err, inspectErr) + } + }) +} + +func TestReconcileNodeLocalCleanupPropagatesWriteErrors(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + + t.Run("completed delete", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + cleanup.Spec.SchedulingGates = nil + cleanup.Spec.NodeName = "node-a" + cleanup.Status.Phase = corev1.PodSucceeded + cleanup.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "lmcache-node-shm-cleanup", + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + }}, + }} + deleteErr := errors.New("delete completed denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return deleteErr + }, + }, backend, cleanup) + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); !errors.Is(err, deleteErr) { + t.Fatalf("completed delete error = %v, want %v", err, deleteErr) + } + }) + + t.Run("retry create", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + cleanup.Spec.SchedulingGates = nil + cleanup.Spec.NodeName = "node-a" + cleanup.Status.Phase = corev1.PodFailed + createErr := errors.New("retry create denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + return createErr + }, + }, backend, cleanup) + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); !errors.Is(err, createErr) { + t.Fatalf("retry create error = %v, want %v", err, createErr) + } + }) + + t.Run("retry occupied", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + cleanup.Spec.SchedulingGates = nil + cleanup.Spec.NodeName = "node-a" + cleanup.Status.Phase = corev1.PodFailed + foreign := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: builtinruntime.NodeLocalCleanupRetryPodName(backend.UID, "node-a", cleanup.Name), + Namespace: backend.Namespace, + }} + r := newReconciler(newScheme(t), backend, cleanup, foreign) + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); err == nil { + t.Fatal("occupied cleanup retry name was accepted") + } + }) + + t.Run("failed delete", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + cleanup.Spec.SchedulingGates = nil + cleanup.Spec.NodeName = "node-a" + cleanup.Status.Phase = corev1.PodFailed + deleteErr := errors.New("delete failed denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Delete: func(_ context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if obj.GetName() == cleanup.Name { + return deleteErr + } + return c.Delete(context.Background(), obj, opts...) + }, + }, backend, cleanup) + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); !errors.Is(err, deleteErr) { + t.Fatalf("failed cleanup delete error = %v, want %v", err, deleteErr) + } + }) + + t.Run("invalid synthetic", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + cleanup.Spec.Containers[0].Command = []string{"/bin/true"} + r := newReconciler(newScheme(t), backend, cleanup) + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); err == nil { + t.Fatal("invalid cleanup Pod did not return a retry error") + } + }) + + t.Run("foreign owner", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + other := backend.DeepCopy() + other.UID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + if err := controllerutil.SetControllerReference(other, cleanup, newScheme(t)); err != nil { + t.Fatal(err) + } + r := newReconciler(newScheme(t), backend, cleanup) + if blocked, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); err != nil || len(blocked) != 0 { + t.Fatalf("foreign cleanup = blocked:%v err:%v", blocked, err) + } + }) + + t.Run("cancel delete", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + deleteErr := errors.New("cancel denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return deleteErr + }, + }, backend, cleanup) + demand := map[string]*corev1.Pod{"node-a": nodeLocalEngine(backend, "engine-a", "node-a")} + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, demand, true); !errors.Is(err, deleteErr) { + t.Fatalf("cancel delete error = %v, want %v", err, deleteErr) + } + }) + + t.Run("gate patch", func(t *testing.T) { + cleanup := nodeLocalCleanupPod(t, backend) + patchErr := errors.New("gate patch denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { + return patchErr + }, + }, backend, cleanup) + if _, err := r.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); !errors.Is(err, patchErr) { + t.Fatalf("gate patch error = %v, want %v", err, patchErr) + } + }) +} + +func TestFinalizeNodeLocalPropagatesPodListErrors(t *testing.T) { + for _, failAt := range []int{1, 2, 3, 4, 5} { + t.Run(fmt.Sprintf("pod list %d", failAt), func(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + listErr := errors.New("list denied") + podLists := 0 + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*corev1.PodList); ok { + podLists++ + if podLists == failAt { + return listErr + } + } + return c.List(ctx, list, opts...) + }, + }, backend) + if _, err := r.finalizeLMCacheNodeLocal(context.Background(), backend); !errors.Is(err, listErr) { + t.Fatalf("finalize Pod list %d error = %v, want %v", failAt, err, listErr) + } + }) + } +} + +func TestFinalizeNodeLocalPropagatesGetAndPatchErrors(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + + t.Run("owned workload get", func(t *testing.T) { + getErr := errors.New("deployment get denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*appsv1.Deployment); ok { + return getErr + } + return c.Get(ctx, key, obj, opts...) + }, + }, backend) + if _, err := r.finalizeLMCacheNodeLocal(context.Background(), backend); !errors.Is(err, getErr) { + t.Fatalf("finalize get error = %v, want %v", err, getErr) + } + }) + + t.Run("finalizer patch", func(t *testing.T) { + patchErr := errors.New("backend patch denied") + r := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Patch: func(_ context.Context, _ client.WithWatch, obj client.Object, _ client.Patch, _ ...client.PatchOption) error { + if _, ok := obj.(*cachev1alpha1.CacheBackend); ok { + return patchErr + } + return nil + }, + }, backend) + if _, err := r.finalizeLMCacheNodeLocal(context.Background(), backend); !errors.Is(err, patchErr) { + t.Fatalf("finalize patch error = %v, want %v", err, patchErr) + } + }) +} + +func markNodeLocalCleanupSucceeded(t *testing.T, reconciler *CacheBackendReconciler, pod *corev1.Pod, nodeName string) { + t.Helper() + pod.Spec.NodeName = nodeName + if err := reconciler.Update(context.Background(), pod); err != nil { + t.Fatalf("bind cleanup Pod to %s: %v", nodeName, err) + } + pod.Status.Phase = corev1.PodSucceeded + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "lmcache-node-shm-cleanup", + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + }}, + }} + if err := reconciler.Status().Update(context.Background(), pod); err != nil { + t.Fatalf("mark cleanup succeeded: %v", err) + } +} + +func TestReconcileNodeLocalFinalizerCompletesEmptyBackendDeletion(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + reconciler := newReconciler(newScheme(t), backend) + reconcile(t, reconciler, backend.Name, backend.Namespace) + live := getBackend(t, reconciler, backend.Name, backend.Namespace) + if !controllerutil.ContainsFinalizer(live, nodeLocalShmCleanupFinalizer) { + t.Fatalf("NodeLocal cleanup finalizer was not installed: %v", live.Finalizers) + } + if err := reconciler.Delete(context.Background(), live); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(backend), &cachev1alpha1.CacheBackend{}); !apierrors.IsNotFound(err) { + t.Fatalf("empty NodeLocal backend after finalization = %v, want NotFound", err) + } +} + func TestReconcileNodeLocalCreatesServersOnlyForEngineNodes(t *testing.T) { backend := nodeLocalBackend("node-cache", "ns1") reconciler := newReconciler(newScheme(t), backend) @@ -187,6 +574,365 @@ func TestReconcileNodeLocalZeroIdleRetentionDeletesImmediately(t *testing.T) { if err := reconciler.Get(context.Background(), serverKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { t.Fatalf("server with zero idle retention = %v, want NotFound", err) } + reconcile(t, reconciler, backend.Name, backend.Namespace) + cleanupKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace} + var cleanup corev1.Pod + if err := reconciler.Get(context.Background(), cleanupKey, &cleanup); err != nil { + t.Fatalf("get SHM cleanup Pod: %v", err) + } + if len(cleanup.Spec.SchedulingGates) != 0 || cleanup.Spec.HostNetwork || cleanup.Spec.HostPID || cleanup.Spec.HostIPC { + t.Fatalf("released cleanup Pod = gates:%+v hostNetwork:%v hostPID:%v hostIPC:%v", cleanup.Spec.SchedulingGates, cleanup.Spec.HostNetwork, cleanup.Spec.HostPID, cleanup.Spec.HostIPC) + } + if cleanup.Spec.Containers[0].Image != reconciler.NodeLocalShmCleanupImage { + t.Fatalf("cleanup image = %q, want platform helper %q", cleanup.Spec.Containers[0].Image, reconciler.NodeLocalShmCleanupImage) + } + markNodeLocalCleanupSucceeded(t, reconciler, &cleanup, "node-a") + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), cleanupKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("completed cleanup Pod = %v, want NotFound", err) + } +} + +func TestReconcileNodeLocalCancelsGatedCleanupWhenDemandReturns(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + backend.Spec.LMCache.NodeLocal.IdleRetentionSeconds = 0 + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Delete(context.Background(), engine); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + replacement := nodeLocalEngine(backend, "engine-b", "node-a") + if err := reconciler.Create(context.Background(), replacement); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + cleanupKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace} + if err := reconciler.Get(context.Background(), cleanupKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("gated cleanup after demand returned = %v, want NotFound", err) + } + serverKey := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + if err := reconciler.Get(context.Background(), serverKey, &corev1.Pod{}); err != nil { + t.Fatalf("server was not recreated after cleanup cancellation: %v", err) + } +} + +func TestReconcileNodeLocalActiveCleanupBlocksServerForNewDemand(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + backend.Spec.LMCache.NodeLocal.IdleRetentionSeconds = 0 + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Delete(context.Background(), engine); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + cleanupKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace} + var cleanup corev1.Pod + if err := reconciler.Get(context.Background(), cleanupKey, &cleanup); err != nil { + t.Fatal(err) + } + if len(cleanup.Spec.SchedulingGates) != 0 { + t.Fatalf("cleanup gate was not released before new demand: %+v", cleanup.Spec.SchedulingGates) + } + + replacement := nodeLocalEngine(backend, "engine-b", "node-a") + if err := builtinruntime.NewVLLMLMCacheMPAdapter(builtinruntime.SubscriberConfig{}).InjectEngineConfig(&replacement.Spec, nil, backend); err != nil { + t.Fatalf("inject replacement Engine startup gate: %v", err) + } + if len(replacement.Spec.InitContainers) != 1 || replacement.Spec.InitContainers[0].Name != "lmcache-node-local-gate" || len(replacement.Spec.InitContainers[0].VolumeMounts) != 0 { + t.Fatalf("replacement Engine does not have an SHM-free startup gate: %+v", replacement.Spec.InitContainers) + } + if err := reconciler.Create(context.Background(), replacement); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + serverKey := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + if err := reconciler.Get(context.Background(), serverKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("server while cleanup is active = %v, want NotFound", err) + } + markNodeLocalCleanupSucceeded(t, reconciler, &cleanup, "node-a") + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), serverKey, &corev1.Pod{}); err != nil { + t.Fatalf("server after cleanup succeeded: %v", err) + } +} + +func TestReconcileNodeLocalCleanupWaitsForExactHostPathConsumer(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + backend.Spec.LMCache.NodeLocal.IdleRetentionSeconds = 0 + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Delete(context.Background(), engine); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + t.Fatal(err) + } + consumer := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "terminating-consumer", Namespace: "other-ns"}, Spec: corev1.PodSpec{ + NodeName: "node-a", Containers: []corev1.Container{{Name: "engine"}}, Volumes: []corev1.Volume{{ + Name: "shm", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: wantPath}}, + }}, + }} + if err := reconciler.Create(context.Background(), consumer); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + cleanupKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace} + var cleanup corev1.Pod + if err := reconciler.Get(context.Background(), cleanupKey, &cleanup); err != nil { + t.Fatal(err) + } + if len(cleanup.Spec.SchedulingGates) != 1 { + t.Fatalf("cleanup gate released while cross-namespace hostPath consumer remained: %+v", cleanup.Spec.SchedulingGates) + } + consumer.Status.Phase = corev1.PodSucceeded + if err := reconciler.Status().Update(context.Background(), consumer); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), cleanupKey, &cleanup); err != nil { + t.Fatal(err) + } + if len(cleanup.Spec.SchedulingGates) != 0 { + t.Fatalf("cleanup gate remained after consumer became terminal: %+v", cleanup.Spec.SchedulingGates) + } +} + +func TestNodeLocalShmConsumersIgnoresUnscheduledPod(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + t.Fatal(err) + } + consumer := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "unscheduled", Namespace: "other-ns"}, Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "engine"}}, + Volumes: []corev1.Volume{{Name: "shm", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: wantPath, + }}}}, + }} + reconciler := newReconciler(newScheme(t), backend, consumer) + consumers, err := reconciler.nodeLocalShmConsumers(context.Background(), backend, "") + if err != nil { + t.Fatal(err) + } + if len(consumers) != 0 { + t.Fatalf("unscheduled Pod reported as SHM consumer: %v", consumers) + } +} + +func TestReconcileNodeLocalRetriesFailedCleanupPod(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + backend.Spec.LMCache.NodeLocal.IdleRetentionSeconds = 0 + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Delete(context.Background(), engine); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + cleanupKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace} + var failed corev1.Pod + if err := reconciler.Get(context.Background(), cleanupKey, &failed); err != nil { + t.Fatal(err) + } + failed.Status.Phase = corev1.PodFailed + if err := reconciler.Status().Update(context.Background(), &failed); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + retryKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupRetryPodName(backend.UID, "node-a", failed.Name), Namespace: backend.Namespace} + var retry corev1.Pod + if err := reconciler.Get(context.Background(), retryKey, &retry); err != nil { + t.Fatalf("get cleanup retry: %v", err) + } + if len(retry.Spec.SchedulingGates) != 1 { + t.Fatalf("cleanup retry did not re-check quiescence: %+v", retry.Spec.SchedulingGates) + } + if retry.Annotations[nodeLocalShmCleanupRetryAnnotation] != "1" { + t.Fatalf("cleanup retry count = %q, want 1", retry.Annotations[nodeLocalShmCleanupRetryAnnotation]) + } + if err := reconciler.Get(context.Background(), cleanupKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("failed cleanup Pod after replacement = %v, want NotFound", err) + } + + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), retryKey, &retry); err != nil { + t.Fatal(err) + } + if len(retry.Spec.SchedulingGates) != 0 { + t.Fatalf("cleanup retry gate remained without consumers: %+v", retry.Spec.SchedulingGates) + } + markNodeLocalCleanupSucceeded(t, reconciler, &retry, "node-a") + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), retryKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("completed cleanup retry = %v, want NotFound", err) + } +} + +func TestReconcileNodeLocalRetriesSucceededPodWithoutHelperStatus(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + cleanup, err := builtinruntime.RenderLMCacheNodeLocalCleanupPod( + backend, + "node-a", + "registry.example/inference-cache-shm-cleanup@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + nodeLocalEngine(backend, "engine-a", "node-a"), + ) + if err != nil { + t.Fatal(err) + } + if err := controllerutil.SetControllerReference(backend, cleanup, newScheme(t)); err != nil { + t.Fatal(err) + } + cleanup.Spec.SchedulingGates = nil + cleanup.Spec.NodeName = "node-a" + cleanup.Status.Phase = corev1.PodSucceeded + cleanup.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "foreign", + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 0, + }}, + }} + reconciler := newReconciler(newScheme(t), backend, cleanup) + if _, err := reconciler.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); err != nil { + t.Fatal(err) + } + + retryKey := types.NamespacedName{ + Name: builtinruntime.NodeLocalCleanupRetryPodName(backend.UID, "node-a", cleanup.Name), + Namespace: backend.Namespace, + } + var retry corev1.Pod + if err := reconciler.Get(context.Background(), retryKey, &retry); err != nil { + t.Fatalf("get retry after untrusted success status: %v", err) + } + if !builtinruntime.LMCacheNodeLocalCleanupIsGated(&retry) { + t.Fatalf("retry after untrusted success status is not gated: %+v", retry.Spec.SchedulingGates) + } + if retry.Annotations[nodeLocalShmCleanupRetryAnnotation] != "1" { + t.Fatalf("cleanup retry count = %q, want 1", retry.Annotations[nodeLocalShmCleanupRetryAnnotation]) + } +} + +func TestReconcileNodeLocalStopsAfterCleanupRetryLimit(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + cleanup, err := builtinruntime.RenderLMCacheNodeLocalCleanupPod( + backend, + "node-a", + "registry.example/inference-cache-shm-cleanup@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + nodeLocalEngine(backend, "engine-a", "node-a"), + ) + if err != nil { + t.Fatal(err) + } + if err := controllerutil.SetControllerReference(backend, cleanup, newScheme(t)); err != nil { + t.Fatal(err) + } + cleanup.Spec.SchedulingGates = nil + cleanup.Spec.NodeName = "node-a" + cleanup.Annotations[nodeLocalShmCleanupRetryAnnotation] = "3" + cleanup.Status.Phase = corev1.PodFailed + reconciler := newReconciler(newScheme(t), backend, cleanup) + if _, err := reconciler.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); err != nil { + t.Fatal(err) + } + + retryKey := types.NamespacedName{ + Name: builtinruntime.NodeLocalCleanupRetryPodName(backend.UID, "node-a", cleanup.Name), + Namespace: backend.Namespace, + } + if err := reconciler.Get(context.Background(), retryKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("cleanup retry beyond limit = %v, want NotFound", err) + } + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(cleanup), &corev1.Pod{}); err != nil { + t.Fatalf("terminal cleanup Pod was not retained for diagnosis: %v", err) + } +} + +func TestReconcileNodeLocalFinalizerSurvivesTopologyChange(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engine := nodeLocalEngine(backend, "engine-a", "node-a") + wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + t.Fatal(err) + } + pathType := corev1.HostPathDirectoryOrCreate + engine.Spec.Volumes = []corev1.Volume{{Name: "shm", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: wantPath, Type: &pathType}}}} + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + live := getBackend(t, reconciler, backend.Name, backend.Namespace) + live.Spec.LMCache = lmcacheBackend("fixture", "ns1").Spec.LMCache.DeepCopy() + if err := reconciler.Update(context.Background(), live); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + cleanupKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace} + var cleanup corev1.Pod + if err := reconciler.Get(context.Background(), cleanupKey, &cleanup); err != nil { + t.Fatalf("cleanup intent after topology change: %v", err) + } + if cleanup.Spec.Containers[0].Image != reconciler.NodeLocalShmCleanupImage || len(cleanup.Spec.SchedulingGates) != 1 { + t.Fatalf("topology-change cleanup = image:%q gates:%+v", cleanup.Spec.Containers[0].Image, cleanup.Spec.SchedulingGates) + } + + live = getBackend(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Delete(context.Background(), live); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Delete(context.Background(), engine); err != nil { + t.Fatal(err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), cleanupKey, &cleanup); err != nil { + t.Fatal(err) + } + if len(cleanup.Spec.SchedulingGates) != 0 { + t.Fatalf("topology-change cleanup gate remained after consumer deletion: %+v", cleanup.Spec.SchedulingGates) + } + markNodeLocalCleanupSucceeded(t, reconciler, &cleanup, "node-a") + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(backend), &cachev1alpha1.CacheBackend{}); !apierrors.IsNotFound(err) { + t.Fatalf("topology-changed backend after cleanup finalization = %v, want NotFound", err) + } +} + +func TestReconcileNodeLocalReturnsInvalidCleanupDeleteError(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + cleanup, err := builtinruntime.RenderLMCacheNodeLocalCleanupPod(backend, "node-a", "registry.example/inference-cache-shm-cleanup@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", nodeLocalEngine(backend, "engine-a", "node-a")) + if err != nil { + t.Fatal(err) + } + if err := controllerutil.SetControllerReference(backend, cleanup, newScheme(t)); err != nil { + t.Fatal(err) + } + cleanup.Spec.Volumes[0].HostPath.Path = "/dev/shm/inference-cache/foreign" + deleteErr := errors.New("cleanup delete denied") + reconciler := newReconcilerWithInterceptor(newScheme(t), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if obj.GetName() == cleanup.Name { + return deleteErr + } + return c.Delete(ctx, obj, opts...) + }, + }, backend, cleanup) + if _, err := reconciler.reconcileLMCacheNodeLocalCleanupPods(context.Background(), backend, nil, false); !errors.Is(err, deleteErr) { + t.Fatalf("invalid cleanup delete error = %v, want %v", err, deleteErr) + } } func TestReconcileNodeLocalIgnoresSelectorMatchOwnedByAnotherBackend(t *testing.T) { @@ -251,10 +997,7 @@ func TestReconcileNodeLocalToPodLocalDeletesServerPods(t *testing.T) { reconcile(t, reconciler, backend.Name, backend.Namespace) live := getBackend(t, reconciler, backend.Name, backend.Namespace) - podLocal := lmcacheBackend("fixture", "ns1").Spec.LMCache.PodLocal.DeepCopy() - live.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyPodLocal - live.Spec.LMCache.NodeLocal = nil - live.Spec.LMCache.PodLocal = podLocal + live.Spec.LMCache = lmcacheBackend("fixture", "ns1").Spec.LMCache.DeepCopy() if err := reconciler.Update(context.Background(), live); err != nil { t.Fatalf("update backend to PodLocal: %v", err) } @@ -291,7 +1034,7 @@ func TestReconcileNodeLocalReplacesServerOnBackendGenerationChange(t *testing.T) } } -func TestReconcileNodeLocalReplacesServerMissingUIDScopedShmIdentity(t *testing.T) { +func TestReconcileNodeLocalCreatesCleanupBeforeReplacingLegacyServer(t *testing.T) { backend := nodeLocalBackend("node-cache", "ns1") engine := nodeLocalEngine(backend, "engine-a", "node-a") reconciler := newReconciler(newScheme(t), backend, engine) @@ -305,13 +1048,12 @@ func TestReconcileNodeLocalReplacesServerMissingUIDScopedShmIdentity(t *testing. if err := reconciler.Delete(context.Background(), &old); err != nil { t.Fatalf("delete original server Pod: %v", err) } - delete(old.Annotations, enginebinding.AnnotationNodeLocalShmName) old.ResourceVersion = "" old.UID = "" old.CreationTimestamp = metav1.Time{} args := old.Spec.Containers[0].Args for i := 0; i+1 < len(args); i++ { - if args[i] == "--shm-name" { + if args[i] == "--supported-transfer-mode" { old.Spec.Containers[0].Args = append(args[:i], args[i+2:]...) break } @@ -320,21 +1062,30 @@ func TestReconcileNodeLocalReplacesServerMissingUIDScopedShmIdentity(t *testing. t.Fatalf("create legacy server Pod: %v", err) } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), key, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("legacy server after cleanup intent = %v, want NotFound", err) + } + cleanupKey := types.NamespacedName{Name: builtinruntime.NodeLocalCleanupPodName(backend.UID, "node-a"), Namespace: backend.Namespace} + var cleanup corev1.Pod + if err := reconciler.Get(context.Background(), cleanupKey, &cleanup); err != nil { + t.Fatalf("cleanup intent for legacy server: %v", err) + } + if len(cleanup.Spec.SchedulingGates) != 1 { + t.Fatalf("legacy server cleanup was not gated: %+v", cleanup.Spec.SchedulingGates) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) var replaced corev1.Pod if err := reconciler.Get(context.Background(), key, &replaced); err != nil { t.Fatalf("get replacement server Pod: %v", err) } - want, err := builtinruntime.NodeLocalServerShmName(backend) - if err != nil { - t.Fatal(err) - } wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) if err != nil { t.Fatal(err) } - if !nodeLocalServerHasShmIdentity(&replaced, want, wantPath) { - t.Fatalf("replacement server lacks UID-scoped shm identity: annotations=%v args=%v", replaced.Annotations, replaced.Spec.Containers[0].Args) + if !nodeLocalServerHasRuntimeIdentity(&replaced, wantPath) { + t.Fatalf("replacement server lacks managed CUDA runtime identity: volumes=%v args=%v", replaced.Spec.Volumes, replaced.Spec.Containers[0].Args) } } @@ -365,15 +1116,11 @@ func TestReconcileNodeLocalReplacesServerWithWrongUIDScopedShmDirectory(t *testi if err := reconciler.Get(context.Background(), key, &replaced); err != nil { t.Fatalf("get replacement server Pod: %v", err) } - wantName, err := builtinruntime.NodeLocalServerShmName(backend) - if err != nil { - t.Fatal(err) - } wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) if err != nil { t.Fatal(err) } - if !nodeLocalServerHasShmIdentity(&replaced, wantName, wantPath) { + if !nodeLocalServerHasRuntimeIdentity(&replaced, wantPath) { t.Fatalf("replacement server lacks UID-scoped SHM directory: volumes=%v", replaced.Spec.Volumes) } } diff --git a/internal/controller/cachebackend_nodelocal_integration_test.go b/internal/controller/cachebackend_nodelocal_integration_test.go index 9863d173..89cddabe 100644 --- a/internal/controller/cachebackend_nodelocal_integration_test.go +++ b/internal/controller/cachebackend_nodelocal_integration_test.go @@ -27,6 +27,7 @@ func TestIntegrationCacheBackendNodeLocalServerPod(t *testing.T) { skipWithoutEnvtest(t) k8s, scheme, _ := startEnv(t) r := &CacheBackendReconciler{Client: k8s, APIReader: k8s, Scheme: scheme, Log: logr.Discard()} + configureTestRegistries(r) ctx := context.Background() ns := freshNS(t, k8s) backend := nodeLocalBackend("node-cache", ns) @@ -65,16 +66,12 @@ func TestIntegrationCacheBackendNodeLocalServerPod(t *testing.T) { if got := server.Labels[enginebinding.LabelCacheBackendUID]; got == "" { t.Fatal("server Pod is missing CacheBackend UID identity") } - wantShmName, err := builtinruntime.NodeLocalServerShmName(&live) - if err != nil { - t.Fatal(err) - } wantShmPath, err := builtinruntime.NodeLocalServerShmHostPath(&live) if err != nil { t.Fatal(err) } - if !nodeLocalServerHasShmIdentity(&server, wantShmName, wantShmPath) { - t.Fatalf("server Pod lacks UID-scoped shm identity: annotations=%v args=%v", server.Annotations, server.Spec.Containers[0].Args) + if !nodeLocalServerHasRuntimeIdentity(&server, wantShmPath) { + t.Fatalf("server Pod lacks managed CUDA runtime identity: volumes=%v args=%v", server.Spec.Volumes, server.Spec.Containers[0].Args) } if err := k8s.Get(ctx, key, &live); err != nil { @@ -95,9 +92,7 @@ func TestIntegrationCacheBackendNodeLocalServerPod(t *testing.T) { if err := k8s.Get(ctx, key, &live); err != nil { t.Fatalf("refresh NodeLocal CacheBackend: %v", err) } - live.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyPodLocal - live.Spec.LMCache.PodLocal = lmcacheBackend("podlocal-shape", ns).Spec.LMCache.PodLocal.DeepCopy() - live.Spec.LMCache.NodeLocal = nil + live.Spec.LMCache = lmcacheBackend("podlocal-shape", ns).Spec.LMCache.DeepCopy() if err := k8s.Update(ctx, &live); err != nil { t.Fatalf("switch NodeLocal to PodLocal: %v", err) } diff --git a/internal/controller/cachebackend_reconciler.go b/internal/controller/cachebackend_reconciler.go index b05d8aca..937a1e69 100644 --- a/internal/controller/cachebackend_reconciler.go +++ b/internal/controller/cachebackend_reconciler.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -64,6 +65,9 @@ type CacheBackendReconciler struct { // BackendRegistry resolves remote provider lifecycle independently from // engine/runtime wiring. The composition root must inject it. BackendRegistry *backendadapter.Registry + // NodeLocalShmCleanupImage is the platform-owned, digest-pinned helper image + // used by one-shot NodeLocal SHM cleanup Pods. + NodeLocalShmCleanupImage string // MatchedEnginePodsRequeueInterval overrides the self-requeue cadence // that keeps status.matchedEnginePods fresh between unrelated reconcile // triggers. Zero means "use [DefaultMatchedEnginePodsRequeueInterval]". @@ -170,6 +174,19 @@ func (r *CacheBackendReconciler) Reconcile(ctx context.Context, req ctrl.Request } return ctrl.Result{}, err } + if backend.DeletionTimestamp.IsZero() && isTypedLMCacheNodeLocal(&backend) && !controllerutil.ContainsFinalizer(&backend, nodeLocalShmCleanupFinalizer) { + before := backend.DeepCopy() + controllerutil.AddFinalizer(&backend, nodeLocalShmCleanupFinalizer) + if err := r.Patch(ctx, &backend, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + } + if !backend.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(&backend, nodeLocalShmCleanupFinalizer) { + return r.finalizeLMCacheNodeLocal(ctx, &backend) + } + return ctrl.Result{}, nil + } before := snapshotState(&backend) result, err := r.dispatch(ctx, logger, &backend) diff --git a/internal/controller/cachebackend_reconciler_test.go b/internal/controller/cachebackend_reconciler_test.go index c70b7613..33659d82 100644 --- a/internal/controller/cachebackend_reconciler_test.go +++ b/internal/controller/cachebackend_reconciler_test.go @@ -50,7 +50,7 @@ func newScheme(t *testing.T) *runtime.Scheme { func newReconciler(scheme *runtime.Scheme, objs ...client.Object) *CacheBackendReconciler { c := fake.NewClientBuilder(). WithScheme(scheme). - WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}). + WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}, &corev1.Pod{}). WithObjects(objs...). Build() r := &CacheBackendReconciler{ @@ -63,6 +63,9 @@ func newReconciler(scheme *runtime.Scheme, objs ...client.Object) *CacheBackendR } func configureTestRegistries(r *CacheBackendReconciler) { + if r.NodeLocalShmCleanupImage == "" { + r.NodeLocalShmCleanupImage = "registry.example/inference-cache-shm-cleanup@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } if r.Registry != nil && r.BackendRegistry != nil { return } diff --git a/internal/enginebinding/metadata.go b/internal/enginebinding/metadata.go index 8a91fea0..01bc9b6c 100644 --- a/internal/enginebinding/metadata.go +++ b/internal/enginebinding/metadata.go @@ -24,6 +24,10 @@ const ( // server Pod. LabelLMCacheNodeLocalServer = "inferencecache.io/lmcache-node-server" + // LabelLMCacheNodeLocalCleanup identifies a controller-owned, one-shot Pod + // that clears one backend UID's auxiliary IPC files on one node. + LabelLMCacheNodeLocalCleanup = "inferencecache.io/lmcache-node-shm-cleanup" + // LabelCacheBackendUID is the immutable, label-safe identity used to list // one CacheBackend's NodeLocal server Pods. LabelCacheBackendUID = "inferencecache.io/cache-backend-uid" @@ -45,10 +49,6 @@ const ( // node affinity rather than spec.nodeName so hostPort conflicts are checked. AnnotationNodeLocalTargetNode = "inferencecache.io/node-local-target-node" - // AnnotationNodeLocalShmName records the controller-derived POSIX shared - // memory object owned by this CacheBackend's NodeLocal server pool. - AnnotationNodeLocalShmName = "inferencecache.io/node-local-shm-name" - // AnnotationNodeLocalIdleSince records when the final active engine left a // node. The controller removes it when demand returns and deletes the server // only after the configured idle-retention window expires. diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index af9579b1..e35e44f0 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -381,7 +381,8 @@ func TestHandle_TypedNodeLocalVLLMGatesOnOwnershipVerifiedSameNodeServer(t *test t.Fatalf("ownership-verifying NodeLocal gate = %+v", gate) } config := testArgValue(mutated.Spec.Containers[0].Args, "--kv-transfer-config") - if !strings.Contains(config, `tcp://$(INFERENCECACHE_NODE_IP)`) { + if !strings.Contains(config, `tcp://$(INFERENCECACHE_NODE_IP)`) || + !strings.Contains(config, `"lmcache.mp.mp_transfer_mode":"lmcache_driven"`) { t.Fatalf("node-derived vLLM config = %q", config) } if mutated.Spec.HostNetwork || mutated.Spec.HostIPC { @@ -426,6 +427,7 @@ func TestHandle_TypedPodLocalVLLMUsesDedicatedMPAdapter(t *testing.T) { `"kv_role":"kv_both"`, `"lmcache.mp.host":"tcp://127.0.0.1"`, `"lmcache.mp.port":"6500"`, + `"lmcache.mp.mp_transfer_mode":"lmcache_driven"`, } { if !strings.Contains(config, want) { t.Fatalf("kv-transfer-config %q missing %q", config, want) diff --git a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go index c06f9e8d..33bc40f1 100644 --- a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go +++ b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go @@ -273,11 +273,6 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{}, } } - toNodeLocal(&persistedMP) - if err := k8s.Update(ctx, &persistedMP); err != nil { - t.Fatalf("valid PodLocal-to-NodeLocal update should be admitted: %v", err) - } - nodeLocalCreate := validPodLocalMPBackend() nodeLocalCreate.Name = "nodelocal-create" nodeLocalCreate.Namespace = "team-a" @@ -290,6 +285,10 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) if err := live.Get(ctx, client.ObjectKeyFromObject(nodeLocalCreate), &persistedNodeLocal); err != nil { t.Fatalf("get back NodeLocal CR: %v", err) } + persistedNodeLocal.Spec.LMCache.NodeLocal.IdleRetentionSeconds++ + if err := k8s.Update(ctx, &persistedNodeLocal); err != nil { + t.Fatalf("NodeLocal operational setting should remain mutable: %v", err) + } persistedNodeLocal.Spec.LMCache.NodeLocal.Server.HTTPPort = persistedNodeLocal.Spec.LMCache.NodeLocal.Server.Port if err := k8s.Update(ctx, &persistedNodeLocal); err == nil { t.Fatal("NodeLocal UPDATE with colliding host ports should be rejected") diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go index feaf990d..ef493511 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go @@ -171,15 +171,7 @@ func validateMPServer( path *field.Path, reservedPorts map[int32]string, ) field.ErrorList { - var errs field.ErrorList - trimmedImage := strings.TrimSpace(image) - switch { - case trimmedImage == "": - errs = append(errs, field.Required(path.Child("image"), "a CacheBackend-owned LMCache MP server image is required")) - case !sha256ImagePattern.MatchString(trimmedImage): - errs = append(errs, field.Invalid(path.Child("image"), image, - "must be pinned by sha256 digest (for example registry.example/lmcache@sha256:<64-hex-digest>)")) - } + errs := validateDigestPinnedImage(image, path.Child("image"), "a CacheBackend-owned LMCache MP server image is required") if port < 1 || port > 65535 { errs = append(errs, field.Invalid(path.Child("port"), port, "must be between 1 and 65535")) @@ -215,7 +207,7 @@ func validateMPServer( "a positive memory request is required for the MP server")) } else if memoryBudget != nil && memoryRequest.Cmp(*memoryBudget) < 0 { errs = append(errs, field.Invalid(path.Child("resources", "requests").Key(string(corev1.ResourceMemory)), - memoryRequest.String(), fmt.Sprintf("must be at least %s (l1Capacity %s + %s headroom) so scheduling accounts for the memory-backed /dev/shm", memoryBudget.String(), l1Capacity.String(), lmcacheMPMemoryHeadroom))) + memoryRequest.String(), fmt.Sprintf("must be at least %s (l1Capacity %s + %s headroom) so scheduling accounts for the eagerly allocated pinned L1 and server overhead", memoryBudget.String(), l1Capacity.String(), lmcacheMPMemoryHeadroom))) } memoryLimit, hasMemoryLimit := resources.Limits[corev1.ResourceMemory] @@ -229,13 +221,25 @@ func validateMPServer( } if memoryBudget != nil && memoryLimit.Cmp(*memoryBudget) < 0 { errs = append(errs, field.Invalid(path.Child("resources", "limits").Key(string(corev1.ResourceMemory)), - memoryLimit.String(), fmt.Sprintf("must be at least %s (l1Capacity %s + %s headroom) to bound the memory-backed /dev/shm", memoryBudget.String(), l1Capacity.String(), lmcacheMPMemoryHeadroom))) + memoryLimit.String(), fmt.Sprintf("must be at least %s (l1Capacity %s + %s headroom) to cover the eagerly allocated pinned L1 and server overhead", memoryBudget.String(), l1Capacity.String(), lmcacheMPMemoryHeadroom))) } } return errs } +func validateDigestPinnedImage(image string, path *field.Path, requiredMessage string) field.ErrorList { + trimmed := strings.TrimSpace(image) + if trimmed == "" { + return field.ErrorList{field.Required(path, requiredMessage)} + } + if !sha256ImagePattern.MatchString(trimmed) { + return field.ErrorList{field.Invalid(path, image, + "must be pinned by sha256 digest (for example registry.example/image@sha256:<64-hex-digest>)")} + } + return nil +} + // validateMPServerResourceRequirements mirrors the generic provider-resource // admission rules for the independently owned MP server resource block. Keep // this at the API boundary: otherwise malformed extended resources are only diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go index e77ffcfd..db11b17d 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go @@ -295,6 +295,9 @@ func TestValidateLMCacheTopologyCurrentMatrix(t *testing.T) { {name: "digest without repository", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.PodLocal.Server.Image = "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, wantField: "spec.lmCache.podLocal.server.image"}, + {name: "missing server image", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Image = "" + }, wantField: "spec.lmCache.podLocal.server.image"}, {name: "KV event port collision", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.PodLocal.Server.Port = lmcacheKVEventPort }, wantField: "spec.lmCache.podLocal.server.port"}, {name: "HTTP health port collision", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.PodLocal.Server.Port = lmcacheMPHTTPPort }, wantField: "spec.lmCache.podLocal.server.port"}, {name: "memory request below shm", mutate: func(cb *cachev1alpha1.CacheBackend) {