Harden NodeLocal LMCache allocation and UID-pool cleanup

.github/workflows/ci.yml

@@ -364,6 +364,9 @@ jobs:
     - name: Test release image digest resolution
       run: make test-release-image-digests
 
+    - name: Test release install rendering
+      run: make test-release-install
+
   tokenize-cgo:
     name: Tokenizer cgo Build Tag (${{ matrix.build-tag }})
     runs-on: ubuntu-latest
@@ -488,7 +491,7 @@ jobs:
     - name: Verify image SBOM outputs
       run: |
         set -euo pipefail
-        for component in controller server subscriber; do
+        for component in controller server subscriber cleanup; do
           sbom="$RUNNER_TEMP/sbom-image-smoke/inference-cache-${component}-${IMAGE_TAG}.spdx.json"
           test -s "$sbom"
           jq -e '.spdxVersion and ((.packages | type) == "array") and ((.packages | length) > 0)' "$sbom" >/dev/null

.github/workflows/release-sbom.yml

@@ -28,6 +28,7 @@ env:
   CONTROLLER_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-controller
   SERVER_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-server
   SUBSCRIBER_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-subscriber
+  CLEANUP_IMAGE_REPO: ghcr.io/cachebox-project/inference-cache-shm-cleanup
 
 jobs:
   secure-release:
@@ -132,6 +133,7 @@ jobs:
         build_image controller "$CONTROLLER_IMAGE_REPO"
         build_image server "$SERVER_IMAGE_REPO"
         build_image subscriber "$SUBSCRIBER_IMAGE_REPO"
+        build_image cleanup "$CLEANUP_IMAGE_REPO"
 
     - name: Resolve published release image digests
       id: published-image-digests
@@ -144,14 +146,31 @@ jobs:
         BUILT_CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }}
         BUILT_SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }}
         BUILT_SUBSCRIBER_DIGEST: ${{ steps.built-image-digests.outputs.subscriber }}
+        BUILT_CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }}
         PUBLISHED_CONTROLLER_DIGEST: ${{ steps.published-image-digests.outputs.controller }}
         PUBLISHED_SERVER_DIGEST: ${{ steps.published-image-digests.outputs.server }}
         PUBLISHED_SUBSCRIBER_DIGEST: ${{ steps.published-image-digests.outputs.subscriber }}
+        PUBLISHED_CLEANUP_DIGEST: ${{ steps.published-image-digests.outputs.cleanup }}
       run: |
         set -euo pipefail
         test "$BUILT_CONTROLLER_DIGEST" = "$PUBLISHED_CONTROLLER_DIGEST"
         test "$BUILT_SERVER_DIGEST" = "$PUBLISHED_SERVER_DIGEST"
         test "$BUILT_SUBSCRIBER_DIGEST" = "$PUBLISHED_SUBSCRIBER_DIGEST"
+        test "$BUILT_CLEANUP_DIGEST" = "$PUBLISHED_CLEANUP_DIGEST"
+
+    - name: Render digest-pinned install manifest
+      env:
+        CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }}
+        SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }}
+        CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }}
+      run: |
+        make -C release-source kustomize
+        CONTROLLER_IMAGE="${CONTROLLER_IMAGE_REPO}@${CONTROLLER_DIGEST}" \
+        SERVER_IMAGE="${SERVER_IMAGE_REPO}@${SERVER_DIGEST}" \
+        CLEANUP_IMAGE="${CLEANUP_IMAGE_REPO}@${CLEANUP_DIGEST}" \
+        OUTPUT_FILE="$GITHUB_WORKSPACE/dist/install/inference-cache-${RELEASE_TAG}.yaml" \
+        KUSTOMIZE_CMD="$GITHUB_WORKSPACE/release-source/bin/kustomize" \
+          release-source/hack/render-release-install.sh
 
     - name: Generate published image SBOMs
       run: make -f "$GITHUB_WORKSPACE/Makefile" -C "$GITHUB_WORKSPACE/release-source" sbom-registry-images TAG="${RELEASE_TAG}" SBOM_DIR="$GITHUB_WORKSPACE/dist/sbom"
@@ -166,11 +185,13 @@ jobs:
         CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }}
         SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }}
         SUBSCRIBER_DIGEST: ${{ steps.built-image-digests.outputs.subscriber }}
+        CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }}
       run: |
         set -euo pipefail
         cosign sign --yes "${CONTROLLER_IMAGE_REPO}@${CONTROLLER_DIGEST}"
         cosign sign --yes "${SERVER_IMAGE_REPO}@${SERVER_DIGEST}"
         cosign sign --yes "${SUBSCRIBER_IMAGE_REPO}@${SUBSCRIBER_DIGEST}"
+        cosign sign --yes "${CLEANUP_IMAGE_REPO}@${CLEANUP_DIGEST}"
 
     - name: Generate controller SLSA provenance
       id: attest-controller
@@ -196,17 +217,27 @@ jobs:
         subject-digest: ${{ steps.built-image-digests.outputs.subscriber }}
         push-to-registry: true
 
+    - name: Generate NodeLocal cleanup SLSA provenance
+      id: attest-cleanup
+      uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8  # v4.2.2
+      with:
+        subject-name: ${{ env.CLEANUP_IMAGE_REPO }}
+        subject-digest: ${{ steps.built-image-digests.outputs.cleanup }}
+        push-to-registry: true
+
     - name: Collect provenance bundles
       env:
         CONTROLLER_BUNDLE: ${{ steps.attest-controller.outputs.bundle-path }}
         SERVER_BUNDLE: ${{ steps.attest-server.outputs.bundle-path }}
         SUBSCRIBER_BUNDLE: ${{ steps.attest-subscriber.outputs.bundle-path }}
+        CLEANUP_BUNDLE: ${{ steps.attest-cleanup.outputs.bundle-path }}
       run: |
         set -euo pipefail
         mkdir -p dist/provenance
         cp -- "$CONTROLLER_BUNDLE" "dist/provenance/inference-cache-controller-${RELEASE_TAG}.provenance.sigstore.json"
         cp -- "$SERVER_BUNDLE" "dist/provenance/inference-cache-server-${RELEASE_TAG}.provenance.sigstore.json"
         cp -- "$SUBSCRIBER_BUNDLE" "dist/provenance/inference-cache-subscriber-${RELEASE_TAG}.provenance.sigstore.json"
+        cp -- "$CLEANUP_BUNDLE" "dist/provenance/inference-cache-cleanup-${RELEASE_TAG}.provenance.sigstore.json"
 
     - name: Verify image signatures and SLSA provenance
       env:
@@ -215,12 +246,14 @@ jobs:
         CONTROLLER_DIGEST: ${{ steps.built-image-digests.outputs.controller }}
         SERVER_DIGEST: ${{ steps.built-image-digests.outputs.server }}
         SUBSCRIBER_DIGEST: ${{ steps.built-image-digests.outputs.subscriber }}
+        CLEANUP_DIGEST: ${{ steps.built-image-digests.outputs.cleanup }}
       run: |
         set -euo pipefail
         for ref in \
           "${CONTROLLER_IMAGE_REPO}@${CONTROLLER_DIGEST}" \
           "${SERVER_IMAGE_REPO}@${SERVER_DIGEST}" \
-          "${SUBSCRIBER_IMAGE_REPO}@${SUBSCRIBER_DIGEST}"; do
+          "${SUBSCRIBER_IMAGE_REPO}@${SUBSCRIBER_DIGEST}" \
+          "${CLEANUP_IMAGE_REPO}@${CLEANUP_DIGEST}"; do
           cosign verify \
             --certificate-identity "$COSIGN_IDENTITY" \
             --certificate-oidc-issuer https://token.actions.githubusercontent.com \
@@ -240,6 +273,7 @@ jobs:
       with:
         name: release-supply-chain
         path: |
+          dist/install/*.yaml
           dist/sbom/*.spdx.json
           dist/provenance/*.sigstore.json
         if-no-files-found: error
@@ -262,4 +296,4 @@ jobs:
       env:
         GH_TOKEN: ${{ github.token }}
         RELEASE_TAG: ${{ env.RELEASE_TAG }}
-      run: gh release upload "${RELEASE_TAG}" dist/sbom/*.spdx.json dist/provenance/*.sigstore.json --clobber
+      run: gh release upload "${RELEASE_TAG}" dist/install/*.yaml dist/sbom/*.spdx.json dist/provenance/*.sigstore.json --clobber

Makefile

@@ -19,9 +19,14 @@ TAG ?= $(shell git describe --tags --dirty --always 2>/dev/null || echo dev)
 CONTROLLER_IMAGE_REPO ?= $(REGISTRY)/inference-cache-controller
 SERVER_IMAGE_REPO ?= $(REGISTRY)/inference-cache-server
 SUBSCRIBER_IMAGE_REPO ?= $(REGISTRY)/inference-cache-subscriber
+CLEANUP_IMAGE_REPO ?= $(REGISTRY)/inference-cache-shm-cleanup
 IMG ?= $(CONTROLLER_IMAGE_REPO):$(TAG)
 SERVER_IMG ?= $(SERVER_IMAGE_REPO):$(TAG)
 SUBSCRIBER_IMG ?= $(SUBSCRIBER_IMAGE_REPO):$(TAG)
+CLEANUP_IMG ?= $(CLEANUP_IMAGE_REPO):$(TAG)
+INSTALL_MANIFEST ?= dist/install/inference-cache.yaml
+INSTALL_KUSTOMIZATION ?= default
+KUBECTL ?= kubectl
 DOCKER_BUILD_CMD ?= docker
 KIND ?= $(shell command -v kind 2>/dev/null || echo $(LOCAL_KIND))
 KIND_CLUSTER ?= inference-cache
@@ -265,10 +270,11 @@ proto-lint: buf ## Lint the gRPC contract with buf (lint-only; codegen stays on
 	$(BUF) lint
 
 .PHONY: build
-build: ## Build controller, server, kvevent-subscriber, and inferencecache binaries.
+build: ## Build controller, server, kvevent-subscriber, NodeLocal cleanup, and inferencecache binaries.
 	$(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/controller ./cmd/controller
 	$(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/server ./cmd/server
 	$(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/kvevent-subscriber ./cmd/kvevent-subscriber
+	$(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/node-local-shm-cleanup ./cmd/node-local-shm-cleanup
 	$(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/inferencecache ./cmd/inferencecache
 
 .PHONY: test
@@ -370,7 +376,7 @@ verify-ranker-calibration: ## Verify the checked-in ranker calibration output ma
 	$(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT) -check
 
 .PHONY: image-build
-image-build: controller-image server-image subscriber-image ## Build controller, server, and kvevent-subscriber images.
+image-build: controller-image server-image subscriber-image cleanup-image ## Build all release images.
 
 .PHONY: controller-image
 controller-image: ## Build the controller container image.
@@ -384,6 +390,10 @@ server-image: ## Build the server container image.
 subscriber-image: ## Build the kvevent-subscriber container image (sidecar auto-attached to engine pods).
 	$(DOCKER_BUILD_CMD) build -f dockerfiles/Dockerfile --target subscriber -t "$(SUBSCRIBER_IMG)" .
 
+.PHONY: cleanup-image
+cleanup-image: ## Build the NodeLocal SHM cleanup helper image.
+	$(DOCKER_BUILD_CMD) build -f dockerfiles/Dockerfile --target cleanup -t "$(CLEANUP_IMG)" .
+
 .PHONY: verify-minimal-base
 verify-minimal-base: ## Verify every shipped runtime stage uses the approved Distroless non-root base.
 	@MINIMAL_IMAGE_DOCKERFILE="$(MINIMAL_IMAGE_DOCKERFILE)" \
@@ -399,7 +409,7 @@ verify-minimal-images: verify-minimal-base ## Inspect built images for non-root
 	@DOCKER="$(DOCKER_BUILD_CMD)" \
 		MINIMAL_IMAGE_DOCKERFILE="$(MINIMAL_IMAGE_DOCKERFILE)" \
 		MINIMAL_RUNTIME_BASE="$(MINIMAL_RUNTIME_BASE)" \
-		IMG="$(IMG)" SERVER_IMG="$(SERVER_IMG)" SUBSCRIBER_IMG="$(SUBSCRIBER_IMG)" \
+		IMG="$(IMG)" SERVER_IMG="$(SERVER_IMG)" SUBSCRIBER_IMG="$(SUBSCRIBER_IMG)" CLEANUP_IMG="$(CLEANUP_IMG)" \
 		bash hack/verify-minimal-images.sh
 
 .PHONY: syft-check
@@ -425,14 +435,15 @@ sbom-release: syft-check ## Generate a source/release SBOM for the checked-out t
 		-o "spdx-json=$(SBOM_DIR)/inference-cache-$(SBOM_TAG).spdx.json"
 
 .PHONY: sbom-images
-sbom-images: syft-check ## Generate SBOMs for controller, server, and kvevent-subscriber images.
+sbom-images: syft-check ## Generate SBOMs for all release images.
 	@if [ "$(SBOM_IMAGE_SOURCE)" = "docker" ] && [ "$(SBOM_IMAGE_BUILD)" = "1" ]; then \
 		$(MAKE) -f "$(MAKEFILE_SELF)" image-build; \
 	fi
 	mkdir -p "$(SBOM_DIR)"
 	"$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-controller-$(SBOM_TAG).spdx.json"
 	"$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(SERVER_IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-server-$(SBOM_TAG).spdx.json"
 	"$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(SUBSCRIBER_IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-subscriber-$(SBOM_TAG).spdx.json"
+	"$(SYFT)" scan "$(SBOM_IMAGE_SOURCE):$(CLEANUP_IMG)" -o "spdx-json=$(SBOM_DIR)/inference-cache-cleanup-$(SBOM_TAG).spdx.json"
 
 .PHONY: sbom-registry-images
 sbom-registry-images: syft-check ## Generate SBOMs for published release images by immutable registry digest.
@@ -441,7 +452,8 @@ sbom-registry-images: syft-check ## Generate SBOMs for published release images
 	for image in \
 		"controller|$(CONTROLLER_IMAGE_REPO)" \
 		"server|$(SERVER_IMAGE_REPO)" \
-		"subscriber|$(SUBSCRIBER_IMAGE_REPO)"; do \
+		"subscriber|$(SUBSCRIBER_IMAGE_REPO)" \
+		"cleanup|$(CLEANUP_IMAGE_REPO)"; do \
 		component="$${image%%|*}"; \
 		repo="$${image#*|}"; \
 		ref="$${repo}:$(TAG)"; \
@@ -529,6 +541,20 @@ sbom-registry-images: syft-check ## Generate SBOMs for published release images
 test-release-image-digests: ## Test fail-closed release image digest resolution with a fake registry client.
 	@bash hack/resolve-release-image-digests_test.sh
 
+.PHONY: test-release-install
+test-release-install: kustomize ## Test digest-pinned release install rendering.
+	@KUSTOMIZE_CMD="$(LOCAL_KUSTOMIZE)" bash hack/render-release-install_test.sh
+
+.PHONY: render-install
+render-install: kustomize ## Render an install manifest; IMG, SERVER_IMG, and CLEANUP_IMG must be digest-pinned.
+	@CONTROLLER_IMAGE="$(IMG)" SERVER_IMAGE="$(SERVER_IMG)" CLEANUP_IMAGE="$(CLEANUP_IMG)" \
+		OUTPUT_FILE="$(INSTALL_MANIFEST)" KUSTOMIZATION_PATH="$(INSTALL_KUSTOMIZATION)" \
+		KUSTOMIZE_CMD="$(LOCAL_KUSTOMIZE)" hack/render-release-install.sh
+
+.PHONY: deploy
+deploy: render-install ## Apply the digest-pinned install manifest to the current Kubernetes context.
+	$(KUBECTL) apply -f "$(INSTALL_MANIFEST)"
+
 .PHONY: dev-cluster
 dev-cluster: kind ## Create a local kind cluster for development.
 	@if $(KIND) get clusters 2>/dev/null | grep -qx "$(KIND_CLUSTER)"; then \

README.md

@@ -195,8 +195,14 @@ The default Kustomize overlay brings up both control-plane components:
   [docs/design/cachebackend-api.md#functional-probe-gate](docs/design/cachebackend-api.md#functional-probe-gate);
   it is intentionally not visible on a default install with no engine workload.
 
+Use the digest-pinned manifest attached to the selected GitHub release. The
+checked-in `config/default` is a rendering template, not a directly installable
+bundle; see [container image operations](docs/operations/container-images.md)
+for the equivalent source-tree `make deploy` command.
+
 ```bash
-kubectl apply -k config/default
+RELEASE_TAG=vX.Y.Z
+kubectl apply -f "inference-cache-${RELEASE_TAG}.yaml"
 kubectl -n inference-cache-system wait --for=condition=Available deployment --all --timeout=180s
 kubectl get cacheindex cluster-default -o yaml
 ```

api/v1alpha1/cachebackend_types.go

@@ -192,9 +192,10 @@ type LMCachePodLocalServerSpec struct {
 	// +kubebuilder:validation:Maximum=65535
 	Port int32 `json:"port"`
 
-	// L1Capacity is the server's host-memory cache capacity. The renderer sizes
-	// /dev/shm to this value plus 1Gi; container memory requests and limits must
-	// each cover that complete budget.
+	// L1Capacity is the server's eagerly allocated private pinned host-memory
+	// cache capacity. The PodLocal renderer also keeps a conservative /dev/shm
+	// IPC budget of this value plus 1Gi; container memory requests and limits
+	// must each cover the same complete budget.
 	// +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero"
 	L1Capacity resource.Quantity `json:"l1Capacity"`
 
@@ -204,7 +205,7 @@ type LMCachePodLocalServerSpec struct {
 
 	// Resources are applied to the injected MP server container. Admission
 	// requires a positive CPU request and requires both the memory request and
-	// memory limit to cover l1Capacity plus 1Gi of /dev/shm headroom.
+	// memory limit to cover l1Capacity plus 1Gi of server headroom.
 	Resources corev1.ResourceRequirements `json:"resources"`
 }
 
@@ -233,8 +234,8 @@ type LMCacheNodeLocalServerSpec struct {
 	// +kubebuilder:validation:Maximum=65535
 	HTTPPort int32 `json:"httpPort"`
 
-	// L1Capacity is one shared host-memory budget per active engine node, not per
-	// selected engine Pod.
+	// L1Capacity is one eagerly allocated private pinned host-memory budget per
+	// active engine node, not per selected engine Pod.
 	// +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero"
 	L1Capacity resource.Quantity `json:"l1Capacity"`
 
@@ -263,6 +264,8 @@ type LMCacheNodeLocalSchedulingSpec struct {
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 
+	// ImagePullSecrets are used by controller-managed Server and cleanup Pods.
+	// The cleanup Pod does not inherit registry credentials from an Engine Pod.
 	// +optional
 	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
 

api/v1alpha1/cachebackend_types_test.go

@@ -74,7 +74,8 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) {
 	requireEnum(t, mustProperty(t, specSchema, "runtime"), []string{"VLLM", "SGLang"})
 	lmCacheSchema := mustProperty(t, specSchema, "lmCache")
 	requireNoProperty(t, lmCacheSchema, "multiprocess")
-	requireEnum(t, mustProperty(t, lmCacheSchema, "topology"), []string{"PodLocal", "NodeLocal"})
+	topologySchema := mustProperty(t, lmCacheSchema, "topology")
+	requireEnum(t, topologySchema, []string{"PodLocal", "NodeLocal"})
 	podLocalSchema := mustProperty(t, lmCacheSchema, "podLocal")
 	requireRequired(t, podLocalSchema, "server")
 	podLocalServerSchema := mustProperty(t, podLocalSchema, "server")
@@ -86,6 +87,7 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) {
 	requireMinimum(t, mustProperty(t, podLocalServerSchema, "maxWorkers"), 1)
 	nodeLocalSchema := mustProperty(t, lmCacheSchema, "nodeLocal")
 	requireRequired(t, nodeLocalSchema, "server")
+	requireNoProperty(t, nodeLocalSchema, "cleanupImage")
 	requireRequired(t, nodeLocalSchema, "idleRetentionSeconds")
 	idleRetentionSchema := mustProperty(t, nodeLocalSchema, "idleRetentionSeconds")
 	requireMinimum(t, idleRetentionSchema, 0)