diff --git a/Makefile b/Makefile index cfb34224..5760f8dc 100644 --- a/Makefile +++ b/Makefile @@ -259,7 +259,8 @@ generate: controller-gen ## Generate Kubernetes deepcopy code. .PHONY: manifests manifests: controller-gen ## Generate CRD, RBAC, and webhook manifests. - $(CONTROLLER_GEN) rbac:roleName=inference-cache-manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases output:rbac:artifacts:config=config/rbac output:webhook:artifacts:config=config/webhook + # CachePolicy.rankerOverrides intentionally exposes bounded float32 fields. + $(CONTROLLER_GEN) rbac:roleName=inference-cache-manager-role crd:allowDangerousTypes=true webhook paths="./..." output:crd:artifacts:config=config/crd/bases output:rbac:artifacts:config=config/rbac output:webhook:artifacts:config=config/webhook .PHONY: proto-gen proto-gen: protoc-gen-go ## Generate protobuf Go code. diff --git a/api/v1alpha1/cachepolicy_types.go b/api/v1alpha1/cachepolicy_types.go index afae0344..23846e1d 100644 --- a/api/v1alpha1/cachepolicy_types.go +++ b/api/v1alpha1/cachepolicy_types.go @@ -147,10 +147,11 @@ type CachePolicySpec struct { // affinityRouting: Disabled. An operator can disable either floor by // setting it to its opt-out value (0 / "0"). // - // Encoded as a stringified float to avoid introducing the first - // float-typed field into the CachePolicy schema (the others are - // int32 / duration). Validated by the +kubebuilder:validation:Pattern - // marker: digits with an optional decimal part, no sign. The pattern + // Encoded as a stringified float for backward compatibility with the + // original v1alpha1 field and its string-based clients. Unlike the bounded + // numeric fields under rankerOverrides, this legacy threshold is validated + // by the +kubebuilder:validation:Pattern marker: digits with an optional + // decimal part, no sign. The pattern // caps the integer part at 8 digits (up to 99,999,999) and the // decimal part at 6 digits — well inside float32 representable // range (~3.4e38), so a value the apiserver admits CANNOT overflow @@ -174,6 +175,12 @@ type CachePolicySpec struct { // +kubebuilder:validation:Minimum=0 LookupTimeoutMs *int32 `json:"lookupTimeoutMs,omitempty"` + // RankerOverrides tunes ranking-v2 for this namespace. Each omitted field + // inherits the server's RankerConfig baseline; explicit zero values retain + // their documented kill-switch behavior. + // +optional + RankerOverrides *CachePolicyRankerOverridesSpec `json:"rankerOverrides,omitempty"` + // Strategy controls which LookupRoute matching strategies may produce a // hint for this namespace. Defaults preserve the historical behavior: // longest-prefix chain matching is enabled, callers are not required to @@ -208,6 +215,41 @@ type CachePolicySpec struct { AffinityRouting *CachePolicyAffinityRouting `json:"affinityRouting,omitempty"` } +// CachePolicyRankerOverridesSpec carries presence-aware per-namespace +// overrides for the ranking-v2 pressure, SLO, and TENANT_HOT knobs. +type CachePolicyRankerOverridesSpec struct { + // PressureWeight controls the replica-pressure penalty. Zero disables it. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=4 + PressureWeight *float32 `json:"pressureWeight,omitempty"` + + // SLOTightTTFTMs is the TTFT threshold below which SLOTightBias applies. + // Zero disables the SLO bias threshold. + // +optional + // +kubebuilder:validation:Minimum=0 + SLOTightTTFTMs *int32 `json:"sloTightTTFTMs,omitempty"` + + // SLOTightBias controls the freshness boost for tight-TTFT requests. Zero + // disables the boost. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=8 + SLOTightBias *float32 `json:"sloTightBias,omitempty"` + + // TenantHotMinHitRate is the minimum reported hit rate for TENANT_HOT. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 + TenantHotMinHitRate *float32 `json:"tenantHotMinHitRate,omitempty"` + + // TenantHotMaxAge is the maximum age of replica stats eligible for + // TENANT_HOT. Zero disables the fallback; negative durations are rejected + // by the validating webhook. + // +optional + TenantHotMaxAge *metav1.Duration `json:"tenantHotMaxAge,omitempty"` +} + // CachePolicyStrategySpec controls per-namespace LookupRoute strategy gates. type CachePolicyStrategySpec struct { // EnableChainMatching allows LookupRoute requests that carry block_hashes diff --git a/api/v1alpha1/remaining_crds_types_test.go b/api/v1alpha1/remaining_crds_types_test.go index 57737cbf..f370e4fe 100644 --- a/api/v1alpha1/remaining_crds_types_test.go +++ b/api/v1alpha1/remaining_crds_types_test.go @@ -34,6 +34,15 @@ func TestRemainingCRDSchemas(t *testing.T) { requireDurationLike(t, mustProperty(t, policySpec, "evictionTTL")) requireMinimum(t, mustProperty(t, policySpec, "minimumPrefixTokens"), 0) requireMinimum(t, mustProperty(t, policySpec, "lookupTimeoutMs"), 0) + rankerOverrides := mustProperty(t, policySpec, "rankerOverrides") + requireMinimum(t, mustProperty(t, rankerOverrides, "pressureWeight"), 0) + requireMaximum(t, mustProperty(t, rankerOverrides, "pressureWeight"), 4) + requireMinimum(t, mustProperty(t, rankerOverrides, "sloTightTTFTMs"), 0) + requireMinimum(t, mustProperty(t, rankerOverrides, "sloTightBias"), 0) + requireMaximum(t, mustProperty(t, rankerOverrides, "sloTightBias"), 8) + requireMinimum(t, mustProperty(t, rankerOverrides, "tenantHotMinHitRate"), 0) + requireMaximum(t, mustProperty(t, rankerOverrides, "tenantHotMinHitRate"), 1) + requireDurationLike(t, mustProperty(t, rankerOverrides, "tenantHotMaxAge")) policyStrategy := mustProperty(t, policySpec, "strategy") requireDefault(t, mustProperty(t, policyStrategy, "enableChainMatching"), true) requireDefault(t, mustProperty(t, policyStrategy, "requireChain"), false) @@ -92,12 +101,24 @@ func TestRemainingCRDDeepCopies(t *testing.T) { enableChainMatching := true requireChain := false enableTenantHot := true + pressureWeight := float32(1) + sloTightTTFTMs := int32(200) + sloTightBias := float32(1) + tenantHotMinHitRate := float32(0.1) + tenantHotMaxAge := metav1.Duration{Duration: 5 * time.Minute} policy := &CachePolicy{ Spec: CachePolicySpec{ Eviction: CachePolicyEvictionAlgorithmLRU, EvictionTTL: &ttl, MinimumPrefixTokens: &minimumPrefixTokens, LookupTimeoutMs: &lookupTimeoutMs, + RankerOverrides: &CachePolicyRankerOverridesSpec{ + PressureWeight: &pressureWeight, + SLOTightTTFTMs: &sloTightTTFTMs, + SLOTightBias: &sloTightBias, + TenantHotMinHitRate: &tenantHotMinHitRate, + TenantHotMaxAge: &tenantHotMaxAge, + }, Strategy: &CachePolicyStrategySpec{ EnableChainMatching: &enableChainMatching, RequireChain: &requireChain, @@ -111,6 +132,11 @@ func TestRemainingCRDDeepCopies(t *testing.T) { *policy.Spec.Strategy.EnableChainMatching = false *policy.Spec.Strategy.RequireChain = true *policy.Spec.Strategy.EnableTenantHot = false + *policy.Spec.RankerOverrides.PressureWeight = 2 + *policy.Spec.RankerOverrides.SLOTightTTFTMs = 100 + *policy.Spec.RankerOverrides.SLOTightBias = 3 + *policy.Spec.RankerOverrides.TenantHotMinHitRate = 0.5 + policy.Spec.RankerOverrides.TenantHotMaxAge.Duration = time.Minute policy.Status.Conditions[0].Message = "changed" if policyCopy.Spec.Eviction != CachePolicyEvictionAlgorithmLRU || policyCopy.Spec.EvictionTTL.Duration != time.Minute || @@ -118,6 +144,12 @@ func TestRemainingCRDDeepCopies(t *testing.T) { policyCopy.Spec.Strategy.EnableChainMatching == nil || !*policyCopy.Spec.Strategy.EnableChainMatching || policyCopy.Spec.Strategy.RequireChain == nil || *policyCopy.Spec.Strategy.RequireChain || policyCopy.Spec.Strategy.EnableTenantHot == nil || !*policyCopy.Spec.Strategy.EnableTenantHot || + policyCopy.Spec.RankerOverrides == nil || + policyCopy.Spec.RankerOverrides.PressureWeight == nil || *policyCopy.Spec.RankerOverrides.PressureWeight != 1 || + policyCopy.Spec.RankerOverrides.SLOTightTTFTMs == nil || *policyCopy.Spec.RankerOverrides.SLOTightTTFTMs != 200 || + policyCopy.Spec.RankerOverrides.SLOTightBias == nil || *policyCopy.Spec.RankerOverrides.SLOTightBias != 1 || + policyCopy.Spec.RankerOverrides.TenantHotMinHitRate == nil || *policyCopy.Spec.RankerOverrides.TenantHotMinHitRate != 0.1 || + policyCopy.Spec.RankerOverrides.TenantHotMaxAge == nil || policyCopy.Spec.RankerOverrides.TenantHotMaxAge.Duration != 5*time.Minute || policyCopy.Status.Conditions[0].Message != "ok" { t.Fatalf("CachePolicy was not deep-copied") } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 75660aca..73f8621a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -571,6 +571,46 @@ func (in *CachePolicyList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CachePolicyRankerOverridesSpec) DeepCopyInto(out *CachePolicyRankerOverridesSpec) { + *out = *in + if in.PressureWeight != nil { + in, out := &in.PressureWeight, &out.PressureWeight + *out = new(float32) + **out = **in + } + if in.SLOTightTTFTMs != nil { + in, out := &in.SLOTightTTFTMs, &out.SLOTightTTFTMs + *out = new(int32) + **out = **in + } + if in.SLOTightBias != nil { + in, out := &in.SLOTightBias, &out.SLOTightBias + *out = new(float32) + **out = **in + } + if in.TenantHotMinHitRate != nil { + in, out := &in.TenantHotMinHitRate, &out.TenantHotMinHitRate + *out = new(float32) + **out = **in + } + if in.TenantHotMaxAge != nil { + in, out := &in.TenantHotMaxAge, &out.TenantHotMaxAge + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CachePolicyRankerOverridesSpec. +func (in *CachePolicyRankerOverridesSpec) DeepCopy() *CachePolicyRankerOverridesSpec { + if in == nil { + return nil + } + out := new(CachePolicyRankerOverridesSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CachePolicySpec) DeepCopyInto(out *CachePolicySpec) { *out = *in @@ -599,6 +639,11 @@ func (in *CachePolicySpec) DeepCopyInto(out *CachePolicySpec) { *out = new(int32) **out = **in } + if in.RankerOverrides != nil { + in, out := &in.RankerOverrides, &out.RankerOverrides + *out = new(CachePolicyRankerOverridesSpec) + (*in).DeepCopyInto(*out) + } if in.Strategy != nil { in, out := &in.Strategy, &out.Strategy *out = new(CachePolicyStrategySpec) diff --git a/config/crd/bases/inferencecache.io_cachepolicies.yaml b/config/crd/bases/inferencecache.io_cachepolicies.yaml index 409efe8a..75e93956 100644 --- a/config/crd/bases/inferencecache.io_cachepolicies.yaml +++ b/config/crd/bases/inferencecache.io_cachepolicies.yaml @@ -136,6 +136,45 @@ spec: format: int32 minimum: 0 type: integer + rankerOverrides: + description: |- + RankerOverrides tunes ranking-v2 for this namespace. Each omitted field + inherits the server's RankerConfig baseline; explicit zero values retain + their documented kill-switch behavior. + properties: + pressureWeight: + description: PressureWeight controls the replica-pressure penalty. + Zero disables it. + maximum: 4 + minimum: 0 + type: number + sloTightBias: + description: |- + SLOTightBias controls the freshness boost for tight-TTFT requests. Zero + disables the boost. + maximum: 8 + minimum: 0 + type: number + sloTightTTFTMs: + description: |- + SLOTightTTFTMs is the TTFT threshold below which SLOTightBias applies. + Zero disables the SLO bias threshold. + format: int32 + minimum: 0 + type: integer + tenantHotMaxAge: + description: |- + TenantHotMaxAge is the maximum age of replica stats eligible for + TENANT_HOT. Zero disables the fallback; negative durations are rejected + by the validating webhook. + type: string + tenantHotMinHitRate: + description: TenantHotMinHitRate is the minimum reported hit rate + for TENANT_HOT. + maximum: 1 + minimum: 0 + type: number + type: object routingFloorScore: default: "0.1" description: |- @@ -176,10 +215,11 @@ spec: affinityRouting: Disabled. An operator can disable either floor by setting it to its opt-out value (0 / "0"). - Encoded as a stringified float to avoid introducing the first - float-typed field into the CachePolicy schema (the others are - int32 / duration). Validated by the +kubebuilder:validation:Pattern - marker: digits with an optional decimal part, no sign. The pattern + Encoded as a stringified float for backward compatibility with the + original v1alpha1 field and its string-based clients. Unlike the bounded + numeric fields under rankerOverrides, this legacy threshold is validated + by the +kubebuilder:validation:Pattern marker: digits with an optional + decimal part, no sign. The pattern caps the integer part at 8 digits (up to 99,999,999) and the decimal part at 6 digits — well inside float32 representable range (~3.4e38), so a value the apiserver admits CANNOT overflow diff --git a/config/samples/cache_v1alpha1_cachepolicy.yaml b/config/samples/cache_v1alpha1_cachepolicy.yaml index ff6f5df2..944110c4 100644 --- a/config/samples/cache_v1alpha1_cachepolicy.yaml +++ b/config/samples/cache_v1alpha1_cachepolicy.yaml @@ -53,6 +53,16 @@ spec: # survivor). routingFloorScore: "0.1" lookupTimeoutMs: 20 + rankerOverrides: + # Per-namespace overrides are optional and merge onto the server baseline. + # pressureWeight=2 is intentionally observable in the install smoke: a + # replica reporting pressure=0.5 is fully demoted (factor 0), proving the + # controller -> /policy -> PolicyStore -> index path adopted this object. + pressureWeight: 2 + sloTightTTFTMs: 150 + sloTightBias: 2 + tenantHotMinHitRate: 0.25 + tenantHotMaxAge: 2m strategy: # Enable longest-common-prefix block-hash chain matching when callers # send block_hashes + block_token_counts. Disable to force legacy exact diff --git a/docs/concepts/cachepolicy-tuning.md b/docs/concepts/cachepolicy-tuning.md index 90911188..49b79bdb 100644 --- a/docs/concepts/cachepolicy-tuning.md +++ b/docs/concepts/cachepolicy-tuning.md @@ -25,7 +25,7 @@ snapshot. The CR is purely declarative: the controller pushes its resolved field server, which is where enforcement actually happens (see [Propagation](#propagation-controller--server) below). -## The seven spec knobs +## The nine spec knobs | Field | Type | Default | When to tune | |---|---|---|---| @@ -36,6 +36,7 @@ server, which is where enforcement actually happens (see | `routingFloorScore` | stringified float, e.g. `"0.1"`, `"5"`, `"0"` | `"0.1"` | Per-replica *score* floor below which a `PREFIX_MATCH` response is downgraded off the prefix-match path. Applied AFTER the lookup runs, against the per-replica score from the distinguishing-power-aware ranker (`matched_tokens × freshness × pressure_factor × slo_bias × distinguishing_power`). Overlaps held by every replica (chat-template framing, RAG corpus headers, custom system prompts shared across the deployment) produce `distinguishing_power = 0` and score = 0 — this floor catches them. The downgrade lands on `StrategyNone`, which surfaces as `AFFINITY_HINT` (default-enabled affinity) or `NO_HINT` (affinity disabled). Composes with `minimumMatchedTokens` — the matched-tokens floor runs first (per-replica), then this score floor gates the top survivor. Set to `"0"` to disable entirely (raw-recall benchmarking / debug). | | `affinityRouting` | enum `Enabled` \| `Disabled` | `Enabled` | Toggles the consistent-hash fallback on the `StrategyNone` branch. With `Enabled` (the default), any `StrategyNone` result with a usable seed + at least one replica in `servingByScope[(tenant, model, hash_scheme)]` surfaces as `AFFINITY_HINT` with a stable single-replica pick (`SHA-256(canonical_seed) mod len(sorted servingByScope)`) — repeat prompts pin to the same replica and warm T1 on diffuse single-turn workloads. With `Disabled`, the same response stays on `NO_HINT` and the gateway round-robins; useful for raw-recall benchmarking and ranker debugging. Diagnostic codes (`UNKNOWN_*`) and `TIMEOUT` keep precedence over `AFFINITY_HINT`. | | `lookupTimeoutMs` | int32 (min `0`) | unset = no deadline | Per-lookup latency budget in milliseconds. A breach returns reason code `TIMEOUT` (still fail-open — empty result, never an error to the gateway). See the foot-gun in [Gotchas](#two-gotchas). | +| `rankerOverrides` | object | server `RankerConfig` baseline | Optional per-namespace overlay for `pressureWeight` (`0..4`), `sloTightTTFTMs` (`>=0`), `sloTightBias` (`0..8`), `tenantHotMinHitRate` (`0..1`), and `tenantHotMaxAge` (non-negative duration). Omitted nested fields inherit the server baseline; explicit zero values retain their kill-switch meaning. | | `strategy` | object | chain matching on, chain not required, tenant-hot on | Per-namespace LookupRoute strategy gates. Use `enableChainMatching: false` to force exact `prefix_hash` matching, `requireChain: true` to reject non-chain callers with `POLICY_REQUIRES_CHAIN`, or `enableTenantHot: false` to suppress soft tenant-hot hints. | `status` carries only `observedGeneration` + `conditions`, and both are **reserved** — the @@ -61,6 +62,12 @@ spec: routingFloorScore: "0.1" # downgrade off PREFIX_MATCH when top score is below the floor — surfaces as AFFINITY_HINT under default-enabled affinity or NO_HINT when disabled (result-side score floor, default "0.1") affinityRouting: Enabled # consistent-hash fallback on the StrategyNone branch; set Disabled for raw-recall benchmarking / debug lookupTimeoutMs: 20 # positive => a real 20ms deadline (NOT 0 — see Gotchas) + rankerOverrides: # optional; omitted fields keep the server baseline + pressureWeight: 1 + sloTightTTFTMs: 200 + sloTightBias: 1 + tenantHotMinHitRate: 0.1 + tenantHotMaxAge: 5m strategy: enableChainMatching: true # default: use block-hash longest-prefix matching when callers send chains requireChain: false # default: legacy exact prefix_hash callers still work diff --git a/docs/design/lookuproute-ranking.md b/docs/design/lookuproute-ranking.md index 33f0b4b5..3f6a25c9 100644 --- a/docs/design/lookuproute-ranking.md +++ b/docs/design/lookuproute-ranking.md @@ -828,6 +828,21 @@ Every factor and threshold is tunable through a `RankerConfig` (in-binary defaults) or a `CachePolicy` CR (per-namespace overrides). Defaults are set so that: +| `CachePolicy.spec.rankerOverrides` field | `RankerConfig` field | Valid range | Inherited default | +|---|---|---|---| +| `pressureWeight` | `PressureWeight` | `0..4` | `1.0` | +| `sloTightTTFTMs` | `SLOTightTTFTMs` | `>= 0` | `200` | +| `sloTightBias` | `SLOTightBias` | `0..8` | `1.0` | +| `tenantHotMinHitRate` | `TenantHotMinHitRate` | `0..1` | `0.1` | +| `tenantHotMaxAge` | `TenantHotMaxAge` | duration `>= 0` | `5m` | + +The object and all five fields are optional. The index resolves one effective +configuration per lookup by copying its server-wide `WithRanker` baseline and +overlaying only non-nil fields. This keeps an omitted field on the calibrated +baseline while preserving explicit zero as a real kill switch. A missing +policy, a missing `rankerOverrides` object, or a deleted policy all use the +server baseline. + - A deployment with **no stats reported** sees pressure factor 1 and no `TENANT_HOT` candidates qualify — those two strategies collapse to the pre-PR baseline. The §2.6 matched-tokens floor and the §2.7 diff --git a/docs/design/policy-crds.md b/docs/design/policy-crds.md index 5d5ac5f1..1bdf0213 100644 --- a/docs/design/policy-crds.md +++ b/docs/design/policy-crds.md @@ -16,6 +16,7 @@ This document tracks the policy-side CRDs that sit beside `CacheBackend`. These | `spec.minimumMatchedTokens` | integer | Minimum *matched* prefix token count required AFTER the index lookup runs for `PREFIX_MATCH` to surface. Replicas whose matched overlap falls below this floor are filtered; if none survive, the response downgrades off the prefix-match path to `StrategyNone`, which surfaces as `AFFINITY_HINT` under `affinityRouting: Enabled` (the kubebuilder default) with a usable seed + serving replica or as `NO_HINT` under `affinityRouting: Disabled`. The CRD field has a `+kubebuilder:default=64` marker, so the apiserver materializes `64` (4 KV blocks at the typical 16-token block size) on any CR that omits the field. The server independently applies the same value as `DefaultMinimumMatchedTokens` when a tenant has *no* `CachePolicy` at all — so trivial 1-block chat-template overlaps are filtered in both shapes. Set to `0` on the CR to disable enforcement for that namespace (e.g. raw-recall benchmarking); the server-side `DefaultMinimumMatchedTokens` is the fallback ONLY for tenants without a CR. Distinct from `minimumPrefixTokens` — that field is a request-side gate; this is a result-side floor. Minimum `0`. | | `spec.routingFloorScore` | stringified float (e.g. `"0.1"`, `"5"`, `"0"`) | Per-replica *score* below which a `PREFIX_MATCH` response is downgraded off the prefix-match path. Applied AFTER the [distinguishing-power-aware ranker](lookuproute-ranking.md) computes scores. Overlaps held by every replica (chat-template framing, RAG corpus headers, custom system prompts) see `distinguishing_power = 0`, score = 0, and this floor catches them. The downgrade lands on `StrategyNone`, which surfaces as `AFFINITY_HINT` under `affinityRouting: Enabled` (the default) with a usable seed + serving replica or as `NO_HINT` under `affinityRouting: Disabled`. The CRD has a `+kubebuilder:default="0.1"` marker so the apiserver materializes `"0.1"` on any CR that omits the field; the server independently applies the same value (as `DefaultRoutingFloorScore`) when a tenant has *no* `CachePolicy` at all. Set to `"0"` on the CR to disable enforcement for that namespace (raw-recall benchmarking). Composes with `minimumMatchedTokens` — the matched-tokens floor is applied first per-replica, then this score floor gates the top survivor's score. Both floors can downgrade independently; an operator can disable either by setting it to its opt-out value. Distinct from `minimumPrefixTokens` — that's a request-side gate; this is a result-side floor on the realized score. Pattern-validated at admission. | | `spec.lookupTimeoutMs` | integer | Lookup latency budget in milliseconds. Minimum `0`. | +| `spec.rankerOverrides` | object | Optional, per-namespace overlay on the server's ranking-v2 baseline. Every nested field is a pointer: omission inherits the baseline while an explicit `0` keeps its documented kill-switch meaning. `pressureWeight` is `0..4`; `sloTightTTFTMs` is `>=0`; `sloTightBias` is `0..8`; `tenantHotMinHitRate` is `0..1`; `tenantHotMaxAge` is a non-negative duration. The controller preserves field presence through `/policy`, and the index resolves one effective config per lookup for both prefix scoring and `TENANT_HOT`. | | `spec.strategy.enableChainMatching` | boolean | Enables the block-hash chain matcher for `LookupRoute` requests that carry `block_hashes` + `block_token_counts`. Default `true` preserves the longest-common-leading-run behavior. When `false`, the handler strips chain fields before lookup and uses the legacy exact `prefix_hash` path. | | `spec.strategy.requireChain` | boolean | Requires callers to provide a valid block-hash chain before the index is touched. Default `false` keeps legacy exact-prefix clients working. When `true` and a request has no chain, the server returns empty scores with `reason_code: POLICY_REQUIRES_CHAIN` (fail-open to normal gateway routing). Admission rejects `requireChain: true` with `enableChainMatching: false`. | | `spec.strategy.enableTenantHot` | boolean | Allows the `TENANT_HOT` soft locality fallback. Default `true` preserves current behavior. When `false`, a tenant-hot result is downgraded to `NO_HINT` while prefix matches and diagnostic misses still behave normally. | @@ -23,7 +24,7 @@ This document tracks the policy-side CRDs that sit beside `CacheBackend`. These `status.conditions` and `status.observedGeneration` are reserved for controller observations. -Runtime propagation (controller → server `/policy`) is described in [policy-propagation.md](policy-propagation.md): `evictionTTL` drives per-tenant index eviction; `minimumPrefixTokens` and `lookupTimeoutMs` are enforced on the `LookupRoute` path before and around the index call respectively (when `affinityRouting: Disabled`, the `minimumPrefixTokens` pre-lookup short-circuit elides the index call entirely; when `affinityRouting: Enabled` — the default — the gate still applies but the request runs the full lookup so the index can classify `UNKNOWN_*` diagnostics before the affinity fallback fires); `minimumMatchedTokens` is enforced on the `LookupRoute` path after the index returns (downgrades sub-floor matches off the prefix-match path to `StrategyNone`, which surfaces as `AFFINITY_HINT` under default-enabled affinity or `NO_HINT` when disabled); `routingFloorScore` is enforced on the `LookupRoute` path after the distinguishing-power-aware ranker scores each candidate (downgrades whole responses whose top score falls below the floor, with the same `AFFINITY_HINT` vs `NO_HINT` split as the matched-tokens row); `affinityRouting` activates on the `StrategyNone` branch after both floors have run, returning `AFFINITY_HINT` instead of `NO_HINT` when enabled; `strategy` gates which lookup strategies may surface; and `eviction` selects the per-namespace cap-based eviction algorithm. The two eviction knobs are orthogonal: `evictionTTL` removes stale entries on the freshness sweep regardless of algorithm, while `eviction` only decides which entries the cap sweep drops when the index is over its entry cap. The three lookup-filtering knobs are orthogonal too: `minimumPrefixTokens` bounds the request, `minimumMatchedTokens` bounds the realized matched-tokens count per replica, and `routingFloorScore` bounds the realized per-replica score; `affinityRouting` is the post-filter fallback that decides whether the otherwise-NO_HINT response carries an `AFFINITY_HINT` stable replica. +Runtime propagation (controller → server `/policy`) is described in [policy-propagation.md](policy-propagation.md): `evictionTTL` drives per-tenant index eviction; `minimumPrefixTokens` and `lookupTimeoutMs` are enforced on the `LookupRoute` path before and around the index call respectively (when `affinityRouting: Disabled`, the `minimumPrefixTokens` pre-lookup short-circuit elides the index call entirely; when `affinityRouting: Enabled` — the default — the gate still applies but the request runs the full lookup so the index can classify `UNKNOWN_*` diagnostics before the affinity fallback fires); `minimumMatchedTokens` is enforced on the `LookupRoute` path after the index returns (downgrades sub-floor matches off the prefix-match path to `StrategyNone`, which surfaces as `AFFINITY_HINT` under default-enabled affinity or `NO_HINT` when disabled); `routingFloorScore` is enforced on the `LookupRoute` path after the distinguishing-power-aware ranker scores each candidate (downgrades whole responses whose top score falls below the floor, with the same `AFFINITY_HINT` vs `NO_HINT` split as the matched-tokens row); `rankerOverrides` overlays the server ranking baseline for prefix and `TENANT_HOT` scoring; `affinityRouting` activates on the `StrategyNone` branch after both floors have run, returning `AFFINITY_HINT` instead of `NO_HINT` when enabled; `strategy` gates which lookup strategies may surface; and `eviction` selects the per-namespace cap-based eviction algorithm. The two eviction knobs are orthogonal: `evictionTTL` removes stale entries on the freshness sweep regardless of algorithm, while `eviction` only decides which entries the cap sweep drops when the index is over its entry cap. The three lookup-filtering knobs are orthogonal too: `minimumPrefixTokens` bounds the request, `minimumMatchedTokens` bounds the realized matched-tokens count per replica, and `routingFloorScore` bounds the realized per-replica score; `affinityRouting` is the post-filter fallback that decides whether the otherwise-NO_HINT response carries an `AFFINITY_HINT` stable replica. ## CacheTenant diff --git a/docs/design/policy-propagation.md b/docs/design/policy-propagation.md index ec46c50c..48cd9227 100644 --- a/docs/design/policy-propagation.md +++ b/docs/design/policy-propagation.md @@ -162,11 +162,11 @@ index). If the server restarts and loses everything, the controller's periodic re-push (default 30s) brings it back into sync without operator intervention. -## Wire schema (v7) +## Wire schema (v8) ```json { - "version": 7, + "version": 8, "policies": [ { "namespace": "team-a", @@ -175,6 +175,13 @@ intervention. "minimumMatchedTokens": 128, "routingFloorScore": 5, "lookupTimeoutMs": 25, + "rankerOverrides": { + "pressureWeight": 0.5, + "sloTightTTFTMs": 200, + "sloTightBias": 1, + "tenantHotMinHitRate": 0.2, + "tenantHotMaxAge": 120000000000 + }, "eviction": "lfu", "strategy": { "enableChainMatching": true, @@ -205,14 +212,15 @@ intervention. skew is observable; **whether the bump is rejected at decode is set separately** by `PolicyMinimumAcceptedVersion` (today `3`) — see the Rollout asymmetry note in §Versioning and forward-compat below. v4, v5, - v6, and v7 are additive and defaultable, so the v7 server still accepts - v3, v4, v5, and v6 bodies; a hypothetical breaking change would bump + v6, v7, and v8 are additive and defaultable, so the v8 server still accepts + v3 through v7 bodies; a hypothetical breaking change would bump `PolicyMinimumAcceptedVersion` in lockstep. The server rejects any value outside `[PolicyMinimumAcceptedVersion, PolicyPropagationVersion]` - (HTTP 400). Currently `7`. Version history: `2` added `tenants`; `3` + (HTTP 400). Currently `8`. Version history: `2` added `tenants`; `3` added `policies[].eviction`; `4` added `policies[].minimumMatchedTokens`; `5` added - `policies[].routingFloorScore`; `6` added `policies[].strategy`; `7` added `policies[].affinityRouting`. + `policies[].routingFloorScore`; `6` added `policies[].strategy`; `7` added + `policies[].affinityRouting`; `8` added `policies[].rankerOverrides`. - `policies[]` — full snapshot of all `CachePolicy` CRs in the cluster. Sorted by `namespace` for deterministic bodies (and for easier diffing in tests). @@ -286,6 +294,19 @@ intervention. survivor's score. See [`lookuproute-ranking.md`](./lookuproute-ranking.md). - `policies[].lookupTimeoutMs` — int32 milliseconds. Optional. `<=0` ⇒ "no deadline". +- `policies[].rankerOverrides` — optional pointer object containing optional + `pressureWeight`, `sloTightTTFTMs`, `sloTightBias`, + `tenantHotMinHitRate`, and `tenantHotMaxAge` fields. Float and integer + fields use JSON numbers; `tenantHotMaxAge` is a Go `time.Duration` + encoded as nanoseconds. The controller preserves nested pointer presence: + omitted fields inherit the index's server-wide `WithRanker` baseline, + while explicit zero values retain their kill-switch behavior. The public + ranges are `0..4`, `>=0`, `0..8`, `0..1`, and duration `>=0`, + respectively. CRD markers plus the validating webhook reject violations; + the `PolicyStore` defensively drops an out-of-range nested value from a + hand-crafted wire body so that field inherits the baseline. A missing + object, including every v3-v7 body, needs no normalization because absence + already means "use the server baseline." - `policies[].eviction` — lower-cased cap-eviction algorithm (`"lru"` / `"lfu"`). Optional. `""` ⇒ "use server default" (`LRU`). The controller lower-cases the CRD's upper-case enum; the index normalizes any @@ -303,19 +324,19 @@ intervention. the shape of `routingFloorScore` exactly: a nil/missing field on a v3/v4/v5/v6 body is normalized to `DefaultAffinityRoutingEnabled` (`true`) (see Rollout asymmetry below) so the in-memory store ends - up byte-for-byte identical to the post-rollout v7 shape, regardless + up byte-for-byte identical to the post-rollout v8 shape, regardless of which side of the rollout boundary the body came from. Note that the wire SHAPE distinguishes `nil` (omitempty drops the field entirely) from `&false` (the literal `"affinityRouting": false`), so the normalizer's input is unambiguous; the choice to normalize nil → `&true` is a SEMANTIC convention so v3/v4/v5/v6 bodies behave - like fresh v7 CRD-defaulted bodies. v7 bodies are NOT normalized; - the v7 wire field can take three shapes: + like fresh v8 CRD-defaulted bodies. v7 and v8 bodies are NOT normalized + for this field; the wire field can take three shapes: - **Normal CRD-defaulted shape (the common case).** The CRD has a `+kubebuilder:default=Enabled` marker, so an admitted CachePolicy always carries a non-nil `spec.affinityRouting`. The controller flattens that to `&true` for `Enabled` (or `&false` for `Disabled`) - and sends it on the wire. v7 bodies in production traffic look + and sends it on the wire. Current bodies in production traffic look like this. - **Explicit opt-out.** An operator setting `spec.affinityRouting: Disabled` reaches the wire as `&false`. The @@ -426,6 +447,7 @@ namespace key `CachePolicy` uses — see the tenant-quota row below. | `minimumMatchedTokens` | Post-lookup floor on each replica's realized `matched_tokens`. The handler resolves the per-tenant floor via `PolicyStore.MinimumMatchedTokens`, which falls back to `DefaultMinimumMatchedTokens` (= 64) for tenants with no `CachePolicy`. Replicas whose `matched_tokens` falls below the floor are filtered from the scored result; if none survive, the response downgrades from `PREFIX_MATCH` to `StrategyNone`, which then surfaces as `reason_code: AFFINITY_HINT` with a stable single replica when `affinityRouting: Enabled` (the default) or as `reason_code: NO_HINT` with empty scores when `affinityRouting: Disabled`. The downgrade runs **before** the LFU `CreditHits` step so a non-delivered hint never bumps the per-entry access counter. See [`lookuproute-ranking.md`](./lookuproute-ranking.md). | | `routingFloorScore` | Post-score floor on the per-replica score from the distinguishing-power-aware ranker. The handler resolves the per-tenant floor via `PolicyStore.RoutingFloorScore`, which falls back to `DefaultRoutingFloorScore` (`0.1`) for tenants with no `CachePolicy`. When the top surviving replica's score falls below the floor, the response downgrades from `PREFIX_MATCH` to `StrategyNone`, which then surfaces as `AFFINITY_HINT` or `NO_HINT` per the `affinityRouting` toggle (same shape as the matched-tokens downgrade row above). Composes with `minimumMatchedTokens` — the matched-tokens floor runs first (per-replica filter), then this score floor checks the top survivor. Both downgrades run **before** the LFU `CreditHits` step so a non-delivered hint never bumps an LFU counter. See [`lookuproute-ranking.md`](./lookuproute-ranking.md). | | `lookupTimeoutMs` | `LookupRoute` derives a `context.WithTimeout`. A breach yields `reason_code: TIMEOUT` (still fail-open: empty scores). `TIMEOUT` keeps absolute precedence over `AFFINITY_HINT`. | +| `rankerOverrides.*` | `internal/index` `RankerResolver`. The index copies its server-wide `WithRanker` baseline, overlays only present fields, and uses that immutable effective config for exact-prefix, chain, and `TENANT_HOT` scoring in one lookup. Missing policy/object/field inherits the baseline; explicit zero preserves the field's kill switch. | | `affinityRouting` | Per-namespace toggle for the consistent-hash fallback on the `StrategyNone` branch. Resolved via `PolicyStore.AffinityRoutingEnabled`, which falls back to `DefaultAffinityRoutingEnabled` (`true`) for tenants with no `CachePolicy`. When enabled (default), `tryAffinityResponse` reads the index-known replica set for the request's `(tenant, model, hash_scheme)` engine domain from `servingByScope` (scheme-aware, mirroring the `TENANT_HOT` Pass 2 check), sorts by `replica_id` for cross-restart determinism, and modulos the SHA-256 of the length-prefixed `block_hashes` (fall-back to `prefix_hash`) against the sorted set — same prompt content → same replica every time. When disabled, the response stays on `NO_HINT`. Diagnostic codes (`UNKNOWN_TENANT` / `UNKNOWN_MODEL` / `UNKNOWN_HASH_SCHEME`) and `TIMEOUT` keep precedence over `AFFINITY_HINT`; affinity never preempts a real `PREFIX_MATCH` or `TENANT_HOT` that cleared the request-side gates (one exception: a tiny request below the per-namespace `minimumPrefixTokens` gate has its positive-hint result — including `TENANT_HOT` — downgraded to StrategyNone, so the affinity fallback can still fire on it; the operator intent "tiny prompts don’t surface a positive hint" outranks the TENANT_HOT-vs-affinity precedence). See [`grpc-contract.md` § "Affinity routing"](./grpc-contract.md). | | `strategy.enableChainMatching` / `strategy.requireChain` / `strategy.enableTenantHot` | Handler-side strategy gates. Chain matching disabled strips request block-hash fields before the index call; chain required rejects non-chain requests with `reason_code: POLICY_REQUIRES_CHAIN` before touching the index; tenant-hot disabled downgrades tenant-hot results to `NO_HINT`. The index remains policy-agnostic. | | `CacheTenant.spec.quota.maxIndexEntries` | `internal/index` `TenantQuotaResolver`. Pushed as a `ResolvedTenant{tenantID, maxIndexEntries, isolationMode}` slice alongside the policies. At ingest, if the tenant's distinct-prefix count exceeds the budget, the index evicts that tenant's oldest prefixes (Fairness) down to budget. Fail-open when no `CacheTenant` matches the ingest's `tenant_id`. | @@ -453,6 +475,11 @@ unbounded (no enforcement). - **Server restart.** The server starts with an empty store (server defaults everywhere). The next periodic tick re-pushes the full snapshot; in steady state this is ≤ 30s. +- **Malformed ranker override bypasses admission.** The authenticated + `/policy` decoder rejects unknown fields and invalid JSON. For a known + field whose numeric value is outside the CRD range, `PolicyStore.Ranker` + omits that field and the index inherits its baseline value; valid sibling + overrides still apply. ## Versioning and forward-compat @@ -462,13 +489,14 @@ the same `version`; load-bearing or semantically breaking changes bump `version` and gate decode on the new value. The controller pushes the constant in `internal/controlplaneapi.PolicyPropagationVersion` on every request. -`version` is `7`: `2` added the `tenants` slice; `3` added +`version` is `8`: `2` added the `tenants` slice; `3` added `policies[].eviction` (the per-namespace cap-eviction algorithm); `4` added `policies[].minimumMatchedTokens` (the result-side matched-tokens floor); `5` added `policies[].routingFloorScore` (the per-namespace post-score floor for the distinguishing-power-aware ranker); `6` added `policies[].strategy` (per-namespace LookupRoute strategy gates); `7` added -`policies[].affinityRouting` (the per-namespace consistent-hash fallback toggle). +`policies[].affinityRouting` (the per-namespace consistent-hash fallback toggle); +`8` added `policies[].rankerOverrides` (presence-aware ranking-v2 overrides). The server decodes with `DisallowUnknownFields`, so an older server receiving a newer body still fails loud on the unknown field even before its version check fires. @@ -478,9 +506,9 @@ deliberately asymmetric so a server-first rollout (newer server, older controller still pushing the prior schema) does NOT drop existing policy state mid-upgrade: -- A v7 server accepts any body whose `version` is in +- A v8 server accepts any body whose `version` is in `[PolicyMinimumAcceptedVersion, PolicyPropagationVersion]` — today - `[3, 7]`. Bodies outside the band are rejected with + `[3, 8]`. Bodies outside the band are rejected with `unsupported policy snapshot version`. - For accepted older bodies, each new field is *normalized* before reaching the store. v3 has none of `minimumMatchedTokens`, `routingFloorScore`, @@ -495,7 +523,7 @@ state mid-upgrade: (omitempty drops the field) from `&0` / `&false` (literal `0` / `false`); the resolver treats `nil` as "use the resolver default", but the normalizer still rewrites `nil` to the default value here so - the in-memory store ends up byte-for-byte identical to a fresh v7 + the in-memory store ends up byte-for-byte identical to a fresh v8 body — a single shape per knob is easier to reason about than two shapes that the resolver papers over. The server fills in `DefaultMinimumMatchedTokens` (`64`) for v3 bodies, `DefaultRoutingFloorScore` (`0.1`) for v3/v4 bodies, @@ -504,9 +532,11 @@ state mid-upgrade: the effective behavior matches the no-CachePolicy fallbacks `PolicyStore.MinimumMatchedTokens` / `PolicyStore.RoutingFloorScore` / `PolicyStore.AffinityRoutingEnabled` apply to tenants without a CR. + Ranker overrides need no synthesized value: a missing object or nested + field is already the explicit instruction to inherit the index baseline. Every other knob (TTL, prefix gate, timeout, eviction, tenant quota) reaches the store byte-for-byte. -- v7 bodies are NOT normalized — an operator's explicit +- v8 bodies are NOT normalized — an operator's explicit `routingFloorScore: 0` opt-out (or `minimumMatchedTokens: 0`, `strategy.enableTenantHot: false`, or `affinityRouting: false`) reaches the store as written. Each normalization fires only when the version says the corresponding new diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index 9950efc1..a6b9ed46 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -484,6 +484,38 @@ kubectl -n "$sample_namespace" get cachetenant cachetenant-sample >/dev/null kubectl -n "$sample_namespace" get prompttemplate prompttemplate-sample >/dev/null kubectl -n "$sample_namespace" get pdtopology pdtopology-sample >/dev/null +ranker_overrides="$(kubectl -n "$sample_namespace" get cachepolicy cachepolicy-sample \ + -o jsonpath='{.spec.rankerOverrides.pressureWeight},{.spec.rankerOverrides.sloTightTTFTMs},{.spec.rankerOverrides.sloTightBias},{.spec.rankerOverrides.tenantHotMinHitRate},{.spec.rankerOverrides.tenantHotMaxAge}')" +[ "$ranker_overrides" = "2,150,2,0.25,2m" ] \ + || fail "CachePolicy rankerOverrides did not round-trip: $ranker_overrides" + +# Seed one pressured holder of ranker-target and one serving peer holding a +# different prefix. With the sample's pressureWeight=2, pressure=0.5 clamps the +# holder's score to zero; routingFloorScore then downgrades PREFIX_MATCH to the +# affinity fallback. The default pressureWeight=1 would leave a positive score, +# so observing AFFINITY_HINT proves the full CRD -> controller -> /policy -> +# PolicyStore -> index path adopted the override without any engine traffic. +grpcurl -plaintext -max-time 5 \ + -import-path proto -proto inferencecache/v1alpha1/inferencecache.proto \ + -d @ "localhost:$GRPC_LOCAL_PORT" \ + inferencecache.v1alpha1.InferenceCache/ReportCacheState >/dev/null < 0 { pressure = s.stats.Pressure } - pressureFactor := pressureFactorAt(pressure, i.ranker.PressureWeight) + pressureFactor := pressureFactorAt(pressure, ranker.PressureWeight) sloBias := 1 + fresh*sloBiasFactor scores = append(scores, ReplicaScore{ ReplicaID: id, @@ -225,7 +225,7 @@ func (i *Index) lookupExact(req LookupRequest) ([]ReplicaScore, map[string][]*re // The pressure and SLO factors from lookupExact compose unchanged: the chain // walk only changes how matched_tokens is derived; the score formula // (matched_tokens × freshness × pressureFactor × sloBias) is the same. -func (i *Index) lookupChain(req LookupRequest) ([]ReplicaScore, map[string][]*replicaEntry) { +func (i *Index) lookupChain(req LookupRequest, ranker RankerConfig) ([]ReplicaScore, map[string][]*replicaEntry) { type running struct { matchedTokens int32 oldestLastSeen time.Time @@ -244,7 +244,7 @@ func (i *Index) lookupChain(req LookupRequest) ([]ReplicaScore, map[string][]*re } now := i.now() ttl := i.ttlFor(req.Tenant) - sloBiasFactor := i.sloTightBiasCoefficient(req.TTFTBudgetMs) + sloBiasFactor := sloTightBiasCoefficient(req.TTFTBudgetMs, ranker) // Resolve the algorithm once, outside the lock (see lookupExact): LFU // tracks the per-block entry pointers so each contributing block's counter // can be bumped; LRU skips both the tracking and the bump. @@ -329,7 +329,7 @@ func (i *Index) lookupChain(req LookupRequest) ([]ReplicaScore, map[string][]*re freshnessAt(now, s.statsReported, ttl) > 0 { pressure = s.stats.Pressure } - pressureFactor := pressureFactorAt(pressure, i.ranker.PressureWeight) + pressureFactor := pressureFactorAt(pressure, ranker.PressureWeight) sloBias := 1 + fresh*sloBiasFactor scores = append(scores, ReplicaScore{ ReplicaID: id, @@ -396,6 +396,7 @@ func (i *Index) LookupRoute(req LookupRequest) LookupResult { if req.Tenant == "" || req.Model == "" || req.HashScheme == "" { return LookupResult{Strategy: StrategyNone} } + ranker := i.rankerFor(req.Tenant) // Chain-bearing requests short-circuit on ANY chain failure (malformed // parallel arrays OR a well-formed chain with zero overlap) — never // fall through to TENANT_HOT. The chain caller is asking specifically @@ -410,17 +411,17 @@ func (i *Index) LookupRoute(req LookupRequest) LookupResult { if len(req.BlockHashes) != len(req.BlockTokenCounts) { return LookupResult{Strategy: StrategyNone} } - if scores, hits := i.lookupWithHits(req); len(scores) > 0 { + if scores, hits := i.lookupWithHits(req, ranker); len(scores) > 0 { return LookupResult{Scores: scores, Strategy: StrategyPrefixMatch, hitsByReplica: hits} } // Chain misses never fall through to TENANT_HOT (by design — see // contract doc), so run the miss classifier directly. return LookupResult{Strategy: i.classifyMiss(req)} } - if scores, hits := i.lookupWithHits(req); len(scores) > 0 { + if scores, hits := i.lookupWithHits(req, ranker); len(scores) > 0 { return LookupResult{Scores: scores, Strategy: StrategyPrefixMatch, hitsByReplica: hits} } - if hot := i.tenantHotCandidates(req); len(hot) > 0 { + if hot := i.tenantHotCandidates(req, ranker); len(hot) > 0 { // TENANT_HOT carries MatchedTokens=0, so no hits to credit — it is a // softer locality nudge, not a prefix HIT. return LookupResult{Scores: hot, Strategy: StrategyTenantHot} @@ -456,8 +457,8 @@ func (i *Index) LookupRoute(req LookupRequest) LookupResult { // by definition here) and reuses the same pressure/SLO factors as the // prefix-match path so a tight-SLO caller still gets a freshness-biased // ranking. -func (i *Index) tenantHotCandidates(req LookupRequest) []ReplicaScore { - if i.ranker.TenantHotMaxAge <= 0 { +func (i *Index) tenantHotCandidates(req LookupRequest, ranker RankerConfig) []ReplicaScore { + if ranker.TenantHotMaxAge <= 0 { return nil } // LookupRoute already short-circuits an empty hash_scheme to NO_HINT, @@ -468,9 +469,9 @@ func (i *Index) tenantHotCandidates(req LookupRequest) []ReplicaScore { return nil } now := i.now() - maxAge := i.ranker.TenantHotMaxAge - minHitRate := i.ranker.TenantHotMinHitRate - sloBiasFactor := i.sloTightBiasCoefficient(req.TTFTBudgetMs) + maxAge := ranker.TenantHotMaxAge + minHitRate := ranker.TenantHotMinHitRate + sloBiasFactor := sloTightBiasCoefficient(req.TTFTBudgetMs, ranker) i.mu.RLock() defer i.mu.RUnlock() @@ -548,7 +549,7 @@ func (i *Index) tenantHotCandidates(req LookupRequest) []ReplicaScore { default: recency = 1 - float32(age)/float32(maxAge) } - pressureFactor := pressureFactorAt(w.pressure, i.ranker.PressureWeight) + pressureFactor := pressureFactorAt(w.pressure, ranker.PressureWeight) sloBias := 1 + recency*sloBiasFactor scores = append(scores, ReplicaScore{ ReplicaID: id, diff --git a/internal/index/ranker_resolver_test.go b/internal/index/ranker_resolver_test.go new file mode 100644 index 00000000..c660542c --- /dev/null +++ b/internal/index/ranker_resolver_test.go @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package index + +import ( + "testing" + "time" +) + +type staticRankerResolver map[string]RankerOverrides + +func (r staticRankerResolver) Ranker(tenant string) (RankerOverrides, bool) { + overrides, ok := r[tenant] + return overrides, ok +} + +func TestRankerResolverAppliesPerTenantOverrides(t *testing.T) { + clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} + zeroFloat := float32(0) + zeroDuration := time.Duration(0) + resolver := staticRankerResolver{ + "pressure-off": {PressureWeight: &zeroFloat}, + "tenant-hot-off": { + TenantHotMaxAge: &zeroDuration, + }, + } + idx := New( + withClock(clk.now), + WithTTL(time.Hour), + WithRanker(DefaultRankerConfig()), + WithRankerResolver(resolver), + ) + + for _, tenant := range []string{"baseline", "pressure-off", "tenant-hot-off"} { + idx.Ingest(Update{ + ReplicaID: "r-" + tenant, Model: "m", Tenant: tenant, HashScheme: "vllm", + Prefixes: []PrefixRef{{PrefixHash: hash("known"), TokenCount: 100}}, + Stats: &ReplicaStats{Pressure: 1, HitRate: 0.9}, + }) + } + + baseline := idx.Lookup(LookupRequest{Tenant: "baseline", Model: "m", HashScheme: "vllm", PrefixHash: hash("known")}) + if len(baseline) != 1 || baseline[0].Score != 0 { + t.Fatalf("baseline pressure penalty = %+v, want one zero-score replica", baseline) + } + overridden := idx.Lookup(LookupRequest{Tenant: "pressure-off", Model: "m", HashScheme: "vllm", PrefixHash: hash("known")}) + if len(overridden) != 1 || overridden[0].Score != 100 { + t.Fatalf("pressure-off override = %+v, want score 100", overridden) + } + + hotBaseline := idx.LookupRoute(LookupRequest{Tenant: "baseline", Model: "m", HashScheme: "vllm", PrefixHash: hash("missing")}) + if hotBaseline.Strategy != StrategyTenantHot { + t.Fatalf("baseline miss strategy = %v, want TENANT_HOT", hotBaseline.Strategy) + } + hotDisabled := idx.LookupRoute(LookupRequest{Tenant: "tenant-hot-off", Model: "m", HashScheme: "vllm", PrefixHash: hash("missing")}) + if hotDisabled.Strategy != StrategyNone { + t.Fatalf("tenant-hot-off miss strategy = %v, want no hint", hotDisabled.Strategy) + } +} + +func TestRankerResolverPreservesBaselineAndExplicitAllZero(t *testing.T) { + zeroFloat := float32(0) + zeroInt := int32(0) + zeroDuration := time.Duration(0) + allZero := RankerOverrides{ + PressureWeight: &zeroFloat, + SLOTightTTFTMs: &zeroInt, + SLOTightBias: &zeroFloat, + TenantHotMinHitRate: &zeroFloat, + TenantHotMaxAge: &zeroDuration, + } + baseline := RankerConfig{ + PressureWeight: 2, SLOTightTTFTMs: 400, SLOTightBias: 3, + TenantHotMinHitRate: 0.5, TenantHotMaxAge: time.Minute, + } + idx := New(WithRanker(baseline), WithRankerResolver(staticRankerResolver{"all-zero": allZero})) + + if got := idx.rankerFor("missing"); got != baseline { + t.Fatalf("missing resolver entry = %+v, want baseline %+v", got, baseline) + } + if got := idx.rankerFor("all-zero"); got != (RankerConfig{}) { + t.Fatalf("explicit all-zero override = %+v, want all zero", got) + } +} diff --git a/internal/index/ranking.go b/internal/index/ranking.go index 8ffd624a..c54bbf33 100644 --- a/internal/index/ranking.go +++ b/internal/index/ranking.go @@ -29,8 +29,43 @@ const ( // TENANT_HOT fallback: stats lastSeen within this window count as // "recent" — anything older is treated as cold for the fallback. DefaultTenantHotMaxAge = 5 * time.Minute + + // Admission and the PolicyStore trust boundary enforce these upper bounds + // so operator input cannot create negative or unbounded score multipliers. + MaxPressureWeight = 4.0 + MaxSLOTightBias = 8.0 ) +// rankerFor overlays the tenant's presence-aware overrides onto the Index's +// server-wide baseline. Resolving once per lookup keeps a concurrent policy +// replacement from mixing two configs inside one routing decision. +func (i *Index) rankerFor(tenant string) RankerConfig { + base := i.ranker + if i.rankerResolver == nil { + return base + } + overrides, ok := i.rankerResolver.Ranker(tenant) + if !ok { + return base + } + if overrides.PressureWeight != nil { + base.PressureWeight = *overrides.PressureWeight + } + if overrides.SLOTightTTFTMs != nil { + base.SLOTightTTFTMs = *overrides.SLOTightTTFTMs + } + if overrides.SLOTightBias != nil { + base.SLOTightBias = *overrides.SLOTightBias + } + if overrides.TenantHotMinHitRate != nil { + base.TenantHotMinHitRate = *overrides.TenantHotMinHitRate + } + if overrides.TenantHotMaxAge != nil { + base.TenantHotMaxAge = *overrides.TenantHotMaxAge + } + return base +} + // applyChainDistinguishingPower folds the depth-aware distinguishing-power // factor into a chain-lookup's per-replica scores in place. Unlike the // exact-match path — where every scored replica shares the same prefix @@ -112,14 +147,14 @@ func sortScoresDescByScoreThenID(scores []ReplicaScore) { // term inside (1 + freshness × coefficient). 0 → no bias (baseline). The // bias only fires when (a) the ranker has SLOTightTTFTMs and SLOTightBias // configured AND (b) the request carries a TTFT budget below the threshold. -func (i *Index) sloTightBiasCoefficient(ttftMs int32) float32 { - if i.ranker.SLOTightTTFTMs <= 0 || i.ranker.SLOTightBias <= 0 { +func sloTightBiasCoefficient(ttftMs int32, ranker RankerConfig) float32 { + if ranker.SLOTightTTFTMs <= 0 || ranker.SLOTightBias <= 0 { return 0 } - if ttftMs <= 0 || ttftMs >= i.ranker.SLOTightTTFTMs { + if ttftMs <= 0 || ttftMs >= ranker.SLOTightTTFTMs { return 0 } - return i.ranker.SLOTightBias + return ranker.SLOTightBias } // pressureFactorAt computes 1 - weight × pressure, clamped to [0, 1]. Kept diff --git a/internal/index/types.go b/internal/index/types.go index 527dcfb1..3eb81036 100644 --- a/internal/index/types.go +++ b/internal/index/types.go @@ -41,6 +41,24 @@ type TTLResolver interface { TTL(tenant string) time.Duration } +// RankerOverrides is a presence-aware set of per-tenant ranking-v2 knobs. +// Nil fields inherit the Index baseline configured through WithRanker; a +// non-nil zero preserves the knob's documented kill-switch behavior. +type RankerOverrides struct { + PressureWeight *float32 + SLOTightTTFTMs *int32 + SLOTightBias *float32 + TenantHotMinHitRate *float32 + TenantHotMaxAge *time.Duration +} + +// RankerResolver returns per-tenant ranker overrides. ok=false (including a +// nil resolver) means the tenant uses the Index's WithRanker baseline. The +// resolver implementation owns its own concurrency. +type RankerResolver interface { + Ranker(tenant string) (overrides RankerOverrides, ok bool) +} + // Eviction algorithm identifiers. The wire form is lower-case ("lru"/"lfu") to // match the casing of ResolvedPolicy.Eviction and reason_code; the CRD enum is // upper-case per K8s convention and the controller lower-cases when flattening. diff --git a/internal/server/policy.go b/internal/server/policy.go index ad6573f1..56923637 100644 --- a/internal/server/policy.go +++ b/internal/server/policy.go @@ -12,13 +12,14 @@ import ( "time" "github.com/cachebox-project/inference-cache/internal/controlplaneapi" + "github.com/cachebox-project/inference-cache/internal/index" ) // PolicyStore is the server-side cache of resolved policies (indexed by // namespace) and resolved tenant quotas (indexed by tenant ID). Reads take // the read lock; pushes from /policy (POST or PUT) take the write lock and // replace the maps atomically. Satisfies index.TTLResolver, -// index.TenantQuotaResolver, and index.EvictionResolver. +// index.TenantQuotaResolver, index.EvictionResolver, and index.RankerResolver. // // The two indices use different keys on purpose: a CachePolicy is keyed by its // namespace (phase-1 tenant boundary for lookups), while a CacheTenant quota is @@ -120,6 +121,36 @@ func (s *PolicyStore) TTL(tenant string) time.Duration { return 0 } +// Ranker satisfies index.RankerResolver. It preserves pointer presence so the +// Index can overlay only configured fields onto its own WithRanker baseline; +// explicit zero values remain valid kill switches. Values that bypass CRD +// admission and fall outside the public ranges are omitted defensively, which +// makes that field inherit the baseline instead of producing an absurd score. +func (s *PolicyStore) Ranker(tenant string) (index.RankerOverrides, bool) { + p, ok := s.Lookup(tenant) + if !ok || p.RankerOverrides == nil { + return index.RankerOverrides{}, false + } + wire := p.RankerOverrides + var out index.RankerOverrides + if wire.PressureWeight != nil && *wire.PressureWeight >= 0 && *wire.PressureWeight <= index.MaxPressureWeight { + out.PressureWeight = wire.PressureWeight + } + if wire.SLOTightTTFTMs != nil && *wire.SLOTightTTFTMs >= 0 { + out.SLOTightTTFTMs = wire.SLOTightTTFTMs + } + if wire.SLOTightBias != nil && *wire.SLOTightBias >= 0 && *wire.SLOTightBias <= index.MaxSLOTightBias { + out.SLOTightBias = wire.SLOTightBias + } + if wire.TenantHotMinHitRate != nil && *wire.TenantHotMinHitRate >= 0 && *wire.TenantHotMinHitRate <= 1 { + out.TenantHotMinHitRate = wire.TenantHotMinHitRate + } + if wire.TenantHotMaxAge != nil && *wire.TenantHotMaxAge >= 0 { + out.TenantHotMaxAge = wire.TenantHotMaxAge + } + return out, true +} + // Eviction satisfies index.EvictionResolver: returns the per-namespace // cap-eviction algorithm in lower-case canonical form ("lru" / "lfu"), or "" // when no policy is configured (the index then defaults to LRU). The index @@ -367,9 +398,9 @@ func policyHandler(store *PolicyStore) http.HandlerFunc { return } // Normalize older bodies so server-first rollouts (newer server, older - // controller still pushing v3/v4/v5/v6) preserve every other knob a CR + // controller still pushing v3-v7) preserve every other knob a CR // carries. Today: older bodies may omit minimumMatchedTokens, - // routingFloorScore, strategy, or affinityRouting. + // routingFloorScore, strategy, affinityRouting, or rankerOverrides. // JSON decodes the missing fields to their zero values // (int32(0) / nil *float32), which would be indistinguishable from // the explicit opt-outs. Fill in the server defaults so the @@ -410,9 +441,10 @@ func policyHandler(store *PolicyStore) http.HandlerFunc { // DefaultAffinityRoutingEnabled so a server-first rollout does not // silently disable the consistent-hash fallback for namespaces with a CR. // -// Bodies already at PolicyPropagationVersion are returned untouched so an -// operator's explicit opt-out (e.g. `routingFloorScore: 0` for raw-recall -// benchmarking, or `enableTenantHot: false`, or `affinityRouting: false`) reaches the store as written. +// RankerOverrides needs no normalization: an absent object or nested pointer +// already means "inherit the index baseline," while non-nil zeroes preserve +// explicit opt-outs. Bodies already at PolicyPropagationVersion are returned +// untouched so every explicit opt-out reaches the store as written. func normalizePolicySnapshotForVersion(snap *controlplaneapi.PolicySnapshot) { if snap.Version >= controlplaneapi.PolicyPropagationVersion { return diff --git a/internal/server/ranker_policy_test.go b/internal/server/ranker_policy_test.go new file mode 100644 index 00000000..8bbc06ac --- /dev/null +++ b/internal/server/ranker_policy_test.go @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/cachebox-project/inference-cache/internal/controlplaneapi" + "github.com/cachebox-project/inference-cache/internal/index" +) + +func TestPolicyStoreRankerPreservesPresenceAndRejectsInvalidWireValues(t *testing.T) { + zero := float32(0) + ttft := int32(150) + maxAge := 2 * time.Minute + tooLarge := float32(index.MaxSLOTightBias + 1) + negativeAge := -time.Second + store := NewPolicyStore() + store.Replace([]controlplaneapi.ResolvedPolicy{ + {Namespace: "configured", RankerOverrides: &controlplaneapi.ResolvedRankerOverrides{ + PressureWeight: &zero, SLOTightTTFTMs: &ttft, TenantHotMaxAge: &maxAge, + }}, + {Namespace: "invalid", RankerOverrides: &controlplaneapi.ResolvedRankerOverrides{ + SLOTightBias: &tooLarge, TenantHotMaxAge: &negativeAge, + }}, + }) + + if _, ok := store.Ranker("missing"); ok { + t.Fatal("missing policy must report no ranker override") + } + configured, ok := store.Ranker("configured") + if !ok || configured.PressureWeight == nil || *configured.PressureWeight != 0 || + configured.SLOTightTTFTMs == nil || *configured.SLOTightTTFTMs != 150 || + configured.TenantHotMaxAge == nil || *configured.TenantHotMaxAge != 2*time.Minute { + t.Fatalf("configured overrides = %+v, ok=%v", configured, ok) + } + invalid, ok := store.Ranker("invalid") + if !ok { + t.Fatal("configured ranker object must remain distinguishable from no policy") + } + if invalid.SLOTightBias != nil || invalid.TenantHotMaxAge != nil { + t.Fatalf("invalid wire values were not dropped: %+v", invalid) + } +} + +func TestPolicySnapshotV7InheritsRankerBaseline(t *testing.T) { + store := NewPolicyStore() + srv := httptest.NewServer(NewPolicyHTTPHandler(store)) + defer srv.Close() + resp, err := http.Post(srv.URL, "application/json", bytes.NewBufferString(`{"version":7,"policies":[{"namespace":"legacy"}]}`)) + if err != nil { + t.Fatalf("post v7 policy: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("v7 policy status = %d, want 204", resp.StatusCode) + } + if _, ok := store.Ranker("legacy"); ok { + t.Fatal("v7 policy without rankerOverrides must inherit the index baseline") + } +} + +func TestServiceWiresPolicyStoreAsRankerResolver(t *testing.T) { + zero := float32(0) + svc := New() + svc.policies.Replace([]controlplaneapi.ResolvedPolicy{{ + Namespace: "pressure-off", + RankerOverrides: &controlplaneapi.ResolvedRankerOverrides{ + PressureWeight: &zero, + }, + }}) + svc.index.Ingest(index.Update{ + ReplicaID: "r", Model: "m", Tenant: "pressure-off", HashScheme: "vllm", + Prefixes: []index.PrefixRef{{PrefixHash: []byte("p"), TokenCount: 100}}, + Stats: &index.ReplicaStats{Pressure: 1}, + }) + scores := svc.index.Lookup(index.LookupRequest{Tenant: "pressure-off", Model: "m", HashScheme: "vllm", PrefixHash: []byte("p")}) + if len(scores) != 1 || scores[0].Score != 100 { + t.Fatalf("service ranker override = %+v, want score 100", scores) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index f562e23c..00059a0f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -141,6 +141,7 @@ func New(opts ...Option) *Service { index.WithTTLResolver(policies), index.WithTenantQuotaResolver(policies), index.WithEvictionResolver(policies), + index.WithRankerResolver(policies), // Reserve the probe tenant from the global cap so a concurrent real- // workload Ingest can never pick a real-workload entry as a victim to // make room for the probe's transient ingest. Probe-tenant entries are diff --git a/internal/server/tenant_quota_test.go b/internal/server/tenant_quota_test.go index 6534c24d..5bcf74f1 100644 --- a/internal/server/tenant_quota_test.go +++ b/internal/server/tenant_quota_test.go @@ -38,7 +38,7 @@ func TestTenantQuotaExemptsProbeTenant(t *testing.T) { } } -// TestPolicyPropagationVersionIsV7 pins the wire-format version. v2 accompanied +// TestPolicyPropagationVersionIsV8 pins the wire-format version. v2 accompanied // the Tenants slice; v3 accompanied ResolvedPolicy.Eviction (per-namespace // cap-eviction algorithm); v4 accompanied ResolvedPolicy.MinimumMatchedTokens // (the result-side matched-tokens floor); v5 accompanied @@ -46,12 +46,13 @@ func TestTenantQuotaExemptsProbeTenant(t *testing.T) { // the distinguishing-power-aware LookupRoute ranker); v6 accompanied // ResolvedPolicy.Strategy (per-namespace LookupRoute strategy gates); v7 // accompanied ResolvedPolicy.AffinityRouting (the per-namespace toggle for -// consistent-hash fallback routing on the NO_HINT path). A controller/server +// consistent-hash fallback routing on the NO_HINT path); v8 accompanied +// ResolvedPolicy.RankerOverrides. A controller/server // version mismatch outside the accepted band is rejected with a clear // "unsupported version" rather than a decode error. -func TestPolicyPropagationVersionIsV7(t *testing.T) { - if controlplaneapi.PolicyPropagationVersion != 7 { - t.Fatalf("controlplaneapi.PolicyPropagationVersion = %d, want 7", controlplaneapi.PolicyPropagationVersion) +func TestPolicyPropagationVersionIsV8(t *testing.T) { + if controlplaneapi.PolicyPropagationVersion != 8 { + t.Fatalf("controlplaneapi.PolicyPropagationVersion = %d, want 8", controlplaneapi.PolicyPropagationVersion) } // PolicyMinimumAcceptedVersion bounds the lenience window for older bodies. // v3, v4, and v5 must be accepted so a server-first rollout does not drop @@ -65,7 +66,7 @@ func TestPolicyPropagationVersionIsV7(t *testing.T) { } // TestPolicySnapshotV3AcceptedWithFloorDefault pins the server-first rollout -// invariant: a v7 server MUST accept a v3 controller's snapshot AND normalize +// invariant: a v8 server MUST accept a v3 controller's snapshot AND normalize // missing fields — minimumMatchedTokens to DefaultMinimumMatchedTokens, // routingFloorScore to DefaultRoutingFloorScore, strategy to its defaults, and // affinityRouting to DefaultAffinityRoutingEnabled — on each policy. @@ -173,13 +174,13 @@ func TestPolicySnapshotV3AcceptedWithFloorDefault(t *testing.T) { // disables the new floor for every namespace. // 3. The missing affinityRouting on the same v4 body MUST be // normalized to DefaultAffinityRoutingEnabled. v4 predates the -// affinity-routing field too, so a v7 server receiving a v4 +// affinity-routing field too, so a v8 server receiving a v4 // body must synthesize the default; otherwise a v4 controller // pushing during a server-first rollout silently flips affinity // off for every namespace. // // Written against a literal v4 body (not PolicyPropagationVersion, which is -// now v7) so the v4-specific behavior under v3→v4→v5→v6→v7 server stays pinned +// now v8) so the v4-specific behavior under later servers stays pinned // even after the constant advances. func TestPolicySnapshotV4ExplicitZeroPreservedAndRoutingFloorNormalized(t *testing.T) { store := NewPolicyStore() @@ -241,7 +242,7 @@ func TestPolicySnapshotV4ExplicitZeroPreservedAndRoutingFloorNormalized(t *testi // normalization tests above: a v5 controller that EXPLICITLY pushes // routingFloorScore=0 (the documented opt-out, useful for raw-recall // benchmarks) must NOT have its zero rewritten to the default when -// processed by a v7 server. The normalization only fires for the +// processed by a v8 server. The normalization only fires for the // fields the body's version says could not have been present // (routingFloorScore was added at v5, so a v5 body's &0 reaches the // store byte-for-byte). The test uses a literal Version: 5 — NOT @@ -255,7 +256,7 @@ func TestPolicySnapshotV5ExplicitRoutingFloorZeroPreserved(t *testing.T) { zero := float32(0) body, err := json.Marshal(controlplaneapi.PolicySnapshot{ - Version: 5, // literal v5 — must reach the store byte-for-byte even on a v7 server. + Version: 5, // literal v5 — must reach the store byte-for-byte even on a v8 server. Policies: []controlplaneapi.ResolvedPolicy{ {Namespace: "raw-recall", RoutingFloorScore: &zero}, }, diff --git a/internal/webhook/v1alpha1/cachepolicy_webhook.go b/internal/webhook/v1alpha1/cachepolicy_webhook.go index ee3d73b5..60329fd9 100644 --- a/internal/webhook/v1alpha1/cachepolicy_webhook.go +++ b/internal/webhook/v1alpha1/cachepolicy_webhook.go @@ -7,6 +7,7 @@ package v1alpha1 import ( "context" "fmt" + "math" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" @@ -79,6 +80,7 @@ type CachePolicyValidationRule func(cp *cachev1alpha1.CachePolicy) field.ErrorLi var DefaultCachePolicyValidationRules = []CachePolicyValidationRule{ rejectNonPositiveEvictionTTL, rejectIncoherentStrategy, + rejectInvalidRankerOverrides, } // SetupCachePolicyWebhookWithManager registers the defaulting and validating @@ -272,3 +274,33 @@ func rejectIncoherentStrategy(cp *cachev1alpha1.CachePolicy) field.ErrorList { ), } } + +// rejectInvalidRankerOverrides mirrors the structural schema bounds in the +// validating webhook and covers metav1.Duration, which has no numeric +// kubebuilder marker. Duplication here is intentional: direct webhook tests pin +// the operator contract, while CRD markers reject malformed objects even when +// the webhook is unavailable during bootstrap. +func rejectInvalidRankerOverrides(cp *cachev1alpha1.CachePolicy) field.ErrorList { + ro := cp.Spec.RankerOverrides + if ro == nil { + return nil + } + path := field.NewPath("spec", "rankerOverrides") + var errs field.ErrorList + if ro.PressureWeight != nil && (math.IsNaN(float64(*ro.PressureWeight)) || math.IsInf(float64(*ro.PressureWeight), 0) || *ro.PressureWeight < 0 || *ro.PressureWeight > 4) { + errs = append(errs, field.Invalid(path.Child("pressureWeight"), *ro.PressureWeight, "must be between 0 and 4")) + } + if ro.SLOTightTTFTMs != nil && *ro.SLOTightTTFTMs < 0 { + errs = append(errs, field.Invalid(path.Child("sloTightTTFTMs"), *ro.SLOTightTTFTMs, "must be greater than or equal to zero")) + } + if ro.SLOTightBias != nil && (math.IsNaN(float64(*ro.SLOTightBias)) || math.IsInf(float64(*ro.SLOTightBias), 0) || *ro.SLOTightBias < 0 || *ro.SLOTightBias > 8) { + errs = append(errs, field.Invalid(path.Child("sloTightBias"), *ro.SLOTightBias, "must be between 0 and 8")) + } + if ro.TenantHotMinHitRate != nil && (math.IsNaN(float64(*ro.TenantHotMinHitRate)) || math.IsInf(float64(*ro.TenantHotMinHitRate), 0) || *ro.TenantHotMinHitRate < 0 || *ro.TenantHotMinHitRate > 1) { + errs = append(errs, field.Invalid(path.Child("tenantHotMinHitRate"), *ro.TenantHotMinHitRate, "must be between 0 and 1")) + } + if ro.TenantHotMaxAge != nil && ro.TenantHotMaxAge.Duration < 0 { + errs = append(errs, field.Invalid(path.Child("tenantHotMaxAge"), ro.TenantHotMaxAge.Duration.String(), "must be greater than or equal to zero")) + } + return errs +} diff --git a/internal/webhook/v1alpha1/cachepolicy_webhook_test.go b/internal/webhook/v1alpha1/cachepolicy_webhook_test.go index d24e49be..7baa9da1 100644 --- a/internal/webhook/v1alpha1/cachepolicy_webhook_test.go +++ b/internal/webhook/v1alpha1/cachepolicy_webhook_test.go @@ -7,6 +7,7 @@ package v1alpha1 import ( "context" "errors" + "math" "strings" "testing" "time" @@ -131,6 +132,48 @@ func TestRejectIncoherentStrategy(t *testing.T) { } } +func TestRejectInvalidRankerOverrides(t *testing.T) { + f32p := func(v float32) *float32 { return &v } + i32p := func(v int32) *int32 { return &v } + tests := []struct { + name string + overrides *cachev1alpha1.CachePolicyRankerOverridesSpec + wantField string + }{ + {name: "nil overrides accepted"}, + {name: "explicit zeroes accepted", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{ + PressureWeight: f32p(0), SLOTightTTFTMs: i32p(0), SLOTightBias: f32p(0), + TenantHotMinHitRate: f32p(0), TenantHotMaxAge: durp(0), + }}, + {name: "upper bounds accepted", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{ + PressureWeight: f32p(4), SLOTightBias: f32p(8), TenantHotMinHitRate: f32p(1), + }}, + {name: "negative pressure rejected", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{PressureWeight: f32p(-0.1)}, wantField: "spec.rankerOverrides.pressureWeight"}, + {name: "large pressure rejected", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{PressureWeight: f32p(4.1)}, wantField: "spec.rankerOverrides.pressureWeight"}, + {name: "nan pressure rejected", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{PressureWeight: f32p(float32(math.NaN()))}, wantField: "spec.rankerOverrides.pressureWeight"}, + {name: "negative ttft rejected", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{SLOTightTTFTMs: i32p(-1)}, wantField: "spec.rankerOverrides.sloTightTTFTMs"}, + {name: "large bias rejected", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{SLOTightBias: f32p(8.1)}, wantField: "spec.rankerOverrides.sloTightBias"}, + {name: "invalid hit rate rejected", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{TenantHotMinHitRate: f32p(1.1)}, wantField: "spec.rankerOverrides.tenantHotMinHitRate"}, + {name: "negative max age rejected", overrides: &cachev1alpha1.CachePolicyRankerOverridesSpec{TenantHotMaxAge: durp(-time.Second)}, wantField: "spec.rankerOverrides.tenantHotMaxAge"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cp := policy("p1", "team-a") + cp.Spec.RankerOverrides = tc.overrides + errs := rejectInvalidRankerOverrides(cp) + if tc.wantField == "" { + if len(errs) != 0 { + t.Fatalf("expected no errors, got %v", errs) + } + return + } + if len(errs) != 1 || errs[0].Field != tc.wantField { + t.Fatalf("errors = %v, want one error for %s", errs, tc.wantField) + } + }) + } +} + // --- ValidateCreate: single-policy-per-namespace ----------------------------- func TestCachePolicyValidateCreate_SinglePerNamespace(t *testing.T) { diff --git a/site/content/en/docs/concepts/cachepolicy.md b/site/content/en/docs/concepts/cachepolicy.md index 519a1dcf..d96b6aa9 100644 --- a/site/content/en/docs/concepts/cachepolicy.md +++ b/site/content/en/docs/concepts/cachepolicy.md @@ -29,6 +29,9 @@ spec: evictionTTL: 30m minimumMatchedTokens: 64 routingFloorScore: "0.1" + rankerOverrides: + pressureWeight: 1 + tenantHotMaxAge: 5m strategy: enableChainMatching: true enableTenantHot: true @@ -51,6 +54,20 @@ and produce a useless hint. The `routingFloorScore` gate catches the mirror case held by *every* replica has zero distinguishing power (see [LookupRoute & ranking]({{< relref "/docs/concepts/lookuproute/" >}}). +## Ranker overrides + +`spec.rankerOverrides` optionally tunes ranking-v2 for one namespace. Every nested field +is optional and inherits the server baseline when omitted; an explicit zero keeps its +kill-switch meaning. + +| Field | Range | Baseline | Zero behavior | +|---|---|---|---| +| `pressureWeight` | `0..4` | `1.0` | disables pressure penalty | +| `sloTightTTFTMs` | `>=0` | `200` | SLO bias never fires | +| `sloTightBias` | `0..8` | `1.0` | disables freshness boost | +| `tenantHotMinHitRate` | `0..1` | `0.1` | every non-negative hit rate qualifies | +| `tenantHotMaxAge` | duration `>=0` | `5m` | disables `TENANT_HOT` | + ## Eviction | Field | Default | Meaning | diff --git a/site/content/en/docs/reference/crd-api.md b/site/content/en/docs/reference/crd-api.md index 54a18d53..b6f2f165 100644 --- a/site/content/en/docs/reference/crd-api.md +++ b/site/content/en/docs/reference/crd-api.md @@ -61,6 +61,11 @@ Full page: [CacheBackend]({{< relref "/docs/concepts/cachebackend/" >}}). | `minimumMatchedTokens` | int32 | `64` (`0` opts out) | | `routingFloorScore` | string (float) | `"0.1"` (`"0"` opts out) | | `lookupTimeoutMs` | int32 | unset (`0`/≤0 = unbounded) | +| `rankerOverrides.pressureWeight` | float32, `0..4` | server baseline `1.0` | +| `rankerOverrides.sloTightTTFTMs` | int32, `>=0` | server baseline `200` | +| `rankerOverrides.sloTightBias` | float32, `0..8` | server baseline `1.0` | +| `rankerOverrides.tenantHotMinHitRate` | float32, `0..1` | server baseline `0.1` | +| `rankerOverrides.tenantHotMaxAge` | non-negative duration | server baseline `5m` | | `strategy.enableChainMatching` | bool | `true` | | `strategy.requireChain` | bool | `false` | | `strategy.enableTenantHot` | bool | `true` | diff --git a/site/content/en/docs/reference/reason-codes.md b/site/content/en/docs/reference/reason-codes.md index 4a032c32..69e9141a 100644 --- a/site/content/en/docs/reference/reason-codes.md +++ b/site/content/en/docs/reference/reason-codes.md @@ -44,11 +44,11 @@ See [LookupRoute & ranking]({{< relref "/docs/concepts/lookuproute/#diagnostics- | `strategy.requireChain` | CachePolicy | false | (n/a) | | `strategy.enableTenantHot` | CachePolicy | true | false | | `affinityRouting` | CachePolicy | `Enabled` | `Disabled` | -| `PressureWeight` | server RankerConfig | 1.0 | 0 | -| `SLOTightTTFTMs` | server RankerConfig | 200ms | 0 | -| `SLOTightBias` | server RankerConfig | 1.0 | 0 | -| `TenantHotMaxAge` | server RankerConfig | 5m | 0 | -| `TenantHotMinHitRate` | server RankerConfig | 0.1 | — | +| `rankerOverrides.pressureWeight` | CachePolicy (inherits server RankerConfig) | 1.0 | 0 | +| `rankerOverrides.sloTightTTFTMs` | CachePolicy (inherits server RankerConfig) | 200ms | 0 | +| `rankerOverrides.sloTightBias` | CachePolicy (inherits server RankerConfig) | 1.0 | 0 | +| `rankerOverrides.tenantHotMaxAge` | CachePolicy (inherits server RankerConfig) | 5m | 0 | +| `rankerOverrides.tenantHotMinHitRate` | CachePolicy (inherits server RankerConfig) | 0.1 | — | ## RenderTemplate