Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions helm/bundles/cortex-nova/templates/pipelines_kvm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,16 @@ spec:
requests without headroom, add `params: [{key: enforce, boolValue: true}]`
to this step. The shadow default is intentional so that newly-rolled-out
releases never silently start rejecting requests.
- name: filter_cr_migration_slot
description: |
During live migrations of VMs that occupy a committed-resource reservation
slot, this filter restricts candidates to hosts that have a ready CR
reservation with sufficient remaining capacity for the full slot size (not
just the VM flavor size). This ensures the reservation slot is migrated
alongside the VM.
If no candidate has a matching slot, all candidates are returned unchanged
so the VM can still migrate using regular (non-slot) capacity.
Only activates for live_migrate requests. All other intents pass through.
weighers:
- name: kvm_prefer_smaller_hosts
params:
Expand Down Expand Up @@ -261,6 +271,16 @@ spec:
requests without headroom, add `params: [{key: enforce, boolValue: true}]`
to this step. The shadow default is intentional so that newly-rolled-out
releases never silently start rejecting requests.
- name: filter_cr_migration_slot
description: |
During live migrations of VMs that occupy a committed-resource reservation
slot, this filter restricts candidates to hosts that have a ready CR
reservation with sufficient remaining capacity for the full slot size (not
just the VM flavor size). This ensures the reservation slot is migrated
alongside the VM.
If no candidate has a matching slot, all candidates are returned unchanged
so the VM can still migrate using regular (non-slot) capacity.
Only activates for live_migrate requests. All other intents pass through.
weighers:
- name: kvm_prefer_smaller_hosts
params:
Expand Down Expand Up @@ -714,6 +734,16 @@ spec:
requests without headroom, add `params: [{key: enforce, boolValue: true}]`
to this step. The shadow default is intentional so that newly-rolled-out
releases never silently start rejecting requests.
- name: filter_cr_migration_slot
description: |
During live migrations of VMs that occupy a committed-resource reservation
slot, this filter restricts candidates to hosts that have a ready CR
reservation with sufficient remaining capacity for the full slot size (not
just the VM flavor size). This ensures the reservation slot is migrated
alongside the VM.
If no candidate has a matching slot, all candidates are returned unchanged
so the VM can still migrate using regular (non-slot) capacity.
Only activates for live_migrate requests. All other intents pass through.
weighers:
- name: kvm_prefer_smaller_hosts
params:
Expand Down Expand Up @@ -866,6 +896,16 @@ spec:
requests without headroom, add `params: [{key: enforce, boolValue: true}]`
to this step. The shadow default is intentional so that newly-rolled-out
releases never silently start rejecting requests.
- name: filter_cr_migration_slot
description: |
During live migrations of VMs that occupy a committed-resource reservation
slot, this filter restricts candidates to hosts that have a ready CR
reservation with sufficient remaining capacity for the full slot size (not
just the VM flavor size). This ensures the reservation slot is migrated
alongside the VM.
If no candidate has a matching slot, all candidates are returned unchanged
so the VM can still migrate using regular (non-slot) capacity.
Only activates for live_migrate requests. All other intents pass through.
weighers:
- name: kvm_prefer_smaller_hosts
params:
Expand Down
35 changes: 28 additions & 7 deletions internal/scheduling/nova/crs/evaluator.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,19 @@ type SlotEvaluator struct {
// BuildSlotEvaluator lists HV CRDs and CR Reservation CRDs once and returns an evaluator
// that can answer slot-usability queries without further K8s reads.
func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, error) {
var resList v1alpha1.ReservationList
if err := c.List(ctx, &resList,
client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource},
); err != nil {
return nil, err
}
return BuildSlotEvaluatorFromReservations(ctx, c, resList.Items)
}

// BuildSlotEvaluatorFromReservations builds a SlotEvaluator from an already-fetched
// reservation slice. Use this when the caller has already listed reservations to avoid
// a redundant K8s read.
func BuildSlotEvaluatorFromReservations(ctx context.Context, c client.Client, reservations []v1alpha1.Reservation) (*SlotEvaluator, error) {
eval := &SlotEvaluator{
hvFreeMemory: make(map[string]int64),
reservationsByHost: make(map[string][]v1alpha1.Reservation),
Expand All @@ -45,13 +58,7 @@ func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, e
eval.hvFreeMemory[hv.Name] = max(effectiveMemQ.Value()-allocMemQ.Value(), 0)
}

var resList v1alpha1.ReservationList
if err := c.List(ctx, &resList,
client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource},
); err != nil {
return nil, err
}
for _, res := range resList.Items {
for _, res := range reservations {
if !res.IsReady() {
continue
}
Expand Down Expand Up @@ -104,6 +111,20 @@ func (e *SlotEvaluator) HasUsableSlot(hostName, projectID, flavorGroup string, v
return false
}

// HasSlotWithCapacity reports whether hostName has at least one ready CR slot
// matching projectID + flavorGroup whose remaining memory is >= requiredBytes.
// Unlike HasUsableSlot, this does not apply the overfill model — it is used
// during migration slot filtering where the full slot size must fit within a
// single reservation on the target host.
func (e *SlotEvaluator) HasSlotWithCapacity(hostName, projectID, flavorGroup string, requiredBytes int64) bool {
for _, slot := range e.SlotsForHost(hostName, projectID, flavorGroup) {
if ReservationRemainingMemory(slot) >= requiredBytes {
return true
}
}
return false
}

// ReservationRemainingMemory returns how many bytes of memory remain
// unallocated in a reservation slot. Returns 0 if the slot is full or nil.
func ReservationRemainingMemory(res v1alpha1.Reservation) int64 {
Expand Down
139 changes: 139 additions & 0 deletions internal/scheduling/nova/plugins/filters/filter_cr_migration_slot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
// Copyright SAP SE
// SPDX-License-Identifier: Apache-2.0

package filters

import (
"context"
"log/slog"

"sigs.k8s.io/controller-runtime/pkg/client"

api "github.com/cobaltcore-dev/cortex/api/external/nova"
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
"github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs"
hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
)

// FilterCRMigrationSlotStep filters live-migration candidates by committed-resource
// slot size rather than VM flavor size.
//
// When a VM that occupies a CR reservation slot is live-migrated, the target host
// must accommodate the full slot, not just the VM's flavor resources. This filter
// removes candidates that lack a ready CR reservation with sufficient remaining
// capacity for the slot.
//
// Placement order in the pipeline: last filter, after all other filters have run.
// Fallback: if no candidate survives the slot-size check, the original candidate
// set is returned unchanged so that the VM can still migrate using flavor-sized
// capacity on the target host.
//
// Only activates for LiveMigrationIntent. All other intents pass through unchanged.
type FilterCRMigrationSlotStep struct {
lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]
}

func (s *FilterCRMigrationSlotStep) Run(
traceLog *slog.Logger,
request api.ExternalSchedulerRequest,
) (*lib.FilterWeigherPipelineStepResult, error) {
result := s.IncludeAllHostsFromRequest(request)

intent, err := request.GetIntent()
if err != nil || intent != api.LiveMigrationIntent {
traceLog.Info("not a live migration, skipping CR slot filter")
return result, nil //nolint:nilerr
}

instanceUUID := request.Spec.Data.InstanceUUID
projectID := request.Spec.Data.ProjectID

// List all CR reservations once. We reuse this list for both finding the
// source slot and building the slot evaluator for target hosts, avoiding
// a second K8s read inside BuildSlotEvaluator.
var allReservations v1alpha1.ReservationList
if err := s.Client.List(context.Background(), &allReservations,
client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource},
); err != nil {
return nil, err
}

// Find the source reservation that currently holds this VM UUID (confirmed).
var sourceSlot *v1alpha1.Reservation
for i := range allReservations.Items {
res := &allReservations.Items[i]
if res.Status.CommittedResourceReservation == nil {
continue
}
if _, confirmed := res.Status.CommittedResourceReservation.Allocations[instanceUUID]; confirmed {
sourceSlot = res
break
}
}

if sourceSlot == nil {
traceLog.Info("migrating VM has no confirmed CR reservation slot, skipping slot filter",
"instanceUUID", instanceUUID)
return result, nil
}

slotMemoryBytes := sourceSlot.Spec.Resources[hv1.ResourceMemory]
if slotMemoryBytes.IsZero() {
traceLog.Info("source CR slot has no memory resource, skipping slot filter",
"instanceUUID", instanceUUID,
"reservation", sourceSlot.Name)
return result, nil
}

resourceGroup := sourceSlot.Spec.CommittedResourceReservation.ResourceGroup

traceLog.Info("found source CR reservation slot for migrating VM",
"instanceUUID", instanceUUID,
"reservation", sourceSlot.Name,
"slotMemoryBytes", slotMemoryBytes.Value(),
"resourceGroup", resourceGroup,
)

// Build the slot evaluator from the already-fetched reservation list so we
// don't issue a second List call. HVs are still fetched once inside the evaluator.
evaluator, err := crs.BuildSlotEvaluatorFromReservations(context.Background(), s.Client, allReservations.Items)
if err != nil {
return nil, err
}

// Filter candidates to those with a ready CR slot that has at least slotMemoryBytes
// remaining. This is a strict check — no overfill model — because the slot must
// fully migrate with the VM.
filtered := make(map[string]float64, len(result.Activations))
for host := range result.Activations {
if evaluator.HasSlotWithCapacity(host, projectID, resourceGroup, slotMemoryBytes.Value()) {
filtered[host] = result.Activations[host]
traceLog.Info("host has usable CR slot for migration",
"host", host, "slotMemoryBytes", slotMemoryBytes.Value())
} else {
traceLog.Info("host has no usable CR slot for migration slot size, excluding",
"host", host, "slotMemoryBytes", slotMemoryBytes.Value())
}
}

// Fallback: if no host has a matching slot, return all candidates so the VM
// can still migrate using regular (non-slot) capacity.
if len(filtered) == 0 {
traceLog.Info("no hosts with matching CR slot found, falling back to all candidates",
"instanceUUID", instanceUUID,
"slotMemoryBytes", slotMemoryBytes.Value(),
"candidateCount", len(result.Activations),
)
return result, nil
}

result.Activations = filtered
return result, nil
}

func init() {
Index["filter_cr_migration_slot"] = func() NovaFilter {
return &FilterCRMigrationSlotStep{}
}
}
Loading
Loading