Daemonset fault tolerance

internal/controller/capacity.go

@@ -87,25 +87,27 @@ func (*InstasliceReconciler) getStartIndexFromPreparedState(instaslice *inferenc
 	for i := range gpuAllocatedIndex {
 		gpuAllocatedIndex[i] = 0
 	}
-	//TODO: remove this once we start using GPU operator with device plugin fix
-	for _, item := range instaslice.Spec.Prepared {
-		if item.Parent == gpuUUID {
-			for i := 0; i < int(item.Size); i++ {
-				gpuAllocatedIndex[int(item.Start)+i] = 1
-			}
-
-		}
-	}
 	// deleted allocations can be reused
 	// ungated allocations are already counted in prepared
 	for _, item := range instaslice.Spec.Allocations {
-		if item.GPUUUID == gpuUUID && item.Allocationstatus != inferencev1alpha1.AllocationStatusDeleted && item.Allocationstatus != inferencev1alpha1.AllocationStatusUngated {
+		if item.GPUUUID == gpuUUID && item.Allocationstatus != inferencev1alpha1.AllocationStatusDeleted {
 			for i := 0; i < int(item.Size); i++ {
 				gpuAllocatedIndex[int(item.Start)+i] = 1
 			}
 		}
 	}
-
+	// Check if all indices are allocated
+	allAllocated := true
+	for _, allocated := range gpuAllocatedIndex {
+		if allocated != 1 {
+			allAllocated = false
+			break
+		}
+	}
+	if allAllocated {
+		// invalid index
+		return uint32(9)
+	}
 	var neededContinousSlot int
 	var possiblePlacements []int
 	for _, placement := range instaslice.Spec.Migplacement {

internal/controller/constants.go

@@ -26,4 +26,5 @@ const (
 	AttributeMediaExtensions    = "me"
 	instaSliceOperatorNamespace = "default"
 	NvidiaMIGPrefix             = "nvidia.com/mig-"
+	NodeLabel                   = "kubernetes.io/hostname"
 )

internal/controller/instaslice_controller_test.go

@@ -33,7 +33,7 @@ import (
 	inferencev1alpha1 "github.com/openshift/instaslice-operator/api/v1alpha1"
 )
 
-func TestChangesAllocationDeletionndFinalizer(t *testing.T) {
+func TestChangesAllocationDeletionAndFinalizer(t *testing.T) {
 	var _ = Describe("InstasliceReconciler processInstasliceAllocation", func() {
 		var (
 			ctx                 context.Context
@@ -105,10 +105,9 @@ func TestChangesAllocationDeletionndFinalizer(t *testing.T) {
 			}
 			Expect(fakeClient.Update(ctx, instaslice)).To(Succeed())
 
-			found, result, err := r.processInstasliceAllocation(ctx, instaslice.Name, podUUID, instaslice.Spec.Allocations[podUUID], req)
+			result, err := r.deleteInstasliceAllocation(ctx, instaslice.Name, instaslice.Spec.Allocations[podUUID])
 
 			Expect(err).NotTo(HaveOccurred())
-			Expect(found).To(BeTrue())
 			Expect(result).To(Equal(ctrl.Result{}))
 
 			updatedInstaSlice := &inferencev1alpha1.Instaslice{}
@@ -125,10 +124,9 @@ func TestChangesAllocationDeletionndFinalizer(t *testing.T) {
 			}
 			Expect(fakeClient.Update(ctx, instaslice)).To(Succeed())
 
-			found, result, err := r.processInstasliceAllocation(ctx, instaslice.Name, podUUID, instaslice.Spec.Allocations[podUUID], req)
+			result, err := r.removeInstaSliceFinalizer(ctx, req)
 
 			Expect(err).NotTo(HaveOccurred())
-			Expect(found).To(BeTrue())
 			Expect(result).To(Equal(ctrl.Result{}))
 
 			updatedPod := &v1.Pod{}
@@ -137,10 +135,9 @@ func TestChangesAllocationDeletionndFinalizer(t *testing.T) {
 		})
 
 		It("should set allocation status to Deleting if status is not Deleted", func() {
-			found, result, err := r.processInstasliceAllocation(ctx, instaslice.Name, podUUID, instaslice.Spec.Allocations[podUUID], req)
+			result, err := r.setInstasliceAllocationToDeleting(ctx, instaslice.Name, podUUID, instaslice.Spec.Allocations[podUUID])
 
 			Expect(err).NotTo(HaveOccurred())
-			Expect(found).To(BeTrue())
 
 			updatedInstaSlice := &inferencev1alpha1.Instaslice{}
 			Expect(fakeClient.Get(ctx, types.NamespacedName{Name: instaslice.Name, Namespace: instasliceNamespace}, updatedInstaSlice)).To(Succeed())
@@ -153,9 +150,8 @@ func TestChangesAllocationDeletionndFinalizer(t *testing.T) {
 		It("should requeue if there is an error updating the instaslice", func() {
 			r.Client = fake.NewClientBuilder().WithScheme(runtime.NewScheme()).Build()
 
-			found, result, err := r.processInstasliceAllocation(ctx, instaslice.Name, podUUID, instaslice.Spec.Allocations[podUUID], req)
+			result, err := r.setInstasliceAllocationToDeleting(ctx, instaslice.Name, podUUID, instaslice.Spec.Allocations[podUUID])
 
-			Expect(found).To(BeTrue())
 			Expect(err).To(HaveOccurred())
 			Expect(result.Requeue).To(BeTrue())
 		})