Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@cheina97
cheina97 committed Sep 12, 2023
1 parent 2ab3638 commit 67ad421
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 116 deletions.
30 changes: 6 additions & 24 deletions pkg/liqo-controller-manager/virtualnode-controller/condition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,7 @@
package virtualnodectrl

import (
"context"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

virtualkubeletv1alpha1 "github.com/liqotech/liqo/apis/virtualkubelet/v1alpha1"
)
Expand All @@ -35,35 +32,34 @@ type VnCondition struct {
// ForgeCondition forges a virtual node condition.
func ForgeCondition(
virtualNode *virtualkubeletv1alpha1.VirtualNode,
vnConditions VnConditionMap) (update bool) {
vnConditions VnConditionMap) {
for nameCondition, vnCondition := range vnConditions {
for i := range virtualNode.Status.Conditions {
if virtualNode.Status.Conditions[i].Type != nameCondition {
continue
}
if virtualNode.Status.Conditions[i].Status == vnCondition.Status {
return false
return
}
if (virtualNode.Status.Conditions[i].Status == virtualkubeletv1alpha1.RunningConditionStatusType) &&
(vnCondition.Status == virtualkubeletv1alpha1.CreatingConditionStatusType) {
return false
return
}
if (virtualNode.Status.Conditions[i].Status == virtualkubeletv1alpha1.NoneConditionStatusType) &&
(vnCondition.Status == virtualkubeletv1alpha1.DrainingConditionStatusType) {
return false
return
}
if (virtualNode.Status.Conditions[i].Status == virtualkubeletv1alpha1.NoneConditionStatusType) &&
(vnCondition.Status == virtualkubeletv1alpha1.DeletingConditionStatusType) {
return false
return
}
if (virtualNode.Status.Conditions[i].Status == virtualkubeletv1alpha1.DeletingConditionStatusType) &&
vnCondition.Status == virtualkubeletv1alpha1.DrainingConditionStatusType {
return false
return
}
virtualNode.Status.Conditions[i].Status = vnCondition.Status
virtualNode.Status.Conditions[i].LastTransitionTime = metav1.Now()
virtualNode.Status.Conditions[i].Message = vnCondition.Message
return true
}
virtualNode.Status.Conditions = append(virtualNode.Status.Conditions,
virtualkubeletv1alpha1.VirtualNodeCondition{
Expand All @@ -73,20 +69,6 @@ func ForgeCondition(
Message: vnCondition.Message,
})
}
return true
}

// UpdateCondition updates the condition of the virtual node.
func UpdateCondition(ctx context.Context, cl client.Client,
virtualNode *virtualkubeletv1alpha1.VirtualNode,
vnConditions VnConditionMap,
) error {
if ForgeCondition(virtualNode, vnConditions) {
if err := cl.Status().Update(ctx, virtualNode); err != nil {
return err
}
}
return nil
}

// GetCondition returns the condition of the virtual node.
Expand Down
89 changes: 50 additions & 39 deletions pkg/liqo-controller-manager/virtualnode-controller/deletion-routine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,74 +103,89 @@ func (dr *DeletionRoutine) processNextItem(ctx context.Context) bool {
return true
}

func (dr *DeletionRoutine) handle(ctx context.Context, key string) error {
namespace, name, err := cache.SplitMetaNamespaceKey(key)
func (dr *DeletionRoutine) handle(ctx context.Context, key string) (err error) {
var namespace, name string
namespace, name, err = cache.SplitMetaNamespaceKey(key)
if err != nil {
return fmt.Errorf("error splitting key: %w", err)
err = fmt.Errorf("error splitting key: %w", err)
return err
}
ref := types.NamespacedName{Namespace: namespace, Name: name}
vn := &virtualkubeletv1alpha1.VirtualNode{}
if err := dr.vnr.Client.Get(ctx, ref, vn); err != nil {
if err = dr.vnr.Client.Get(ctx, ref, vn); err != nil {
if k8serrors.IsNotFound(err) {
return nil
err = nil
return err
}
return fmt.Errorf("error getting virtual node: %w", err)
err = fmt.Errorf("error getting virtual node: %w", err)
return err
}

defer func() {
if interr := dr.vnr.Client.Status().Update(ctx, vn); interr != nil {
if err != nil {
klog.Error(err)
}
err = fmt.Errorf("error updating virtual node status: %w", interr)
}
}()

klog.Infof("Deletion routine started for virtual node %s", vn.Name)
if err := UpdateCondition(ctx, dr.vnr.Client, vn,
ForgeCondition(vn,
VnConditionMap{
virtualkubeletv1alpha1.NodeConditionType: VnCondition{
Status: virtualkubeletv1alpha1.DrainingConditionStatusType,
},
}); err != nil {
return fmt.Errorf("error updating condition: %w", err)
}
}})

node, err := getters.GetNodeFromVirtualNode(ctx, dr.vnr.Client, vn)
var node *corev1.Node
node, err = getters.GetNodeFromVirtualNode(ctx, dr.vnr.Client, vn)
if client.IgnoreNotFound(err) != nil {
return fmt.Errorf("error getting node: %w", err)
err = fmt.Errorf("error getting node: %w", err)
return err
}

if node != nil {
if !*vn.Spec.CreateNode {
// We need to ensure that the current pods will no recreate the node after deleting it.
if found, err := vkutils.CheckVirtualKubeletFlagsConsistence(
var found bool
if found, err = vkutils.CheckVirtualKubeletFlagsConsistence(
ctx, dr.vnr.ClientVK, vn, dr.vnr.VirtualKubeletOptions, createNodeFalseFlag); err != nil || !found {
if err == nil {
err = fmt.Errorf("virtual kubelet pods are still running with arg %s", createNodeFalseFlag.String())
return err
}
return fmt.Errorf("error checking virtual kubelet pods: %w", err)
err = fmt.Errorf("error checking virtual kubelet pods: %w", err)
return err
}
}
if err := dr.deleteNode(ctx, node, vn); err != nil {
return fmt.Errorf("error deleting node: %w", err)
if err = dr.deleteNode(ctx, node, vn); err != nil {
err = fmt.Errorf("error deleting node: %w", err)
return err
}
}

if !vn.DeletionTimestamp.IsZero() {
// VirtualNode resource is being deleted.
if err := dr.vnr.ensureNamespaceMapAbsence(ctx, vn); err != nil {
return fmt.Errorf("error deleting namespace map: %w", err)
if err = dr.vnr.ensureNamespaceMapAbsence(ctx, vn); err != nil {
err = fmt.Errorf("error deleting namespace map: %w", err)
return err
}
err := dr.vnr.removeVirtualNodeFinalizer(ctx, vn)
err = dr.vnr.removeVirtualNodeFinalizer(ctx, vn)
if err != nil {
return fmt.Errorf("error removing finalizer: %w", err)
err = fmt.Errorf("error removing finalizer: %w", err)
return err
}
} else {
// Node is being deleted, but the VirtualNode resource is not.
// The VirtualNode .Spec.CreateNode field is set to false.
if err := UpdateCondition(ctx, dr.vnr.Client, vn,
VnConditionMap{
virtualkubeletv1alpha1.NodeConditionType: VnCondition{
Status: virtualkubeletv1alpha1.NoneConditionStatusType,
},
}); err != nil {
return fmt.Errorf("error updating condition: %w", err)
}
ForgeCondition(vn, VnConditionMap{
virtualkubeletv1alpha1.NodeConditionType: VnCondition{
Status: virtualkubeletv1alpha1.NoneConditionStatusType,
}})
}

klog.Infof("Deletion routine completed for virtual node %s", vn.Name)
return nil
return err
}

// deleteNode deletes the Node created by VirtualNode.
Expand All @@ -188,29 +203,25 @@ func (dr *DeletionRoutine) deleteNode(ctx context.Context, node *corev1.Node, vn
klog.Infof("Node %s drained", node.Name)

if !vn.DeletionTimestamp.IsZero() {
if err := UpdateCondition(ctx, dr.vnr.Client, vn,
ForgeCondition(vn,
VnConditionMap{
virtualkubeletv1alpha1.VirtualKubeletConditionType: VnCondition{
Status: virtualkubeletv1alpha1.DeletingConditionStatusType,
},
},
); err != nil {
return fmt.Errorf("error updating condition: %w", err)
}
if reEnque, err := dr.vnr.ensureVirtualKubeletDeploymentAbsence(ctx, vn); reEnque {
)
if err := dr.vnr.ensureVirtualKubeletDeploymentAbsence(ctx, vn); err != nil {
return fmt.Errorf("error deleting virtual kubelet deployment: %w", err)
}
}
klog.Infof("VirtualKubelet deployment %s deleted", vn.Name)

if err := UpdateCondition(ctx, dr.vnr.Client, vn,
ForgeCondition(vn,
VnConditionMap{
virtualkubeletv1alpha1.NodeConditionType: VnCondition{
Status: virtualkubeletv1alpha1.DeletingConditionStatusType,
},
}); err != nil {
return fmt.Errorf("error updating condition: %w", err)
}
})
if err := client.IgnoreNotFound(dr.vnr.Client.Delete(ctx, node, &client.DeleteOptions{})); err != nil {
return fmt.Errorf("error deleting node: %w", err)
}
Expand Down
78 changes: 38 additions & 40 deletions pkg/liqo-controller-manager/virtualnode-controller/virtualkubelet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,34 +35,40 @@ import (

// createVirtualKubeletDeployment creates the VirtualKubelet Deployment.
func (r *VirtualNodeReconciler) ensureVirtualKubeletDeploymentPresence(
ctx context.Context, cl client.Client, virtualNode *virtualkubeletv1alpha1.VirtualNode) error {
ctx context.Context, virtualNode *virtualkubeletv1alpha1.VirtualNode) (err error) {
var nodeStatusInitial virtualkubeletv1alpha1.VirtualNodeConditionStatusType
if *virtualNode.Spec.CreateNode {
nodeStatusInitial = virtualkubeletv1alpha1.CreatingConditionStatusType
} else {
nodeStatusInitial = virtualkubeletv1alpha1.NoneConditionStatusType
}
if err := UpdateCondition(ctx, cl, virtualNode,
defer func() {
if interr := r.Client.Status().Update(ctx, virtualNode); interr != nil {
if err != nil {
klog.Error(err)
}
err = fmt.Errorf("error updating virtual node status: %w", interr)
}
}()
ForgeCondition(virtualNode,
VnConditionMap{
virtualkubeletv1alpha1.VirtualKubeletConditionType: VnCondition{
Status: virtualkubeletv1alpha1.CreatingConditionStatusType,
},
virtualkubeletv1alpha1.NodeConditionType: VnCondition{Status: nodeStatusInitial},
},
); err != nil {
return err
}
)

namespace := virtualNode.Namespace
name := virtualNode.Name
remoteClusterIdentity := virtualNode.Spec.ClusterIdentity
// create the base resources
vkServiceAccount := vkforge.VirtualKubeletServiceAccount(namespace, name)
op, err := controllerutil.CreateOrUpdate(ctx, r.Client, vkServiceAccount, func() error {
var op controllerutil.OperationResult
op, err = controllerutil.CreateOrUpdate(ctx, r.Client, vkServiceAccount, func() error {
return nil
})
if err != nil {
klog.Error(err)
return err
}
klog.V(5).Infof("[%v] ServiceAccount %s/%s reconciled: %s",
Expand All @@ -73,7 +79,6 @@ func (r *VirtualNodeReconciler) ensureVirtualKubeletDeploymentPresence(
return nil
})
if err != nil {
klog.Error(err)
return err
}

Expand All @@ -89,7 +94,6 @@ func (r *VirtualNodeReconciler) ensureVirtualKubeletDeploymentPresence(
return nil
})
if err != nil {
klog.Error(err)
return err
}
klog.V(5).Infof("[%v] Deployment %s/%s reconciled: %s",
Expand All @@ -102,63 +106,57 @@ func (r *VirtualNodeReconciler) ensureVirtualKubeletDeploymentPresence(
r.EventsRecorder.Event(virtualNode, "Normal", "VkCreated", msg)
}

if err := UpdateCondition(ctx, cl, virtualNode,
ForgeCondition(virtualNode,
VnConditionMap{
virtualkubeletv1alpha1.VirtualKubeletConditionType: VnCondition{
Status: virtualkubeletv1alpha1.RunningConditionStatusType,
},
}); err != nil {
return err
}
}})
if *virtualNode.Spec.CreateNode {
return UpdateCondition(ctx, cl, virtualNode,
ForgeCondition(virtualNode,
VnConditionMap{
virtualkubeletv1alpha1.NodeConditionType: VnCondition{
Status: virtualkubeletv1alpha1.RunningConditionStatusType,
},
})
}
return nil
return err
}

// ensureVirtualKubeletDeploymentAbsence deletes the VirtualKubelet Deployment.
// It checks if the VirtualKubelet Pods have been deleted.
func (r *VirtualNodeReconciler) ensureVirtualKubeletDeploymentAbsence(
ctx context.Context, virtualNode *virtualkubeletv1alpha1.VirtualNode) (reEnque bool, err error) {
ctx context.Context, virtualNode *virtualkubeletv1alpha1.VirtualNode) error {
virtualKubeletDeployment, err := vkutils.GetVirtualKubeletDeployment(ctx, r.Client, virtualNode, r.VirtualKubeletOptions)
if err != nil {
klog.Error(err)
return true, err
}
if virtualKubeletDeployment == nil {
return false, nil
return err
}
if virtualKubeletDeployment != nil {
msg := fmt.Sprintf("[%v] Deleting virtual-kubelet in namespace %v", virtualNode.Spec.ClusterIdentity.ClusterID, virtualNode.Namespace)
klog.Info(msg)
r.EventsRecorder.Event(virtualNode, "Normal", "VkDeleted", msg)

msg := fmt.Sprintf("[%v] Deleting virtual-kubelet in namespace %v", virtualNode.Spec.ClusterIdentity.ClusterID, virtualNode.Namespace)
klog.Info(msg)
r.EventsRecorder.Event(virtualNode, "Normal", "VkDeleted", msg)

if err := r.Client.Delete(ctx, virtualKubeletDeployment); err != nil {
klog.Error(err)
return true, err
if err := r.Client.Delete(ctx, virtualKubeletDeployment); err != nil {
return err
}
}

if ok, err := vkutils.CheckVirtualKubeletPodAbsence(ctx, r.ClientVK, virtualNode, r.VirtualKubeletOptions); err != nil || !ok {
return true, err
if err := vkutils.CheckVirtualKubeletPodAbsence(ctx, r.ClientVK, virtualNode, r.VirtualKubeletOptions); err != nil {
return err
}

if err := r.Client.Delete(ctx, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{
err = r.Client.Delete(ctx, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{
Name: k8strings.ShortenString(fmt.Sprintf("%s%s", vkMachinery.CRBPrefix, virtualNode.Name), 253),
}}); err != nil {
klog.Error(err)
return true, err
}})
if client.IgnoreNotFound(err) != nil {
return err
}

if err := r.Client.Delete(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{
err = r.Client.Delete(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{
Name: virtualNode.Name, Namespace: virtualNode.Namespace,
}}); err != nil {
klog.Error(err)
return true, err
}})
if client.IgnoreNotFound(err) != nil {
return err
}

return false, nil
return nil
}
Loading

0 comments on commit 67ad421

Please sign in to comment.