feat(hostpath): allow custom node affinity label (#15)

Ref: openebs/openebs#2875

provide a feature for administrators to configure a
custom node affinity label in place of hostnames.

This will help in scenarios, where hostnames can change
when node are removed and added back to the cluster with
the underlying disks intact.

cluster admin can setup custom labels to the nodes and
provide this information to Local PV hostpath provisioner
to use via StorageClass config key called `NodeAffinityLabel`

```
+ //Example: Local PV device StorageClass for using a custom
+ //node label as: openebs.io/node-affinity-value
+ //will be as follows
+ //
+ // kind: StorageClass
+ // metadata:
+ //   name: openebs-hostpath
+ //   annotations:
+ //     openebs.io/cas-type: local
+ //     cas.openebs.io/config: |
+ //       - name: StorageType
+ //         value: "device"
+ //       - name: NodeAffinityLabel
+ //         value: "openebs.io/node-affinity-value"
+ // provisioner: openebs.io/local
+ // volumeBindingMode: WaitForFirstConsumer
+ // reclaimPolicy: Delete
+ //
```


Signed-off-by: kmova <[email protected]>

cmd/provisioner-localpv/app/config.go

@@ -78,6 +78,29 @@ const (
 	//
 	KeyBDTag = "BlockDeviceTag"
 
+	//KeyNodeAffinityLabel defines the label key that should be
+	//used in the nodeAffinitySpec. Default is to use "kubernetes.io/hostname"
+	//
+	//Example: Local PV device StorageClass for using a custom
+	//node label as: openebs.io/node-affinity-value
+	//will be as follows
+	//
+	// kind: StorageClass
+	// metadata:
+	//   name: openebs-device-tag-x
+	//   annotations:
+	//     openebs.io/cas-type: local
+	//     cas.openebs.io/config: |
+	//       - name: StorageType
+	//         value: "device"
+	//       - name: NodeAffinityLabel
+	//         value: "openebs.io/node-affinity-value"
+	// provisioner: openebs.io/local
+	// volumeBindingMode: WaitForFirstConsumer
+	// reclaimPolicy: Delete
+	//
+	KeyNodeAffinityLabel = "NodeAffinityLabel"
+
 	//KeyPVRelativePath defines the alternate folder name under the BasePath
 	// By default, the pv name will be used as the folder name.
 	// KeyPVBasePath can be useful for providing the same underlying folder
@@ -188,6 +211,18 @@ func (c *VolumeConfig) GetBDTagValue() string {
 	return bdTagValue
 }
 
+//GetNodeAffinityLabelKey returns the custom node affinity
+//label key as configured in StorageClass.
+//
+//Default is "", use the standard kubernetes.io/hostname label.
+func (c *VolumeConfig) GetNodeAffinityLabelKey() string {
+	nodeAffinityLabelKey := c.getValue(KeyNodeAffinityLabel)
+	if len(strings.TrimSpace(nodeAffinityLabelKey)) == 0 {
+		return ""
+	}
+	return nodeAffinityLabelKey
+}
+
 //GetPath returns a valid PV path based on the configuration
 // or an error. The Path is constructed using the following rules:
 // If AbsolutePath is specified return it. (Future)
@@ -277,6 +312,16 @@ func GetNodeHostname(n *v1.Node) string {
 	return hostname
 }
 
+// GetNodeLabelValue extracts the value from the given label on the Node
+// If specificed label is not present an empty string is returned.
+func GetNodeLabelValue(n *v1.Node, labelKey string) string {
+	labelValue, found := n.Labels[labelKey]
+	if !found {
+		return ""
+	}
+	return labelValue
+}
+
 // GetTaints extracts the Taints from the Spec on the node
 // If Taints are empty, it just returns empty structure of corev1.Taints
 func GetTaints(n *v1.Node) []v1.Taint {

cmd/provisioner-localpv/app/helper_hostpath.go

@@ -28,9 +28,9 @@ import (
 
 	hostpath "github.com/openebs/maya/pkg/hostpath/v1alpha1"
 
-	container "github.com/openebs/maya/pkg/kubernetes/container/v1alpha1"
-	pod "github.com/openebs/maya/pkg/kubernetes/pod/v1alpha1"
-	volume "github.com/openebs/maya/pkg/kubernetes/volume/v1alpha1"
+	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/container"
+	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/pod"
+	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/volume"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -52,8 +52,11 @@ var (
 // to execute a command (cmdsForPath) on a given
 // volume path (path)
 type HelperPodOptions struct {
-	//nodeHostname represents the hostname of the node where pod should be launched.
-	nodeHostname string
+	//nodeAffinityLabelKey represents the label key of the node where pod should be launched.
+	nodeAffinityLabelKey string
+
+	//nodeAffinityLabelValue represents the label value of the node where pod should be launched.
+	nodeAffinityLabelValue string
 
 	//name is the name of the PV for which the pod is being launched
 	name string
@@ -78,7 +81,8 @@ type HelperPodOptions struct {
 func (pOpts *HelperPodOptions) validate() error {
 	if pOpts.name == "" ||
 		pOpts.path == "" ||
-		pOpts.nodeHostname == "" ||
+		pOpts.nodeAffinityLabelKey == "" ||
+		pOpts.nodeAffinityLabelValue == "" ||
 		pOpts.serviceAccountName == "" {
 		return errors.Errorf("invalid empty name or hostpath or hostname or service account name")
 	}
@@ -165,9 +169,10 @@ func (p *Provisioner) launchPod(config podConfig) (*corev1.Pod, error) {
 	privileged := true
 
 	helperPod, err := pod.NewBuilder().
-		WithName(config.podName + "-" + config.pOpts.name).
+		WithName(config.podName+"-"+config.pOpts.name).
 		WithRestartPolicy(corev1.RestartPolicyNever).
-		WithNodeSelectorHostnameNew(config.pOpts.nodeHostname).
+		//WithNodeSelectorHostnameNew(config.pOpts.nodeHostname).
+		WithNodeAffinityNew(config.pOpts.nodeAffinityLabelKey, config.pOpts.nodeAffinityLabelValue).
 		WithServiceAccountName(config.pOpts.serviceAccountName).
 		WithTolerationsForTaints(config.taints...).
 		WithContainerBuilder(

cmd/provisioner-localpv/app/provisioner_hostpath.go

@@ -26,20 +26,25 @@ import (
 
 	pvController "sigs.k8s.io/sig-storage-lib-external-provisioner/controller"
 	//pvController "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller"
+	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/persistentvolume"
 	mconfig "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
-	persistentvolume "github.com/openebs/maya/pkg/kubernetes/persistentvolume/v1alpha1"
 )
 
 // ProvisionHostPath is invoked by the Provisioner which expect HostPath PV
 //  to be provisioned and a valid PV spec returned.
 func (p *Provisioner) ProvisionHostPath(opts pvController.ProvisionOptions, volumeConfig *VolumeConfig) (*v1.PersistentVolume, error) {
 	pvc := opts.PVC
-	nodeHostname := GetNodeHostname(opts.SelectedNode)
 	taints := GetTaints(opts.SelectedNode)
 	name := opts.PVName
 	stgType := volumeConfig.GetStorageType()
 	saName := getOpenEBSServiceAccountName()
 
+	nodeAffinityKey := volumeConfig.GetNodeAffinityLabelKey()
+	if len(nodeAffinityKey) == 0 {
+		nodeAffinityKey = k8sNodeLabelKeyHostname
+	}
+	nodeAffinityValue := GetNodeLabelValue(opts.SelectedNode, nodeAffinityKey)
+
 	path, err := volumeConfig.GetPath()
 	if err != nil {
 		alertlog.Logger.Errorw("",
@@ -52,17 +57,18 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.ProvisionOptions, volu
 		return nil, err
 	}
 
-	klog.Infof("Creating volume %v at %v:%v", name, nodeHostname, path)
+	klog.Infof("Creating volume %v at node with label %v=%v, path:%v", name, nodeAffinityKey, nodeAffinityValue, path)
 
 	//Before using the path for local PV, make sure it is created.
 	initCmdsForPath := []string{"mkdir", "-m", "0777", "-p"}
 	podOpts := &HelperPodOptions{
-		cmdsForPath:        initCmdsForPath,
-		name:               name,
-		path:               path,
-		nodeHostname:       nodeHostname,
-		serviceAccountName: saName,
-		selectedNodeTaints: taints,
+		cmdsForPath:            initCmdsForPath,
+		name:                   name,
+		path:                   path,
+		nodeAffinityLabelKey:   nodeAffinityKey,
+		nodeAffinityLabelValue: nodeAffinityValue,
+		serviceAccountName:     saName,
+		selectedNodeTaints:     taints,
 	}
 	iErr := p.createInitPod(podOpts)
 	if iErr != nil {
@@ -104,7 +110,7 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.ProvisionOptions, volu
 		WithVolumeMode(fs).
 		WithCapacityQty(pvc.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]).
 		WithLocalHostDirectory(path).
-		WithNodeAffinity(nodeHostname).
+		WithNodeAffinity(nodeAffinityKey, nodeAffinityValue).
 		Build()
 
 	if err != nil {
@@ -126,20 +132,25 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.ProvisionOptions, volu
 	return pvObj, nil
 }
 
-// GetNodeObjectFromHostName returns the Node Object with matching NodeHostName.
-func (p *Provisioner) GetNodeObjectFromHostName(hostName string) (*v1.Node, error) {
-	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{persistentvolume.KeyNode: hostName}}
+// GetNodeObjectFromLabels returns the Node Object with matching label key and value
+func (p *Provisioner) GetNodeObjectFromLabels(key, value string) (*v1.Node, error) {
+	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{key: value}}
 	listOptions := metav1.ListOptions{
 		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
-		Limit:         1,
 	}
 	nodeList, err := p.kubeClient.CoreV1().Nodes().List(listOptions)
 	if err != nil || len(nodeList.Items) == 0 {
 		// After the PV is created and node affinity is set
 		// based on kubernetes.io/hostname label, either:
 		// - hostname label changed on the node or
 		// - the node is deleted from the cluster.
-		return nil, errors.Errorf("Unable to get the Node with the NodeHostName [%s]", hostName)
+		return nil, errors.Errorf("Unable to get the Node with the Node Label %s [%s]", key, value)
+	}
+	if len(nodeList.Items) != 1 {
+		// After the PV is created and node affinity is set
+		// on a custom affinity label, there may be a transitory state
+		// with two nodes matching (old and new) label.
+		return nil, errors.Errorf("Unable to determine the Node. Found multiple nodes matching the labels %s [%s].", key, value)
 	}
 	return &nodeList.Items[0], nil
 
@@ -162,28 +173,29 @@ func (p *Provisioner) DeleteHostPath(pv *v1.PersistentVolume) (err error) {
 		return errors.Errorf("no HostPath set")
 	}
 
-	hostname := pvObj.GetAffinitedNodeHostname()
-	if hostname == "" {
-		return errors.Errorf("cannot find affinited node hostname")
+	nodeAffinityKey, nodeAffinityValue := pvObj.GetAffinitedNodeLabelKeyAndValue()
+	if nodeAffinityValue == "" {
+		return errors.Errorf("cannot find affinited node details")
 	}
-	alertlog.Logger.Infof("Get the Node Object from hostName: %v", hostname)
+	alertlog.Logger.Infof("Get the Node Object with label %v : %v", nodeAffinityKey, nodeAffinityValue)
 
 	//Get the node Object once again to get updated Taints.
-	nodeObject, err := p.GetNodeObjectFromHostName(hostname)
+	nodeObject, err := p.GetNodeObjectFromLabels(nodeAffinityKey, nodeAffinityValue)
 	if err != nil {
 		return err
 	}
 	taints := GetTaints(nodeObject)
 	//Initiate clean up only when reclaim policy is not retain.
-	klog.Infof("Deleting volume %v at %v:%v", pv.Name, hostname, path)
+	klog.Infof("Deleting volume %v at %v:%v", pv.Name, GetNodeHostname(nodeObject), path)
 	cleanupCmdsForPath := []string{"rm", "-rf"}
 	podOpts := &HelperPodOptions{
-		cmdsForPath:        cleanupCmdsForPath,
-		name:               pv.Name,
-		path:               path,
-		nodeHostname:       hostname,
-		serviceAccountName: saName,
-		selectedNodeTaints: taints,
+		cmdsForPath:            cleanupCmdsForPath,
+		name:                   pv.Name,
+		path:                   path,
+		nodeAffinityLabelKey:   nodeAffinityKey,
+		nodeAffinityLabelValue: nodeAffinityValue,
+		serviceAccountName:     saName,
+		selectedNodeTaints:     taints,
 	}
 
 	if err := p.createCleanupPod(podOpts); err != nil {

deploy/kubectl/busybox-localpv-path.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: busybox
+  namespace: default
+spec:
+  containers:
+  - command:
+       - sh
+       - -c
+       - 'date >> /mnt/store1/date.txt; hostname >> /mnt/store1/hostname.txt; sync; sleep 5; sync; tail -f /dev/null;'
+    image: busybox
+    imagePullPolicy: Always
+    name: busybox
+    volumeMounts:
+    - mountPath: /mnt/store1
+      name: demo-vol1
+  volumes:
+  - name: demo-vol1
+    persistentVolumeClaim:
+      claimName: demo-vol1-claim
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: demo-vol1-claim
+spec:
+  storageClassName: openebs-hostpath
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5G

deploy/kubectl/fillup-localpv-hostpath.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: fillup
+  namespace: default
+spec:
+  containers:
+  - command:
+       - sh
+       - -c
+       - 'dd if=/dev/zero of=/mnt/store1/dump.dd bs=1M; sync; sleep 5; sync; tail -f /dev/null;'
+    image: busybox
+    imagePullPolicy: Always
+    name: fillup-bb
+    volumeMounts:
+    - mountPath: /mnt/store1
+      name: fillup
+  volumes:
+  - name: fillup
+    persistentVolumeClaim:
+      claimName: fillup-claim
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: fillup-claim
+spec:
+  storageClassName: openebs-hostpath
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1G

deploy/kubectl/openebs-lite-sc.yaml

@@ -0,0 +1,29 @@
+#Sample storage classes for OpenEBS Local PV
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: openebs-hostpath
+  annotations:
+    openebs.io/cas-type: local
+    cas.openebs.io/config: |
+      #hostpath type will create a PV by 
+      # creating a sub-directory under the
+      # BASEPATH provided below.
+      - name: StorageType
+        value: "hostpath"
+      #Specify the location (directory) where
+      # where PV(volume) data will be saved. 
+      # A sub-directory with pv-name will be 
+      # created. When the volume is deleted, 
+      # the PV sub-directory will be deleted.
+      #Default value is /var/openebs/local
+      - name: BasePath
+        value: "/var/openebs/local/"
+      #Specify the node affinity label 
+      # to be added to the PV
+      #Default: kubernetes.io/hostname
+      #- name: NodeAffinityLabel
+      #  value: "openebs.io/stg-node-name"
+provisioner: openebs.io/local
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete