fix ce mountpod recreated & missing uuid (#1014)

* fix ce mountpod created Signed-off-by: Xuhui zhang <[email protected]> * fix ce mountpod clean-cache Signed-off-by: Xuhui zhang <[email protected]> * Add more test Signed-off-by: Xuhui zhang <[email protected]> --------- Signed-off-by: Xuhui zhang <[email protected]>
juicedata · Jul 4, 2024 · 7c1b20c · 7c1b20c
1 parent d60de1f
commit 7c1b20c
Show file tree

Hide file tree

Showing 4 changed files with 171 additions and 1 deletion.
diff --git a/.github/scripts/e2e-test.py b/.github/scripts/e2e-test.py
@@ -51,6 +51,7 @@
     test_mountpod_recreated,
     test_validate_pv,
     test_config,
+    test_recreate_mountpod_reload_config,
 )
 from util import die, mount_on_host, umount, clean_juicefs_volume, deploy_secret_and_sc, check_do_test
 
@@ -87,6 +88,7 @@
                 test_multi_pvc()
                 test_mountpod_recreated()
                 test_config()
+                test_recreate_mountpod_reload_config()
                 if without_kubelet:
                     test_pod_resource_err()
 
@@ -139,6 +141,7 @@
                 test_dynamic_expand()
                 test_multi_pvc()
                 test_config()
+                test_recreate_mountpod_reload_config()
                 if without_kubelet:
                     test_pod_resource_err()
 

diff --git a/.github/scripts/test_case.py b/.github/scripts/test_case.py
@@ -2736,6 +2736,161 @@ def test_config():
     LOG.info("Remove pvc 2 {}".format(pvc2.name))
     pvc2.delete()
 
+    # reset to empty config
+    update_config({})
+
+    # patch all csi-node pods anno to make cfg update faster
+    subprocess.check_call(["kubectl", "annotate", "pods", "--overwrite", "-n", KUBE_SYSTEM, "-l", "app=juicefs-csi-node", "updatedAt=" + str(int(time.time()))])
+
+
     LOG.info("Test pass.")
 
 
+def test_recreate_mountpod_reload_config():
+    LOG.info("[test case] Test recreate mountpod need reload config begin..")
+
+    init_config = {
+        "mountPodPatch": [
+            {
+                "labels": {
+                    "apply": "global_labels"
+                },
+                "hostNetwork": True,
+            },
+            {
+                "ceMountImage": "juicedata/mount:ce-nightly",
+                "eeMountImage": "juicedata/mount:ee-nightly",
+            },
+            {
+                "resources": {
+                    "requests": {
+                        "cpu": "100m",
+                        "memory": "128Mi"
+                    },
+                    "limits": {
+                        "cpu": "2",
+                        "memory": "2Gi"
+                    }
+                }
+            }
+        ]
+    }
+    update_config(init_config)
+    # patch all csi-node pods anno to make cfg update faster
+    subprocess.check_call(["kubectl", "annotate", "pods", "--overwrite", "-n", KUBE_SYSTEM, "-l", "app=juicefs-csi-node", "updatedAt=" + str(int(time.time()))])
+
+    pvc = PVC(name="pvc-mountpod-recreated-reload-config", access_mode="ReadWriteMany", storage_name=STORAGECLASS_NAME, pv="")
+    LOG.info("Deploy pvc {}".format(pvc.name))
+    pvc.create()
+
+    # wait for pvc bound
+    for i in range(0, 60):
+        if pvc.check_is_bound():
+            break
+        time.sleep(1)
+
+    # deploy pod
+    deployment = Deployment(name="app-recreated-reload-config", pvc=pvc.name, replicas=1)
+    LOG.info("Deploy deployment {}".format(deployment.name))
+    deployment.create()
+
+    pod = Pod(name="", deployment_name=deployment.name, replicas=deployment.replicas)
+    LOG.info("Watch for pods of {} for success.".format(deployment.name))
+    result = pod.watch_for_success()
+    if not result:
+        raise Exception("Pods of deployment {} are not ready within 10 min.".format(deployment.name))
+
+    LOG.info("Check mount point..")
+    volume_id = pvc.get_volume_id()
+    LOG.info("Get volume_id {}".format(volume_id))
+    check_path = volume_id + "/out.txt"
+    result = check_mount_point(check_path)
+    if not result:
+        raise Exception("mount Point of /jfs/{}/out.txt are not ready within 5 min.".format(volume_id))
+
+    # update cfg
+    new_cfg = {
+        "mountPodPatch": [
+            {
+                "labels": {
+                    "apply": "updated_config",
+                    "volume_id": r"${VOLUME_ID}",
+                    "volume_name": r"${VOLUME_NAME}",
+                },
+                "hostNetwork": False,
+            },
+            {
+                "ceMountImage": "juicedata/mount:ce-v1.2.0",
+                "eeMountImage": "juicedata/mount:ee-5.0.20-c87a555",
+            },
+            {
+                "resources": {
+                    "requests": {
+                        "cpu": "200m",
+                        "memory": "128Mi"
+                    },
+                    "limits": {
+                        "cpu": "2",
+                        "memory": "2Gi"
+                    }
+                }
+            }
+        ]
+    }
+    update_config(new_cfg)
+
+    # patch all csi-node pods anno to make cfg update faster
+    subprocess.check_call(["kubectl", "annotate", "pods", "--overwrite", "-n", KUBE_SYSTEM, "-l", "app=juicefs-csi-node", "updatedAt=" + str(int(time.time()))])
+
+    LOG.info("Start to delete mountpod..")
+    mount_pod = Pod(name=get_only_mount_pod_name(volume_id), deployment_name="", replicas=1, namespace=KUBE_SYSTEM)
+    mount_pod.delete()
+
+    # wait for mountpod recreated
+    LOG.info("Wait for mountpod recreated..")
+    time.sleep(20)
+    for i in range(0, 60):
+        if mount_pod.watch_for_success():
+            break
+        time.sleep(5)
+
+    # check mount point
+    LOG.info("Check mount point..")
+    result = check_mount_point(check_path)
+    if not result:
+        raise Exception("mount Point of /jfs/{}/out.txt are not ready within 5 min.".format(volume_id))
+
+    mount_pod = Pod(name=get_only_mount_pod_name(volume_id), deployment_name="", replicas=1, namespace=KUBE_SYSTEM)
+    if mount_pod.get_metadata().labels.get("apply") != "updated_config":
+        raise Exception("mountpod config labels not set")
+    if mount_pod.get_metadata().labels.get("volume_id") != volume_id:
+        raise Exception("mountpod config volume id not set")
+    if mount_pod.get_metadata().labels.get("volume_name") == "":
+        raise Exception("mountpod config volume name not set")
+
+    if mount_pod.get_spec().host_network == True:
+        raise Exception("mountpod config hostNetwork not set to false")
+
+    updated_image = "juicedata/mount:ee-5.0.20-c87a555"
+    if IS_CE:
+        updated_image = "juicedata/mount:ce-v1.2.0"
+    if mount_pod.get_spec().containers[0].image != updated_image:
+        raise Exception("mountpod config image not set")
+
+    if mount_pod.get_spec().containers[0].resources.requests["cpu"] != "200m":
+        raise Exception("mountpod config resources not set")
+
+    # delete test resources
+    LOG.info("Remove deployment {}".format(deployment.name))
+    deployment.delete()
+
+    LOG.info("Remove pvc {}".format(pvc.name))
+    pvc.delete()
+
+    # reset to empty config
+    update_config({})
+
+    # patch all csi-node pods anno to make cfg update faster
+    subprocess.check_call(["kubectl", "annotate", "pods", "--overwrite", "-n", KUBE_SYSTEM, "-l", "app=juicefs-csi-node", "updatedAt=" + str(int(time.time()))])
+
+    LOG.info("Test pass.")
diff --git a/pkg/config/setting.go b/pkg/config/setting.go
@@ -480,6 +480,7 @@ func GenPodAttrWithMountPod(ctx context.Context, client *k8sclient.K8sClient, mo
 	}
 	attr.Resources = resources
 	setting := &JfsSetting{
+		IsCe:      IsCEMountPod(mountPod),
 		PV:        pv,
 		PVC:       pvc,
 		Name:      mountPod.Annotations[JuiceFSUUID],
@@ -683,3 +684,14 @@ func applyAttrPatch(attr *PodAttr, setting *JfsSetting) {
 	attr.StartupProbe = patch.StartupProbe
 	attr.TerminationGracePeriodSeconds = patch.TerminationGracePeriodSeconds
 }
+
+// IsCEMountPod check if the pod is a mount pod of CE
+// check mountpod command's has metaurl
+func IsCEMountPod(pod *corev1.Pod) bool {
+	for _, cmd := range pod.Spec.Containers[0].Command {
+		if strings.Contains(cmd, "metaurl") {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/juicefs/mount/pod_mount.go b/pkg/juicefs/mount/pod_mount.go
@@ -76,7 +76,7 @@ func (p *PodMount) JMount(ctx context.Context, appInfo *jfsConfig.AppInfo, jfsSe
 	if err != nil {
 		return err
 	}
-	if jfsSetting.CleanCache && jfsSetting.UUID == "" {
+	if jfsSetting.UUID == "" {
 		// need set uuid as label in mount pod for clean cache
 		uuid, err := p.GetJfsVolUUID(ctx, jfsSetting.Source)
 		if err != nil {