From 5f63e9d8ef9f3f5dea7a9ec2f7ad0c489a7f60bc Mon Sep 17 00:00:00 2001 From: cgw Date: Tue, 26 Sep 2023 19:01:03 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E6=A0=87=E7=AD=BE=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/EnvReplicaApplicationService.java | 4 +- .../strategy/cluster/K8sClusterHelper.java | 4 +- .../strategy/cluster/K8sClusterStrategy.java | 61 +++++++++++++------ .../cluster/k8s/K8sClusterStrategy.java | 2 +- .../dhorse/infrastructure/utils/K8sUtils.java | 40 ++++++++++-- static/page/replica/list.html | 2 +- 6 files changed, 83 insertions(+), 30 deletions(-) diff --git a/dhorse-application/src/main/java/org/dhorse/application/service/EnvReplicaApplicationService.java b/dhorse-application/src/main/java/org/dhorse/application/service/EnvReplicaApplicationService.java index 4fe81dfe..0567e1f3 100644 --- a/dhorse-application/src/main/java/org/dhorse/application/service/EnvReplicaApplicationService.java +++ b/dhorse-application/src/main/java/org/dhorse/application/service/EnvReplicaApplicationService.java @@ -255,7 +255,9 @@ public void collectReplicaMetrics() { } for(ReplicaMetrics metric : replicaMetrics) { String replicaName = metric.getReplicaName(); - AppEnvPO appEnvPO = envMap.get(replicaName.substring(0, replicaName.indexOf("-dhorse-") + 7)); + //解析deployment名字的方式不优雅，后续可以考虑其他方案 + String[] appNameTag = K8sUtils.appNameAndEnvTag(replicaName); + AppEnvPO appEnvPO = envMap.get(K8sUtils.getReplicaAppName(appNameTag[0], appNameTag[1])); if(appEnvPO == null) { continue; } diff --git a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterHelper.java b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterHelper.java index 04c9886f..0a9fc45c 100644 --- a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterHelper.java +++ b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterHelper.java @@ -454,7 +454,7 @@ public static void createDHorseConfig(ClusterPO clusterPO, CoreV1Api coreApi) { continue; } V1ConfigMapList list = coreApi.listNamespacedConfigMap(namespace, null, null, null, null, - K8sUtils.DHORSE_SELECTOR_KEY + K8sUtils.DHORSE_CONFIGMAP_NAME, null, null, null, null, null); + K8sUtils.getSelectorKey(K8sUtils.DHORSE_CONFIGMAP_NAME), null, null, null, null, null); V1ConfigMap configMap = K8sClusterHelper.dhorseConfigMap(); if(CollectionUtils.isEmpty(list.getItems())) { @@ -510,7 +510,7 @@ public static void deleteDHorseConfig(ClusterPO clusterPO, CoreV1Api coreApi) { if(CollectionUtils.isEmpty(namespaceList.getItems())) { return; } - String selector = K8sUtils.DHORSE_SELECTOR_KEY + K8sUtils.DHORSE_CONFIGMAP_NAME; + String selector = K8sUtils.getSelectorKey(K8sUtils.DHORSE_CONFIGMAP_NAME); for(V1Namespace n : namespaceList.getItems()) { String namespace = n.getMetadata().getName(); if(!K8sUtils.DHORSE_NAMESPACE.equals(namespace) diff --git a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterStrategy.java b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterStrategy.java index 94e3c408..714b0a76 100644 --- a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterStrategy.java +++ b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterStrategy.java @@ -183,7 +183,7 @@ public Void createSecret(ClusterPO clusterPO, ImageRepo imageRepo) { if(CollectionUtils.isEmpty(namespaceList.getItems())) { return null; } - String selector = K8sUtils.DHORSE_SELECTOR_KEY + K8sUtils.DOCKER_REGISTRY_KEY; + String selector = K8sUtils.getSelectorKey(K8sUtils.DOCKER_REGISTRY_KEY); for(V1Namespace n : namespaceList.getItems()) { String namespace = n.getMetadata().getName(); if(!K8sUtils.DHORSE_NAMESPACE.equals(namespace) @@ -250,14 +250,14 @@ public boolean createDeployment(DeploymentContext context) { V1Deployment deployment = new V1Deployment(); deployment.apiVersion("apps/v1"); deployment.setKind("Deployment"); - deployment.setMetadata(deploymentMetaData(context.getDeploymentName(), context.getAppEnv().getTag())); + deployment.setMetadata(deploymentMetaData(context.getDeploymentName())); deployment.setSpec(deploymentSpec(context)); ApiClient apiClient = this.apiClient(context.getCluster().getClusterUrl(), context.getCluster().getAuthToken()); AppsV1Api api = new AppsV1Api(apiClient); CoreV1Api coreApi = new CoreV1Api(apiClient); String namespace = context.getAppEnv().getNamespaceName(); - String labelSelector = K8sUtils.getDeploymentLabelSelector(context.getDeploymentName()); + String labelSelector = K8sUtils.getSelectorKey(context.getDeploymentName()); try { V1DeploymentList oldDeployment = api.listNamespacedDeployment(namespace, null, null, null, null, labelSelector, null, null, null, null, null); @@ -268,8 +268,15 @@ public boolean createDeployment(DeploymentContext context) { null, null); } + //v1.4.1的兼容代码，后续版本删除 + try { + String name = K8sUtils.getReplicaAppName2(context.getApp().getAppName(), context.getAppEnv().getTag()); + api.deleteNamespacedDeployment(name, namespace, null, null, null, null, null, null); + }catch(Exception e) { + logger.error("Failed to delete deployment", e); + } // 自动扩容任务 - createAutoScaling(context.getAppEnv(), context.getDeploymentName(), apiClient); + createAutoScaling(context.getApp().getAppName(), context.getAppEnv(), context.getDeploymentName(), apiClient); if(!checkHealthOfAll(coreApi, namespace, labelSelector)) { logger.error("Failed to create k8s deployment, because the replica is not fully started"); @@ -302,7 +309,7 @@ private boolean createService(DeploymentContext context) throws ApiException { String namespace = context.getAppEnv().getNamespaceName(); String serviceName = K8sUtils.getServiceName(context.getApp().getAppName(), context.getAppEnv().getTag()); V1ServiceList serviceList = coreApi.listNamespacedService(namespace, null, null, null, null, - K8sUtils.DHORSE_SELECTOR_KEY + serviceName, 1, null, null, null, null); + K8sUtils.getSelectorKey(serviceName), 1, null, null, null, null); if (CollectionUtils.isEmpty(serviceList.getItems())) { logger.info("Start to create service"); V1Service service = new V1Service(); @@ -314,6 +321,14 @@ private boolean createService(DeploymentContext context) throws ApiException { }else{ List patchs = K8sClusterHelper.updatePrometheus("Service", serviceList.getItems().get(0) .getMetadata().getAnnotations(), context); + + //兼容逻辑，v1.4.1以后应该删除 + JsonPatch path = new JsonPatch(); + path.setOp("replace"); + path.setPath("/spec/selector/dhorse-app"); + path.setValue(serviceName); + patchs.add(path); + if(!CollectionUtils.isEmpty(patchs)) { logger.info("Start to update service"); V1Patch patch = new V1Patch(JsonUtils.toJsonString(patchs)); @@ -334,7 +349,7 @@ private boolean createIngress(DeploymentContext context) throws ApiException { String ingressName = K8sUtils.getServiceName(context.getApp().getAppName(), context.getAppEnv().getTag()); V1IngressList list = networkingApi.listNamespacedIngress(namespace, null, null, null, null, - K8sUtils.DHORSE_SELECTOR_KEY + ingressName, 1, null, null, null, null); + K8sUtils.getSelectorKey(ingressName), 1, null, null, null, null); V1Ingress ingress = null; if (!StringUtils.isBlank(ingressHost) && CollectionUtils.isEmpty(list.getItems())) { logger.info("Start to create ingress"); @@ -406,7 +421,7 @@ public boolean deleteDeployment(ClusterPO clusterPO, AppPO appPO, AppEnvPO appEn AppsV1Api api = new AppsV1Api(apiClient); String namespace = appEnvPO.getNamespaceName(); String depolymentName = K8sUtils.getDeploymentName(appPO.getAppName(), appEnvPO.getTag()); - String labelSelector = K8sUtils.getDeploymentLabelSelector(depolymentName); + String labelSelector = K8sUtils.getSelectorKey(depolymentName); try { V1DeploymentList oldDeployment = api.listNamespacedDeployment(namespace, null, null, null, null, labelSelector, null, null, null, null, null); @@ -437,10 +452,10 @@ public boolean deleteDeployment(ClusterPO clusterPO, AppPO appPO, AppEnvPO appEn public boolean autoScaling(AppPO appPO, AppEnvPO appEnvPO, ClusterPO clusterPO) { ApiClient apiClient = this.apiClient(clusterPO.getClusterUrl(), clusterPO.getAuthToken()); String deploymentName = K8sUtils.getReplicaAppName(appPO.getAppName(), appEnvPO.getTag()); - return createAutoScaling(appEnvPO, deploymentName, apiClient); + return createAutoScaling(appPO.getAppName(), appEnvPO, deploymentName, apiClient); } - private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiClient apiClient) { + private boolean createAutoScaling(String appName, AppEnvPO appEnvPO, String deploymentName, ApiClient apiClient) { AutoscalingV1Api autoscalingApi = new AutoscalingV1Api(apiClient); V1HorizontalPodAutoscaler body = new V1HorizontalPodAutoscaler(); body.setKind("HorizontalPodAutoscaler"); @@ -454,9 +469,9 @@ private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiC scaleTargetRef.setName(deploymentName); spec.setScaleTargetRef(scaleTargetRef); spec.setTargetCPUUtilizationPercentage(appEnvPO.getAutoScalingCpu()); - body.setMetadata(deploymentMetaData(deploymentName, appEnvPO.getTag())); + body.setMetadata(deploymentMetaData(deploymentName)); body.setSpec(spec); - String labelSelector = K8sUtils.getDeploymentLabelSelector(deploymentName); + String labelSelector = K8sUtils.getSelectorKey(deploymentName); try { V1HorizontalPodAutoscalerList autoscalerList = autoscalingApi.listNamespacedHorizontalPodAutoscaler( appEnvPO.getNamespaceName(), null, null, null, null, labelSelector, 1, null, null, null, @@ -468,6 +483,14 @@ private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiC autoscalingApi.replaceNamespacedHorizontalPodAutoscaler(deploymentName, appEnvPO.getNamespaceName(), body, null, null, null, null); } + + //v1.4.1的兼容代码，后续版本删除 + try { + String name = K8sUtils.getReplicaAppName2(appName, appEnvPO.getTag()); + autoscalingApi.deleteNamespacedHorizontalPodAutoscaler(name, appEnvPO.getNamespaceName(), null, null, null, null, null, null); + }catch(Exception e) { + logger.error("Failed to delete hpa", e); + } } catch (ApiException e) { String message = e.getResponseBody() == null ? e.getMessage() : e.getResponseBody(); LogUtils.throwException(logger, message, MessageCodeEnum.REPLICA_RESTARTED_FAILURE); @@ -477,7 +500,7 @@ private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiC private boolean deleteAutoScaling(String namespace, String deploymentName, ApiClient apiClient) { AutoscalingV1Api autoscalingApi = new AutoscalingV1Api(apiClient); - String labelSelector = K8sUtils.getDeploymentLabelSelector(deploymentName); + String labelSelector = K8sUtils.getSelectorKey(deploymentName); try { V1HorizontalPodAutoscalerList autoscalerList = autoscalingApi.listNamespacedHorizontalPodAutoscaler( namespace, null, null, null, null, labelSelector, 1, null, null, null, null); @@ -506,7 +529,7 @@ private boolean deleteService(String namespace, String serviceName, ApiClient ap CoreV1Api coreApi = new CoreV1Api(apiClient); try { V1ServiceList serviceList = coreApi.listNamespacedService(namespace, null, null, null, null, - K8sUtils.DHORSE_SELECTOR_KEY + serviceName, 1, null, null, null, null); + K8sUtils.getSelectorKey(serviceName), 1, null, null, null, null); if (CollectionUtils.isEmpty(serviceList.getItems())) { return true; } @@ -530,7 +553,7 @@ private boolean deleteIngress(String namespace, String serviceName, ApiClient ap NetworkingV1Api networkingApi = new NetworkingV1Api(apiClient); try { V1IngressList list = networkingApi.listNamespacedIngress(namespace, null, null, null, null, - K8sUtils.DHORSE_SELECTOR_KEY + serviceName, 1, null, null, null, null); + K8sUtils.getSelectorKey(serviceName), 1, null, null, null, null); if (CollectionUtils.isEmpty(list.getItems())) { return true; } @@ -588,10 +611,10 @@ private List servicePorts(DeploymentContext context){ return ports; } - private V1ObjectMeta deploymentMetaData(String appName, String envTag) { + private V1ObjectMeta deploymentMetaData(String deploymentName) { V1ObjectMeta metadata = new V1ObjectMeta(); - metadata.setName(appName); - metadata.setLabels(K8sClusterHelper.dhorseLabel(appName)); + metadata.setName(deploymentName); + metadata.setLabels(K8sClusterHelper.dhorseLabel(deploymentName)); return metadata; } @@ -612,9 +635,7 @@ private V1LabelSelector specSelector(String deploymentName) { private V1PodTemplateSpec specTemplate(DeploymentContext context) { V1ObjectMeta specMetadata = new V1ObjectMeta(); - Map labels = K8sClusterHelper.dhorseLabel(context.getDeploymentName()); - labels.put("version", String.valueOf(System.currentTimeMillis())); - specMetadata.setLabels(labels); + specMetadata.setLabels(K8sClusterHelper.deploymentLabel(context)); specMetadata.setAnnotations(K8sClusterHelper.addPrometheus("Pod", context)); V1PodTemplateSpec template = new V1PodTemplateSpec(); template.setMetadata(specMetadata); diff --git a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/k8s/K8sClusterStrategy.java b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/k8s/K8sClusterStrategy.java index 87b63ca2..a9f79475 100644 --- a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/k8s/K8sClusterStrategy.java +++ b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/k8s/K8sClusterStrategy.java @@ -177,7 +177,7 @@ public Replica readDeployment(ClusterPO clusterPO, AppEnv appEnv, AppPO appPO) { public boolean createDeployment(DeploymentContext context) { logger.info("Start to deploy k8s server"); String namespace = context.getAppEnv().getNamespaceName(); - String labelSelector = K8sUtils.getDeploymentLabelSelector(context.getDeploymentName()); + String labelSelector = K8sUtils.getSelectorKey(context.getDeploymentName()); try(KubernetesClient client = client(context.getCluster().getClusterUrl(), context.getCluster().getAuthToken())){ diff --git a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/utils/K8sUtils.java b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/utils/K8sUtils.java index 2805deac..fa7992d0 100644 --- a/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/utils/K8sUtils.java +++ b/dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/utils/K8sUtils.java @@ -46,7 +46,7 @@ public class K8sUtils { public static final String DHORSE_LABEL_KEY = Constants.DHORSE_TAG + "-app"; - public static final String DHORSE_SELECTOR_KEY = DHORSE_LABEL_KEY + "="; + public static final String DHORSE_SELECTOR_KEY = DHORSE_LABEL_KEY + " in ("; private static final Set SYSTEM_NAMESPACES = new HashSet<>(); @@ -63,6 +63,10 @@ public static String getDeploymentName(String appName, String appEnvTag) { } public static String getReplicaAppName(String appName, String appEnvTag) { + return getServiceName(appName, appEnvTag); + } + + public static String getReplicaAppName2(String appName, String appEnvTag) { return new StringBuilder() .append(appName).append("-") .append("1").append("-") @@ -79,19 +83,45 @@ public static String getServiceName(String appName, String appEnvTag) { } public static String[] appNameAndEnvTag(String podName) { - String nameAndeEnv = podName.split("-dhorse-")[0]; + String appName = ""; + String envTag = ""; + //兼容逻辑，v1.4.1以后的版本应该删除 + if(podName.contains("-dhorse-")) { + String nameAndeEnv = podName.split("-dhorse-")[0]; + int offset = nameAndeEnv.lastIndexOf("-1-"); + appName = nameAndeEnv.substring(0, offset); + envTag = nameAndeEnv.substring(offset + 3); + }else { + String[] segments = podName.split("-"); + envTag = segments[segments.length-3]; + for(int i=0; i < segments.length-3; i++) { + appName += segments[i] + "-"; + } + appName = appName.substring(0, appName.length() - 1); + } + return new String[]{appName, envTag}; + } + + public static String[] appNameAndEnvTag2(String podName) { + String nameAndeEnv = podName.split("-dhorse")[0]; int offset = nameAndeEnv.lastIndexOf("-1-"); String appName = nameAndeEnv.substring(0, offset); String envTag = nameAndeEnv.substring(offset + 3); return new String[]{appName, envTag}; } - public static String getDeploymentLabelSelector(String appName) { - return DHORSE_SELECTOR_KEY + appName; + public static String getSelectorKey(String appValue) { + if(appValue.contains("-dhorse")) { + String[] nameTag = appNameAndEnvTag2(appValue); + String newName = nameTag[0] + "-" + nameTag[1]; + return DHORSE_SELECTOR_KEY + appValue + "," + newName +")"; + } + return DHORSE_SELECTOR_KEY + appValue + ")"; } public static String getDeploymentLabelSelector(String appName, String appEnvTag) { - return DHORSE_SELECTOR_KEY + getReplicaAppName(appName, appEnvTag); + return DHORSE_SELECTOR_KEY + getReplicaAppName(appName, appEnvTag) + + "," + getServiceName(appName, appEnvTag) + ")"; } public static Set getSystemNamspaces() { diff --git a/static/page/replica/list.html b/static/page/replica/list.html index 5b429767..7dc96cfd 100644 --- a/static/page/replica/list.html +++ b/static/page/replica/list.html @@ -183,7 +183,7 @@ //提示 $('#replicaMetrics').on('mouseover', function(){ - layer.tips('集群需要安装Metrics Server，参考文档', this, {tips:[3, '#fff'], time:1000}); + layer.tips('集群需要安装Metrics Server，参考文档', this, {tips:[3, '#fff'], time:1000}); }); dropdown.render({