Skip to content

Commit

Permalink
统一标签值
Browse files Browse the repository at this point in the history
  • Loading branch information
cgw committed Sep 26, 2023
1 parent 4843a37 commit 5f63e9d
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 30 deletions.
4 changes: 3 additions & 1 deletion ...pplication/src/main/java/org/dhorse/application/service/EnvReplicaApplicationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,9 @@ public void collectReplicaMetrics() {
}
for(ReplicaMetrics metric : replicaMetrics) {
String replicaName = metric.getReplicaName();
AppEnvPO appEnvPO = envMap.get(replicaName.substring(0, replicaName.indexOf("-dhorse-") + 7));
//解析deployment名字的方式不优雅，后续可以考虑其他方案
String[] appNameTag = K8sUtils.appNameAndEnvTag(replicaName);
AppEnvPO appEnvPO = envMap.get(K8sUtils.getReplicaAppName(appNameTag[0], appNameTag[1]));
if(appEnvPO == null) {
continue;
}
Expand Down
4 changes: 2 additions & 2 deletions ...astructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@ public static void createDHorseConfig(ClusterPO clusterPO, CoreV1Api coreApi) {
continue;
}
V1ConfigMapList list = coreApi.listNamespacedConfigMap(namespace, null, null, null, null,
K8sUtils.DHORSE_SELECTOR_KEY + K8sUtils.DHORSE_CONFIGMAP_NAME, null, null, null, null, null);
K8sUtils.getSelectorKey(K8sUtils.DHORSE_CONFIGMAP_NAME), null, null, null, null, null);

V1ConfigMap configMap = K8sClusterHelper.dhorseConfigMap();
if(CollectionUtils.isEmpty(list.getItems())) {
Expand Down Expand Up @@ -510,7 +510,7 @@ public static void deleteDHorseConfig(ClusterPO clusterPO, CoreV1Api coreApi) {
if(CollectionUtils.isEmpty(namespaceList.getItems())) {
return;
}
String selector = K8sUtils.DHORSE_SELECTOR_KEY + K8sUtils.DHORSE_CONFIGMAP_NAME;
String selector = K8sUtils.getSelectorKey(K8sUtils.DHORSE_CONFIGMAP_NAME);
for(V1Namespace n : namespaceList.getItems()) {
String namespace = n.getMetadata().getName();
if(!K8sUtils.DHORSE_NAMESPACE.equals(namespace)
Expand Down
61 changes: 41 additions & 20 deletions ...tructure/src/main/java/org/dhorse/infrastructure/strategy/cluster/K8sClusterStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ public Void createSecret(ClusterPO clusterPO, ImageRepo imageRepo) {
if(CollectionUtils.isEmpty(namespaceList.getItems())) {
return null;
}
String selector = K8sUtils.DHORSE_SELECTOR_KEY + K8sUtils.DOCKER_REGISTRY_KEY;
String selector = K8sUtils.getSelectorKey(K8sUtils.DOCKER_REGISTRY_KEY);
for(V1Namespace n : namespaceList.getItems()) {
String namespace = n.getMetadata().getName();
if(!K8sUtils.DHORSE_NAMESPACE.equals(namespace)
Expand Down Expand Up @@ -250,14 +250,14 @@ public boolean createDeployment(DeploymentContext context) {
V1Deployment deployment = new V1Deployment();
deployment.apiVersion("apps/v1");
deployment.setKind("Deployment");
deployment.setMetadata(deploymentMetaData(context.getDeploymentName(), context.getAppEnv().getTag()));
deployment.setMetadata(deploymentMetaData(context.getDeploymentName()));
deployment.setSpec(deploymentSpec(context));
ApiClient apiClient = this.apiClient(context.getCluster().getClusterUrl(),
context.getCluster().getAuthToken());
AppsV1Api api = new AppsV1Api(apiClient);
CoreV1Api coreApi = new CoreV1Api(apiClient);
String namespace = context.getAppEnv().getNamespaceName();
String labelSelector = K8sUtils.getDeploymentLabelSelector(context.getDeploymentName());
String labelSelector = K8sUtils.getSelectorKey(context.getDeploymentName());
try {
V1DeploymentList oldDeployment = api.listNamespacedDeployment(namespace, null, null, null, null,
labelSelector, null, null, null, null, null);
Expand All @@ -268,8 +268,15 @@ public boolean createDeployment(DeploymentContext context) {
null, null);
}

//v1.4.1的兼容代码，后续版本删除
try {
String name = K8sUtils.getReplicaAppName2(context.getApp().getAppName(), context.getAppEnv().getTag());
api.deleteNamespacedDeployment(name, namespace, null, null, null, null, null, null);
}catch(Exception e) {
logger.error("Failed to delete deployment", e);
}
// 自动扩容任务
createAutoScaling(context.getAppEnv(), context.getDeploymentName(), apiClient);
createAutoScaling(context.getApp().getAppName(), context.getAppEnv(), context.getDeploymentName(), apiClient);

if(!checkHealthOfAll(coreApi, namespace, labelSelector)) {
logger.error("Failed to create k8s deployment, because the replica is not fully started");
Expand Down Expand Up @@ -302,7 +309,7 @@ private boolean createService(DeploymentContext context) throws ApiException {
String namespace = context.getAppEnv().getNamespaceName();
String serviceName = K8sUtils.getServiceName(context.getApp().getAppName(), context.getAppEnv().getTag());
V1ServiceList serviceList = coreApi.listNamespacedService(namespace, null, null, null, null,
K8sUtils.DHORSE_SELECTOR_KEY + serviceName, 1, null, null, null, null);
K8sUtils.getSelectorKey(serviceName), 1, null, null, null, null);
if (CollectionUtils.isEmpty(serviceList.getItems())) {
logger.info("Start to create service");
V1Service service = new V1Service();
Expand All @@ -314,6 +321,14 @@ private boolean createService(DeploymentContext context) throws ApiException {
}else{
List<JsonPatch> patchs = K8sClusterHelper.updatePrometheus("Service", serviceList.getItems().get(0)
.getMetadata().getAnnotations(), context);

//兼容逻辑，v1.4.1以后应该删除
JsonPatch path = new JsonPatch();
path.setOp("replace");
path.setPath("/spec/selector/dhorse-app");
path.setValue(serviceName);
patchs.add(path);

if(!CollectionUtils.isEmpty(patchs)) {
logger.info("Start to update service");
V1Patch patch = new V1Patch(JsonUtils.toJsonString(patchs));
Expand All @@ -334,7 +349,7 @@ private boolean createIngress(DeploymentContext context) throws ApiException {
String ingressName = K8sUtils.getServiceName(context.getApp().getAppName(), context.getAppEnv().getTag());

V1IngressList list = networkingApi.listNamespacedIngress(namespace, null, null, null, null,
K8sUtils.DHORSE_SELECTOR_KEY + ingressName, 1, null, null, null, null);
K8sUtils.getSelectorKey(ingressName), 1, null, null, null, null);
V1Ingress ingress = null;
if (!StringUtils.isBlank(ingressHost) && CollectionUtils.isEmpty(list.getItems())) {
logger.info("Start to create ingress");
Expand Down Expand Up @@ -406,7 +421,7 @@ public boolean deleteDeployment(ClusterPO clusterPO, AppPO appPO, AppEnvPO appEn
AppsV1Api api = new AppsV1Api(apiClient);
String namespace = appEnvPO.getNamespaceName();
String depolymentName = K8sUtils.getDeploymentName(appPO.getAppName(), appEnvPO.getTag());
String labelSelector = K8sUtils.getDeploymentLabelSelector(depolymentName);
String labelSelector = K8sUtils.getSelectorKey(depolymentName);
try {
V1DeploymentList oldDeployment = api.listNamespacedDeployment(namespace, null, null, null, null,
labelSelector, null, null, null, null, null);
Expand Down Expand Up @@ -437,10 +452,10 @@ public boolean deleteDeployment(ClusterPO clusterPO, AppPO appPO, AppEnvPO appEn
public boolean autoScaling(AppPO appPO, AppEnvPO appEnvPO, ClusterPO clusterPO) {
ApiClient apiClient = this.apiClient(clusterPO.getClusterUrl(), clusterPO.getAuthToken());
String deploymentName = K8sUtils.getReplicaAppName(appPO.getAppName(), appEnvPO.getTag());
return createAutoScaling(appEnvPO, deploymentName, apiClient);
return createAutoScaling(appPO.getAppName(), appEnvPO, deploymentName, apiClient);
}

private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiClient apiClient) {
private boolean createAutoScaling(String appName, AppEnvPO appEnvPO, String deploymentName, ApiClient apiClient) {
AutoscalingV1Api autoscalingApi = new AutoscalingV1Api(apiClient);
V1HorizontalPodAutoscaler body = new V1HorizontalPodAutoscaler();
body.setKind("HorizontalPodAutoscaler");
Expand All @@ -454,9 +469,9 @@ private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiC
scaleTargetRef.setName(deploymentName);
spec.setScaleTargetRef(scaleTargetRef);
spec.setTargetCPUUtilizationPercentage(appEnvPO.getAutoScalingCpu());
body.setMetadata(deploymentMetaData(deploymentName, appEnvPO.getTag()));
body.setMetadata(deploymentMetaData(deploymentName));
body.setSpec(spec);
String labelSelector = K8sUtils.getDeploymentLabelSelector(deploymentName);
String labelSelector = K8sUtils.getSelectorKey(deploymentName);
try {
V1HorizontalPodAutoscalerList autoscalerList = autoscalingApi.listNamespacedHorizontalPodAutoscaler(
appEnvPO.getNamespaceName(), null, null, null, null, labelSelector, 1, null, null, null,
Expand All @@ -468,6 +483,14 @@ private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiC
autoscalingApi.replaceNamespacedHorizontalPodAutoscaler(deploymentName, appEnvPO.getNamespaceName(),
body, null, null, null, null);
}

//v1.4.1的兼容代码，后续版本删除
try {
String name = K8sUtils.getReplicaAppName2(appName, appEnvPO.getTag());
autoscalingApi.deleteNamespacedHorizontalPodAutoscaler(name, appEnvPO.getNamespaceName(), null, null, null, null, null, null);
}catch(Exception e) {
logger.error("Failed to delete hpa", e);
}
} catch (ApiException e) {
String message = e.getResponseBody() == null ? e.getMessage() : e.getResponseBody();
LogUtils.throwException(logger, message, MessageCodeEnum.REPLICA_RESTARTED_FAILURE);
Expand All @@ -477,7 +500,7 @@ private boolean createAutoScaling(AppEnvPO appEnvPO, String deploymentName, ApiC

private boolean deleteAutoScaling(String namespace, String deploymentName, ApiClient apiClient) {
AutoscalingV1Api autoscalingApi = new AutoscalingV1Api(apiClient);
String labelSelector = K8sUtils.getDeploymentLabelSelector(deploymentName);
String labelSelector = K8sUtils.getSelectorKey(deploymentName);
try {
V1HorizontalPodAutoscalerList autoscalerList = autoscalingApi.listNamespacedHorizontalPodAutoscaler(
namespace, null, null, null, null, labelSelector, 1, null, null, null, null);
Expand Down Expand Up @@ -506,7 +529,7 @@ private boolean deleteService(String namespace, String serviceName, ApiClient ap
CoreV1Api coreApi = new CoreV1Api(apiClient);
try {
V1ServiceList serviceList = coreApi.listNamespacedService(namespace, null, null, null, null,
K8sUtils.DHORSE_SELECTOR_KEY + serviceName, 1, null, null, null, null);
K8sUtils.getSelectorKey(serviceName), 1, null, null, null, null);
if (CollectionUtils.isEmpty(serviceList.getItems())) {
return true;
}
Expand All @@ -530,7 +553,7 @@ private boolean deleteIngress(String namespace, String serviceName, ApiClient ap
NetworkingV1Api networkingApi = new NetworkingV1Api(apiClient);
try {
V1IngressList list = networkingApi.listNamespacedIngress(namespace, null, null, null, null,
K8sUtils.DHORSE_SELECTOR_KEY + serviceName, 1, null, null, null, null);
K8sUtils.getSelectorKey(serviceName), 1, null, null, null, null);
if (CollectionUtils.isEmpty(list.getItems())) {
return true;
}
Expand Down Expand Up @@ -588,10 +611,10 @@ private List<V1ServicePort> servicePorts(DeploymentContext context){
return ports;
}

private V1ObjectMeta deploymentMetaData(String appName, String envTag) {
private V1ObjectMeta deploymentMetaData(String deploymentName) {
V1ObjectMeta metadata = new V1ObjectMeta();
metadata.setName(appName);
metadata.setLabels(K8sClusterHelper.dhorseLabel(appName));
metadata.setName(deploymentName);
metadata.setLabels(K8sClusterHelper.dhorseLabel(deploymentName));
return metadata;
}

Expand All @@ -612,9 +635,7 @@ private V1LabelSelector specSelector(String deploymentName) {

private V1PodTemplateSpec specTemplate(DeploymentContext context) {
V1ObjectMeta specMetadata = new V1ObjectMeta();
Map<String, String> labels = K8sClusterHelper.dhorseLabel(context.getDeploymentName());
labels.put("version", String.valueOf(System.currentTimeMillis()));
specMetadata.setLabels(labels);
specMetadata.setLabels(K8sClusterHelper.deploymentLabel(context));
specMetadata.setAnnotations(K8sClusterHelper.addPrometheus("Pod", context));
V1PodTemplateSpec template = new V1PodTemplateSpec();
template.setMetadata(specMetadata);
Expand Down
2 changes: 1 addition & 1 deletion ...ture/src/main/java/org/dhorse/infrastructure/strategy/cluster/k8s/K8sClusterStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ public Replica readDeployment(ClusterPO clusterPO, AppEnv appEnv, AppPO appPO) {
public boolean createDeployment(DeploymentContext context) {
logger.info("Start to deploy k8s server");
String namespace = context.getAppEnv().getNamespaceName();
String labelSelector = K8sUtils.getDeploymentLabelSelector(context.getDeploymentName());
String labelSelector = K8sUtils.getSelectorKey(context.getDeploymentName());
try(KubernetesClient client = client(context.getCluster().getClusterUrl(),
context.getCluster().getAuthToken())){

Expand Down
40 changes: 35 additions & 5 deletions dhorse-infrastructure/src/main/java/org/dhorse/infrastructure/utils/K8sUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public class K8sUtils {

public static final String DHORSE_LABEL_KEY = Constants.DHORSE_TAG + "-app";

public static final String DHORSE_SELECTOR_KEY = DHORSE_LABEL_KEY + "=";
public static final String DHORSE_SELECTOR_KEY = DHORSE_LABEL_KEY + " in (";

private static final Set<String> SYSTEM_NAMESPACES = new HashSet<>();

Expand All @@ -63,6 +63,10 @@ public static String getDeploymentName(String appName, String appEnvTag) {
}

public static String getReplicaAppName(String appName, String appEnvTag) {
return getServiceName(appName, appEnvTag);
}

public static String getReplicaAppName2(String appName, String appEnvTag) {
return new StringBuilder()
.append(appName).append("-")
.append("1").append("-")
Expand All @@ -79,19 +83,45 @@ public static String getServiceName(String appName, String appEnvTag) {
}

public static String[] appNameAndEnvTag(String podName) {
String nameAndeEnv = podName.split("-dhorse-")[0];
String appName = "";
String envTag = "";
//兼容逻辑，v1.4.1以后的版本应该删除
if(podName.contains("-dhorse-")) {
String nameAndeEnv = podName.split("-dhorse-")[0];
int offset = nameAndeEnv.lastIndexOf("-1-");
appName = nameAndeEnv.substring(0, offset);
envTag = nameAndeEnv.substring(offset + 3);
}else {
String[] segments = podName.split("-");
envTag = segments[segments.length-3];
for(int i=0; i < segments.length-3; i++) {
appName += segments[i] + "-";
}
appName = appName.substring(0, appName.length() - 1);
}
return new String[]{appName, envTag};
}

public static String[] appNameAndEnvTag2(String podName) {
String nameAndeEnv = podName.split("-dhorse")[0];
int offset = nameAndeEnv.lastIndexOf("-1-");
String appName = nameAndeEnv.substring(0, offset);
String envTag = nameAndeEnv.substring(offset + 3);
return new String[]{appName, envTag};
}

public static String getDeploymentLabelSelector(String appName) {
return DHORSE_SELECTOR_KEY + appName;
public static String getSelectorKey(String appValue) {
if(appValue.contains("-dhorse")) {
String[] nameTag = appNameAndEnvTag2(appValue);
String newName = nameTag[0] + "-" + nameTag[1];
return DHORSE_SELECTOR_KEY + appValue + "," + newName +")";
}
return DHORSE_SELECTOR_KEY + appValue + ")";
}

public static String getDeploymentLabelSelector(String appName, String appEnvTag) {
return DHORSE_SELECTOR_KEY + getReplicaAppName(appName, appEnvTag);
return DHORSE_SELECTOR_KEY + getReplicaAppName(appName, appEnvTag)
+ "," + getServiceName(appName, appEnvTag) + ")";
}

public static Set<String> getSystemNamspaces() {
Expand Down
2 changes: 1 addition & 1 deletion static/page/replica/list.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@

//提示
$('#replicaMetrics').on('mouseover', function(){
layer.tips('<span style="color:#000;">集群需要安装Metrics Server，<a class="layui-font-blue" target="_blank" href="https://github.com/tiandizhiguai/dhorse-doc/blob/main/guide/%E5%AE%89%E8%A3%85Metrics%20Server.md">参考文档</a></span>', this, {tips:[3, '#fff'], time:1000});
layer.tips('<span style="color:#000;">集群需要安装Metrics Server，<a class="layui-font-blue" target="_blank" href="https://gitee.com/i512team/dhorse-doc/blob/main/guide/%E5%AE%89%E8%A3%85Metrics%20Server.md">参考文档</a></span>', this, {tips:[3, '#fff'], time:1000});
});

dropdown.render({
Expand Down

0 comments on commit 5f63e9d

Please sign in to comment.