Skip to content

Commit

Permalink
refactor: add monitorparams in ula (#806)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsy1001de
jsy1001de authored Feb 27, 2024
1 parent 82d8297 commit 7588050
Show file tree
Hide file tree
Showing 17 changed files with 204 additions and 87 deletions.
7 changes: 7 additions & 0 deletions ...boot/src/main/java/io/holoinsight/server/home/bootstrap/HoloinsightHomeConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import io.holoinsight.server.home.biz.service.EnvironmentService;
import io.holoinsight.server.home.biz.service.TenantInitService;
import io.holoinsight.server.home.biz.service.UserinfoVerificationService;
import io.holoinsight.server.home.biz.service.impl.AlarmHistoryServiceImpl;
import io.holoinsight.server.home.biz.service.impl.AlertRuleServiceImpl;
import io.holoinsight.server.home.biz.service.impl.DefaultEnvironmentServiceImpl;
import io.holoinsight.server.home.biz.service.impl.DefaultTenantInitServiceImpl;
import io.holoinsight.server.home.biz.service.impl.UserinfoVerificationServiceImpl;
Expand Down Expand Up @@ -110,6 +112,11 @@ public ProductCtlService productCtlService() {
return new ProductCtlServiceImpl();
}

@Bean
public AlertRuleServiceImpl alertRuleService() {
return new AlertRuleServiceImpl();
}

@Bean
public ParameterSecurityService alertSecurityService() {
return new ParameterSecurityServiceImpl();
Expand Down
2 changes: 2 additions & 0 deletions ...common/src/main/java/io/holoinsight/server/home/common/service/RequestContextAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ public interface RequestContextAdapter {

<T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String tenant, String workspace);

void tenantAdapt(String tenant, String workspace);

<T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String tenant);

<T> void queryWrapperWorkspaceAdapt(QueryWrapper<T> queryWrapper, String workspace);
Expand Down
5 changes: 5 additions & 0 deletions ...on/src/main/java/io/holoinsight/server/home/common/service/RequestContextAdapterImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ public <T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String ten
}
}

@Override
public void tenantAdapt(String tenant, String workspace) {

}

@Override
public <T> void queryWrapperTenantAdapt(QueryWrapper<T> queryWrapper, String tenant) {
if (queryWrapper != null && StringUtils.isNotBlank(tenant)) {
Expand Down
42 changes: 38 additions & 4 deletions ...e/home-dal/src/main/java/io/holoinsight/server/home/dal/model/dto/conf/CollectMetric.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@
*/
package io.holoinsight.server.home.dal.model.dto.conf;

import io.holoinsight.server.agg.v1.core.conf.FillZero;
import io.holoinsight.server.agg.v1.core.conf.JoinMeta;
import io.holoinsight.server.agg.v1.core.conf.OutputItem.Topn;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.util.CollectionUtils;

import java.io.Serializable;
Expand Down Expand Up @@ -44,13 +48,18 @@ public class CollectMetric implements Serializable {

public List<String> tags;

public List<String> refTags;


public List<Metric> metrics;

/****************************** 后置过滤 ******************************/
/**
* after filters
*/
public List<AfterFilter> afterFilters;

/****************************** 日志采样 ******************************/
public Boolean logSample;
public List<LogSampleRule> logSampleRules;

Expand All @@ -60,13 +69,11 @@ public class CollectMetric implements Serializable {
// Logs that exceed 4096 are truncated
public Integer sampleMaxLength = 4096;

/****************************** 预计算 ******************************/
// pre calculate, 是否开启预计算
public Boolean calculate;

/**
* agg 预聚合指标表名称，非前端传入，是后端自动生成
*/
public String aggTableName;
public LogCalculate logCalculate;

// 单机明细数据是否存储
public Boolean notStorage;
Expand Down Expand Up @@ -112,6 +119,33 @@ public static class LogSampleRule implements Serializable {

}

@Data
public static class LogCalculate {

/**
* agg 预聚合指标表名称，非前端传入，是后端自动生成
*/
public String aggTableName;

public FillZero fillZero;

/**
* topN 指标计算
*/
public Topn topn;

// join 元数据
public Boolean joinMeta;
public JoinMetaConfig joinMetaConfig;

}

@EqualsAndHashCode(callSuper = true)
@Data
public static class JoinMetaConfig extends JoinMeta {

}

public boolean checkLogPattern() {
if (!CollectionUtils.isEmpty(metrics)) {
for (Metric metric : metrics) {
Expand Down
16 changes: 15 additions & 1 deletion ...er/home/home-service/src/main/java/io/holoinsight/server/home/biz/common/AggTaskUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.holoinsight.server.agg.v1.core.conf.AggFunc;
import io.holoinsight.server.agg.v1.core.conf.CompletenessConfig;
import io.holoinsight.server.agg.v1.core.conf.CompletenessConfig.Mode;
import io.holoinsight.server.agg.v1.core.conf.FillZero;
import io.holoinsight.server.agg.v1.core.conf.From;
import io.holoinsight.server.agg.v1.core.conf.FromConfigs;
import io.holoinsight.server.agg.v1.core.conf.FromMetrics;
Expand Down Expand Up @@ -145,7 +146,7 @@ public static Output buildOutput(CollectMetric collectMetric) {

OutputItem outputItem = new OutputItem();
outputItem.setType(OUTPUT_STORAGE_ENGINE);
outputItem.setName(collectMetric.getAggTableName());
outputItem.setName(collectMetric.getLogCalculate().getAggTableName());

List<OutputField> fields = new ArrayList<>();
OutputField outputField = new OutputField();
Expand All @@ -154,6 +155,11 @@ public static Output buildOutput(CollectMetric collectMetric) {
fields.add(outputField);
outputItem.setFields(fields);

if (null != collectMetric.getLogCalculate().getTopn()
&& collectMetric.getLogCalculate().getTopn().isEnabled()) {
outputItem.setTopn(collectMetric.getLogCalculate().getTopn());
}

List<OutputItem> items = new ArrayList<>();
items.add(outputItem);

Expand All @@ -164,4 +170,12 @@ public static Output buildOutput(CollectMetric collectMetric) {
public static List<PartitionKey> buildPartition(CollectMetric collectMetric) {
return new ArrayList<>();
}

public static FillZero buildFillZero(CollectMetric collectMetric) {
if (null != collectMetric.getLogCalculate().getFillZero()
&& collectMetric.getLogCalculate().getFillZero().isEnabled()) {
return collectMetric.getLogCalculate().getFillZero();
}
return new FillZero();
}
}
130 changes: 76 additions & 54 deletions ...ome/home-service/src/main/java/io/holoinsight/server/home/biz/common/GaeaSqlTaskUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import io.holoinsight.server.home.dal.model.dto.conf.Translate.TranslateTransform;
import io.holoinsight.server.registry.model.Elect;
import io.holoinsight.server.registry.model.Elect.RefIndex;
import io.holoinsight.server.registry.model.Elect.RefMeta;
import io.holoinsight.server.registry.model.Elect.RefName;
import io.holoinsight.server.registry.model.Elect.TransFormFilter;
import io.holoinsight.server.registry.model.Elect.TransFormFilterAppend;
Expand Down Expand Up @@ -665,76 +666,97 @@ public static GroupBy buildGroupBy(LogParse logParse, ExtraConfig extraConfig,
return buildLogPatternParse(logParse, extraConfig);
}
List<String> tags = collectMetric.getTags();
List<String> refTags = collectMetric.getRefTags();

GroupBy groupBy = new GroupBy();
groupBy.setGroups(new ArrayList<>());
if (CollectionUtils.isEmpty(tags)) {
if (CollectionUtils.isEmpty(tags) && CollectionUtils.isEmpty(refTags)) {
return groupBy;
}
tags.forEach(t -> {
if (!CollectionUtils.isEmpty(tags)) {
tags.forEach(t -> {

SplitCol dim = splitColMap.get(dimColType).get(t);
if (null == dim) {
return;
}
Rule rule = dim.rule;
SplitCol dim = splitColMap.get(dimColType).get(t);
if (null == dim) {
return;
}
Rule rule = dim.rule;

Elect elect = new Elect();
Elect elect = new Elect();

switch (logParse.splitType) {
case leftRight:
Elect.LeftRight leftRight = new Elect.LeftRight();
leftRight.setLeft(rule.left);
leftRight.setLeftIndex(rule.leftIndex);
leftRight.setRight(rule.right);
switch (logParse.splitType) {
case leftRight:
Elect.LeftRight leftRight = new Elect.LeftRight();
leftRight.setLeft(rule.left);
leftRight.setLeftIndex(rule.leftIndex);
leftRight.setRight(rule.right);

elect.setType("leftRight");
elect.setLeftRight(leftRight);
break;
elect.setType("leftRight");
elect.setLeftRight(leftRight);
break;

case separator:
elect.setType("refIndex");
elect.setRefIndex(new RefIndex());
elect.getRefIndex().setIndex(rule.pos);
break;
case regexp:
if (StringUtils.isNotEmpty(rule.regexpName)) {
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.regexpName);
} else if (rule.pos != null) {
case separator:
elect.setType("refIndex");
elect.setRefIndex(new RefIndex());
elect.getRefIndex().setIndex(rule.pos);
}
break;
case json:
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.jsonPathSyntax);
break;
default:
break;
}
break;
case regexp:
if (StringUtils.isNotEmpty(rule.regexpName)) {
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.regexpName);
} else if (rule.pos != null) {
elect.setType("refIndex");
elect.setRefIndex(new RefIndex());
elect.getRefIndex().setIndex(rule.pos);
}
break;
case json:
elect.setType("refName");
elect.setRefName(new RefName());
elect.getRefName().setName(rule.jsonPathSyntax);
break;
default:
break;
}

if (null != rule.translate && !CollectionUtils.isEmpty(rule.translate.transforms)) {
Transform transform = new Transform();
transform.setFilters(convertTransFormFilters(rule.translate.transforms));
elect.setTransform(transform);
}
if (null != rule.translate && !CollectionUtils.isEmpty(rule.translate.transforms)) {
Transform transform = new Transform();
transform.setFilters(convertTransFormFilters(rule.translate.transforms));
elect.setTransform(transform);
}

if (StringUtils.isNotEmpty(rule.defaultValue)) {
elect.setDefaultValue(rule.defaultValue);
}
if (StringUtils.isNotEmpty(rule.defaultValue)) {
elect.setDefaultValue(rule.defaultValue);
}

GroupBy.Group group = new GroupBy.Group();
{
group.setName(t);
group.setElect(elect);
}
groupBy.setMaxKeySize(extraConfig.getMaxKeySize());
groupBy.getGroups().add(group);
});
GroupBy.Group group = new GroupBy.Group();
{
group.setName(t);
group.setElect(elect);
}
groupBy.setMaxKeySize(extraConfig.getMaxKeySize());
groupBy.getGroups().add(group);
});
}

if (!CollectionUtils.isEmpty(refTags)) {
refTags.forEach(t -> {

Elect elect = new Elect();
elect.setType("refMeta");
elect.setRefMeta(new RefMeta());
elect.getRefMeta().setName(t);

GroupBy.Group group = new GroupBy.Group();
{
group.setName(t);
group.setElect(elect);
}
groupBy.setMaxKeySize(extraConfig.getMaxKeySize());
groupBy.getGroups().add(group);
});
}

Map<String, SplitCol> extendColTypes = splitColMap.get(extendColType);
if (!CollectionUtils.isEmpty(extendColTypes)) {
Expand Down
12 changes: 8 additions & 4 deletions ...ice/src/main/java/io/holoinsight/server/home/biz/listener/CustomPluginUpdateListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,9 @@ public void onEvent(CustomPluginDTO customPluginDTO) {
String tableName = String.format("%s_%s", name, customPluginDTO.id);
sqlTaskMaps.put(tableName, sqlTask);

if (null != collectMetric.getCalculate() && Boolean.TRUE == collectMetric.calculate) {
if (null != collectMetric.getCalculate() && Boolean.TRUE == collectMetric.getCalculate()) {
AggTask aggTask = buildAggTask(collectMetric, customPluginDTO);
aggTaskMaps.put(collectMetric.aggTableName, aggTask);
aggTaskMaps.put(collectMetric.logCalculate.aggTableName, aggTask);
}
}

Expand Down Expand Up @@ -221,6 +221,7 @@ private AggTask buildAggTask(CollectMetric collectMetric, CustomPluginDTO custom
AggTaskUtil.buildGroupBy(collectMetric, tenantInitService.getAggDefaultGroupByTags()));
aggTask.setWindow(AggTaskUtil.buildWindow(customPluginDTO.getPeriodType().dataUnitMs));
aggTask.setOutput(AggTaskUtil.buildOutput(collectMetric));
aggTask.setFillZero(AggTaskUtil.buildFillZero(collectMetric));
return aggTask;
}

Expand Down Expand Up @@ -277,7 +278,7 @@ private void saveMetricInfo(CustomPluginDTO customPluginDTO) {

for (CollectMetric collectMetric : collectMetrics) {
saveMetricByCollectMetric(customPluginDTO, collectMetric, conf.spm, false);
if (Boolean.TRUE == collectMetric.calculate) {
if (null != collectMetric.getCalculate() && Boolean.TRUE == collectMetric.getCalculate()) {
saveMetricByCollectMetric(customPluginDTO, collectMetric, conf.spm, true);
}
}
Expand All @@ -289,7 +290,7 @@ private void saveMetricByCollectMetric(CustomPluginDTO customPluginDTO,
String tableName = collectMetric.getTableName();
String targetTable = collectMetric.getTargetTable();
if (isAgg) {
targetTable = collectMetric.getAggTableName();
targetTable = collectMetric.logCalculate.getAggTableName();
}

try {
Expand Down Expand Up @@ -325,6 +326,9 @@ private void saveMetricByCollectMetric(CustomPluginDTO customPluginDTO,
if (!isAgg && !CollectionUtils.isEmpty(collectMetric.tags)) {
tags.addAll(collectMetric.tags);
}
if (!isAgg && !CollectionUtils.isEmpty(collectMetric.refTags)) {
tags.addAll(collectMetric.refTags);
}
metricInfoDTO.setTags(tags);
metricInfoDTO.setRef(String.valueOf(customPluginDTO.getId()));
MetricInfoDTO db = metricInfoService.queryByMetric(metricInfoDTO.getTenant(),
Expand Down
Loading

0 comments on commit 7588050

Please sign in to comment.