feat: loganalysis agg (#807)

server/agg/agg-core/pom.xml

@@ -61,6 +61,10 @@
 			<groupId>com.alibaba</groupId>
 			<artifactId>fastjson</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-annotations</artifactId>
+		</dependency>
 	</dependencies>
 
 	<build>

server/agg/agg-core/src/main/java/io/holoinsight/server/agg/v1/core/conf/OutputField.java

@@ -5,6 +5,7 @@
 
 import javax.annotation.Nonnull;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.googlecode.aviator.AviatorEvaluator;
 import com.googlecode.aviator.Expression;
 
@@ -58,6 +59,7 @@ public static OutputField create(String name, String expr) {
     return new OutputField(name, expr);
   }
 
+  @JsonIgnore
   public Expression getCompiledExpression() {
     if (compiledExpression == null) {
       // TODO compile error?

server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/AggDispatcherAutoConfiguration.java

@@ -11,6 +11,7 @@
 import org.springframework.context.annotation.Import;
 
 import io.holoinsight.server.agg.v1.core.AggProperties;
+import io.holoinsight.server.agg.v1.dispatcher.mock.AggDispatcherMockDataGenerator;
 import io.holoinsight.server.common.dao.CommonDaoConfiguration;
 import io.holoinsight.server.common.springboot.ConditionalOnRole;
 

server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AggDispatcherMockDataGenerator.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+package io.holoinsight.server.agg.v1.dispatcher.mock;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+
+import io.holoinsight.server.agg.v1.dispatcher.AggDispatcher;
+import io.holoinsight.server.common.JsonUtils;
+import io.holoinsight.server.common.auth.AuthInfo;
+import io.holoinsight.server.extension.model.Header;
+import io.holoinsight.server.extension.model.Record;
+import io.holoinsight.server.extension.model.Row;
+import io.holoinsight.server.extension.model.Table;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * <p>
+ * created at 2023/10/17
+ *
+ * @author xzchaoo
+ */
+@Slf4j
+public class AggDispatcherMockDataGenerator {
+  @Autowired
+  private AggDispatcher aggDispatcher;
+
+  @Scheduled(initialDelay = 1000L, fixedDelay = 1000L)
+  public void execute() {
+    AuthInfo ai = new AuthInfo();
+    ai.setTenant("monitor");
+
+    long ts = System.currentTimeMillis() / 1000 * 1000;
+
+    Random r = new Random();
+
+    Table table = new Table();
+
+    Header header = new Header();
+    header.setTagKeys(Arrays.asList("app", "userId"));
+    header.setFieldKeys(Arrays.asList("count", "count1"));
+    table.setHeader(header);
+
+    table.setRows(new ArrayList<>(100));
+
+    table.setName("test");
+    table.setTimestamp(ts);
+
+    for (int i = 0; i < 100; i++) {
+      Row row = new Row();
+      row.setTimestamp(ts);
+
+      String app = "foo" + (r.nextInt(5));
+      String userId = "user" + (r.nextInt(100));
+      row.setTagValues(Arrays.asList(app, userId));
+
+      row.setFieldValues(Arrays.asList(1D, 1D));
+
+      table.getRows().add(row);
+    }
+
+    aggDispatcher.dispatchDetailData(ai, table);
+  }
+
+  @Scheduled(initialDelay = 1000L, fixedDelay = 1000L)
+  public void execute_analysis_known() {
+    AuthInfo ai = new AuthInfo();
+    ai.setTenant("monitor");
+
+    List<Record> records = new ArrayList<>();
+    String name = "loganalysis";
+    {
+      Record r = new Record();
+      r.setName(name);
+      r.setTimestamp(System.currentTimeMillis());
+      Map<String, String> tags = new HashMap<>();
+      tags.put("hostname", "hostname-1");
+      tags.put("eventName", "test-1");
+      r.setTags(tags);
+
+      String value;
+      {
+        List<AnalyzedLog> logs = new ArrayList<>();
+
+        AnalyzedLog log1 = new AnalyzedLog();
+        log1.setCount(2);
+        log1.setSample("2024-02-26 18:18:18 [INFO] hello world, cost=[100]");
+
+        logs.add(log1);
+
+        value = JsonUtils.toJson(Collections.singletonMap("analyzedLogs", logs));
+      }
+
+      Map<String, Object> fields = new HashMap<>();
+      fields.put("value", value);
+      r.setFields(fields);
+      records.add(r);
+    }
+    {
+      Record r = new Record();
+      r.setName(name);
+      r.setTimestamp(System.currentTimeMillis());
+      Map<String, String> tags = new HashMap<>();
+      tags.put("hostname", "hostname-2");
+      tags.put("eventName", "test-1");
+      r.setTags(tags);
+
+      String value;
+      {
+        List<AnalyzedLog> logs = new ArrayList<>();
+
+        AnalyzedLog log1 = new AnalyzedLog();
+        log1.setCount(3);
+        log1.setSample("2024-02-26 18:18:18 [INFO] hello world, cost=[100]");
+
+        logs.add(log1);
+
+        value = JsonUtils.toJson(Collections.singletonMap("analyzedLogs", logs));
+      }
+
+      Map<String, Object> fields = new HashMap<>();
+      fields.put("value", value);
+      r.setFields(fields);
+      records.add(r);
+    }
+    {
+      Record r = new Record();
+      r.setName(name);
+      r.setTimestamp(System.currentTimeMillis());
+      Map<String, String> tags = new HashMap<>();
+      tags.put("app", "testapp");
+      tags.put("hostname", "hostname-3");
+      tags.put("eventName", "__analysis");
+      r.setTags(tags);
+
+      String value;
+      {
+        List<AnalyzedLog> logs = new ArrayList<>();
+
+        AnalyzedLog log1 = new AnalyzedLog();
+        List<LAPart> parts = new ArrayList<>();
+        parts.add(new LAPart("UNKNOWN", false, false, 1));
+        parts.add(new LAPart("hello world", false, true, 1));
+        parts.add(new LAPart("cost", false, false, 1));
+        parts.add(new LAPart("100", false, false, 1));
+        log1.setParts(parts);
+        log1.setCount(3);
+        List<SourceWord> sourceWords = new ArrayList<>();
+        sourceWords.add(new SourceWord("1.1.1.1", 1));
+        sourceWords.add(new SourceWord("2.2.2.2", 1));
+        log1.setSourceWords(sourceWords);
+        log1.setSample("2024-02-26 18:18:18 [UNKNOWN] hello world, cost=[100]");
+
+        logs.add(log1);
+
+        value = JsonUtils.toJson(Collections.singletonMap("analyzedLogs", logs));
+      }
+
+      Map<String, Object> fields = new HashMap<>();
+      fields.put("value", value);
+      r.setFields(fields);
+      records.add(r);
+    }
+
+    aggDispatcher.dispatchRecords(ai, name, records);
+  }
+}

server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/AnalyzedLog.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+package io.holoinsight.server.agg.v1.dispatcher.mock;
+
+import java.util.List;
+
+import lombok.Data;
+
+/**
+ * <p>
+ * created at 2024/2/26
+ *
+ * @author xzchaoo
+ */
+@Data
+class AnalyzedLog {
+  private List<LAPart> parts;
+  private String sample;
+  private int count;
+  private List<SourceWord> sourceWords;
+}

server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/LAPart.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+package io.holoinsight.server.agg.v1.dispatcher.mock;
+
+import lombok.Data;
+
+/**
+ * <p>
+ * created at 2024/2/26
+ *
+ * @author xzchaoo
+ */
+@Data
+class LAPart {
+  private String content;
+  private boolean source;
+  private boolean important;
+  private int count;
+
+  public LAPart() {}
+
+  public LAPart(String content, boolean source, boolean important, int count) {
+    this.content = content;
+    this.source = source;
+    this.important = important;
+    this.count = count;
+  }
+}

server/agg/agg-dispatcher/src/main/java/io/holoinsight/server/agg/v1/dispatcher/mock/SourceWord.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0.
+ */
+package io.holoinsight.server.agg.v1.dispatcher.mock;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * <p>
+ * created at 2024/2/26
+ *
+ * @author xzchaoo
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+class SourceWord {
+  private String source;
+  private int count;
+}

server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/executor/AggTaskExecutor.java

@@ -98,6 +98,12 @@ public AggTaskExecutor(AggTaskState state, CompletenessService completenessServi
    * @param aggTaskValue
    */
   public void process(XAggTask latestAggTask, AggProtos.AggTaskValue aggTaskValue) {
+    boolean debug = latestAggTask.getInner().getExtension().isDebug();
+    if (debug) {
+      log.info("[agg] [debug] [{}] input={}", key(), ProtoJsonUtils.toJson(aggTaskValue));
+    }
+
+
     switch (aggTaskValue.getType()) {
       case AggTaskValueTypes.COMPLETENESS_INFO:
         processCompletenessInfo(latestAggTask, aggTaskValue);