Merge pull request #103 from scottsut/master

chore: alpha.1 release
running-elephant · Oct 25, 2021 · f626a07 · f626a07
2 parents 01bf976 + 9736f48
commit f626a07
Show file tree

Hide file tree

Showing 143 changed files with 3,199 additions and 3,126 deletions.
diff --git a/config/application-config.yml.example b/config/application-config.yml.example
@@ -6,26 +6,33 @@ spring:
     username: { USERNAME }
     password: { PASSWORD }
 
-  mail:
-    host: { 邮箱服务地址 }
-    port: { 端口号 }
-    username: { 邮箱地址 }
-    fromAddress:
-    password: { 邮箱服务密码 }
-    senderName: { 发送者昵称 }
-
-    properties:
-      smtp:
-        starttls:
-          enable: true
-          required: true
-        auth: true
-      mail:
-        smtp:
-          ssl:
-            enable: true
-  redis:
-    host: { HOST }
+# mail config
+
+  #  mail:
+  #    host: { 邮箱服务地址 }
+  #    port: { 端口号 }
+  #    username: { 邮箱地址 }
+  #    fromAddress:
+  #    password: { 邮箱服务密码 }
+  #    senderName: { 发送者昵称 }
+  #
+  #    properties:
+  #      smtp:
+  #        starttls:
+  #          enable: true
+  #          required: true
+  #        auth: true
+  #      mail:
+  #        smtp:
+  #          ssl:
+  #            enable: true
+
+
+# redis config
+
+#  redis:
+#    port: 6379
+#    host: { HOST }
 
 
 server:
@@ -51,4 +58,4 @@ datart:
   screenshot:
     timeout-seconds: 60
     webdriver-type: CHROME
-    webdriver-path: {Web Driver Path}
+    webdriver-path: { Web Driver Path }
diff --git a/config/jdbc-driver-ext.yml b/config/jdbc-driver-ext.yml
@@ -0,0 +1,16 @@
+#EXAPLE:
+#  db-type: "example"
+#  name: "example"
+#  driver-class: "com.example.jdbc.Driver"
+#  literal-quote: "'"
+#  identifier-quote: "`"
+#  url-prefix: jdbc:example://
+
+IMPALA:
+  db-type: "impala"
+  name: "impala"
+  driver-class: "com.mysql.cj.jdbc.Driver"
+  literal-quote: "'"
+  identifier-quote: "`"
+  adapter-class: "datart.data.provider.jdbc.adapters.ImpalaDataProviderAdapter"
+  url-prefix: "jdbc:impala://"
diff --git a/core/pom.xml b/core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-alpha.0</version>
+        <version>1.0.0-alpha.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -50,10 +50,6 @@
             <version>1.9.8.M1</version>
         </dependency>
 
-        <!--        <dependency>-->
-        <!--            <groupId>com.alibaba.cloud</groupId>-->
-        <!--            <artifactId>spring-cloud-starter-dubbo</artifactId>-->
-        <!--        </dependency>-->
         <dependency>
             <groupId>com.h2database</groupId>
             <artifactId>h2</artifactId>
@@ -92,11 +88,6 @@
             <version>1.3.1</version>
         </dependency>
 
-<!--        <dependency>-->
-<!--            <groupId>org.reflections</groupId>-->
-<!--            <artifactId>reflections</artifactId>-->
-<!--            <version>0.9.11</version>-->
-<!--        </dependency>-->
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
@@ -146,6 +137,14 @@
             <artifactId>fastjson</artifactId>
             <version>1.2.73</version>
         </dependency>
+        <dependency>
+            <groupId>org.hibernate.validator</groupId>
+            <artifactId>hibernate-validator</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>jakarta.validation</groupId>
+            <artifactId>jakarta.validation-api</artifactId>
+        </dependency>
 
     </dependencies>
 
@@ -180,7 +179,7 @@
                     <dependency>
                         <groupId>datart</groupId>
                         <artifactId>datart-core</artifactId>
-                        <version>${datart.version}</version>
+                        <version>${project.version}</version>
                     </dependency>
                 </dependencies>
             </plugin>

diff --git a/core/src/main/java/datart/core/common/BeanUtils.java b/core/src/main/java/datart/core/common/BeanUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Datart
+ * <p>
+ * Copyright 2021
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package datart.core.common;
+
+import org.springframework.util.CollectionUtils;
+
+import javax.validation.ConstraintViolation;
+import javax.validation.Validation;
+import javax.validation.ValidationException;
+import javax.validation.ValidatorFactory;
+import java.util.Set;
+import java.util.StringJoiner;
+
+public class BeanUtils {
+
+    private static final ValidatorFactory validatorFactory = Validation.buildDefaultValidatorFactory();
+
+    public static void requireNotNull(Object obj, String... fields) {
+        for (String field : fields) {
+            Object fieldValue = ReflectUtils.getFieldValue(obj, field);
+            if (fieldValue == null) {
+                throw new RuntimeException("field " + field + " can not be null");
+            }
+        }
+    }
+
+    public static void validate(Object obj, Class<?>... groups) {
+        Set<ConstraintViolation<Object>> validate = validatorFactory.getValidator().validate(obj, groups);
+        if (!CollectionUtils.isEmpty(validate)) {
+            StringJoiner message = new StringJoiner(",");
+            for (ConstraintViolation<Object> v : validate) {
+                message.add(v.getPropertyPath() + ":" + v.getMessage());
+            }
+            throw new ValidationException(message.toString());
+        }
+    }
+
+}
diff --git a/core/src/main/java/datart/core/common/FileUtils.java b/core/src/main/java/datart/core/common/FileUtils.java
@@ -21,6 +21,11 @@
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.File;
+import java.io.FileFilter;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.Set;
 
 public class FileUtils {
 
@@ -65,5 +70,28 @@ public static void delete(File file) {
         }
     }
 
+    public static Set<String> walkDir(File file, String extension, boolean recursion) {
+        if (file == null || !file.exists()) {
+            return Collections.emptySet();
+        }
+        if (file.isFile()) {
+            return Collections.singleton(file.getName());
+        } else {
+            File[] files = file.listFiles(pathname -> extension == null || pathname.getName().endsWith(extension));
+            if (files == null) {
+                return Collections.emptySet();
+            }
+            Set<String> names = new LinkedHashSet<>();
+            for (File f : files) {
+                if (f.isFile()) {
+                    names.add(f.getName());
+                } else if (recursion) {
+                    names.addAll(walkDir(f, extension, recursion));
+                }
+            }
+            return names;
+        }
+    }
+
 
 }
diff --git a/core/src/main/java/datart/core/data/provider/DataProvider.java b/core/src/main/java/datart/core/data/provider/DataProvider.java
@@ -112,4 +112,12 @@ public Set<StdSqlOperator> supportedStdFunctions(DataProviderSource source) {
 
     public abstract boolean validateFunction(DataProviderSource source, String snippet);
 
+    /**
+     * 数据源被修改，重置数据源缓存
+     *
+     * @param source
+     */
+    public void resetSource(DataProviderSource source) {
+    }
+
 }
diff --git a/core/src/main/java/datart/core/data/provider/DataProviderManager.java b/core/src/main/java/datart/core/data/provider/DataProviderManager.java
@@ -42,4 +42,6 @@ public interface DataProviderManager {
 
     boolean validateFunction(DataProviderSource source, String snippet);
 
+    void updateSource(DataProviderSource source);
+
 }
diff --git a/core/src/main/java/datart/core/mappers/ext/DatachartMapperExt.java b/core/src/main/java/datart/core/mappers/ext/DatachartMapperExt.java
@@ -20,7 +20,7 @@ public interface DatachartMapperExt extends DatachartMapper {
 
     @Select({
             "<script>",
-            "SELECT * FROM `datachart` WHERE `status`= 1 AND `id` IN ",
+            "SELECT * FROM `datachart` WHERE `status`!= 0 AND `id` IN ",
             "<foreach collection='datachartIds' item='item' index='index' open='(' close=')' separator=','>  #{item} </foreach> ;",
             "</script>"
     })

diff --git a/core/src/main/java/datart/core/mappers/ext/OrganizationMapperExt.java b/core/src/main/java/datart/core/mappers/ext/OrganizationMapperExt.java
@@ -31,7 +31,7 @@ public interface OrganizationMapperExt extends OrganizationMapper {
     List<User> listOrgMembers(@Param("orgId") String orgId);
 
     @Delete({
-            "DELETE FROM rel_user_organization WHERE user_id=#{userId};",
+            "DELETE FROM rel_user_organization WHERE user_id=#{userId} AND org_id=#{orgId};",
             "DELETE FROM rel_role_user WHERE user_id=#{userId} AND role_id IN (SELECT DISTINCT id FROM role r WHERE r.org_id=#{orgId});",
             "DELETE FROM role WHERE `type`='PER_USER' AND org_id=#{orgId} AND create_by=#{userId};",
     })

diff --git a/core/src/main/java/datart/core/mappers/ext/VariableMapperExt.java b/core/src/main/java/datart/core/mappers/ext/VariableMapperExt.java
@@ -26,6 +26,11 @@ public interface VariableMapperExt extends VariableMapper {
     })
     List<Variable> selectViewVariables(String viewId);
 
+    @Select({
+            "SELECT * FROM variable WHERE view_id = #{viewId} AND `type`='QUERY'"
+    })
+    List<Variable> selectViewQueryVariables(String viewId);
+
     @Select({
             "SELECT * FROM variable WHERE org_id = #{orgId} AND view_id is NULL"
     })

diff --git a/data-providers/file-data-provider/pom.xml b/data-providers/file-data-provider/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-alpha.0</version>
+        <version>1.0.0-alpha.1</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -20,7 +20,7 @@
         <dependency>
             <groupId>datart</groupId>
             <artifactId>datart-data-provider</artifactId>
-            <version>${datart.version}</version>
+            <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
     </dependencies>

diff --git a/data-providers/file-data-provider/src/main/java/datart/data/provider/FileDataProvider.java b/data-providers/file-data-provider/src/main/java/datart/data/provider/FileDataProvider.java
@@ -28,6 +28,8 @@
 import datart.core.data.provider.Dataframe;
 import datart.data.provider.base.DataProviderException;
 import datart.data.provider.jdbc.DataTypeUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
 
 import java.io.File;
@@ -39,6 +41,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+@Slf4j
 public class FileDataProvider extends DefaultDataProvider {
 
     public static final String FILE_FORMAT = "format";
@@ -60,17 +63,19 @@ public List<Dataframe> loadFullDataFromSource(DataProviderSource config) {
             for (Map<String, Object> schema : schemas) {
                 String path = schema.get(FILE_PATH).toString();
                 FileFormat fileFormat = FileFormat.valueOf(schema.get(FILE_FORMAT).toString().toUpperCase());
-
-                List<Map<String, String>> columnConfig = (List<Map<String, String>>) schema.get(COLUMNS);
                 List<Column> columns = null;
-                if (!CollectionUtils.isEmpty(columnConfig)) {
-                    columns = columnConfig
-                            .stream()
-                            .map(c -> new Column(c.get(COLUMN_NAME), ValueType.valueOf(c.get(COLUMN_TYPE))))
-                            .collect(Collectors.toList());
+                try {
+                    List<Map<String, String>> columnConfig = (List<Map<String, String>>) schema.get(COLUMNS);
+                    if (!CollectionUtils.isEmpty(columnConfig)) {
+                        columns = columnConfig
+                                .stream()
+                                .map(c -> new Column(c.get(COLUMN_NAME), ValueType.valueOf(c.get(COLUMN_TYPE))))
+                                .collect(Collectors.toList());
+                    }
+                } catch (ClassCastException ignored) {
                 }
                 Dataframe dataframe = loadFromPath(FileUtils.withBasePath(path), fileFormat, columns);
-                dataframe.setName(schema.containsKey(TABLE) ? schema.get(TABLE).toString() : "TEST" + UUIDGenerator.generate());
+                dataframe.setName(StringUtils.isNoneBlank(schema.getOrDefault(TABLE, "").toString()) ? schema.get(TABLE).toString() : "TEST" + UUIDGenerator.generate());
                 dataframes.add(dataframe);
             }
             return dataframes;

diff --git a/data-providers/file-data-provider/src/main/resources/file-data-provider.json b/data-providers/file-data-provider/src/main/resources/file-data-provider.json
@@ -7,7 +7,14 @@
     {
       "name": "schemas",
       "type": "files",
+      "key": "tableName",
       "children": [
+        {
+          "name": "tableName",
+          "required": true,
+          "defaultValue": "",
+          "type": "string"
+        },
         {
           "name": "format",
           "required": true,
@@ -27,8 +34,7 @@
           "description": "file path"
         },
         {
-          "name": "schema",
-          "required": true,
+          "name": "columns",
           "defaultValue": "",
           "type": "schema"
         }

diff --git a/data-providers/http-data-provider/pom.xml b/data-providers/http-data-provider/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-alpha.0</version>
+        <version>1.0.0-alpha.1</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
@@ -17,7 +17,7 @@
         <dependency>
             <groupId>datart</groupId>
             <artifactId>datart-data-provider</artifactId>
-            <version>${datart.version}</version>
+            <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
Original file line number	Diff line number	Diff line change
Expand Up		@@ -42,4 +42,6 @@ public interface DataProviderManager {

		boolean validateFunction(DataProviderSource source, String snippet);

		void updateSource(DataProviderSource source);

		}