Merge pull request kruize#1168 from shreyabiradar07/dsmetadata_db

DB integration for Datasource Metadata
shreyabiradar07 · Apr 19, 2024 · bbedf08 · bbedf08
2 parents d9f3b3c + 061b6ff
commit bbedf08
Show file tree

Hide file tree

Showing 8 changed files with 487 additions and 63 deletions.
diff --git a/migrations/kruize_local_ddl.sql b/migrations/kruize_local_ddl.sql
@@ -1,3 +1,3 @@
 create table IF NOT EXISTS kruize_datasources (version varchar(255), name varchar(255), provider varchar(255), serviceName varchar(255), namespace varchar(255), url varchar(255), primary key (name));
-create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), cluster_group_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id));
+create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), datasource_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id));
 alter table kruize_experiments add column metadata_id bigint references kruize_dsmetadata(id), alter column datasource type varchar(255);
diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java
@@ -1,10 +1,10 @@
 package com.autotune.common.datasource;
 
+import com.autotune.common.data.ValidationOutputData;
 import com.autotune.common.exceptions.datasource.DataSourceDoesNotExist;
-import com.autotune.common.data.dataSourceMetadata.DataSourceMetadataInfo;
+import com.autotune.common.data.dataSourceMetadata.*;
+import com.autotune.database.service.ExperimentDBService;
 import com.autotune.utils.KruizeConstants;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -16,7 +16,7 @@
  * Currently Supported Implementations:
  *  - importMetadataFromDataSource
  *  - getMetadataFromDataSource
- *  TODO - Implement update and delete functionalities
+ *  TODO - DB integration for update and delete functionalities
  */
 public class DataSourceManager {
     private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceManager.class);
@@ -28,12 +28,19 @@ public DataSourceManager() {
     /**
      * Imports Metadata for a specific data source using associated DataSourceInfo.
      */
-    public void importMetadataFromDataSource(DataSourceInfo dataSource) {
+    public void importMetadataFromDataSource(DataSourceInfo dataSourceInfo) {
         try {
-            if (null == dataSource) {
+            if (null == dataSourceInfo) {
                 throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO);
             }
-            dataSourceMetadataOperator.createDataSourceMetadata(dataSource);
+            String dataSourceName = dataSourceInfo.getName();
+            if(checkIfDataSourceMetadataExists(dataSourceName)) {
+                LOGGER.error("Metadata already exists for datasource: {}!", dataSourceName);
+                return;
+            }
+            dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo);
+            // save the metadata to DB
+            saveMetadataFromDataSourceToDB(dataSourceInfo);
         } catch (Exception e) {
             LOGGER.error(e.getMessage());
         }
@@ -51,9 +58,17 @@ public DataSourceMetadataInfo getMetadataFromDataSource(DataSourceInfo dataSourc
             if (null == dataSource) {
                 throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO);
             }
-            return dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSource);
-        } catch (Exception e) {
+            String dataSourceName = dataSource.getName();
+            DataSourceMetadataInfo dataSourceMetadataInfo = new ExperimentDBService().loadMetadataFromDBByName(dataSourceName, "true");
+            if (null == dataSourceMetadataInfo) {
+                LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE, "for datasource {}" + dataSourceName);
+                return null;
+            }
+            return dataSourceMetadataInfo;
+        } catch (DataSourceDoesNotExist e) {
             LOGGER.error(e.getMessage());
+        }catch (Exception e) {
+            LOGGER.error("Loading saved datasource metadata failed: {} ", e.getMessage());
         }
         return null;
     }
@@ -93,4 +108,57 @@ public void deleteMetadataFromDataSource(DataSourceInfo dataSource) {
             LOGGER.error(e.getMessage());
         }
     }
+
+    /**
+     * retrieves and adds Metadata object from the specified data source
+     * @param dataSourceInfo DataSourceInfo object
+     */
+    public void saveMetadataFromDataSourceToDB(DataSourceInfo dataSourceInfo) {
+        try {
+            DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSourceInfo);
+            if (null == dataSourceMetadataInfo) {
+                LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE, "for datasource {}" + dataSourceInfo.getName());
+                return;
+            }
+            // add the metadata to DB
+            addMetadataToDB(dataSourceMetadataInfo);
+
+        } catch (Exception e) {
+            LOGGER.error(e.getMessage());
+        }
+    }
+
+    /**
+     * Adds Metadata object to DB
+     * @param dataSourceMetadataInfo DataSourceMetadataInfo object
+     */
+    public void addMetadataToDB(DataSourceMetadataInfo dataSourceMetadataInfo) {
+        ValidationOutputData addedToDB = null;
+        try {
+            // add the data source to DB
+            addedToDB = new ExperimentDBService().addMetadataToDB(dataSourceMetadataInfo);
+            if (addedToDB.isSuccess()) {
+                LOGGER.info("Metadata added to the DB successfully.");
+            } else {
+                LOGGER.error("Failed to add metadata to DB: {}", addedToDB.getMessage());
+            }
+        } catch (Exception e) {
+            LOGGER.error("Exception occurred while adding metadata : {} ", e.getMessage());
+        }
+
+    }
+
+    private boolean checkIfDataSourceMetadataExists(String dataSourceName) {
+        boolean isPresent = false;
+        try {
+            DataSourceMetadataInfo dataSourceMetadataInfo = new ExperimentDBService().loadMetadataFromDBByName(dataSourceName,"false");
+            if (null != dataSourceMetadataInfo) {
+                LOGGER.error("Metadata already exists for datasource: {}!", dataSourceName);
+                isPresent = true;
+            }
+        } catch (Exception e) {
+            LOGGER.error("Failed to load metadata for the datasource: {}: {} ", dataSourceName, e.getMessage());
+        }
+        return isPresent;
+    }
 }
diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAO.java b/src/main/java/com/autotune/database/dao/ExperimentDAO.java
@@ -80,7 +80,7 @@ public interface ExperimentDAO {
     List<KruizeDataSourceEntry> loadAllDataSources() throws Exception;
 
     // Load data source cluster group by name
-    List<KruizeDSMetadataEntry> loadDataSourceClusterGroupByName(String clusterGroupName) throws Exception;
+    List<KruizeDSMetadataEntry> loadMetadataByName(String clusterGroupName) throws Exception;
 
     // add metadata
     ValidationOutputData addMetadataToDB(KruizeDSMetadataEntry kruizeDSMetadataEntry);

diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java
@@ -810,17 +810,17 @@ public YearMonth buildDateForNextMonth(YearMonth yearMonth) {
     }
 
     /**
-     * @param clusterGroupName
+     * @param dataSourceName
      * @return
      */
     @Override
-    public List<KruizeDSMetadataEntry> loadDataSourceClusterGroupByName(String clusterGroupName) throws Exception {
+    public List<KruizeDSMetadataEntry> loadMetadataByName(String dataSourceName) throws Exception {
         List<KruizeDSMetadataEntry> kruizeMetadataList;
         try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
-            kruizeMetadataList = session.createQuery(SELECT_FROM_METADATA_BY_CLUSTER_GROUP_NAME, KruizeDSMetadataEntry.class)
-                    .setParameter("cluster_group_name", clusterGroupName).list();
+            kruizeMetadataList = session.createQuery(SELECT_FROM_METADATA_BY_DATASOURCE_NAME, KruizeDSMetadataEntry.class)
+                    .setParameter("dataSourceName", dataSourceName).list();
         } catch (Exception e) {
-            LOGGER.error("Unable to load metadata with clusterGroupName: {} : {}", clusterGroupName, e.getMessage());
+            LOGGER.error("Unable to load metadata with dataSourceName: {} : {}", dataSourceName, e.getMessage());
             throw new Exception("Error while loading existing metadata object from database : " + e.getMessage());
         }
         return kruizeMetadataList;

diff --git a/src/main/java/com/autotune/database/helper/DBConstants.java b/src/main/java/com/autotune/database/helper/DBConstants.java
@@ -12,7 +12,7 @@ public static final class SQLQUERY {
         public static final String SELECT_FROM_DATASOURCE = "from KruizeDataSourceEntry";
         public static final String SELECT_FROM_DATASOURCE_BY_NAME = "from KruizeDataSourceEntry kd WHERE kd.name = :name";
         public static final String SELECT_FROM_METADATA = "from KruizeDSMetadataEntry";
-        public static final String SELECT_FROM_METADATA_BY_CLUSTER_GROUP_NAME = "from KruizeDSMetadataEntry km WHERE km.cluster_group_name = :clusterGroupName";
+        public static final String SELECT_FROM_METADATA_BY_DATASOURCE_NAME = "from KruizeDSMetadataEntry km WHERE km.datasource_name = :dataSourceName";
         public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_DATE_RANGE_AND_LIMIT =
                 String.format("from KruizeResultsEntry k " +
                                 "WHERE k.experiment_name = :%s and " +