From ad08bb54d5e985fefa480ff4c5b5228d5b23d1f2 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 18 Apr 2024 14:20:45 +0530 Subject: [PATCH 1/6] Update naming convention of metadata DB fields --- migrations/kruize_local_ddl.sql | 2 +- .../database/table/KruizeDSMetadataEntry.java | 90 +++++++++---------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/migrations/kruize_local_ddl.sql b/migrations/kruize_local_ddl.sql index dbe9970b4..bdfc0aaf4 100644 --- a/migrations/kruize_local_ddl.sql +++ b/migrations/kruize_local_ddl.sql @@ -1,3 +1,3 @@ create table IF NOT EXISTS kruize_datasources (version varchar(255), name varchar(255), provider varchar(255), serviceName varchar(255), namespace varchar(255), url varchar(255), primary key (name)); -create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), cluster_group_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id)); +create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), datasource_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id)); alter table kruize_experiments add column metadata_id bigint references kruize_dsmetadata(id), alter column datasource type varchar(255); diff --git a/src/main/java/com/autotune/database/table/KruizeDSMetadataEntry.java b/src/main/java/com/autotune/database/table/KruizeDSMetadataEntry.java index fdb86dd62..7927dcaa7 100644 --- a/src/main/java/com/autotune/database/table/KruizeDSMetadataEntry.java +++ b/src/main/java/com/autotune/database/table/KruizeDSMetadataEntry.java @@ -20,19 +20,19 @@ /** * This is a Java class named KruizeDSMetadataEntry annotated with JPA annotations. - * It represents a table named kruize_metadata in a relational database. + * It represents a table named kruize_dsmetadata in a relational database. *

* The class has the following fields: *

* id: A unique identifier for each experiment detail. * version: A String representing the version of the experiment. - * clusterGroupName: A String representing the datasource name. - * clusterName: A String representing the cluster name. + * datasource_name: A String representing the datasource name. + * cluster_name: A String representing the cluster name. * namespace: A String representing the namespace name. - * workloadType: A String representing the type of workload. - * workloadName: A String representing the workload name. - * containerName: A String representing the container name. - * containerImageName: A String representing the container image name. + * workload_type: A String representing the type of workload. + * workload_name: A String representing the workload name. + * container_name: A String representing the container name. + * container_image_name: A String representing the container image name. */ @Entity @Table(name = "kruize_dsmetadata") @@ -41,13 +41,13 @@ public class KruizeDSMetadataEntry { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; private String version; - private String clusterGroupName; - private String clusterName; + private String datasource_name; + private String cluster_name; private String namespace; - private String workloadType; - private String workloadName; - private String containerName; - private String containerImageName; + private String workload_type; + private String workload_name; + private String container_name; + private String container_image_name; // TODO: Relation mapping needs to be done later // @OneToMany(mappedBy = "metadata", cascade = CascadeType.ALL) // private List experimentEntries; @@ -55,15 +55,15 @@ public class KruizeDSMetadataEntry { public KruizeDSMetadataEntry() { } - public KruizeDSMetadataEntry(String version, String clusterGroupName, String clusterName, String namespace, String workloadType, String workloadName, String containerName, String containerImageName) { + public KruizeDSMetadataEntry(String version, String datasource_name, String cluster_name, String namespace, String workload_type, String workload_name, String container_name, String container_image_name) { this.version = version; - this.clusterGroupName = clusterGroupName; - this.clusterName = clusterName; + this.datasource_name = datasource_name; + this.cluster_name = cluster_name; this.namespace = namespace; - this.workloadType = workloadType; - this.workloadName = workloadName; - this.containerName = containerName; - this.containerImageName = containerImageName; + this.workload_type = workload_type; + this.workload_name = workload_name; + this.container_name = container_name; + this.container_image_name = container_image_name; } public Long getId() { @@ -78,20 +78,20 @@ public void setVersion(String version) { this.version = version; } - public String getClusterGroupName() { - return clusterGroupName; + public String getDataSourceName() { + return datasource_name; } - public void setClusterGroupName(String clusterGroupName) { - this.clusterGroupName = clusterGroupName; + public void setDataSourceName(String datasource_name) { + this.datasource_name = datasource_name; } public String getClusterName() { - return clusterName; + return cluster_name; } - public void setClusterName(String clusterName) { - this.clusterName = clusterName; + public void setClusterName(String cluster_name) { + this.cluster_name = cluster_name; } public String getNamespace() { @@ -103,35 +103,35 @@ public void setNamespace(String namespace) { } public String getWorkloadType() { - return workloadType; + return workload_type; } - public void setWorkloadType(String workloadType) { - this.workloadType = workloadType; + public void setWorkloadType(String workload_type) { + this.workload_type = workload_type; } public String getWorkloadName() { - return workloadName; + return workload_name; } - public void setWorkloadName(String workloadName) { - this.workloadName = workloadName; + public void setWorkloadName(String workload_name) { + this.workload_name = workload_name; } public String getContainerName() { - return containerName; + return container_name; } - public void setContainerName(String containerName) { - this.containerName = containerName; + public void setContainerName(String container_name) { + this.container_name = container_name; } public String getContainerImageName() { - return containerImageName; + return container_image_name; } - public void setContainerImageName(String containerImageName) { - this.containerImageName = containerImageName; + public void setContainerImageName(String container_image_name) { + this.container_image_name = container_image_name; } @Override @@ -139,13 +139,13 @@ public String toString() { return "KruizeDSMetadataEntry{" + "id=" + id + ", version='" + version + '\'' + - ", clusterGroupName='" + clusterGroupName + '\'' + - ", clusterName='" + clusterName + '\'' + + ", datasource_name='" + datasource_name + '\'' + + ", cluster_name='" + cluster_name + '\'' + ", namespace='" + namespace + '\'' + - ", workloadType='" + workloadType + '\'' + - ", workloadName='" + workloadName + '\'' + - ", containerName='" + containerName + '\'' + - ", containerImageName='" + containerImageName + '\'' + + ", workload_type='" + workload_type + '\'' + + ", workload_name='" + workload_name + '\'' + + ", container_name='" + container_name + '\'' + + ", container_image_name='" + container_image_name + '\'' + '}'; } } From 0cb8133ade5185324d9333be19bfb132f3e2f384 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 18 Apr 2024 14:23:13 +0530 Subject: [PATCH 2/6] Add dsmetadata DB methods and converters --- .../autotune/database/dao/ExperimentDAO.java | 2 +- .../database/dao/ExperimentDAOImpl.java | 10 +- .../autotune/database/helper/DBConstants.java | 2 +- .../autotune/database/helper/DBHelpers.java | 256 ++++++++++++++++++ .../database/service/ExperimentDBService.java | 44 +++ 5 files changed, 307 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAO.java b/src/main/java/com/autotune/database/dao/ExperimentDAO.java index 1694b127f..fcd4fcc8e 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAO.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAO.java @@ -80,7 +80,7 @@ public interface ExperimentDAO { List loadAllDataSources() throws Exception; // Load data source cluster group by name - List loadDataSourceClusterGroupByName(String clusterGroupName) throws Exception; + List loadMetadataByName(String clusterGroupName) throws Exception; // add metadata ValidationOutputData addMetadataToDB(KruizeDSMetadataEntry kruizeDSMetadataEntry); diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java index b567b31be..0b216b6e6 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java @@ -810,17 +810,17 @@ public YearMonth buildDateForNextMonth(YearMonth yearMonth) { } /** - * @param clusterGroupName + * @param dataSourceName * @return */ @Override - public List loadDataSourceClusterGroupByName(String clusterGroupName) throws Exception { + public List loadMetadataByName(String dataSourceName) throws Exception { List kruizeMetadataList; try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { - kruizeMetadataList = session.createQuery(SELECT_FROM_METADATA_BY_CLUSTER_GROUP_NAME, KruizeDSMetadataEntry.class) - .setParameter("cluster_group_name", clusterGroupName).list(); + kruizeMetadataList = session.createQuery(SELECT_FROM_METADATA_BY_DATASOURCE_NAME, KruizeDSMetadataEntry.class) + .setParameter("dataSourceName", dataSourceName).list(); } catch (Exception e) { - LOGGER.error("Unable to load metadata with clusterGroupName: {} : {}", clusterGroupName, e.getMessage()); + LOGGER.error("Unable to load metadata with dataSourceName: {} : {}", dataSourceName, e.getMessage()); throw new Exception("Error while loading existing metadata object from database : " + e.getMessage()); } return kruizeMetadataList; diff --git a/src/main/java/com/autotune/database/helper/DBConstants.java b/src/main/java/com/autotune/database/helper/DBConstants.java index 9747a297a..de69b667f 100644 --- a/src/main/java/com/autotune/database/helper/DBConstants.java +++ b/src/main/java/com/autotune/database/helper/DBConstants.java @@ -12,7 +12,7 @@ public static final class SQLQUERY { public static final String SELECT_FROM_DATASOURCE = "from KruizeDataSourceEntry"; public static final String SELECT_FROM_DATASOURCE_BY_NAME = "from KruizeDataSourceEntry kd WHERE kd.name = :name"; public static final String SELECT_FROM_METADATA = "from KruizeDSMetadataEntry"; - public static final String SELECT_FROM_METADATA_BY_CLUSTER_GROUP_NAME = "from KruizeDSMetadataEntry km WHERE km.cluster_group_name = :clusterGroupName"; + public static final String SELECT_FROM_METADATA_BY_DATASOURCE_NAME = "from KruizeDSMetadataEntry km WHERE km.datasource_name = :dataSourceName"; public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_DATE_RANGE_AND_LIMIT = String.format("from KruizeResultsEntry k " + "WHERE k.experiment_name = :%s and " + diff --git a/src/main/java/com/autotune/database/helper/DBHelpers.java b/src/main/java/com/autotune/database/helper/DBHelpers.java index cd36f8b15..893ffa427 100644 --- a/src/main/java/com/autotune/database/helper/DBHelpers.java +++ b/src/main/java/com/autotune/database/helper/DBHelpers.java @@ -26,9 +26,12 @@ import com.autotune.analyzer.utils.AnalyzerConstants; import com.autotune.analyzer.utils.AnalyzerErrorConstants; import com.autotune.analyzer.utils.GsonUTCDateAdapter; +import com.autotune.common.data.dataSourceMetadata.*; import com.autotune.common.data.result.ContainerData; import com.autotune.common.data.result.ExperimentResultData; +import com.autotune.common.datasource.DataSourceCollection; import com.autotune.common.datasource.DataSourceInfo; +import com.autotune.common.datasource.DataSourceMetadataOperator; import com.autotune.common.k8sObjects.K8sObject; import com.autotune.database.table.*; import com.autotune.utils.KruizeConstants; @@ -706,7 +709,260 @@ public static KruizeDataSourceEntry convertDataSourceToDataSourceDBObj(DataSourc } return kruizeDataSource; } + + public static List convertKruizeMetadataToDataSourceMetadataObject(List kruizeMetadataList) throws Exception { + List dataSourceMetadataInfoList = new ArrayList<>(); + int failureThreshHold = kruizeMetadataList.size(); + int failureCount = 0; + DataSourceMetadataInfo dataSourceMetadataInfo = new DataSourceMetadataInfo(new HashMap<>()); + + for (KruizeDSMetadataEntry kruizeMetadata : kruizeMetadataList) { + try { + DataSource dataSource = getDataSource(kruizeMetadata, dataSourceMetadataInfo); + DataSourceCluster dataSourceCluster = getDataSourceCluster(kruizeMetadata, dataSource); + DataSourceNamespace dataSourceNamespace = getDataSourceNamespace(kruizeMetadata, dataSourceCluster); + DataSourceWorkload dataSourceWorkload = getDataSourceWorkload(kruizeMetadata, dataSourceNamespace); + DataSourceContainer dataSourceContainer = getDataSourceContainer(kruizeMetadata, dataSourceWorkload); + + // Update DataSourceDetailsInfo with the DataSourceClusterGroup, DataSourceCluster, DataSourceNamespace, DataSourceWorkload, and DataSourceContainer + dataSourceMetadataInfo.getDataSourceHashMap().put(kruizeMetadata.getDataSourceName(), dataSource); + dataSource.getDataSourceClusterHashMap().put(kruizeMetadata.getClusterName(), dataSourceCluster); + dataSourceCluster.getDataSourceNamespaceHashMap().put(kruizeMetadata.getNamespace(), dataSourceNamespace); + if (null == dataSourceWorkload) { + dataSourceNamespace.setDataSourceWorkloadHashMap(null); + continue; + } + dataSourceNamespace.getDataSourceWorkloadHashMap().put(kruizeMetadata.getWorkloadName(), dataSourceWorkload); + dataSourceWorkload.getDataSourceContainerHashMap().put(kruizeMetadata.getContainerName(), dataSourceContainer); + } catch (Exception e) { + LOGGER.error("Error occurred while converting to dataSourceMetadataInfo from DB object : {}", e.getMessage()); + LOGGER.error(e.getMessage()); + failureCount++; + } + } + if (failureThreshHold > 0 && failureCount == failureThreshHold) + throw new Exception("None of the Metadata loaded from DB."); + + dataSourceMetadataInfoList.add(dataSourceMetadataInfo); + return dataSourceMetadataInfoList; + } + public static List convertKruizeMetadataToClusterLevelDataSourceMetadata(List kruizeMetadataList) throws Exception { + List dataSourceMetadataInfoList = new ArrayList<>(); + int failureThreshHold = kruizeMetadataList.size(); + int failureCount = 0; + + // Create a single instance of DataSourceDetailsInfo + DataSourceMetadataInfo dataSourceMetadataInfo = new DataSourceMetadataInfo(new HashMap<>()); + + for (KruizeDSMetadataEntry kruizeMetadata : kruizeMetadataList) { + try { + DataSource dataSourceClusterGroup = getDataSource(kruizeMetadata, dataSourceMetadataInfo); + DataSourceCluster dataSourceCluster = getDataSourceCluster(kruizeMetadata, dataSourceClusterGroup); + dataSourceCluster.setDataSourceNamespaceHashMap(null); + + // Update DataSourceDetailsInfo with the DataSourceClusterGroup and DataSourceCluster + dataSourceMetadataInfo.getDataSourceHashMap() + .put(kruizeMetadata.getDataSourceName(), dataSourceClusterGroup); + + dataSourceClusterGroup.getDataSourceClusterHashMap() + .put(kruizeMetadata.getClusterName(), dataSourceCluster); + + } catch (Exception e) { + LOGGER.error("Error occurred while converting to dataSourceMetadataInfo from DB object : {}", e.getMessage()); + LOGGER.error(e.getMessage()); + failureCount++; + } + } + + if (failureThreshHold > 0 && failureCount == failureThreshHold) + throw new Exception("None of the Metadata loaded from DB."); + + dataSourceMetadataInfoList.add(dataSourceMetadataInfo); + return dataSourceMetadataInfoList; + } + + public static List convertKruizeMetadataToNamespaceLevelDataSourceMetadata(List kruizeMetadataList) throws Exception { + List dataSourceMetadataInfoList = new ArrayList<>(); + int failureThreshHold = kruizeMetadataList.size(); + int failureCount = 0; + + // Create a single instance of DataSourceMetadataInfo + DataSourceMetadataInfo dataSourceMetadataInfo = new DataSourceMetadataInfo(new HashMap<>()); + + for (KruizeDSMetadataEntry kruizeMetadata : kruizeMetadataList) { + try { + DataSource dataSource = getDataSource(kruizeMetadata, dataSourceMetadataInfo); + DataSourceCluster dataSourceCluster = getDataSourceCluster(kruizeMetadata, dataSource); + DataSourceNamespace dataSourceNamespace = getDataSourceNamespace(kruizeMetadata, dataSourceCluster); + dataSourceNamespace.setDataSourceWorkloadHashMap(null); + + // Update DataSourceMetadataInfo with the DataSource and DataSourceCluster + dataSourceMetadataInfo.getDataSourceHashMap() + .put(kruizeMetadata.getDataSourceName(), dataSource); + + dataSource.getDataSourceClusterHashMap() + .put(kruizeMetadata.getClusterName(), dataSourceCluster); + + dataSourceCluster.getDataSourceNamespaceHashMap() + .put(kruizeMetadata.getNamespace(), dataSourceNamespace); + + } catch (Exception e) { + LOGGER.error("Error occurred while converting to dataSourceMetadataInfo from DB object : {}", e.getMessage()); + LOGGER.error(e.getMessage()); + failureCount++; + } + } + + if (failureThreshHold > 0 && failureCount == failureThreshHold) + throw new Exception("None of the Metadata loaded from DB."); + + dataSourceMetadataInfoList.add(dataSourceMetadataInfo); + return dataSourceMetadataInfoList; + } + + public static List convertDataSourceMetadataToMetadataObj(DataSourceMetadataInfo dataSourceMetadataInfo) { + List kruizeMetadataList = new ArrayList<>(); + try { + + for (DataSource dataSource : dataSourceMetadataInfo.getDataSourceHashMap().values()) { + String dataSourceName = dataSource.getDataSourceName(); + + for (DataSourceCluster dataSourceCluster : dataSource.getDataSourceClusterHashMap().values()) { + String dataSourceClusterName = dataSourceCluster.getDataSourceClusterName(); + + for (DataSourceNamespace dataSourceNamespace : dataSourceCluster.getDataSourceNamespaceHashMap().values()) { + String namespaceName = dataSourceNamespace.getDataSourceNamespaceName(); + + if (null == dataSourceNamespace.getDataSourceWorkloadHashMap()) { + KruizeDSMetadataEntry kruizeMetadata = new KruizeDSMetadataEntry(); + kruizeMetadata.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); + kruizeMetadata.setDataSourceName(dataSourceName); + kruizeMetadata.setClusterName(dataSourceClusterName); + kruizeMetadata.setNamespace(namespaceName); + kruizeMetadata.setWorkloadName(null); + kruizeMetadata.setWorkloadType(null); + kruizeMetadata.setContainerName(null); + kruizeMetadata.setContainerImageName(null); + kruizeMetadataList.add(kruizeMetadata); + continue; + } + + for (DataSourceWorkload dataSourceWorkload : dataSourceNamespace.getDataSourceWorkloadHashMap().values()) { + + for (DataSourceContainer dataSourceContainer : dataSourceWorkload.getDataSourceContainerHashMap().values()) { + KruizeDSMetadataEntry kruizeMetadata = new KruizeDSMetadataEntry(); + kruizeMetadata.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); + + kruizeMetadata.setDataSourceName(dataSourceName); + kruizeMetadata.setClusterName(dataSourceClusterName); + kruizeMetadata.setNamespace(namespaceName); + kruizeMetadata.setWorkloadType(dataSourceWorkload.getDataSourceWorkloadType()); + kruizeMetadata.setWorkloadName(dataSourceWorkload.getDataSourceWorkloadName()); + + kruizeMetadata.setContainerName(dataSourceContainer.getDataSourceContainerName()); + kruizeMetadata.setContainerImageName(dataSourceContainer.getDataSourceContainerImageName()); + + kruizeMetadataList.add(kruizeMetadata); + } + } + } + } + } + } catch (Exception e) { + LOGGER.error("Error while converting DataSourceMetadata Object to KruizeDSMetadataEntry table due to {}", e.getMessage()); + e.printStackTrace(); + } + return kruizeMetadataList; + } + + } + + } + private static DataSource getDataSource(KruizeDSMetadataEntry kruizeMetadata, DataSourceMetadataInfo dataSourceMetadataInfo) { + String dataSourceName = kruizeMetadata.getDataSourceName(); + + // Check if the data source already exists + if (dataSourceMetadataInfo.getDataSourceHashMap().containsKey(dataSourceName)) { + return dataSourceMetadataInfo.getDataSourceHashMap().get(dataSourceName); + } + + DataSource dataSource = new DataSource(dataSourceName, new HashMap<>()); + dataSourceMetadataInfo.getDataSourceHashMap().put(dataSourceName, dataSource); + + return dataSource; + } + + private static DataSourceCluster getDataSourceCluster(KruizeDSMetadataEntry kruizeMetadata, DataSource dataSourceClusterGroup) { + String clusterName = kruizeMetadata.getClusterName(); + + // Check if the cluster already exists in the DataSourceClusterGroup + if (dataSourceClusterGroup.getDataSourceClusterHashMap().containsKey(clusterName)) { + return dataSourceClusterGroup.getDataSourceClusterHashMap().get(clusterName); + } + + DataSourceCluster dataSourceCluster = new DataSourceCluster(clusterName, new HashMap<>()); + dataSourceClusterGroup.getDataSourceClusterHashMap().put(clusterName, dataSourceCluster); + + return dataSourceCluster; + } + + private static DataSourceNamespace getDataSourceNamespace(KruizeDSMetadataEntry kruizeMetadata, DataSourceCluster dataSourceCluster) { + String namespaceName = kruizeMetadata.getNamespace(); + + // Check if the namespace already exists in the DataSourceCluster + if (dataSourceCluster.getDataSourceNamespaceHashMap().containsKey(namespaceName)) { + return dataSourceCluster.getDataSourceNamespaceHashMap().get(namespaceName); + } + + DataSourceNamespace dataSourceNamespace = new DataSourceNamespace(namespaceName, new HashMap<>()); + dataSourceCluster.getDataSourceNamespaceHashMap().put(namespaceName, dataSourceNamespace); + + return dataSourceNamespace; + } + + private static DataSourceWorkload getDataSourceWorkload(KruizeDSMetadataEntry kruizeMetadata, DataSourceNamespace dataSourceNamespace) { + String workloadName = kruizeMetadata.getWorkloadName(); + + if (null == workloadName) { + return null; } + // Check if the workload already exists in the DataSourceNamespace + if (dataSourceNamespace.getDataSourceWorkloadHashMap().containsKey(workloadName)) { + return dataSourceNamespace.getDataSourceWorkloadHashMap().get(workloadName); + } + + DataSourceWorkload dataSourceWorkload = new DataSourceWorkload(workloadName, kruizeMetadata.getWorkloadType(), new HashMap<>()); + dataSourceNamespace.getDataSourceWorkloadHashMap().put(workloadName, dataSourceWorkload); + + return dataSourceWorkload; + } + private static DataSourceContainer getDataSourceContainer(KruizeDSMetadataEntry kruizeMetadata, DataSourceWorkload dataSourceWorkload) { + String containerName = kruizeMetadata.getContainerName(); + + if (null == containerName) { + return null; + } + + // Check if the container already exists in the DataSourceWorkload + if (dataSourceWorkload.getDataSourceContainerHashMap().containsKey(containerName)) { + return dataSourceWorkload.getDataSourceContainerHashMap().get(containerName); + } + + DataSourceContainer dataSourceContainer = new DataSourceContainer(containerName, kruizeMetadata.getContainerImageName()); + dataSourceWorkload.getDataSourceContainerHashMap().put(containerName, dataSourceContainer); + + return dataSourceContainer; + } + + private static KruizeDSMetadataEntry getMetadata(String datasource) { + DataSourceMetadataOperator dataSourceDetailsOperator = DataSourceMetadataOperator.getInstance(); + HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); + DataSourceMetadataInfo dataSourceDetailsInfo = dataSourceDetailsOperator.getDataSourceMetadataInfo(dataSources.get(datasource)); + List kruizeMetadataList = Converters.KruizeObjectConverters.convertDataSourceMetadataToMetadataObj(dataSourceDetailsInfo); + if (kruizeMetadataList.isEmpty()) + return null; + else + return kruizeMetadataList.get(0); } } diff --git a/src/main/java/com/autotune/database/service/ExperimentDBService.java b/src/main/java/com/autotune/database/service/ExperimentDBService.java index f65ea0f1f..7827a979a 100644 --- a/src/main/java/com/autotune/database/service/ExperimentDBService.java +++ b/src/main/java/com/autotune/database/service/ExperimentDBService.java @@ -24,6 +24,7 @@ import com.autotune.analyzer.serviceObjects.*; import com.autotune.analyzer.utils.AnalyzerConstants; import com.autotune.common.data.ValidationOutputData; +import com.autotune.common.data.dataSourceMetadata.DataSourceMetadataInfo; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.datasource.DataSourceInfo; import com.autotune.database.dao.ExperimentDAO; @@ -420,4 +421,47 @@ public List loadAllDataSources() throws Exception { } return dataSourceInfoList; } + + /** + * adds metadata to database table + * @param dataSourceMetadataInfo DataSourceMetadataInfo object + * @return + */ + public ValidationOutputData addMetadataToDB(DataSourceMetadataInfo dataSourceMetadataInfo) { + ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null); + try { + List kruizeMetadataList = DBHelpers.Converters.KruizeObjectConverters.convertDataSourceMetadataToMetadataObj(dataSourceMetadataInfo); + for(KruizeDSMetadataEntry kruizeMetadata : kruizeMetadataList) { + validationOutputData = this.experimentDAO.addMetadataToDB(kruizeMetadata); + } + } catch (Exception e) { + LOGGER.error("Not able to save metadata due to {}", e.getMessage()); + } + return validationOutputData; + } + + /** + * fetches metadata of specified datasource name from database + * @param dataSourceName String containing the name of datasource + * @param verbose + * @return DataSourceMetadataInfo object containing metadata + */ + public DataSourceMetadataInfo loadMetadataFromDBByName(String dataSourceName, String verbose) throws Exception { + List kruizeMetadataList = experimentDAO.loadMetadataByName(dataSourceName); + List dataSourceDetailsInfoList = new ArrayList<>(); + if (null != kruizeMetadataList && !kruizeMetadataList.isEmpty()) { + if (verbose.equals("false")) { + dataSourceDetailsInfoList = DBHelpers.Converters.KruizeObjectConverters + .convertKruizeMetadataToClusterLevelDataSourceMetadata(kruizeMetadataList); + } else { + dataSourceDetailsInfoList = DBHelpers.Converters.KruizeObjectConverters + .convertKruizeMetadataToDataSourceMetadataObject(kruizeMetadataList); + } + } + if (dataSourceDetailsInfoList.isEmpty()) + return null; + else + return dataSourceDetailsInfoList.get(0); + } + } From 7d8a1e78a7b03d9d1accb2a34ba6402fbbd091d9 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 18 Apr 2024 14:27:34 +0530 Subject: [PATCH 3/6] Integration of DB for import metadata --- .../common/datasource/DataSourceManager.java | 55 +++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index 177bc2333..2ade814f0 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -1,7 +1,9 @@ package com.autotune.common.datasource; +import com.autotune.common.data.ValidationOutputData; import com.autotune.common.exceptions.datasource.DataSourceDoesNotExist; -import com.autotune.common.data.dataSourceMetadata.DataSourceMetadataInfo; +import com.autotune.common.data.dataSourceMetadata.*; +import com.autotune.database.service.ExperimentDBService; import com.autotune.utils.KruizeConstants; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -28,12 +30,43 @@ public DataSourceManager() { /** * Imports Metadata for a specific data source using associated DataSourceInfo. */ - public void importMetadataFromDataSource(DataSourceInfo dataSource) { + public void importMetadataFromDataSource(DataSourceInfo dataSourceInfo) { try { - if (null == dataSource) { + if (null == dataSourceInfo) { throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } - dataSourceMetadataOperator.createDataSourceMetadata(dataSource); + dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo); + + try { + DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSourceInfo); + + if (null == dataSourceMetadataInfo) { + LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE); + return; + } + // save the metadata to DB + ValidationOutputData addedToDB = null; + for (DataSource dataSource : dataSourceMetadataInfo.getDataSourceHashMap().values()) { + String dataSourceName = dataSource.getDataSourceName(); + // check if dataSource already exists in the DB and proceed to add accordingly + if (!checkIfDataSourceExists(dataSourceName)) { + try { + // add the data source to DB + addedToDB = new ExperimentDBService().addMetadataToDB(dataSourceMetadataInfo); + if (addedToDB.isSuccess()) { + LOGGER.info("Metadata added to the DB successfully."); + } else { + LOGGER.error("Failed to add metadata to DB: {}", addedToDB.getMessage()); + } + } catch (Exception e) { + LOGGER.error("Exception occurred while adding metadata : {} ", e.getMessage()); + } + } + } + + } catch (Exception e) { + LOGGER.error(e.getMessage()); + } } catch (Exception e) { LOGGER.error(e.getMessage()); } @@ -93,4 +126,18 @@ public void deleteMetadataFromDataSource(DataSourceInfo dataSource) { LOGGER.error(e.getMessage()); } } + + private boolean checkIfDataSourceExists(String dataSourceName) { + boolean isPresent = false; + try { + DataSourceMetadataInfo dataSourceMetadataInfo = new ExperimentDBService().loadMetadataFromDBByName(dataSourceName,"false"); + if (null != dataSourceMetadataInfo) { + LOGGER.warn("Cluster group: {} already exists!", dataSourceName); + isPresent = true; + } + } catch (Exception e) { + LOGGER.error("Failed to load metadata for the cluster group: {}: {} ", dataSourceName, e.getMessage()); + } + return isPresent; + } } From 76545d50daa2738affa3fc469fe3c58fe42bc784 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 18 Apr 2024 17:59:04 +0530 Subject: [PATCH 4/6] Refactor import and get metadata --- .../common/datasource/DataSourceManager.java | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index 2ade814f0..5e207af2d 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -35,33 +35,32 @@ public void importMetadataFromDataSource(DataSourceInfo dataSourceInfo) { if (null == dataSourceInfo) { throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } - dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo); + String dataSourceName = dataSourceInfo.getName(); + if(checkIfDataSourceMetadataExists(dataSourceName)) { + LOGGER.error("Metadata already exists for datasource: {}!", dataSourceName); + return; + } try { + dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo); DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSourceInfo); if (null == dataSourceMetadataInfo) { - LOGGER.debug(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE); + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE); return; } // save the metadata to DB ValidationOutputData addedToDB = null; - for (DataSource dataSource : dataSourceMetadataInfo.getDataSourceHashMap().values()) { - String dataSourceName = dataSource.getDataSourceName(); - // check if dataSource already exists in the DB and proceed to add accordingly - if (!checkIfDataSourceExists(dataSourceName)) { - try { - // add the data source to DB - addedToDB = new ExperimentDBService().addMetadataToDB(dataSourceMetadataInfo); - if (addedToDB.isSuccess()) { - LOGGER.info("Metadata added to the DB successfully."); - } else { - LOGGER.error("Failed to add metadata to DB: {}", addedToDB.getMessage()); - } - } catch (Exception e) { - LOGGER.error("Exception occurred while adding metadata : {} ", e.getMessage()); - } + try { + // add the data source metadata to DB + addedToDB = new ExperimentDBService().addMetadataToDB(dataSourceMetadataInfo); + if (addedToDB.isSuccess()) { + LOGGER.info("Metadata added to the DB successfully."); + } else { + LOGGER.error("Failed to add metadata to DB: {}", addedToDB.getMessage()); } + } catch (Exception e) { + LOGGER.error("Exception occurred while adding metadata : {} ", e.getMessage()); } } catch (Exception e) { @@ -84,9 +83,17 @@ public DataSourceMetadataInfo getMetadataFromDataSource(DataSourceInfo dataSourc if (null == dataSource) { throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } - return dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSource); - } catch (Exception e) { + String dataSourceName = dataSource.getName(); + DataSourceMetadataInfo dataSourceMetadataInfo = new ExperimentDBService().loadMetadataFromDBByName(dataSource.getName(), "true"); + if (null == dataSourceMetadataInfo) { + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE, "for datasource {}" + dataSourceName); + return null; + } + return dataSourceMetadataInfo; + } catch (DataSourceDoesNotExist e) { LOGGER.error(e.getMessage()); + }catch (Exception e) { + LOGGER.error("Loading saved datasource metadata failed: {} ", e.getMessage()); } return null; } @@ -127,16 +134,16 @@ public void deleteMetadataFromDataSource(DataSourceInfo dataSource) { } } - private boolean checkIfDataSourceExists(String dataSourceName) { + private boolean checkIfDataSourceMetadataExists(String dataSourceName) { boolean isPresent = false; try { DataSourceMetadataInfo dataSourceMetadataInfo = new ExperimentDBService().loadMetadataFromDBByName(dataSourceName,"false"); if (null != dataSourceMetadataInfo) { - LOGGER.warn("Cluster group: {} already exists!", dataSourceName); + LOGGER.error("Metadata already exists for datasource: {}!", dataSourceName); isPresent = true; } } catch (Exception e) { - LOGGER.error("Failed to load metadata for the cluster group: {}: {} ", dataSourceName, e.getMessage()); + LOGGER.error("Failed to load metadata for the datasource: {}: {} ", dataSourceName, e.getMessage()); } return isPresent; } From 56d7dabde1318d755b09c27eb4cc956da9e77c6e Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 18 Apr 2024 21:47:32 +0530 Subject: [PATCH 5/6] Refactor DataSourceManager encapsulating database operations --- .../common/datasource/DataSourceManager.java | 74 +++++++++++-------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index 5e207af2d..f06012983 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -5,8 +5,6 @@ import com.autotune.common.data.dataSourceMetadata.*; import com.autotune.database.service.ExperimentDBService; import com.autotune.utils.KruizeConstants; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +16,7 @@ * Currently Supported Implementations: * - importMetadataFromDataSource * - getMetadataFromDataSource - * TODO - Implement update and delete functionalities + * TODO - DB integration for update and delete functionalities */ public class DataSourceManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceManager.class); @@ -40,32 +38,9 @@ public void importMetadataFromDataSource(DataSourceInfo dataSourceInfo) { LOGGER.error("Metadata already exists for datasource: {}!", dataSourceName); return; } - - try { - dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo); - DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSourceInfo); - - if (null == dataSourceMetadataInfo) { - LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE); - return; - } - // save the metadata to DB - ValidationOutputData addedToDB = null; - try { - // add the data source metadata to DB - addedToDB = new ExperimentDBService().addMetadataToDB(dataSourceMetadataInfo); - if (addedToDB.isSuccess()) { - LOGGER.info("Metadata added to the DB successfully."); - } else { - LOGGER.error("Failed to add metadata to DB: {}", addedToDB.getMessage()); - } - } catch (Exception e) { - LOGGER.error("Exception occurred while adding metadata : {} ", e.getMessage()); - } - - } catch (Exception e) { - LOGGER.error(e.getMessage()); - } + dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo); + // save the metadata to DB + saveMetadataFromDataSourceToDB(dataSourceInfo); } catch (Exception e) { LOGGER.error(e.getMessage()); } @@ -84,7 +59,7 @@ public DataSourceMetadataInfo getMetadataFromDataSource(DataSourceInfo dataSourc throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } String dataSourceName = dataSource.getName(); - DataSourceMetadataInfo dataSourceMetadataInfo = new ExperimentDBService().loadMetadataFromDBByName(dataSource.getName(), "true"); + DataSourceMetadataInfo dataSourceMetadataInfo = new ExperimentDBService().loadMetadataFromDBByName(dataSourceName, "true"); if (null == dataSourceMetadataInfo) { LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE, "for datasource {}" + dataSourceName); return null; @@ -134,6 +109,45 @@ public void deleteMetadataFromDataSource(DataSourceInfo dataSource) { } } + /** + * retrieves and adds Metadata object from the specified data source + * @param dataSourceInfo DataSourceInfo object + */ + public void saveMetadataFromDataSourceToDB(DataSourceInfo dataSourceInfo) { + try { + DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSourceInfo); + if (null == dataSourceMetadataInfo) { + LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE, "for datasource {}" + dataSourceInfo.getName()); + return; + } + // add the metadata to DB + addMetadataToDB(dataSourceMetadataInfo); + + } catch (Exception e) { + LOGGER.error(e.getMessage()); + } + } + + /** + * Adds Metadata object to DB + * @param dataSourceMetadataInfo DataSourceMetadataInfo object + */ + public void addMetadataToDB(DataSourceMetadataInfo dataSourceMetadataInfo) { + ValidationOutputData addedToDB = null; + try { + // add the data source to DB + addedToDB = new ExperimentDBService().addMetadataToDB(dataSourceMetadataInfo); + if (addedToDB.isSuccess()) { + LOGGER.info("Metadata added to the DB successfully."); + } else { + LOGGER.error("Failed to add metadata to DB: {}", addedToDB.getMessage()); + } + } catch (Exception e) { + LOGGER.error("Exception occurred while adding metadata : {} ", e.getMessage()); + } + + } + private boolean checkIfDataSourceMetadataExists(String dataSourceName) { boolean isPresent = false; try { From 061b6fff4a34ea8293a5f05aa9d7a27a6b57d900 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 18 Apr 2024 22:10:37 +0530 Subject: [PATCH 6/6] Add javadocs and refactor DBHelpers --- .../autotune/database/helper/DBHelpers.java | 116 +++++++++++++----- 1 file changed, 86 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/autotune/database/helper/DBHelpers.java b/src/main/java/com/autotune/database/helper/DBHelpers.java index 893ffa427..8bae35c82 100644 --- a/src/main/java/com/autotune/database/helper/DBHelpers.java +++ b/src/main/java/com/autotune/database/helper/DBHelpers.java @@ -696,7 +696,7 @@ public static KruizeDataSourceEntry convertDataSourceToDataSourceDBObj(DataSourc KruizeDataSourceEntry kruizeDataSource; try { kruizeDataSource = new KruizeDataSourceEntry(); - kruizeDataSource.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); + kruizeDataSource.setVersion(KruizeConstants.DataSourceConstants.DataSourceMetadataInfoConstants.version); kruizeDataSource.setName(dataSourceInfo.getName()); kruizeDataSource.setProvider(dataSourceInfo.getProvider()); kruizeDataSource.setServiceName(dataSourceInfo.getServiceName()); @@ -710,6 +710,12 @@ public static KruizeDataSourceEntry convertDataSourceToDataSourceDBObj(DataSourc return kruizeDataSource; } + /** + * converts DataSourceMetadataInfo objects to KruizeDSMetadataEntry table objects + * @param kruizeMetadataList List of KruizeDSMetadataEntry objects + * @return DataSourceMetadataInfo object + * @throws Exception + */ public static List convertKruizeMetadataToDataSourceMetadataObject(List kruizeMetadataList) throws Exception { List dataSourceMetadataInfoList = new ArrayList<>(); int failureThreshHold = kruizeMetadataList.size(); @@ -718,13 +724,13 @@ public static List convertKruizeMetadataToDataSourceMeta for (KruizeDSMetadataEntry kruizeMetadata : kruizeMetadataList) { try { - DataSource dataSource = getDataSource(kruizeMetadata, dataSourceMetadataInfo); - DataSourceCluster dataSourceCluster = getDataSourceCluster(kruizeMetadata, dataSource); - DataSourceNamespace dataSourceNamespace = getDataSourceNamespace(kruizeMetadata, dataSourceCluster); - DataSourceWorkload dataSourceWorkload = getDataSourceWorkload(kruizeMetadata, dataSourceNamespace); - DataSourceContainer dataSourceContainer = getDataSourceContainer(kruizeMetadata, dataSourceWorkload); + DataSource dataSource = getOrCreateDataSourceFromDB(kruizeMetadata, dataSourceMetadataInfo); + DataSourceCluster dataSourceCluster = getOrCreateDataSourceClusterFromDB(kruizeMetadata, dataSource); + DataSourceNamespace dataSourceNamespace = getOrCreateDataSourceNamespaceFromDB(kruizeMetadata, dataSourceCluster); + DataSourceWorkload dataSourceWorkload = getOrCreateDataSourceWorkloadFromDB(kruizeMetadata, dataSourceNamespace); + DataSourceContainer dataSourceContainer = getOrCreateDataSourceContainerFromDB(kruizeMetadata, dataSourceWorkload); - // Update DataSourceDetailsInfo with the DataSourceClusterGroup, DataSourceCluster, DataSourceNamespace, DataSourceWorkload, and DataSourceContainer + // Update DataSourceMetadataInfo with the DataSource, DataSourceCluster, DataSourceNamespace, DataSourceWorkload, and DataSourceContainer dataSourceMetadataInfo.getDataSourceHashMap().put(kruizeMetadata.getDataSourceName(), dataSource); dataSource.getDataSourceClusterHashMap().put(kruizeMetadata.getClusterName(), dataSourceCluster); dataSourceCluster.getDataSourceNamespaceHashMap().put(kruizeMetadata.getNamespace(), dataSourceNamespace); @@ -746,25 +752,32 @@ public static List convertKruizeMetadataToDataSourceMeta dataSourceMetadataInfoList.add(dataSourceMetadataInfo); return dataSourceMetadataInfoList; } + + /** + * Converts KruizeDSMetadataEntry table objects to DataSourceMetadataInfo with only cluster-level metadata + * @param kruizeMetadataList KruizeDSMetadataEntry objects + * @return DataSourceMetadataInfo object with only cluster-level metadata + * @throws Exception + */ public static List convertKruizeMetadataToClusterLevelDataSourceMetadata(List kruizeMetadataList) throws Exception { List dataSourceMetadataInfoList = new ArrayList<>(); int failureThreshHold = kruizeMetadataList.size(); int failureCount = 0; - // Create a single instance of DataSourceDetailsInfo + // Create a single instance of DataSourceMetadataInfo DataSourceMetadataInfo dataSourceMetadataInfo = new DataSourceMetadataInfo(new HashMap<>()); for (KruizeDSMetadataEntry kruizeMetadata : kruizeMetadataList) { try { - DataSource dataSourceClusterGroup = getDataSource(kruizeMetadata, dataSourceMetadataInfo); - DataSourceCluster dataSourceCluster = getDataSourceCluster(kruizeMetadata, dataSourceClusterGroup); + DataSource dataSource = getOrCreateDataSourceFromDB(kruizeMetadata, dataSourceMetadataInfo); + DataSourceCluster dataSourceCluster = getOrCreateDataSourceClusterFromDB(kruizeMetadata, dataSource); dataSourceCluster.setDataSourceNamespaceHashMap(null); - // Update DataSourceDetailsInfo with the DataSourceClusterGroup and DataSourceCluster + // Update DataSourceMetadataInfo with the DataSource and DataSourceCluster dataSourceMetadataInfo.getDataSourceHashMap() - .put(kruizeMetadata.getDataSourceName(), dataSourceClusterGroup); + .put(kruizeMetadata.getDataSourceName(), dataSource); - dataSourceClusterGroup.getDataSourceClusterHashMap() + dataSource.getDataSourceClusterHashMap() .put(kruizeMetadata.getClusterName(), dataSourceCluster); } catch (Exception e) { @@ -781,6 +794,12 @@ public static List convertKruizeMetadataToClusterLevelDa return dataSourceMetadataInfoList; } + /** + * Converts KruizeDSMetadataEntry table objects to DataSourceMetadataInfo with only namespace-level metadata + * @param kruizeMetadataList List of KruizeDSMetadataEntry objects + * @return DataSourceMetadataInfo with only namespace-level metadata + * @throws Exception + */ public static List convertKruizeMetadataToNamespaceLevelDataSourceMetadata(List kruizeMetadataList) throws Exception { List dataSourceMetadataInfoList = new ArrayList<>(); int failureThreshHold = kruizeMetadataList.size(); @@ -791,9 +810,9 @@ public static List convertKruizeMetadataToNamespaceLevel for (KruizeDSMetadataEntry kruizeMetadata : kruizeMetadataList) { try { - DataSource dataSource = getDataSource(kruizeMetadata, dataSourceMetadataInfo); - DataSourceCluster dataSourceCluster = getDataSourceCluster(kruizeMetadata, dataSource); - DataSourceNamespace dataSourceNamespace = getDataSourceNamespace(kruizeMetadata, dataSourceCluster); + DataSource dataSource = getOrCreateDataSourceFromDB(kruizeMetadata, dataSourceMetadataInfo); + DataSourceCluster dataSourceCluster = getOrCreateDataSourceClusterFromDB(kruizeMetadata, dataSource); + DataSourceNamespace dataSourceNamespace = getOrCreateDataSourceNamespaceFromDB(kruizeMetadata, dataSourceCluster); dataSourceNamespace.setDataSourceWorkloadHashMap(null); // Update DataSourceMetadataInfo with the DataSource and DataSourceCluster @@ -820,6 +839,11 @@ public static List convertKruizeMetadataToNamespaceLevel return dataSourceMetadataInfoList; } + /** + * Converts DataSourceMetadataInfo object to KruizeDSMetadataEntry objects + * @param dataSourceMetadataInfo DataSourceMetadataInfo object + * @return List of KruizeDSMetadataEntry objects + */ public static List convertDataSourceMetadataToMetadataObj(DataSourceMetadataInfo dataSourceMetadataInfo) { List kruizeMetadataList = new ArrayList<>(); try { @@ -835,7 +859,7 @@ public static List convertDataSourceMetadataToMetadataObj if (null == dataSourceNamespace.getDataSourceWorkloadHashMap()) { KruizeDSMetadataEntry kruizeMetadata = new KruizeDSMetadataEntry(); - kruizeMetadata.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); + kruizeMetadata.setVersion(KruizeConstants.DataSourceConstants.DataSourceMetadataInfoConstants.version); kruizeMetadata.setDataSourceName(dataSourceName); kruizeMetadata.setClusterName(dataSourceClusterName); kruizeMetadata.setNamespace(namespaceName); @@ -851,7 +875,7 @@ public static List convertDataSourceMetadataToMetadataObj for (DataSourceContainer dataSourceContainer : dataSourceWorkload.getDataSourceContainerHashMap().values()) { KruizeDSMetadataEntry kruizeMetadata = new KruizeDSMetadataEntry(); - kruizeMetadata.setVersion(KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version); + kruizeMetadata.setVersion(KruizeConstants.DataSourceConstants.DataSourceMetadataInfoConstants.version); kruizeMetadata.setDataSourceName(dataSourceName); kruizeMetadata.setClusterName(dataSourceClusterName); @@ -878,7 +902,14 @@ public static List convertDataSourceMetadataToMetadataObj } } - private static DataSource getDataSource(KruizeDSMetadataEntry kruizeMetadata, DataSourceMetadataInfo dataSourceMetadataInfo) { + + /** + * Retrieves an existing DataSource from the DB entry or creates a new one if not found. + * @param kruizeMetadata KruizeDSMetadataEntry object + * @param dataSourceMetadataInfo DataSourceMetadataInfo object + * @return The DataSource instance associated with the DB entry. + */ + private static DataSource getOrCreateDataSourceFromDB(KruizeDSMetadataEntry kruizeMetadata, DataSourceMetadataInfo dataSourceMetadataInfo) { String dataSourceName = kruizeMetadata.getDataSourceName(); // Check if the data source already exists @@ -892,21 +923,33 @@ private static DataSource getDataSource(KruizeDSMetadataEntry kruizeMetadata, D return dataSource; } - private static DataSourceCluster getDataSourceCluster(KruizeDSMetadataEntry kruizeMetadata, DataSource dataSourceClusterGroup) { + /** + * Retrieves an existing DataSourceCluster from the DB entry or creates a new one if not found. + * @param kruizeMetadata KruizeDSMetadataEntry object + * @param dataSource DataSource object + * @return The DataSourceCluster instance associated with the DB entry + */ + private static DataSourceCluster getOrCreateDataSourceClusterFromDB(KruizeDSMetadataEntry kruizeMetadata, DataSource dataSource) { String clusterName = kruizeMetadata.getClusterName(); - // Check if the cluster already exists in the DataSourceClusterGroup - if (dataSourceClusterGroup.getDataSourceClusterHashMap().containsKey(clusterName)) { - return dataSourceClusterGroup.getDataSourceClusterHashMap().get(clusterName); + // Check if the cluster already exists in the DataSource + if (dataSource.getDataSourceClusterHashMap().containsKey(clusterName)) { + return dataSource.getDataSourceClusterHashMap().get(clusterName); } DataSourceCluster dataSourceCluster = new DataSourceCluster(clusterName, new HashMap<>()); - dataSourceClusterGroup.getDataSourceClusterHashMap().put(clusterName, dataSourceCluster); + dataSource.getDataSourceClusterHashMap().put(clusterName, dataSourceCluster); return dataSourceCluster; } - private static DataSourceNamespace getDataSourceNamespace(KruizeDSMetadataEntry kruizeMetadata, DataSourceCluster dataSourceCluster) { + /** + * Retrieves an existing DataSourceNamespace from the DB entry or creates a new one if not found. + * @param kruizeMetadata KruizeDSMetadataEntry object + * @param dataSourceCluster DataSourceCluster object + * @return The DataSourceNamespace instance associated with the DB entry + */ + private static DataSourceNamespace getOrCreateDataSourceNamespaceFromDB(KruizeDSMetadataEntry kruizeMetadata, DataSourceCluster dataSourceCluster) { String namespaceName = kruizeMetadata.getNamespace(); // Check if the namespace already exists in the DataSourceCluster @@ -920,7 +963,13 @@ private static DataSourceNamespace getDataSourceNamespace(KruizeDSMetadataEntry return dataSourceNamespace; } - private static DataSourceWorkload getDataSourceWorkload(KruizeDSMetadataEntry kruizeMetadata, DataSourceNamespace dataSourceNamespace) { + /** + * Retrieves an existing DataSourceWorkload from the DB entry or creates a new one if not found. + * @param kruizeMetadata KruizeDSMetadataEntry object + * @param dataSourceNamespace DataSourceNamespace object + * @return The DataSourceWorkload instance associated with the DB entry + */ + private static DataSourceWorkload getOrCreateDataSourceWorkloadFromDB(KruizeDSMetadataEntry kruizeMetadata, DataSourceNamespace dataSourceNamespace) { String workloadName = kruizeMetadata.getWorkloadName(); if (null == workloadName) { @@ -937,7 +986,14 @@ private static DataSourceWorkload getDataSourceWorkload(KruizeDSMetadataEntry kr return dataSourceWorkload; } - private static DataSourceContainer getDataSourceContainer(KruizeDSMetadataEntry kruizeMetadata, DataSourceWorkload dataSourceWorkload) { + + /** + * Retrieves an existing DataSourceContainer from the DB entry or creates a new one if not found. + * @param kruizeMetadata KruizeDSMetadataEntry object + * @param dataSourceWorkload DataSourceWorkload object + * @return The DataSourceContainer instance associated with the DB entry + */ + private static DataSourceContainer getOrCreateDataSourceContainerFromDB(KruizeDSMetadataEntry kruizeMetadata, DataSourceWorkload dataSourceWorkload) { String containerName = kruizeMetadata.getContainerName(); if (null == containerName) { @@ -956,10 +1012,10 @@ private static DataSourceContainer getDataSourceContainer(KruizeDSMetadataEntry } private static KruizeDSMetadataEntry getMetadata(String datasource) { - DataSourceMetadataOperator dataSourceDetailsOperator = DataSourceMetadataOperator.getInstance(); + DataSourceMetadataOperator dataSourceMetadataOperator = DataSourceMetadataOperator.getInstance(); HashMap dataSources = DataSourceCollection.getInstance().getDataSourcesCollection(); - DataSourceMetadataInfo dataSourceDetailsInfo = dataSourceDetailsOperator.getDataSourceMetadataInfo(dataSources.get(datasource)); - List kruizeMetadataList = Converters.KruizeObjectConverters.convertDataSourceMetadataToMetadataObj(dataSourceDetailsInfo); + DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.getDataSourceMetadataInfo(dataSources.get(datasource)); + List kruizeMetadataList = Converters.KruizeObjectConverters.convertDataSourceMetadataToMetadataObj(dataSourceMetadataInfo); if (kruizeMetadataList.isEmpty()) return null; else