Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

List metric profiles with name and verbose query parameters #1240

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
71f21f1
Add metric profile database table
shreyabiradar07 Jul 4, 2024
a4f263a
Add create and list metricProfile APIs with database integration
shreyabiradar07 Jul 5, 2024
e915d89
Add missing EOF
shreyabiradar07 Jul 5, 2024
10d3ed8
Add metric profile documentation
shreyabiradar07 Jul 5, 2024
33fc237
Fix logic to fetch all the metric profiles created
shreyabiradar07 Jul 9, 2024
6caabb5
Add string constants
shreyabiradar07 Jul 9, 2024
edfcbf1
Validate missing mandatory values
shreyabiradar07 Jul 9, 2024
55cf6d5
Refactor loop logic to include all the specified aggregation functions
shreyabiradar07 Jul 10, 2024
4731ecf
Remove MetricProfile metrics
shreyabiradar07 Jul 11, 2024
858306e
Remove builder constants
shreyabiradar07 Jul 11, 2024
2cae219
Refactor metric profile validation
shreyabiradar07 Jul 12, 2024
91cacce
Update documentation
shreyabiradar07 Jul 12, 2024
cb7da69
Update KruizeLocalAPI.md to include metric profile APIs
shreyabiradar07 Jul 12, 2024
4a2a30e
Add MetricProfileCollection
shreyabiradar07 Jul 15, 2024
4857c29
Revert "Add MetricProfileCollection"
shreyabiradar07 Jul 15, 2024
6741d76
Add MetricProfileCollection
shreyabiradar07 Jul 15, 2024
fea236e
Integrate metric profile with local experiments
shreyabiradar07 Jul 15, 2024
9664b8a
Update generateRecommendations to use metric profile queries
shreyabiradar07 Jul 16, 2024
a534bc7
Add constants for query parameter placeholders
shreyabiradar07 Jul 16, 2024
d345eae
Add name and verbose query parameters
shreyabiradar07 Jul 24, 2024
c7742cd
Validate if metric profiles are present
shreyabiradar07 Jul 25, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
484 changes: 484 additions & 0 deletions design/KruizeLocalAPI.md

Large diffs are not rendered by default.

577 changes: 577 additions & 0 deletions design/MetricProfileAPI.md

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions migrations/kruize_local_ddl.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
create table IF NOT EXISTS kruize_datasources (version varchar(255), name varchar(255), provider varchar(255), serviceName varchar(255), namespace varchar(255), url varchar(255), primary key (name));
create table IF NOT EXISTS kruize_dsmetadata (id serial, version varchar(255), datasource_name varchar(255), cluster_name varchar(255), namespace varchar(255), workload_type varchar(255), workload_name varchar(255), container_name varchar(255), container_image_name varchar(255), primary key (id));
alter table kruize_experiments add column metadata_id bigint references kruize_dsmetadata(id), alter column datasource type varchar(255);
create table IF NOT EXISTS kruize_metric_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, slo jsonb, primary key (name));
11 changes: 11 additions & 0 deletions src/main/java/com/autotune/Autotune.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.autotune.analyzer.exceptions.MonitoringAgentNotFoundException;
import com.autotune.analyzer.exceptions.MonitoringAgentNotSupportedException;
import com.autotune.analyzer.utils.AnalyzerConstants;
import com.autotune.analyzer.performanceProfiles.MetricProfileCollection;
import com.autotune.common.datasource.DataSourceCollection;
import com.autotune.common.datasource.DataSourceInfo;
import com.autotune.database.helper.DBConstants;
Expand Down Expand Up @@ -114,6 +115,8 @@ public static void main(String[] args) {
setUpDataSources();
// checking available DataSources
checkAvailableDataSources();
// load available metric profiles from db
loadMetricProfilesFromDB();

}
// close the existing session factory before recreating
Expand Down Expand Up @@ -195,6 +198,14 @@ private static void checkAvailableDataSources() {
}
}

/**
* loads metric profiles from database
*/
private static void loadMetricProfilesFromDB() {
MetricProfileCollection metricProfileCollection = MetricProfileCollection.getInstance();
metricProfileCollection.loadMetricProfilesFromDB();
}

private static void addAutotuneServlets(ServletContextHandler context) {
context.addServlet(HealthService.class, HEALTH_SERVICE);
// Start the Prometheus end point (/metrics) for Autotune
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/autotune/analyzer/Analyzer.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ public static void addServlets(ServletContextHandler context) {
context.addServlet(ListRecommendations.class, ServerContext.RECOMMEND_RESULTS);
context.addServlet(PerformanceProfileService.class, ServerContext.CREATE_PERF_PROFILE);
context.addServlet(PerformanceProfileService.class, ServerContext.LIST_PERF_PROFILES);
context.addServlet(MetricProfileService.class, ServerContext.CREATE_METRIC_PROFILE);
context.addServlet(MetricProfileService.class, ServerContext.LIST_METRIC_PROFILES);
context.addServlet(ListDatasources.class, ServerContext.LIST_DATASOURCES);
context.addServlet(DSMetadataService.class, ServerContext.DATASOURCE_METADATA);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.autotune.common.data.result.ContainerData;
import com.autotune.common.k8sObjects.K8sObject;
import com.autotune.database.service.ExperimentDBService;
import com.autotune.operator.KruizeDeploymentInfo;
import com.autotune.operator.KruizeOperator;
import com.autotune.utils.KruizeConstants;
import org.slf4j.Logger;
Expand Down Expand Up @@ -105,9 +106,13 @@ public void validate(List<KruizeObject> kruizeExptList) {
errorMsg = AnalyzerErrorConstants.AutotuneObjectErrors.SLO_REDUNDANCY_ERROR;
validationOutputData.setErrorCode(HttpServletResponse.SC_BAD_REQUEST);
} else {
// fetch the Performance Profile from the DB
// fetch the Performance/Metric Profile from the DB
try {
new ExperimentDBService().loadPerformanceProfileFromDBByName(performanceProfilesMap, kruizeObject.getPerformanceProfile());
if (!KruizeDeploymentInfo.local) {
new ExperimentDBService().loadPerformanceProfileFromDBByName(performanceProfilesMap, kruizeObject.getPerformanceProfile());
} else {
new ExperimentDBService().loadMetricProfileFromDBByName(performanceProfilesMap, kruizeObject.getPerformanceProfile());
}
} catch (Exception e) {
LOGGER.error("Loading saved Performance Profile {} failed: {} ", expName, e.getMessage());
}
Expand All @@ -122,7 +127,13 @@ public void validate(List<KruizeObject> kruizeExptList) {
errorMsg = AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_SLO_DATA;
validationOutputData.setErrorCode(HttpServletResponse.SC_BAD_REQUEST);
} else {
String perfProfileName = KruizeOperator.setDefaultPerformanceProfile(kruizeObject.getSloInfo(), mode, target_cluster);
// TODO - set metric profile when local=true
String perfProfileName;
if (!KruizeDeploymentInfo.local) {
perfProfileName = KruizeOperator.setDefaultPerformanceProfile(kruizeObject.getSloInfo(), mode, target_cluster);
} else {
perfProfileName = KruizeOperator.setDefaultMetricProfile(kruizeObject.getSloInfo(), mode, target_cluster);
}
kruizeObject.setPerformanceProfile(perfProfileName);
proceed = true;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package com.autotune.analyzer.performanceProfiles;

import com.autotune.database.service.ExperimentDBService;
import com.autotune.utils.KruizeConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;

public class MetricProfileCollection {
private static final Logger LOGGER = LoggerFactory.getLogger(MetricProfileCollection.class);
private static MetricProfileCollection metricProfileCollectionInstance = new MetricProfileCollection();
private HashMap<String, PerformanceProfile> metricProfileCollection;

private MetricProfileCollection() {
this.metricProfileCollection = new HashMap<>();
}

public static MetricProfileCollection getInstance() {
return metricProfileCollectionInstance;
}

public HashMap<String, PerformanceProfile> getMetricProfileCollection() {
return metricProfileCollection;
}

public void loadMetricProfilesFromDB() {
try {
LOGGER.info(KruizeConstants.MetricProfileConstants.CHECKING_AVAILABLE_METRIC_PROFILE_FROM_DB);
Map<String, PerformanceProfile> availableMetricProfiles = new HashMap<>();
new ExperimentDBService().loadAllMetricProfiles(availableMetricProfiles);
if (availableMetricProfiles.isEmpty()) {
LOGGER.info(KruizeConstants.MetricProfileConstants.NO_METRIC_PROFILE_FOUND_IN_DB);
}else {
for (Map.Entry<String, PerformanceProfile> metricProfile : availableMetricProfiles.entrySet()) {
LOGGER.info(KruizeConstants.MetricProfileConstants.METRIC_PROFILE_FOUND, metricProfile.getKey());
metricProfileCollection.put(metricProfile.getKey(), metricProfile.getValue());
}
}

} catch (Exception e) {
LOGGER.error(e.getMessage());
}
}


public void addMetricProfile(PerformanceProfile metricProfile) {
String metricProfileName = metricProfile.getMetadata().get("name").asText();

LOGGER.info(KruizeConstants.MetricProfileConstants.ADDING_METRIC_PROFILE + "{}", metricProfileName);

if(metricProfileCollection.containsKey(metricProfileName)) {
LOGGER.error(KruizeConstants.MetricProfileConstants.METRIC_PROFILE_ALREADY_EXISTS + "{}", metricProfileName);
} else {
LOGGER.info(KruizeConstants.MetricProfileConstants.METRIC_PROFILE_ADDED + "{}", metricProfileName);
metricProfileCollection.put(metricProfileName, metricProfile);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import com.autotune.analyzer.kruizeObject.SloInfo;
import com.autotune.analyzer.recommendations.term.Terms;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.gson.annotations.SerializedName;

import java.util.Map;
Expand All @@ -25,9 +26,17 @@
* Container class for the PerformanceProfile kubernetes kind, which is used to define
* a profile
*
* This class provides a direct representation of MetricProfile CRD in JSON format,
* corresponding to the structure of PerformanceProfile YAML file. It includes mandatory fields
* for API version, kind, metadata and additional custom fields - profile_version, k8s_type and sloInfo
*/

public class PerformanceProfile {
private String apiVersion;

private String kind;

private JsonNode metadata;

private String name;

Expand All @@ -41,6 +50,30 @@ public class PerformanceProfile {

private Map<String, Terms> terms;

public String getApiVersion() {
return apiVersion;
}

public void setApiVersion(String apiVersion) {
this.apiVersion = apiVersion;
}

public String getKind() {
return kind;
}

public void setKind(String kind) {
this.kind = kind;
}

public JsonNode getMetadata() {
return metadata;
}

public void setMetadata(JsonNode metadata) {
this.metadata = metadata;
}

public void setName(String name) {
this.name = name;
}
Expand Down Expand Up @@ -68,6 +101,17 @@ public PerformanceProfile(String name, double profile_version, String k8s_type,
this.sloInfo = sloInfo;
}

// Constructor for MetricProfile
public PerformanceProfile(String apiVersion, String kind, JsonNode metadata,
double profile_version, String k8s_type, SloInfo sloInfo) {
this.apiVersion = apiVersion;
this.kind = kind;
this.metadata = metadata;
this.profile_version = profile_version;
this.k8s_type = k8s_type;
this.sloInfo = sloInfo;
}

public String getName() {
return name;
}
Expand Down Expand Up @@ -95,7 +139,10 @@ public void setTerms(Map<String, Terms> terms) {
@Override
public String toString() {
return "PerformanceProfile{" +
"name='" + name + '\'' +
"apiVersion='" + apiVersion + '\'' +
", kind='" + kind + '\'' +
", metadata=" + metadata +
", name='" + name + '\'' +
", profile_version=" + profile_version +
", k8s_type='" + k8s_type + '\'' +
", sloInfo=" + sloInfo +
Expand Down
Loading
Loading