AUT-16502-New-Elasticsearch-Initial-work-on-new-repo

build.gradle

@@ -18,15 +18,15 @@ java {
 repositories {
     mavenCentral()
     maven {
-        url 'https://artifactory.spot.io/artifactory/spotinst-repository'
+        url 'https://artifactory.internal.spot.io/artifactory/spotinst-repository'
     }
     maven {
-        url 'https://artifactory.spot.io/artifactory/third-party'
+        url 'https://artifactory.internal.spot.io/artifactory/third-party'
     }
 }
 
 // The main class of the application
-mainClassName = 'com.spotinst.service.SampleApplication'
+mainClassName = 'com.spotinst.service.MetricsApplication'
 
 dependencies {
 
@@ -40,13 +40,19 @@ dependencies {
 //    compile project(":spotinst-messaging")
     // TODO: instead of blindly taking the version here, check if a newer version is available
     implementation(
-        'com.spotinst:spotinst-dropwizard-v4:4.0.3-SNAPSHOT',
-        'com.spotinst:spotinst-messaging:2.0.1',
-        'redis.clients:jedis:5.1.0',
+//            'com.spotinst:spotinst-dropwizard-v4:4.0.3-SNAPSHOT',
+            'com.spotinst:spotinst-dropwizard-v4:4.0.15',
+//            'co.elastic.clients:elasticsearch-java:8.17.0',
+            'co.elastic.clients:elasticsearch-java:8.16.1',
+//            'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.17.0',
+            'vc.inreach.aws:aws-signing-request-interceptor:0.0.22',
+            'com.spotinst:spotinst-messaging:2.0.1',
+            'org.hibernate:hibernate-validator:8.0.0.Final',
+            'redis.clients:jedis:5.1.0',
 
-        // Swagger - optional
-        'com.smoketurner:dropwizard-swagger:4.0.5-1',
-        'io.swagger:swagger-jersey2-jaxrs:1.6.13'
+            // Swagger - optional
+            'com.smoketurner:dropwizard-swagger:4.0.5-1',
+            'io.swagger:swagger-jersey2-jaxrs:1.6.13'
     )
 
     testImplementation(

src/dist/config/dev.yml

@@ -8,16 +8,64 @@ coreDbToken: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzcG90aW5zdCIsImV4cC
 ##############################################
 #                SWAGGER
 ##############################################
-swagger:
-  resourcePackage: com.spotinst.service.api.resources.app
+#swagger:
+#  resourcePackage: com.spotinst.service.api.resources.app
 
 ##############################################
 #                SERVICES
 ##############################################
-services:
-  eventsServiceUrl: http://events-public.dev.spotinst.com:2300
-  dbServiceUrl: http://db-java-public.dev.spotinst.com:2100
-  umsServiceUrl: http://ums-public.dev.spotinst.com:4100
+#services:
+##  eventsServiceUrl: http://events-public.dev.spotinst.com:2300
+#  dbServiceUrl: http://db-java-public.dev.spotinst.com:2100
+#  umsServiceUrl: http://ums-public.dev.spotinst.com:4100
+
+##############################################
+#                ELASTIC
+##############################################
+elastic:
+  scheme: https
+  host: vpc-es-aws-statistics-dev-cl2wtyjjsmb67aia2teaama6eq.us-east-2.es.amazonaws.com
+  port: 443
+  # Values in ms
+  connectionTimeout: 3000
+  socketTimeout: 90000
+  clusterRegion: us-east-2
+  indexNameOverriddenPatterns: [ "spot_storage", "azure_stateful_opt", "azure_ocean_opt" ]
+
+  indexNamePatterns:
+    # Add an index name pattern to be used by the get statistics requests
+    readPattern: "opt_%s"
+
+    # Add an index name pattern to be used by the get aggregative statistics requests
+    aggregationReadPattern: "agg_%s"
+
+    # Add an index name pattern to be used in the report metrics
+    # For optimized index, prefix MUST BE ALIGNED WITH CLUSTER TEMPLATE of 'opt_*'
+#    writePattern: "opt_%s_test"
+    writePattern: "opt_%s"
+
+  bulkProcessorIndexer:
+
+    # Sets when to flush a new bulk request based on the number of actions currently added
+    bufferedActions: 1000
+
+    # Sets when to flush a new bulk request based on the size of actions currently added
+    sizeInMB: 5
+
+    # Sets a flush interval flushing *any* bulk actions pending if the interval passes
+    flushIntervalInSeconds: 5
+
+    # Sets the number of concurrent requests allowed to be executed
+    concurrentRequests: 16
+
+    # Provides a backoff policy for bulk requests for whenever a bulk request is rejected due to resource constraints
+    backoffPolicyConfig:
+
+      # The initial delay defines how long to wait for the first retry attempt
+      initialDelayMs: 100
+
+      # The maximum number of retries. Must be a non-negative number
+      numOfRetries: 3
 
 
 ##############################################
@@ -33,14 +81,14 @@ httpClientDefaultRetryCount: 3
 ##############################################
 #                RATE LIMIT
 ##############################################
-rateLimit:
-  byOrganization:
-    organizationDefault: 300
-    organizations:
-      606079861234: 400
-
-  byMySql:
-    mySqlDefault: 1000
+#rateLimit:
+#  byOrganization:
+#    organizationDefault: 300
+#    organizations:
+#      606079861234: 400
+#
+#  byMySql:
+#    mySqlDefault: 1000
 
 
 ##############################################
@@ -91,6 +139,20 @@ redisConfiguration:
   poolMinIdle: 100 #Minimum number of resources to keep in pool at any given time
 
 
+queryConfig:
+  # Limit the amount of dimension keys that can be asked for in a single request
+  # Every dimensions has its own terms aggregation on the elastic search, must be limited
+  dimensionsKeysRequestLimit: 50
+
+  # Indicate how many values should be returned for each dimension key
+  # This is the maximum cap for items in each terms aggregation bucket
+  dimensionsValuesResultPerBucketLimit: 100
+
+  # Limit the amount of indices we're enabling to query for
+  # If query limit is exceeded, we'll set a new date range of NOW - (last 30 days)
+  queryTimeLimitInDays: 32
+
+
 ##############################################
 #                MESSAGING
 ##############################################

src/main/java/com/spotinst/metrics/MetricsAppContext.java

@@ -0,0 +1,196 @@
+//package com.spotinst.metrics;
+//
+//import com.spotinst.dropwizard.common.context.BaseAppContext;
+//import com.spotinst.metrics.bl.model.BlOrganization;
+//import com.spotinst.metrics.commons.configuration.MetricsConfiguration;
+//import com.spotinst.metrics.dal.services.elastic.ElasticSearchPercentilesService;
+//import com.spotinst.metrics.dal.services.elastic.IElasticSearchPercentilesService;
+//import com.spotinst.metrics.dal.services.elastic.infra.ElasticSearchService;
+//import com.spotinst.metrics.dal.services.elastic.infra.IElasticSearchService;
+////import org.elasticsearch.client.RestHighLevelClient;
+//import co.elastic.clients.transport.rest_client.RestClientTransport;
+//
+//import java.math.BigInteger;
+//import java.util.Map;
+//
+//public class MetricsAppContext extends BaseAppContext<MetricsConfiguration, BlOrganization> {
+//
+//    private static MetricsAppContext instance;
+//
+//    private MetricsAppContext() {
+//    }
+//
+//    public static MetricsAppContext getInstance() {
+//
+//        if (instance == null) {
+//            synchronized (MetricsAppContext.class) {
+//                if (instance == null) {
+//                    instance = new MetricsAppContext();
+//                }
+//            }
+//        }
+//        return instance;
+//    }
+//
+//    //region Members
+//    private Map<BigInteger, BlOrganization> organizations;
+//    private RestHighLevelClient             elasticClient;
+//    private IElasticSearchService            elasticSearchService;
+//    private IElasticSearchPercentilesService elasticSearchPercentilesService;
+//
+//    //endregion
+//
+//    // Method to get the ElasticSearchService
+//    public IElasticSearchService getElasticSearchService() {
+//        if (elasticSearchService == null) {
+//            // Initialize the elasticSearchService here
+//            elasticSearchService = new ElasticSearchService(); // Ensure this is a valid instance
+//        }
+//        return elasticSearchService;
+//    }
+//
+//    // Method to get the ElasticSearchService
+//    public IElasticSearchPercentilesService getElasticSearchPercentilesService() {
+//        if (elasticSearchPercentilesService == null) {
+//            // Initialize the elasticSearchService here
+//            elasticSearchPercentilesService = new ElasticSearchPercentilesService(); // Ensure this is a valid instance
+//        }
+//        return elasticSearchPercentilesService;
+//    }
+//
+//    //region Getters and Setters
+//
+//    @Override
+//    public Map<BigInteger, BlOrganization> getOrganizations() {
+//        return organizations;
+//    }
+//
+//    @Override
+//    public void setOrganizations(Map<BigInteger, BlOrganization> organizations) {
+//        this.organizations = organizations;
+//    }
+//
+//    public RestHighLevelClient getElasticClient() {
+//        return elasticClient;
+//    }
+//
+//    public void setElasticClient(RestHighLevelClient elasticClient) {
+//        this.elasticClient = elasticClient;
+//    }
+//
+////    public IElasticSearchService getElasticSearchService() {
+////        return elasticSearchService;
+////    }
+//
+//    public void setElasticSearchService(IElasticSearchService elasticSearchService) {
+//        this.elasticSearchService = elasticSearchService;
+//    }
+//
+//    public void setElasticSearchPercentilesService(IElasticSearchPercentilesService elasticSearchPercentilesService) {
+//        this.elasticSearchPercentilesService = elasticSearchPercentilesService;
+//    }
+//
+//    //endregion
+//}
+
+package com.spotinst.metrics;
+
+import com.spotinst.dropwizard.common.context.BaseAppContext;
+import com.spotinst.metrics.bl.model.BlOrganization;
+import com.spotinst.metrics.commons.configuration.MetricsConfiguration;
+import com.spotinst.metrics.dal.services.elastic.ElasticSearchPercentilesService;
+import com.spotinst.metrics.dal.services.elastic.IElasticSearchPercentilesService;
+import com.spotinst.metrics.dal.services.elastic.infra.ElasticSearchService;
+import com.spotinst.metrics.dal.services.elastic.infra.IElasticSearchService;
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.transport.ElasticsearchTransport;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
+import co.elastic.clients.json.jackson.JacksonJsonpMapper;
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.RestClient;
+
+import java.math.BigInteger;
+import java.util.Map;
+
+public class MetricsAppContext extends BaseAppContext<MetricsConfiguration, BlOrganization> {
+
+    private static MetricsAppContext instance;
+
+    private MetricsAppContext() {
+    }
+
+    public static MetricsAppContext getInstance() {
+
+        if (instance == null) {
+            synchronized (MetricsAppContext.class) {
+                if (instance == null) {
+                    instance = new MetricsAppContext();
+                }
+            }
+        }
+        return instance;
+    }
+
+    //region Members
+    private Map<BigInteger, BlOrganization> organizations;
+    private ElasticsearchClient elasticClient;
+    private IElasticSearchService elasticSearchService;
+    private IElasticSearchPercentilesService elasticSearchPercentilesService;
+
+    //endregion
+
+    // Method to get the ElasticSearchService
+    public IElasticSearchService getElasticSearchService() {
+        if (elasticSearchService == null) {
+            // Initialize the elasticSearchService here
+            elasticSearchService = new ElasticSearchService(); // Ensure this is a valid instance
+        }
+        return elasticSearchService;
+    }
+
+    // Method to get the ElasticSearchPercentilesService
+    public IElasticSearchPercentilesService getElasticSearchPercentilesService() {
+        if (elasticSearchPercentilesService == null) {
+            // Initialize the elasticSearchPercentilesService here
+            elasticSearchPercentilesService = new ElasticSearchPercentilesService(); // Ensure this is a valid instance
+        }
+        return elasticSearchPercentilesService;
+    }
+
+    //region Getters and Setters
+
+    @Override
+    public Map<BigInteger, BlOrganization> getOrganizations() {
+        return organizations;
+    }
+
+    @Override
+    public void setOrganizations(Map<BigInteger, BlOrganization> organizations) {
+        this.organizations = organizations;
+    }
+
+    public ElasticsearchClient getElasticClient() {
+        if (elasticClient == null) {
+            // Initialize the elasticClient here
+            RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build();
+            ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
+            elasticClient = new ElasticsearchClient(transport);
+        }
+        return elasticClient;
+    }
+
+    public void setElasticClient(ElasticsearchClient elasticClient) {
+        this.elasticClient = elasticClient;
+    }
+
+    public void setElasticSearchService(IElasticSearchService elasticSearchService) {
+        this.elasticSearchService = elasticSearchService;
+    }
+
+    public void setElasticSearchPercentilesService(IElasticSearchPercentilesService elasticSearchPercentilesService) {
+        this.elasticSearchPercentilesService = elasticSearchPercentilesService;
+    }
+
+    //endregion
+}
+