Skip to content

Commit a5cf0e6

Browse files
AmiableAnilAmiableAnil
authored
Issue #LR-583 merge: Merge release-5.4.0 to master (#162)
1 parent 3ee2f8c commit a5cf0e6

File tree

30 files changed

+150
-87
lines changed

30 files changed

+150
-87
lines changed

‎.circleci/config.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
- run:
2121
name: Execute coverage report
2222
command: |
23-
mvn clean scoverage:report
23+
mvn clean scoverage:report -DCLOUD_STORE_GROUP_ID=$CLOUD_STORE_GROUP_ID -DCLOUD_STORE_ARTIFACT_ID=$CLOUD_STORE_ARTIFACT_ID -DCLOUD_STORE_VERSION=$CLOUD_STORE_VERSION
2424
- run:
2525
name: Save test results
2626
command: |
@@ -36,7 +36,7 @@ jobs:
3636
- run:
3737
name: sonar
3838
command: |
39-
mvn verify -DskipTests=true sonar:sonar -Dlog4j.configuration=./logs sonar:sonar -Dsonar.projectKey=Sunbird-Lern_data-pipeline -Dsonar.organization=sunbird-lern -Dsonar.host.url=https://sonarcloud.io -Dsonar.java.binaries=$(find . -path '*/build/classes/java' | xargs | tr ' ' ',') -Dsonar.coverage.exclusions=**/notification-sdk/**,**/notification/domain/** -Dsonar.test.exclusions=**/notification-sdk/** -Dsonar.exclusions=**/**.java -Dsonar.scala.coverage.reportPaths=/home/circleci/lern/target/scoverage.xml
39+
mvn verify -DskipTests=true sonar:sonar -Dlog4j.configuration=./logs sonar:sonar -Dsonar.projectKey=Sunbird-Lern_data-pipeline -Dsonar.organization=sunbird-lern -Dsonar.host.url=https://sonarcloud.io -Dsonar.java.binaries=$(find . -path '*/build/classes/java' | xargs | tr ' ' ',') -Dsonar.coverage.exclusions=**/notification-sdk/**,**/notification/domain/** -Dsonar.test.exclusions=**/notification-sdk/** -Dsonar.exclusions=**/**.java -Dsonar.scala.coverage.reportPaths=/home/circleci/lern/target/scoverage.xml -DCLOUD_STORE_GROUP_ID=$CLOUD_STORE_GROUP_ID -DCLOUD_STORE_ARTIFACT_ID=$CLOUD_STORE_ARTIFACT_ID -DCLOUD_STORE_VERSION=$CLOUD_STORE_VERSION
4040
4141
workflows:
4242
version: 2

‎ansible/inventory/env/group_vars/all.yml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ search_es7_host: "{{ groups['es7']|join(':9200,')}}:9200"
8686
mlworkbench: "{{ groups['mlworkbench'][0]}}"
8787

8888
cloud_account: "{{ cloud_public_storage_accountname }}"
89-
cloud_secret: "{{ cloud_public_storage_accountname }}"
89+
cloud_secret: "{{ cloud_public_storage_secret }}"
9090
dedup_redis_host: "{{ dp_redis_host }}"
9191
kp_redis_host: "{{ groups['redisall'][0] }}"
9292
neo4j_route_path: "bolt://{{ groups['learning-neo4j-node1'][0] }}:7687"
@@ -98,7 +98,7 @@ kp_print_service_base_url: "http://{{private_ingressgateway_ip}}/print"
9898
cert_reg_service_base_url: "http://{{private_ingressgateway_ip}}/certreg"
9999
kp_search_service_base_url: "http://{{private_ingressgateway_ip}}/search"
100100
lms_service_base_url: "http://{{private_ingressgateway_ip}}/lms"
101-
learner_service_base_url: "http://{{private_ingressgateway_ip}}/learner"
101+
learner_service_base_url: "http://{{private_ingressgateway_ip}}/userorg"
102102
sourcing_content_service_base_url: "http://{{dock_private_ingressgateway_ip | default('localhost') }}/content"
103103
content_service_base_url: "http://{{private_ingressgateway_ip}}/content"
104104
kp_content_service_base_url: "http://{{private_ingressgateway_ip}}/content"
@@ -112,13 +112,13 @@ cert_container_name: "{{ cloud_storage_certservice_bucketname }}"
112112
cert_cloud_storage_type: "{{cloud_service_provider}}"
113113
cert_cloud_storage_secret: "{{cloud_private_storage_secret}}"
114114
cert_cloud_storage_key: "{{cloud_private_storage_accountname}}"
115+
cert_cloud_storage_endpoint: "{{cloud_private_storage_endpoint | default('')}}"
115116
cloud_storage_base_url: "{{cloud_storage_base_url}}"
116117
content_read_api_host: "{{content_read_api_host}}"
117118
content_read_api_endpoint: "{{content_read_api_endpoint}}"
118119
content_cloud_storage_container: "{{cloud_storage_content_bucketname}}"
119120
cloud_store_base_path_placeholder: "CLOUD_BASE_PATH"
120121

121-
122122
default_channel: "org.sunbird"
123123
download_neo4j: true
124124
neo4j_upstream_download: false
@@ -164,7 +164,6 @@ kafka_group_lms_notification: "{{env_name}}-lms-notification"
164164
kafka_group_programuser_info: "{{env_name}}-programuser-group"
165165

166166
## Checkpoint Config
167-
checkpoint_store_type: "{{cloud_service_provider}}"
168167
cloud_storage_key: "{{ cloud_public_storage_accountname }}"
169168
cloud_storage_secret: "{{ cloud_public_storage_secret }}"
170169
cloud_storage_endpoint: ""

‎ansible/roles/setup-lern-kafka/defaults/main.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ processing_lern_kafka_overriden_topics:
5757
- name: prom.monitoring.metrics
5858
retention_time: 172800000
5959
replication_factor: 1
60+
- name: telemetry.audit
61+
retention_time: 172800000
62+
replication_factor: 1
6063

6164
processing_lern_kafka_topics:
6265
- name: telemetry.assess
@@ -113,3 +116,6 @@ processing_lern_kafka_topics:
113116
- name: prom.monitoring.metrics
114117
num_of_partitions: 1
115118
replication_factor: 1
119+
- name: telemetry.audit
120+
num_of_partitions: 2
121+
replication_factor: 1

‎jobs-core/pom.xml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
<properties>
1717
<encoding>UTF-8</encoding>
1818
<scoverage.plugin.version>1.4.0</scoverage.plugin.version>
19+
<CLOUD_STORE_GROUP_ID>org.sunbird</CLOUD_STORE_GROUP_ID>
20+
<CLOUD_STORE_ARTIFACT_ID>cloud-store-sdk_2.12</CLOUD_STORE_ARTIFACT_ID>
21+
<CLOUD_STORE_VERSION>1.4.6</CLOUD_STORE_VERSION>
1922
</properties>
2023
<dependencies>
2124
<dependency>
@@ -79,9 +82,9 @@
7982
<version>0.1.1</version>
8083
</dependency>
8184
<dependency>
82-
<groupId>org.sunbird</groupId>
83-
<artifactId>cloud-store-sdk_2.12</artifactId>
84-
<version>1.4.3</version>
85+
<groupId>${CLOUD_STORE_GROUP_ID}</groupId>
86+
<artifactId>${CLOUD_STORE_ARTIFACT_ID}</artifactId>
87+
<version>${CLOUD_STORE_VERSION}</version>
8588
<exclusions>
8689
<exclusion>
8790
<artifactId>log4j</artifactId>

‎jobs-distribution/Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ RUN apt-get install -y imagemagick
77
COPY target/jobs-distribution-1.0.tar.gz /tmp
88
USER flink
99
RUN tar -xvf /tmp/jobs-distribution-1.0.tar.gz -C $FLINK_HOME/lib/
10+
RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/lib/flink-aaa-s3-fs-presto-1.13.5.jar
1011
USER root
1112
RUN rm -f /tmp/jobs-distribution-1.0.tar.gz
1213
USER flink

‎kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -110,16 +110,16 @@ spec:
110110
command: ["/opt/flink/bin/standalone-job.sh"]
111111
args: ["start-foreground",
112112
"--job-classname={{ .Values.job_classname }}",
113-
{{- if eq .Values.checkpoint_store_type "azure" }}
113+
{{- if eq .Values.cloud_service_provider "azure" }}
114114
"-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}",
115115
{{- end }}
116-
{{- if eq .Values.checkpoint_store_type "s3" }}
117-
"-Ds3.access-key={{ .Values.cloud_storage_key }}",
118-
"-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
119-
"-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
120-
"-Ds3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
116+
{{- if or (eq .Values.cloud_service_provider "aws") (eq .Values.cloud_service_provider "oci") }}
117+
"-Dpresto.s3.access-key={{ .Values.cloud_storage_key }}",
118+
"-Dpresto.s3.secret-key={{ .Values.cloud_storage_secret }}",
119+
"-Dpresto.s3.endpoint={{ .Values.cloud_storage_endpoint }}",
120+
"-Dpresto.s3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
121121
{{- end }}
122-
{{- if eq .Values.checkpoint_store_type "gcloud" }}
122+
{{- if eq .Values.cloud_service_provider "gcloud" }}
123123
"-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
124124
"-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
125125
"-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
@@ -196,16 +196,16 @@ spec:
196196
workingDir: {{ .Values.taskmanager.flink_work_dir }}
197197
command: ["/opt/flink/bin/taskmanager.sh"]
198198
args: ["start-foreground",
199-
{{- if eq .Values.checkpoint_store_type "azure" }}
199+
{{- if eq .Values.cloud_service_provider "azure" }}
200200
"-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}",
201201
{{- end }}
202-
{{- if eq .Values.checkpoint_store_type "s3" }}
203-
"-Ds3.access-key={{ .Values.cloud_storage_key }}",
204-
"-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
205-
"-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
206-
"-Ds3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
202+
{{- if or (eq .Values.cloud_service_provider "aws") (eq .Values.cloud_service_provider "oci") }}
203+
"-Dpresto.s3.access-key={{ .Values.cloud_storage_key }}",
204+
"-Dpresto.s3.secret-key={{ .Values.cloud_storage_secret }}",
205+
"-Dpresto.s3.endpoint={{ .Values.cloud_storage_endpoint }}",
206+
"-Dpresto.s3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
207207
{{- end }}
208-
{{- if eq .Values.checkpoint_store_type "gcloud" }}
208+
{{- if eq .Values.cloud_service_provider "gcloud" }}
209209
"-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
210210
"-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
211211
"-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",

‎kubernetes/helm_charts/datapipeline_jobs/values.j2

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ imagepullsecrets: {{ imagepullsecrets }}
33
dockerhub: {{ dockerhub }}
44
repository: {{flink_repository|default('data-pipeline')}}
55
image_tag: {{ image_tag }}
6-
checkpoint_store_type: {{ checkpoint_store_type }}
6+
cloud_service_provider: {{ cloud_service_provider }}
77
cloud_storage_key: {{ cloud_storage_key }}
88
cloud_storage_secret: {{ cloud_storage_secret }}
99
cloud_storage_endpoint: {{ cloud_private_storage_endpoint }}
@@ -97,15 +97,15 @@ base_config: |
9797
statebackend {
9898
blob {
9999
storage {
100-
account = "{% if checkpoint_store_type == "azure" %}{{ cloud_storage_key }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% endif %}"
100+
account = "{% if cloud_service_provider == "azure" %}{{ cloud_storage_key }}.blob.core.windows.net{% elif cloud_service_provider in ["aws", "oci"] %}{{ cloud_storage_flink_bucketname }}{% endif %}"
101101
container = "{{ cloud_storage_flink_bucketname }}"
102102
checkpointing.dir = "checkpoint"
103103
}
104104
}
105-
{% if checkpoint_store_type == "azure" %}
105+
{% if cloud_service_provider == "azure" %}
106106
base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir}
107-
{% elif checkpoint_store_type == "s3" %}
108-
base.url = "s3://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
107+
{% elif cloud_service_provider in ["aws", "oci"] %}
108+
base.url = "s3://"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
109109
{% endif %}
110110
}
111111
}
@@ -184,7 +184,7 @@ activity-aggregate-updater:
184184
}
185185
redis {
186186
database {
187-
relationCache.id = 10
187+
relationCache.id = {{ redis_database_relation_cache_index | default(10) }}
188188
}
189189
}
190190
dedup-redis {
@@ -231,12 +231,12 @@ relation-cache-updater:
231231
table = "{{ middleware_content_hierarchy_table }}"
232232
}
233233
redis {
234-
database.index = 10
234+
database.index = {{ redis_database_index | default(10) }}
235235
}
236236
dp-redis {
237237
host = {{ dp_redis_host }}
238238
port = 6379
239-
database.index = 5
239+
database.index = {{ redis_dp_redis_database_index | default(5) }}
240240
}
241241

242242
flink-conf: |+
@@ -272,7 +272,7 @@ enrolment-reconciliation:
272272
}
273273
redis {
274274
database {
275-
relationCache.id = 10
275+
relationCache.id = {{ redis_database_relation_cache_index | default(10) }}
276276
}
277277
}
278278
threshold.batch.write.size = {{ enrolment_reconciliation_batch_write_size }}
@@ -317,6 +317,12 @@ collection-cert-pre-processor:
317317
assessment_aggregator.table = "{{ middleware_assessment_aggregator_table }}"
318318
user_activity_agg.table = "{{ middleware_user_activity_agg_table }}"
319319
}
320+
redis {
321+
database {
322+
contentCache.id = {{ redis_database_content_cache_index | default(5) }}
323+
collectionCache.id = {{ redis_collection_cache_index | default(0) }}
324+
}
325+
}
320326
cert_domain_url = "{{ cert_domain_url }}"
321327
user_read_api = "/private/user/v1/read"
322328
content_read_api = "/content/v3/read"
@@ -334,8 +340,11 @@ collection-cert-pre-processor:
334340
port = 6379
335341
}
336342
assessment.metrics.supported.contenttype = ["SelfAssess"]
343+
cert_cloud_storage_type = "{{ cert_cloud_storage_type }}"
344+
cert_cloud_storage_secret = "{{ cert_cloud_storage_secret }}"
345+
cert_cloud_storage_key = "{{ cert_cloud_storage_key }}"
337346
cloud_storage_base_url = "{{ cloud_storage_base_url }}"
338-
cloud_store_base_path_placeholder = "{{ cloud_store_base_path_placeholder }}"
347+
cloud_store_base_path_placeholder = "{{ cloud_store_base_path_placeholder | default('CLOUD_BASE_PATH')}}"
339348
content_cloud_storage_container = "{{ cloud_storage_content_bucketname }}"
340349
cloud_storage_cname_url = "{{ cloud_storage_cname_url }}"
341350

@@ -346,7 +355,7 @@ collection-cert-pre-processor:
346355
parallelism.default: 1
347356
jobmanager.execution.failover-strategy: region
348357
taskmanager.memory.network.fraction: 0.1
349-
358+
350359
collection-certificate-generator:
351360
collection-certificate-generator: |+
352361
include file("/data/flink/conf/base-config.conf")
@@ -372,9 +381,10 @@ collection-certificate-generator:
372381
cert_container_name = "{{ cert_container_name }}"
373382
cert_cloud_storage_type = "{{ cert_cloud_storage_type }}"
374383
cert_cloud_storage_secret = "{{ cert_cloud_storage_secret }}"
384+
cert_cloud_storage_endpoint = "{{cert_cloud_storage_endpoint}}"
375385
cert_cloud_storage_key = "{{ cert_cloud_storage_key }}"
376386
cloud_storage_base_url = "{{ cloud_storage_base_url }}"
377-
cloud_store_base_path_placeholder = "{{ cloud_store_base_path_placeholder }}"
387+
cloud_store_base_path_placeholder = "{{ cloud_store_base_path_placeholder | default('CLOUD_BASE_PATH')}}"
378388
content_cloud_storage_container = "{{ cloud_storage_content_bucketname }}"
379389
cloud_storage_cname_url = "{{ cloud_storage_cname_url }}"
380390
service {
@@ -508,8 +518,8 @@ assessment-aggregator:
508518
}
509519
redis {
510520
database {
511-
relationCache.id = 10
512-
contentCache.id = 5
521+
relationCache.id = {{ redis_database_relation_cache_index | default(10) }}
522+
contentCache.id = {{ redis_database_content_cache_index | default(5) }}
513523
}
514524
}
515525
assessment.skip.missingRecords = true
@@ -580,7 +590,7 @@ user-cache-updater-v2:
580590
# redis-metadata
581591
redis-meta {
582592
database {
583-
userstore.id = 12
593+
userstore.id = {{ redis_meta_database_user_index | default(12) }}
584594
key.expiry.seconds = 3600
585595
}
586596
}

‎kubernetes/pipelines/build/Jenkinsfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ node('build-slave') {
2525
env.NODE_ENV = "build"
2626
print "Environment will be : ${env.NODE_ENV}"
2727
sh '/opt/apache-maven-3.6.3/bin/mvn3.6 -v'
28-
sh '/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests'
29-
28+
sh "/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests -DCLOUD_STORE_GROUP_ID=${params.cloud_store_group_id} -DCLOUD_STORE_ARTIFACT_ID=${params.cloud_store_artifact_id} -DCLOUD_STORE_VERSION=${params.cloud_store_version}"
29+
3030
}
3131

3232
stage('Package') {

‎kubernetes/pipelines/build/auto_build_deploy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ node('build-slave') {
2929
// stage Build
3030
env.NODE_ENV = "build"
3131
print "Environment will be : ${env.NODE_ENV}"
32-
sh '/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests'
32+
sh "/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests -DCLOUD_STORE_GROUP_ID=${params.cloud_store_group_id} -DCLOUD_STORE_ARTIFACT_ID=${params.cloud_store_artifact_id} -DCLOUD_STORE_VERSION=${params.cloud_store_version}"
3333

3434
// stage Package
3535
dir('jobs-distribution') {

‎lms-jobs/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,21 @@ class ContentConsumptionDeDupFunction(config: ActivityAggregateUpdaterConfig)(im
4343
val contents = eData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala
4444
val filteredContents = contents.filter(x => x.get("status") == 2).toList
4545
if (filteredContents.size == 0)
46-
metrics.incCounter(config.skipEventsCount)
46+
{
47+
metrics.incCounter(config.skipEventsCount)
48+
}
4749
else
48-
metrics.incCounter(config.batchEnrolmentUpdateEventCount)
49-
filteredContents.map(c => {
50+
{
51+
metrics.incCounter(config.batchEnrolmentUpdateEventCount)
52+
}
53+
val fc = filteredContents.map(c => {
5054
(eData + ("contents" -> List(Map("contentId" -> c.get("contentId"), "status" -> c.get("status"))))).toMap
51-
}).filter(e => discardDuplicates(e)).foreach(d => context.output(config.uniqueConsumptionOutput, d))
52-
} else metrics.incCounter(config.skipEventsCount)
55+
}).filter(e => discardDuplicates(e))
56+
fc.foreach(d => context.output(config.uniqueConsumptionOutput, d))
57+
} else
58+
{
59+
metrics.incCounter(config.skipEventsCount)
60+
}
5361
}
5462

5563
override def metricsList(): List[String] = {

0 commit comments

Comments
 (0)