chore: CWSM export job: support running with on demand nodes.

edx · Nov 28, 2023 · d91f879 · d91f879
1 parent 0f82b6a
commit d91f879
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/dataeng/resources/database-export-courseware-studentmodule.sh b/dataeng/resources/database-export-courseware-studentmodule.sh
@@ -4,7 +4,9 @@ env
 
 DAY_OF_MONTH=$(date +%d)
 OUTPUT_URL=$BASE_OUTPUT_URL/$DAY_OF_MONTH/$OUTPUT_DIR
-NUM_REDUCE_TASKS=$(( $NUM_TASK_CAPACITY + 1 ))
+NUM_CAPACITY=$(( $NUM_TASK_CAPACITY + $ON_DEMAND_CAPACITY ))
+NUM_MAPPERS=$(( ($NUM_CAPACITY + 1) * 2 ))
+NUM_REDUCE_TASKS=$(( $NUM_CAPACITY + 1 ))
 
 ${WORKSPACE}/analytics-configuration/automation/run-automated-task.sh \
   StudentModulePerCourseAfterImportWorkflow --local-scheduler \
@@ -13,7 +15,7 @@ ${WORKSPACE}/analytics-configuration/automation/run-automated-task.sh \
   --output-root $OUTPUT_URL/ \
   --delete-output-root \
   --output-suffix $OUTPUT_SUFFIX \
-  --num-mappers $(( ($NUM_TASK_CAPACITY + 1) * 2 )) \
+  --num-mappers $NUM_MAPPERS \
   --n-reduce-tasks $NUM_REDUCE_TASKS \
   --verbose \
   $EXTRA_ARGS