Merge branch 'spark2a-template' into 'main'

Update to spark2a templates

Closes #2 and #1

See merge request nvspark/nds!5

Author: wjxiz1992

power_run_cpu.template

@@ -0,0 +1,34 @@
+#
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+export CUDF_JAR=cudf-22.02.0.jar
+export SPARK_RAPIDS_PLUGIN_JAR=rapids-4-spark_2.12-22.02.0.jar
+export SPARK_HOME=/usr/lib/spark
+$SPARK_HOME/bin/spark-shell \
+--master yarn \
+--conf spark.rapids.sql.enabled=false \
+--conf spark.executor.cores=16 \
+--conf spark.executor.memory=16G \
+--conf spark.driver.memory=50G \
+--conf spark.sql.adaptive.enabled=true \
+--conf spark.locality.wait=0 \
+--conf spark.sql.shuffle.partitions=128 \
+--conf spark.executor.instances=8 \
+--conf spark.dynamicAllocation.enabled=false \
+--conf spark.scheduler.mode=FIFO \
+--conf spark.sql.autoBroadcastJoinThreshold=10m \
+--conf spark.scheduler.minRegisteredResourcesRatio=1.0 \
+--conf spark.sql.broadcastTimeout=1200 \

power_run_gpu.template

@@ -16,28 +16,29 @@
 #
 export CUDF_JAR=cudf-22.02.0.jar
 export SPARK_RAPIDS_PLUGIN_JAR=rapids-4-spark_2.12-22.02.0.jar
-export SPARK_HOME=YOUR_SPARK_HOME
+export SPARK_HOME=/usr/lib/spark
 $SPARK_HOME/bin/spark-shell \
-  --master YOUR_SPARK_MASTER \
-  --driver-memory 10G \
-  --num-executors 1 \
-  --executor-memory 40G \
-  --executor-cores 12 \
-  --conf spark.task.cpus=1 \
-  --conf spark.task.resource.gpu.amount=0.05 \
-  --conf spark.rapids.sql.concurrentGpuTasks=2 \
-  --conf spark.plugins=com.nvidia.spark.SQLPlugin \
-  --conf spark.sql.files.maxPartitionBytes=2g \
-  --conf spark.rapids.memory.pinnedPool.size=8g \
-  --conf spark.rapids.sql.explain=NOT_ON_GPU \
-  --conf spark.rapids.sql.incompatibleOps.enabled=true \
-  --conf spark.rapids.sql.variableFloatAgg.enabled=true \
-  --conf spark.rapids.sql.csv.read.date.enabled=true \
-  --conf spark.rapids.sql.csvTimestamps.enabled=false \
-  --conf spark.rapids.sql.csv.read.integer.enabled=true \
-  --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=CORRECTED \
-  --conf spark.rapids.sql.castFloatToDecimal.enabled=true \
-  --conf spark.executor.resource.gpu.amount=1 \
-  --conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \
-  --files $SPARK_HOME/examples/src/main/scripts/getGpusResources.sh \
-  --jars $SPARK_RAPIDS_PLUGIN_JAR,$CUDF_JAR \
+--master yarn \
+--conf spark.rapids.memory.gpu.pool=ARENA \
+--conf spark.rapids.sql.batchSizeBytes=1GB \
+--conf spark.driver.maxResultSize=2GB \
+--conf spark.executor.instances=8 \
+--conf spark.executor.cores=16 \
+--conf spark.locality.wait=0 \
+--conf spark.rapids.sql.concurrentGpuTasks=2 \
+--conf spark.executor.resource.gpu.amount=1 \
+--conf spark.task.resource.gpu.amount=0.0625 \
+--conf spark.executor.memory=16G \
+--conf spark.driver.memory=50G \
+--conf spark.sql.shuffle.partitions=200 \
+--conf spark.sql.files.maxPartitionBytes=2gb \
+--conf spark.driver.extraClassPath=$SPARK_RAPIDS_PLUGIN_JAR:$CUDF_JAR \
+--conf spark.executor.extraClassPath=$SPARK_RAPIDS_PLUGIN_JAR:$CUDF_JAR \
+--conf spark.executor.extraJavaOptions=-Dai.rapids.cudf.nvtx.enabled=true \
+--conf spark.rapids.memory.host.spillStorageSize=32G \
+--conf spark.sql.adaptive.enabled=true \
+--conf spark.plugins=com.nvidia.spark.SQLPlugin \
+--conf spark.rapids.memory.pinnedPool.size=8g \
+--conf spark.rapids.shuffle.maxMetadataSize=512KB \
+--conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \
+--files $SPARK_HOME/examples/src/main/scripts/getGpusResources.sh \