From 4fe47b3a57f52f112cc4eed78cd0c4d663512c1c Mon Sep 17 00:00:00 2001
From: Matt Denno <mgdenno@gmail.com>
Date: Thu, 6 Jun 2024 16:15:09 -0400
Subject: [PATCH] add spark conf

---
 Dockerfile          | 3 +++
 spark-defaults.conf | 1 +
 2 files changed, 4 insertions(+)
 create mode 100644 spark-defaults.conf

diff --git a/Dockerfile b/Dockerfile
index f441f503..d3b680b9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,6 +34,9 @@ ENV SPARK_HOME ${NB_PYTHON_PREFIX}/lib/python3.11/site-packages/pyspark
 RUN curl -s https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar -Lo ${SPARK_HOME}/jars/aws-java-sdk-bundle-1.12.262.jar
 RUN curl -s https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar -Lo ${SPARK_HOME}/jars/hadoop-aws-3.3.4.jar
 
+RUN mkdir -p ${SPARK_HOME}/conf
+COPY spark-defaults.conf ${SPARK_HOME}/conf/spark-defaults.conf
+
 WORKDIR /teehr
 
 RUN pip install duckdb spatialpandas easydev colormap colorcet hydrotools
diff --git a/spark-defaults.conf b/spark-defaults.conf
new file mode 100644
index 00000000..91b77ea6
--- /dev/null
+++ b/spark-defaults.conf
@@ -0,0 +1 @@
+spark.driver.memory                    96g
\ No newline at end of file