diff --git a/site/docs/libraries/installation/spark.md b/site/docs/libraries/installation/spark.md index 38f538319d..3ad8d7edb2 100644 --- a/site/docs/libraries/installation/spark.md +++ b/site/docs/libraries/installation/spark.md @@ -28,22 +28,45 @@ import TabItem from "@theme/TabItem"; ```python -from pathling import PathlingContext, find_jar +from pathling import PathlingContext from pyspark.sql import SparkSession spark = ( - SparkSession.builder - .config("spark.jars", find_jar()) - .config("spark.jars.packages", "io.delta:delta-core_2.12:2.2.0") - .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .config("spark.sql.catalog.spark_catalog", - "org.apache.spark.sql.delta.catalog.DeltaCatalog") - .getOrCreate() + SparkSession.builder.config( + "spark.jars.packages", + "au.csiro.pathling:library-runtime:6.4.2," + "io.delta:delta-core_2.12:2.4.0," + ) + .config( + "spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension" + ) + .config( + "spark.sql.catalog.spark_catalog", + "org.apache.spark.sql.delta.catalog.DeltaCatalog", + ) ) pc = PathlingContext.create(spark) ``` + + + +```r +library(sparklyr) +library(pathling) + +sc <- spark_connect(master = "local", + packages = c(paste("au.csiro.pathling:library-runtime:", pathling_version()), + "io.delta:delta-core_2.12:2.4.0"), + config = list("sparklyr.shell.conf" = c( + "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension", + "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" + )), version = "3.4.0") + +pc <- pathling_connect(sc) +``` + @@ -51,7 +74,8 @@ pc = PathlingContext.create(spark) import au.csiro.pathling.library.PathlingContext val spark = SparkSession.builder - .config("spark.jars.packages", "io.delta:delta-core_2.12:2.2.0") + .config("spark.jars.packages", "au.csiro.pathling:library-runtime:6.4.2," + + "io.delta:delta-core_2.12:2.4.0") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") @@ -71,7 +95,9 @@ class MyApp { public static void main(String[] args) { SparkSession spark = SparkSession.builder() - .config("spark.jars.packages", "io.delta:delta-core_2.12:2.2.0") + .config("spark.jars.packages", + "au.csiro.pathling:library-runtime:6.4.2," + + "io.delta:delta-core_2.12:2.4.0") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .config("spark.sql.catalog.spark_catalog",