From 7b454e61ec88176468163dfabfd8c8fc4e9834b8 Mon Sep 17 00:00:00 2001
From: vaibhj <vaibhj@microsoft.com>
Date: Mon, 2 Dec 2024 14:31:03 -0800
Subject: [PATCH] Passing Default Storage for avoiding workspace network
setting policies (#3449)
* Passing Default Storage for avoiding workspace network setting policies
* trying quick fix
* fix
* fix
---
sdk/python/jobs/spark/setup_spark.sh | 10 +++++++---
.../submit_spark_standalone_jobs_managed_vnet.ipynb | 4 ++++
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/sdk/python/jobs/spark/setup_spark.sh b/sdk/python/jobs/spark/setup_spark.sh
index faea23617d9..9b6b5d4505a 100644
--- a/sdk/python/jobs/spark/setup_spark.sh
+++ b/sdk/python/jobs/spark/setup_spark.sh
@@ -1,6 +1,6 @@
# <create_variables>
SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-LOCATION=$(az ml workspace show --query location -o tsv)
+LOCATION="eastus"
RESOURCE_GROUP=$(az group show --query name -o tsv)
AML_WORKSPACE_NAME=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
API_VERSION="2022-05-01"
@@ -48,13 +48,16 @@ AML_USER_MANAGED_ID_OID=$(az identity show --resource-group $RESOURCE_GROUP -n $
#<setup_vnet_resources>
if [[ "$2" == *"managed_vnet"* ]]
then
- TIMESTAMP=`date +%m%d%H%M%S`
+ TIMESTAMP=`date +%m%d%H%M`
AML_WORKSPACE_NAME=${AML_WORKSPACE_NAME}-vnet-$TIMESTAMP
AZURE_STORAGE_ACCOUNT=${RESOURCE_GROUP}blobvnet
+ DEFAULT_STORAGE_ACCOUNT="sparkdefaultvnet"
BLOB_CONTAINER_NAME="blobstoragevnetcontainer"
GEN2_STORAGE_ACCOUNT_NAME=${RESOURCE_GROUP}gen2vnet
ADLS_CONTAINER_NAME="gen2containervnet"
+ az storage account create -n $DEFAULT_STORAGE_ACCOUNT -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS
+
az storage account create -n $AZURE_STORAGE_ACCOUNT -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS
az storage container create -n $BLOB_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT
@@ -80,7 +83,8 @@ then
s/<ACCESS_KEY_SECRET_NAME>/$ACCESS_KEY_SECRET_NAME/g;
s/<BLOB_CONTAINER_NAME>/$BLOB_CONTAINER_NAME/g;
s/<GEN2_STORAGE_ACCOUNT_NAME>/$GEN2_STORAGE_ACCOUNT_NAME/g;
- s/<ADLS_CONTAINER_NAME>/$ADLS_CONTAINER_NAME/g;" $2
+ s/<ADLS_CONTAINER_NAME>/$ADLS_CONTAINER_NAME/g
+ s/<DEFAULT_STORAGE_ACCOUNT>/$DEFAULT_STORAGE_ACCOUNT/g;" $2
#</setup_vnet_resources>
#<setup_interactive_session_resources>
elif [[ "$2" == *"run_interactive_session_notebook"* ]]
diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb
index cfa920dfae9..eace1dce406 100644
--- a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb
+++ b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb
@@ -114,6 +114,9 @@
"region = \"<AZURE_REGION_NAME>\"\n",
"# Enter Azure Blob storage account name for the outbound rule\n",
"blob_storage_account = \"<STORAGE_ACCOUNT_NAME>\"\n",
+ "default_storage_account = \"<DEFAULT_STORAGE_ACCOUNT>\"\n",
+ "\n",
+ "default_storage_resource_id = f\"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Storage/storageAccounts/{default_storage_account}\"\n",
"\n",
"ws_mvnet = Workspace(\n",
" name=ws_name,\n",
@@ -121,6 +124,7 @@
" hbi_workspace=False,\n",
" public_network_access=\"Disabled\", # Comment this out to enable Public Network Access\n",
" tags=dict(purpose=\"demo\"),\n",
+ " storage_account=default_storage_resource_id,\n",
")\n",
"\n",
"ws_mvnet.managed_network = ManagedNetwork(\n",