Enable create new ws for OAI v2 and changed chat dataset format

.github/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml

@@ -34,6 +34,7 @@ jobs:
           creds: ${{secrets.AZUREML_CREDENTIALS}}
       - name: bootstrap resources
         run: |
+          export RUN_BOOTSTRAP='true'
           echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
           bash bootstrap_oai_v2.sh
         working-directory: infra/bootstrapping

.github/workflows/sdk-foundation-models-azure_openai-oai-v2-openai_chat_finetune_pipeline.yml

@@ -40,6 +40,7 @@ jobs:
         creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: bootstrap resources
       run: |
+          export RUN_BOOTSTRAP='true'
           echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
           bash bootstrap_oai_v2.sh
       working-directory: infra/bootstrapping

cli/foundation-models/azure_openai/oai-v1/dataset-create.yaml

@@ -1,5 +1,5 @@
 $schema: https://azuremlschemas.azureedge.net/latest/data.schema.json
 type: uri_folder
-name: oai_sample_training_data
+name: oai_sample_training_data_chat
 description: sample training data
 path: data/

cli/foundation-models/azure_openai/oai-v1/openai_completions_finetune/openai_completions_finetune_spec.yaml

@@ -15,11 +15,11 @@ jobs:
     inputs:
       train_dataset: 
         type: uri_folder
-        path: azureml:oai_sample_training_data:1
+        path: azureml:oai_sample_training_data_chat:1
         mode: ro_mount
       validation_dataset: 
         type: uri_folder
-        path: azureml:oai_sample_training_data:1
+        path: azureml:oai_sample_training_data_chat:1
         mode: ro_mount
       registered_model_name: ft_ada_model_from_cli_doc
       model: ${{parent.inputs.model_name}}

cli/foundation-models/azure_openai/oai-v2/dataset-create.yaml

@@ -1,5 +1,5 @@
 $schema: https://azuremlschemas.azureedge.net/latest/data.schema.json
 type: uri_folder
-name: oai_sample_training_data
+name: oai_sample_training_data_chat
 description: sample training data
 path: data/

cli/foundation-models/azure_openai/oai-v2/openai_completions_finetune_pipeline/openai_completions_finetune_pipeline_spec.yaml

@@ -20,10 +20,10 @@ jobs:
     inputs:
       train_dataset:
         type: uri_folder
-        path: azureml:oai_sample_training_data:1
+        path: azureml:oai_sample_training_data_chat:1
       validation_dataset:
         type: uri_folder
-        path: azureml:oai_sample_training_data:1
+        path: azureml:oai_sample_training_data_chat:1
       registered_model_name: ${{parent.inputs.registered_model_name}}
       model: ${{parent.inputs.model}}
       task_type: ${{parent.inputs.task_type}}

infra/bootstrapping/bootstrap_oai_v2.sh

@@ -71,6 +71,7 @@ echo_title "RESOURCE_GROUP_NAME = \"${RESOURCE_GROUP_NAME}\" & LOCATION=\"${LOCA
 az configure --defaults group="${RESOURCE_GROUP_NAME}" workspace="${WORKSPACE_NAME}" location="${LOCATION}"  # for subsequent commands.
 az account set -s "${SUBSCRIPTION_ID}" || exit 1
 
+LOCATION=${LOCATION:-"northcentralus"}
 
 # RUN_BOOTSTRAP=1
 if [[ ! -z "${RUN_BOOTSTRAP:-}" ]]; then
@@ -82,81 +83,10 @@ if [[ ! -z "${RUN_BOOTSTRAP:-}" ]]; then
     "$SCRIPT_DIR"/sdk_helpers.sh ensure_ml_workspace "mlw-mevnet"
     "$SCRIPT_DIR"/sdk_helpers.sh ensure_vnet "vnet-mevnet"
     "$SCRIPT_DIR"/sdk_helpers.sh ensure_subnet "vnet-mevnet" "snet-scoring"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_identity "uaimevnet"
-    "$SCRIPT_DIR"/sdk_helpers.sh grant_permission_identity_on_acr "uaimevnet"
 
     echo_title "Ensuring Permissions on RG"
     "$SCRIPT_DIR"/sdk_helpers.sh grant_permission_app_id_on_rg "${APP_NAME}"
 
-    echo_title "Ensuring Registry ${REGISTRY_NAME}"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_registry "${REGISTRY_NAME}"
-    echo_title "Ensuring Registry of tomorrow ${REGISTRY_NAME_TOMORROW}"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_registry "${REGISTRY_NAME_TOMORROW}"
-
-    echo_title "Ensuring CPU compute"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "cpu-cluster" 0 20 "Standard_DS3_v2"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "automl-cpu-cluster" 0 4 "Standard_DS3_v2"
-    # Larger CPU cluster for Dask and Spark examples
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "cpu-cluster-lg" 0 4 "Standard_DS15_v2"
-
-    echo_title "Ensuring GPU compute"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "gpu-cluster" 0 20 "STANDARD_NC6s_v3"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "automl-gpu-cluster" 0 4 "STANDARD_NC6s_v3"
-    # v100 single GPU cluster for pytorch 2.0 based notebooks
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "gpu-v100-1GPU-cluster" 0 4 "Standard_NC6s_v3"
-    # v100 GPU cluster for deepspeed cli examples
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "gpu-v100-cluster" 0 2 "Standard_ND40rs_v2"
-
-    echo_title "Running prerequisites"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_prerequisites_in_workspace
-    "$SCRIPT_DIR"/sdk_helpers.sh update_dataset
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_prerequisites_in_registry
-
-    "$SCRIPT_DIR"/sdk_helpers.sh register_providers
-
-    echo_title "Creating AKS clusters."
-    configure_aks_cluster=(
-      aks-cpu-is
-      aks-cpu-ml
-      aks-cpu-od
-      aks-cpu-mc
-      scoring-explain
-    )
-    for aks_compute in "${configure_aks_cluster[@]}"; do
-      (
-        echo_info "Creating AKS cluster: '$aks_compute'"
-        "$SCRIPT_DIR"/sdk_helpers.sh ensure_aks_compute "${aks_compute}" 1 3 "STANDARD_D3_V2"
-      ) &
-    done
-    wait # until all AKS are created
-    for aks_compute in "${configure_aks_cluster[@]}"; do
-      (
-        echo_info "Attaching AKS cluster: '$aks_compute'"
-        "$SCRIPT_DIR"/sdk_helpers.sh install_k8s_extension "${aks_compute}" "managedClusters" "Microsoft.ContainerService/managedClusters"
-        "$SCRIPT_DIR"/sdk_helpers.sh setup_compute "${aks_compute}" "${aks_compute}" "managedClusters" "azureml"
-      )
-    done
-    echo_info ">>> Done creating AKS clusters"
-
-    # Arc cluster configuration
-    configure_arc_cluster=(
-      ${ARC_CLUSTER_NAME}
-    )
-    for arc_compute in "${configure_arc_cluster[@]}"; do
-      (
-        echo_info "Creating amlarc cluster: '$arc_compute'"
-        "$SCRIPT_DIR"/sdk_helpers.sh ensure_aks_compute "${arc_compute}" 1 3 "STANDARD_D3_V2"
-        "$SCRIPT_DIR"/sdk_helpers.sh install_k8s_extension "${arc_compute}" "connectedClusters" "Microsoft.Kubernetes/connectedClusters"
-        "$SCRIPT_DIR"/sdk_helpers.sh setup_compute "${arc_compute}-arc" "${ARC_COMPUTE_NAME}" "connectedClusters" "azureml"
-        "$SCRIPT_DIR"/sdk_helpers.sh setup_instance_type_aml_arc "${arc_compute}"
-      )
-    done
-    echo_info ">>> Done creating amlarc clusters"
-    "$SCRIPT_DIR"/sdk_helpers.sh vmss_upgrade_policy_all_rg
-    # echo_title "Copying data"
-    # "$SCRIPT_DIR"/sdk_helpers.sh install_azcopy
-    # "$SCRIPT_DIR"/sdk_helpers.sh copy_dataset
-
 else
     "$SCRIPT_DIR"/sdk_helpers.sh update_dataset
     echo_info "Skipping Bootstrapping. Set the RUN_BOOTSTRAP environment variable to enable bootstrapping."