06C - Parallel Run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://docs.microsoft.com/zh-tw/azure/machine-learning/how-to-use-parallel-run-step\n",
    "from azureml.core import Datastore\n",
    "from azureml.core import Workspace\n",
    "ws = Workspace.from_config()\n",
    "\n",
    "mnist_blob = Datastore.register_azure_blob_container(ws, \n",
    "                      datastore_name=\"mnistdatastore\", \n",
    "                      container_name=\"mnistdata\", \n",
    "                      account_name=\"lab04astorage\",\n",
    "                      account_key='q46wVHJlcwlfeab2yg2dqPePM4RmEq8J26Bp4fN86luRKr/ign0SSBcYbVBui0fHrQ5tkVxY+naBC/2OLdsBkA==',\n",
    "                      overwrite=True\n",
    "                      )\n",
    "def_data_store = Datastore.get(ws, 'mnistdatastore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core.dataset import Dataset\n",
    "\n",
    "path_on_datastore = mnist_blob.path('mnist/')\n",
    "input_mnist_ds = Dataset.File.from_files(path=path_on_datastore, validate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.data.dataset_consumption_config import DatasetConsumptionConfig\n",
    "from azureml.pipeline.core import PipelineParameter\n",
    "\n",
    "pipeline_param = PipelineParameter(name=\"mnist_param\", default_value=input_mnist_ds)\n",
    "input_mnist_ds_consumption = DatasetConsumptionConfig(\"minist_param_config\", pipeline_param).as_mount()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.pipeline.core import Pipeline, PipelineData\n",
    "\n",
    "output_dir = PipelineData(name=\"inferences\", datastore=def_data_store)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "import urllib.request\n",
    "\n",
    "model_dir = 'models'\n",
    "if not os.path.isdir(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "\n",
    "url=\"https://pipelinedata.blob.core.windows.net/mnist-model/mnist-tf.tar.gz\"\n",
    "response = urllib.request.urlretrieve(url, \"model.tar.gz\")\n",
    "tar = tarfile.open(\"model.tar.gz\", \"r:gz\")\n",
    "tar.extractall(model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Registering model mnist\n"
     ]
    }
   ],
   "source": [
    "from azureml.core.model import Model\n",
    "\n",
    "# Register the downloaded model \n",
    "model = Model.register(model_path=\"models/\",\n",
    "                       model_name=\"mnist\",\n",
    "                       tags={'pretrained': \"mnist\"},\n",
    "                       description=\"Mnist trained tensorflow model\",\n",
    "                       workspace=ws)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting digit_identification.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile digit_identification.py\n",
    "\n",
    "# Snippets from a sample script.\n",
    "# Refer to the accompanying digit_identification.py\n",
    "# (https://aka.ms/batch-inference-notebooks)\n",
    "# for the implementation script.\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from PIL import Image\n",
    "from azureml.core import Model\n",
    "\n",
    "\n",
    "def init():\n",
    "    global g_tf_sess\n",
    "\n",
    "    # Pull down the model from the workspace\n",
    "    model_path = Model.get_model_path(\"mnist\")\n",
    "\n",
    "    # Construct a graph to execute\n",
    "    tf.reset_default_graph()\n",
    "    saver = tf.train.import_meta_graph(os.path.join(model_path, 'mnist-tf.model.meta'))\n",
    "    g_tf_sess = tf.Session()\n",
    "    saver.restore(g_tf_sess, os.path.join(model_path, 'mnist-tf.model'))\n",
    "\n",
    "\n",
    "def run(mini_batch):\n",
    "#     print(f'run method start: {__file__}, run({mini_batch})')\n",
    "    print(f'run method start')\n",
    "    resultList = []\n",
    "    in_tensor = g_tf_sess.graph.get_tensor_by_name(\"network/X:0\")\n",
    "    output = g_tf_sess.graph.get_tensor_by_name(\"network/output/MatMul:0\")\n",
    "\n",
    "    for image in mini_batch:\n",
    "        # Prepare each image\n",
    "        data = Image.open(image)\n",
    "        np_im = np.array(data).reshape((1, 784))\n",
    "        # Perform inference\n",
    "        inference_result = output.eval(feed_dict={in_tensor: np_im}, session=g_tf_sess)\n",
    "        # Find the best probability, and add it to the result list\n",
    "        best_result = np.argmax(inference_result)\n",
    "        resultList.append(\"{}: {}\".format(os.path.basename(image), best_result))\n",
    "\n",
    "    return resultList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core.environment import Environment\n",
    "from azureml.core.conda_dependencies import CondaDependencies\n",
    "from azureml.core.runconfig import DEFAULT_CPU_IMAGE\n",
    "\n",
    "batch_conda_deps = CondaDependencies.create(pip_packages=[\"tensorflow==1.15.2\", \"pillow\", \n",
    "                                                          \"azureml-core\", \"azureml-dataset-runtime[pandas, fuse]\"])\n",
    "\n",
    "batch_env = Environment(name=\"batch_environment\")\n",
    "batch_env.python.conda_dependencies = batch_conda_deps\n",
    "batch_env.docker.enabled = True\n",
    "batch_env.docker.base_image = DEFAULT_CPU_IMAGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing cluster, use it.\n",
      "Succeeded\n",
      "AmlCompute wait for completion finished\n",
      "\n",
      "Minimum number of nodes requested have been provisioned\n",
      "{'currentNodeCount': 0, 'targetNodeCount': 0, 'nodeStateCounts': {'preparingNodeCount': 0, 'runningNodeCount': 0, 'idleNodeCount': 0, 'unusableNodeCount': 0, 'leavingNodeCount': 0, 'preemptedNodeCount': 0}, 'allocationState': 'Steady', 'allocationStateTransitionTime': '2020-08-02T08:26:08.535000+00:00', 'errors': None, 'creationTime': '2020-08-02T06:28:59.179593+00:00', 'modifiedTime': '2020-08-02T06:29:16.479532+00:00', 'provisioningState': 'Succeeded', 'provisioningStateTransitionTime': None, 'scaleSettings': {'minNodeCount': 0, 'maxNodeCount': 2, 'nodeIdleTimeBeforeScaleDown': 'PT120S'}, 'vmPriority': 'Dedicated', 'vmSize': 'STANDARD_DS2_V2'}\n"
     ]
    }
   ],
   "source": [
    "from azureml.core.compute import AmlCompute, ComputeTarget\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "\n",
    "# choose a name for your cluster\n",
    "cluster_name = \"LAB05B-Cluster\" # change to your compute cluster name\n",
    "\n",
    "# Verify that cluster exists\n",
    "try:\n",
    "    compute_target = ComputeTarget(workspace=ws, name=cluster_name)\n",
    "    print('Found existing cluster, use it.')\n",
    "except ComputeTargetException:\n",
    "    # If not, create it\n",
    "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_DS2_V2', max_nodes=2)\n",
    "    compute_target = ComputeTarget.create(ws, cluster_name, compute_config)\n",
    "    \n",
    "compute_target.wait_for_completion(show_output=True)\n",
    "print(compute_target.get_status().serialize())\n",
    "\n",
    "#if compute_name in ws.compute_targets:\n",
    "#    compute_target = ws.compute_targets[compute_name]\n",
    "#    if compute_target and type(compute_target) is AmlCompute:\n",
    "#        print('found compute target. just use it. ' + compute_name)\n",
    "#else:\n",
    "#    print('creating a new compute target...')\n",
    "#    provisioning_config = AmlCompute.provisioning_configuration(vm_size = vm_size,\n",
    "#                                                                min_nodes = compute_min_nodes, \n",
    "#                                                                max_nodes = compute_max_nodes)\n",
    "#\n",
    "#    # create the cluster\n",
    "#    compute_target = ComputeTarget.create(ws, compute_name, provisioning_config)\n",
    "#    \n",
    "#    # can poll for a minimum number of nodes and for a specific timeout. \n",
    "#    # if no min node count is provided it will use the scale settings for the cluster\n",
    "#    compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
    "#    \n",
    "#     # For a more detailed view of current AmlCompute status, use get_status()\n",
    "#    print(compute_target.get_status().serialize())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.pipeline.core import PipelineParameter\n",
    "from azureml.pipeline.steps import ParallelRunConfig\n",
    "\n",
    "parallel_run_config = ParallelRunConfig(\n",
    "    source_directory='.',\n",
    "    entry_script=\"digit_identification.py\",\n",
    "    mini_batch_size=PipelineParameter(name=\"batch_size_param\", default_value=\"5\"),\n",
    "    error_threshold=10,\n",
    "    output_action=\"append_row\",\n",
    "    append_row_file_name=\"mnist_outputs.txt\",\n",
    "    environment=batch_env,\n",
    "    compute_target=compute_target,\n",
    "    process_count_per_node=PipelineParameter(name=\"process_count_param\", default_value=2),\n",
    "    node_count=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/_parallel_run_step_base.py:555: UserWarning: \n",
      "ParallelRunStep requires azureml-dataset-runtime[fuse] for file dataset.\n",
      "Please add relevant package in CondaDependencies.\n",
      "  Please add relevant package in CondaDependencies.\"\"\".format(extra, self._input_ds_type), UserWarning)\n"
     ]
    }
   ],
   "source": [
    "from azureml.pipeline.steps import ParallelRunStep\n",
    "\n",
    "parallelrun_step = ParallelRunStep(\n",
    "    name=\"predict-digits-mnist\",\n",
    "    parallel_run_config=parallel_run_config,\n",
    "    inputs=[input_mnist_ds_consumption],\n",
    "    output=output_dir,\n",
    "    allow_reuse=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING - 'azureml-defaults' is not included in the pip package list in the environment definition in RunConfiguration. This may result in job failures during execution.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created step predict-digits-mnist [b181fa79][697b107a-ba59-47b5-964d-1e32ba217f70], (This step will run and generate new outputs)\n",
      "Submitted PipelineRun 3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/digit_identification/runs/3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2?wsid=/subscriptions/35241d74-3b9e-4778-bb92-4bb15e7b0410/resourcegroups/DP-100/workspaces/Lab01A\n"
     ]
    }
   ],
   "source": [
    "from azureml.pipeline.core import Pipeline\n",
    "from azureml.core.experiment import Experiment\n",
    "\n",
    "pipeline = Pipeline(workspace=ws, steps=[parallelrun_step])\n",
    "experiment = Experiment(ws, 'digit_identification')\n",
    "pipeline_run = experiment.submit(pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49183c5fdad04164957412196ef2911c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_PipelineWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Running\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/digit_identification/runs/3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2?wsid=/subscriptions/35241d74-3b9e-4778-bb92-4bb15e7b0410/resourcegroups/DP-100/workspaces/Lab01A\", \"run_id\": \"3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2\", \"run_properties\": {\"run_id\": \"3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2\", \"created_utc\": \"2020-08-02T09:09:10.88899Z\", \"properties\": {\"azureml.runsource\": \"azureml.PipelineRun\", \"runSource\": \"SDK\", \"runType\": \"SDK\", \"azureml.parameters\": \"{\\\"batch_size_param\\\":\\\"5\\\",\\\"process_count_param\\\":\\\"2\\\"}\"}, \"tags\": {\"azureml.pipelineComponent\": \"pipelinerun\"}, \"end_time_utc\": null, \"status\": \"Running\", \"log_files\": {\"logs/azureml/executionlogs.txt\": \"https://lab01a2002232825.blob.core.windows.net/azureml/ExperimentRun/dcid.3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2/logs/azureml/executionlogs.txt?sv=2019-02-02&sr=b&sig=1l2Jwb%2BUjimt6JHlRxNqP3CkzAIff73un2koFduDZpY%3D&st=2020-08-02T09%3A08%3A28Z&se=2020-08-02T17%3A18%3A28Z&sp=r\", \"logs/azureml/stderrlogs.txt\": \"https://lab01a2002232825.blob.core.windows.net/azureml/ExperimentRun/dcid.3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2/logs/azureml/stderrlogs.txt?sv=2019-02-02&sr=b&sig=%2Bg%2FAx1fDZpksQ54kv7aidSuRTM6QDMN6Xfiv5zOX2nY%3D&st=2020-08-02T09%3A08%3A28Z&se=2020-08-02T17%3A18%3A28Z&sp=r\", \"logs/azureml/stdoutlogs.txt\": \"https://lab01a2002232825.blob.core.windows.net/azureml/ExperimentRun/dcid.3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2/logs/azureml/stdoutlogs.txt?sv=2019-02-02&sr=b&sig=83%2FdHrdfFtbljWPKBrudv2LvxmHw4VjKC%2FGo%2BXUTQ48%3D&st=2020-08-02T09%3A08%3A28Z&se=2020-08-02T17%3A18%3A28Z&sp=r\"}, \"log_groups\": [[\"logs/azureml/executionlogs.txt\", \"logs/azureml/stderrlogs.txt\", \"logs/azureml/stdoutlogs.txt\"]], \"run_duration\": \"0:09:17\"}, \"child_runs\": [{\"run_id\": \"11e7ea3f-3758-4224-935a-614a78e5a267\", \"name\": \"predict-digits-mnist\", \"status\": \"Running\", \"start_time\": \"\", \"created_time\": \"2020-08-02T09:09:22.932621Z\", \"end_time\": \"\", \"duration\": \"0:09:06\", \"run_number\": 2, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-08-02T09:09:22.932621Z\", \"is_reused\": \"\"}], \"children_metrics\": {\"categories\": null, \"series\": null, \"metricName\": null}, \"run_metrics\": [], \"run_logs\": \"[2020-08-02 09:09:22Z] Submitting 1 runs, first five are: b181fa79:11e7ea3f-3758-4224-935a-614a78e5a267\\n\", \"graph\": {\"datasource_nodes\": {\"e594d6f4\": {\"node_id\": \"e594d6f4\", \"name\": \"mnist_param\"}}, \"module_nodes\": {\"b181fa79\": {\"node_id\": \"b181fa79\", \"name\": \"predict-digits-mnist\", \"status\": \"Running\", \"_is_reused\": false, \"run_id\": \"11e7ea3f-3758-4224-935a-614a78e5a267\"}}, \"edges\": [{\"source_node_id\": \"e594d6f4\", \"source_node_name\": \"mnist_param\", \"source_name\": \"data\", \"target_name\": \"minist_param_config\", \"dst_node_id\": \"b181fa79\", \"dst_node_name\": \"predict-digits-mnist\"}], \"child_runs\": [{\"run_id\": \"11e7ea3f-3758-4224-935a-614a78e5a267\", \"name\": \"predict-digits-mnist\", \"status\": \"Running\", \"start_time\": \"\", \"created_time\": \"2020-08-02T09:09:22.932621Z\", \"end_time\": \"\", \"duration\": \"0:09:06\", \"run_number\": 2, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-08-02T09:09:22.932621Z\", \"is_reused\": \"\"}]}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.10.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PipelineRunId: 3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/digit_identification/runs/3e9a0fd4-d35e-42c3-bc08-2788dc7ebab2?wsid=/subscriptions/35241d74-3b9e-4778-bb92-4bb15e7b0410/resourcegroups/DP-100/workspaces/Lab01A\n",
      "PipelineRun Status: Running\n",
      "\n",
      "\n",
      "StepRunId: 11e7ea3f-3758-4224-935a-614a78e5a267\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/digit_identification/runs/11e7ea3f-3758-4224-935a-614a78e5a267?wsid=/subscriptions/35241d74-3b9e-4778-bb92-4bb15e7b0410/resourcegroups/DP-100/workspaces/Lab01A\n",
      "StepRun( predict-digits-mnist ) Status: NotStarted\n",
      "StepRun( predict-digits-mnist ) Status: Queued\n",
      "\n",
      "Streaming azureml-logs/20_image_build_log.txt\n",
      "=============================================\n",
      "2020/08/02 09:11:14 Downloading source code...\n",
      "2020/08/02 09:11:15 Finished downloading source code\n",
      "StepRun( predict-digits-mnist ) Status: Running\n",
      "2020/08/02 09:11:16 Creating Docker network: acb_default_network, driver: 'bridge'\n",
      "2020/08/02 09:11:16 Successfully set up Docker network: acb_default_network\n",
      "2020/08/02 09:11:16 Setting up Docker configuration...\n",
      "2020/08/02 09:11:17 Successfully set up Docker configuration\n",
      "2020/08/02 09:11:17 Logging in to registry: lab01a91a38f72.azurecr.io\n",
      "2020/08/02 09:11:18 Successfully logged into lab01a91a38f72.azurecr.io\n",
      "2020/08/02 09:11:18 Executing step ID: acb_step_0. Timeout(sec): 5400, Working directory: '', Network: 'acb_default_network'\n",
      "2020/08/02 09:11:18 Scanning for dependencies...\n",
      "2020/08/02 09:11:19 Successfully scanned dependencies\n",
      "2020/08/02 09:11:19 Launching container with name: acb_step_0\n",
      "Sending build context to Docker daemon  60.93kB\n",
      "\n",
      "Step 1/15 : FROM mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200423.v1@sha256:a8f6491296cbc183e95e9de29b59098e97d50ea87b4f1afa93e2afd43afeaf6d\n",
      "sha256:a8f6491296cbc183e95e9de29b59098e97d50ea87b4f1afa93e2afd43afeaf6d: Pulling from azureml/intelmpi2018.3-ubuntu16.04\n",
      "fe703b657a32: Pulling fs layer\n",
      "f9df1fafd224: Pulling fs layer\n",
      "a645a4b887f9: Pulling fs layer\n",
      "57db7fe0b522: Pulling fs layer\n",
      "20b5fabe4f63: Pulling fs layer\n",
      "22898513a7dc: Pulling fs layer\n",
      "b77f65fcd9d7: Pulling fs layer\n",
      "132ebd5cd5ca: Pulling fs layer\n",
      "01991399be72: Pulling fs layer\n",
      "60c58ca14ef7: Pulling fs layer\n",
      "ca339bb1ce1b: Pulling fs layer\n",
      "22898513a7dc: Waiting\n",
      "b77f65fcd9d7: Waiting\n",
      "132ebd5cd5ca: Waiting\n",
      "01991399be72: Waiting\n",
      "60c58ca14ef7: Waiting\n",
      "ca339bb1ce1b: Waiting\n",
      "57db7fe0b522: Waiting\n",
      "20b5fabe4f63: Waiting\n",
      "f9df1fafd224: Verifying Checksum\n",
      "f9df1fafd224: Download complete\n",
      "a645a4b887f9: Verifying Checksum\n",
      "a645a4b887f9: Download complete\n",
      "57db7fe0b522: Verifying Checksum\n",
      "57db7fe0b522: Download complete\n",
      "fe703b657a32: Verifying Checksum\n",
      "fe703b657a32: Download complete\n",
      "22898513a7dc: Verifying Checksum\n",
      "22898513a7dc: Download complete\n",
      "b77f65fcd9d7: Verifying Checksum\n",
      "b77f65fcd9d7: Download complete\n",
      "20b5fabe4f63: Verifying Checksum\n",
      "20b5fabe4f63: Download complete\n",
      "132ebd5cd5ca: Verifying Checksum\n",
      "132ebd5cd5ca: Download complete\n",
      "60c58ca14ef7: Download complete\n",
      "ca339bb1ce1b: Verifying Checksum\n",
      "ca339bb1ce1b: Download complete\n",
      "01991399be72: Verifying Checksum\n",
      "01991399be72: Download complete\n",
      "fe703b657a32: Pull complete\n",
      "f9df1fafd224: Pull complete\n",
      "a645a4b887f9: Pull complete\n",
      "57db7fe0b522: Pull complete\n",
      "20b5fabe4f63: Pull complete\n",
      "22898513a7dc: Pull complete\n",
      "b77f65fcd9d7: Pull complete\n",
      "132ebd5cd5ca: Pull complete\n",
      "01991399be72: Pull complete\n",
      "60c58ca14ef7: Pull complete\n",
      "ca339bb1ce1b: Pull complete\n",
      "Digest: sha256:a8f6491296cbc183e95e9de29b59098e97d50ea87b4f1afa93e2afd43afeaf6d\n",
      "Status: Downloaded newer image for mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200423.v1@sha256:a8f6491296cbc183e95e9de29b59098e97d50ea87b4f1afa93e2afd43afeaf6d\n",
      " ---> a0dab4f7c804\n",
      "Step 2/15 : USER root\n",
      " ---> Running in c51a5c1427e7\n",
      "Removing intermediate container c51a5c1427e7\n",
      " ---> 64c9c0779f02\n",
      "Step 3/15 : RUN mkdir -p $HOME/.cache\n",
      " ---> Running in 672866c6e425\n",
      "Removing intermediate container 672866c6e425\n",
      " ---> 8979a26ee740\n",
      "Step 4/15 : WORKDIR /\n",
      " ---> Running in baa7543b3af4\n",
      "Removing intermediate container baa7543b3af4\n",
      " ---> 100e4ee15e33\n",
      "Step 5/15 : COPY azureml-environment-setup/99brokenproxy /etc/apt/apt.conf.d/\n",
      " ---> e84ab874a063\n",
      "Step 6/15 : RUN if dpkg --compare-versions `conda --version | grep -oE '[^ ]+$'` lt 4.4.11; then conda install conda==4.4.11; fi\n",
      " ---> Running in 7fd5941d4947\n",
      "Removing intermediate container 7fd5941d4947\n",
      " ---> 9630a6d22b54\n",
      "Step 7/15 : COPY azureml-environment-setup/mutated_conda_dependencies.yml azureml-environment-setup/mutated_conda_dependencies.yml\n",
      " ---> 7c8e7c3c5159\n",
      "Step 8/15 : RUN ldconfig /usr/local/cuda/lib64/stubs && conda env create -p /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e -f azureml-environment-setup/mutated_conda_dependencies.yml && rm -rf \"$HOME/.cache/pip\" && conda clean -aqy && CONDA_ROOT_DIR=$(conda info --root) && rm -rf \"$CONDA_ROOT_DIR/pkgs\" && find \"$CONDA_ROOT_DIR\" -type d -name __pycache__ -exec rm -rf {} + && ldconfig\n",
      " ---> Running in f5fbd106f4c6\n",
      "Warning: you have pip-installed dependencies in your environment file, but you do not list pip itself as one of your conda dependencies.  Conda may not use the correct pip to install your packages, and they may end up in the wrong place.  Please add an explicit pip dependency.  I'm adding one for you, but still nagging you.\n",
      "Collecting package metadata (repodata.json): ...working... \n",
      "done\n",
      "Solving environment: ...working... done\n",
      "\n",
      "Downloading and Extracting Packages\n",
      "\n",
      "python-3.6.2         | 27.0 MB   |            |   0% \n",
      "python-3.6.2         | 27.0 MB   | #1         |  12% \n",
      "python-3.6.2         | 27.0 MB   | ####2      |  43% \n",
      "python-3.6.2         | 27.0 MB   | #######1   |  71% \n",
      "python-3.6.2         | 27.0 MB   | ########## | 100% \n",
      "\n",
      "sqlite-3.23.1        | 1.5 MB    |            |   0% \n",
      "sqlite-3.23.1        | 1.5 MB    | ########## | 100% \n",
      "\n",
      "xz-5.2.5             | 438 KB    |            |   0% \n",
      "xz-5.2.5             | 438 KB    | ########## | 100% \n",
      "\n",
      "ca-certificates-2020 | 133 KB    |            |   0% \n",
      "ca-certificates-2020 | 133 KB    | ########## | 100% \n",
      "\n",
      "openssl-1.0.2u       | 3.1 MB    |            |   0% \n",
      "openssl-1.0.2u       | 3.1 MB    | ########## | 100% \n",
      "\n",
      "libstdcxx-ng-9.1.0   | 4.0 MB    |            |   0% \n",
      "libstdcxx-ng-9.1.0   | 4.0 MB    | ########## | 100% \n",
      "\n",
      "certifi-2020.6.20    | 160 KB    |            |   0% \n",
      "certifi-2020.6.20    | 160 KB    | ########## | 100% \n",
      "\n",
      "zlib-1.2.11          | 120 KB    |            |   0% \n",
      "zlib-1.2.11          | 120 KB    | ########## | 100% \n",
      "\n",
      "tk-8.6.10            | 3.2 MB    |            |   0% \n",
      "tk-8.6.10            | 3.2 MB    | ########## | 100% \n",
      "\n",
      "ncurses-6.0          | 907 KB    |            |   0% \n",
      "ncurses-6.0          | 907 KB    | ########## | 100% \n",
      "\n",
      "pip-20.1.1           | 2.0 MB    |            |   0% \n",
      "pip-20.1.1           | 2.0 MB    | ########## | 100% \n",
      "\n",
      "wheel-0.34.2         | 49 KB     |            |   0% \n",
      "wheel-0.34.2         | 49 KB     | ########## | 100% \n",
      "\n",
      "setuptools-49.2.0    | 929 KB    |            |   0% \n",
      "setuptools-49.2.0    | 929 KB    | ########## | 100% \n",
      "\n",
      "readline-7.0         | 387 KB    |            |   0% \n",
      "readline-7.0         | 387 KB    | ########## | 100% \n",
      "\n",
      "libffi-3.2.1         | 43 KB     |            |   0% \n",
      "libffi-3.2.1         | 43 KB     | ########## | 100% \n",
      "\n",
      "libedit-3.1          | 171 KB    |            |   0% \n",
      "libedit-3.1          | 171 KB    | ########## | 100% \n",
      "\n",
      "libgcc-ng-9.1.0      | 8.1 MB    |            |   0% \n",
      "libgcc-ng-9.1.0      | 8.1 MB    | ########   |  81% \n",
      "libgcc-ng-9.1.0      | 8.1 MB    | ########## | 100% \n",
      "Preparing transaction: ...working... done\n",
      "Verifying transaction: ...working... done\n",
      "Executing transaction: ...working... done\n",
      "Ran pip subprocess with arguments:\n",
      "['/azureml-envs/azureml_40740abd0db39f33e22f8442e072813e/bin/python', '-m', 'pip', 'install', '-U', '-r', '/azureml-environment-setup/condaenv.8jaujdn9.requirements.txt']\n",
      "Pip subprocess output:\n",
      "Collecting tensorflow==1.15.2\n",
      "  Downloading tensorflow-1.15.2-cp36-cp36m-manylinux2010_x86_64.whl (110.5 MB)\n",
      "Collecting pillow\n",
      "  Downloading Pillow-7.2.0-cp36-cp36m-manylinux1_x86_64.whl (2.2 MB)\n",
      "Collecting azureml-core~=1.10.0\n",
      "  Downloading azureml_core-1.10.0.post1-py3-none-any.whl (2.0 MB)\n",
      "Collecting azureml-dataset-runtime[fuse,pandas]~=1.10.0\n",
      "  Downloading azureml_dataset_runtime-1.10.0-py3-none-any.whl (3.2 kB)\n",
      "Collecting gast==0.2.2\n",
      "  Downloading gast-0.2.2.tar.gz (10 kB)\n",
      "Collecting wrapt>=1.11.1\n",
      "  Downloading wrapt-1.12.1.tar.gz (27 kB)\n",
      "Collecting six>=1.10.0\n",
      "  Downloading six-1.15.0-py2.py3-none-any.whl (10 kB)\n",
      "Collecting keras-preprocessing>=1.0.5\n",
      "  Downloading Keras_Preprocessing-1.1.2-py2.py3-none-any.whl (42 kB)\n",
      "Collecting tensorflow-estimator==1.15.1\n",
      "  Downloading tensorflow_estimator-1.15.1-py2.py3-none-any.whl (503 kB)\n",
      "Requirement already satisfied, skipping upgrade: wheel>=0.26; python_version >= \"3\" in /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e/lib/python3.6/site-packages (from tensorflow==1.15.2->-r /azureml-environment-setup/condaenv.8jaujdn9.requirements.txt (line 1)) (0.34.2)\n",
      "Collecting grpcio>=1.8.6\n",
      "  Downloading grpcio-1.30.0-cp36-cp36m-manylinux2010_x86_64.whl (3.0 MB)\n",
      "Collecting google-pasta>=0.1.6\n",
      "  Downloading google_pasta-0.2.0-py3-none-any.whl (57 kB)\n",
      "Collecting tensorboard<1.16.0,>=1.15.0\n",
      "  Downloading tensorboard-1.15.0-py3-none-any.whl (3.8 MB)\n",
      "Collecting absl-py>=0.7.0\n",
      "  Downloading absl-py-0.9.0.tar.gz (104 kB)\n",
      "Collecting opt-einsum>=2.3.2\n",
      "  Downloading opt_einsum-3.3.0-py3-none-any.whl (65 kB)\n",
      "Collecting protobuf>=3.6.1\n",
      "  Downloading protobuf-3.12.4-cp36-cp36m-manylinux1_x86_64.whl (1.3 MB)\n",
      "Collecting termcolor>=1.1.0\n",
      "  Downloading termcolor-1.1.0.tar.gz (3.9 kB)\n",
      "Collecting astor>=0.6.0\n",
      "  Downloading astor-0.8.1-py2.py3-none-any.whl (27 kB)\n",
      "Collecting keras-applications>=1.0.8\n",
      "  Downloading Keras_Applications-1.0.8-py3-none-any.whl (50 kB)\n",
      "Collecting numpy<2.0,>=1.16.0\n",
      "  Downloading numpy-1.19.1-cp36-cp36m-manylinux2010_x86_64.whl (14.5 MB)\n",
      "Collecting msrestazure>=0.4.33\n",
      "  Downloading msrestazure-0.6.4-py2.py3-none-any.whl (40 kB)\n",
      "Collecting SecretStorage\n",
      "  Downloading SecretStorage-3.1.2-py3-none-any.whl (14 kB)\n",
      "Collecting jsonpickle\n",
      "  Downloading jsonpickle-1.4.1-py2.py3-none-any.whl (36 kB)\n",
      "Collecting ruamel.yaml>0.16.7\n",
      "  Downloading ruamel.yaml-0.16.10-py2.py3-none-any.whl (111 kB)\n",
      "Collecting pyopenssl\n",
      "  Downloading pyOpenSSL-19.1.0-py2.py3-none-any.whl (53 kB)\n",
      "Collecting backports.tempfile\n",
      "  Downloading backports.tempfile-1.0-py2.py3-none-any.whl (4.4 kB)\n",
      "Collecting docker\n",
      "  Downloading docker-4.2.2-py2.py3-none-any.whl (144 kB)\n",
      "Collecting azure-graphrbac>=0.40.0\n",
      "  Downloading azure_graphrbac-0.61.1-py2.py3-none-any.whl (141 kB)\n",
      "Collecting cryptography!=1.9,!=2.0.*,!=2.1.*,!=2.2.*\n",
      "  Downloading cryptography-3.0-cp35-abi3-manylinux2010_x86_64.whl (2.7 MB)\n",
      "Collecting pytz\n",
      "  Downloading pytz-2020.1-py2.py3-none-any.whl (510 kB)\n",
      "Collecting azure-mgmt-containerregistry>=2.0.0\n",
      "  Downloading azure_mgmt_containerregistry-2.8.0-py2.py3-none-any.whl (718 kB)\n",
      "Collecting python-dateutil>=2.7.3\n",
      "  Downloading python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)\n",
      "Collecting azure-mgmt-authorization>=0.40.0\n",
      "  Downloading azure_mgmt_authorization-0.60.0-py2.py3-none-any.whl (82 kB)\n",
      "Collecting urllib3>=1.23\n",
      "  Downloading urllib3-1.25.10-py2.py3-none-any.whl (127 kB)\n",
      "Collecting adal>=1.2.0\n",
      "  Downloading adal-1.2.4-py2.py3-none-any.whl (55 kB)\n",
      "Collecting ndg-httpsclient\n",
      "  Downloading ndg_httpsclient-0.5.1-py3-none-any.whl (34 kB)\n",
      "Collecting msrest>=0.5.1\n",
      "  Downloading msrest-0.6.18-py2.py3-none-any.whl (84 kB)\n",
      "Collecting azure-common>=1.1.12\n",
      "  Downloading azure_common-1.1.25-py2.py3-none-any.whl (12 kB)\n",
      "Collecting PyJWT\n",
      "  Downloading PyJWT-1.7.1-py2.py3-none-any.whl (18 kB)\n",
      "Collecting requests>=2.19.1\n",
      "  Downloading requests-2.24.0-py2.py3-none-any.whl (61 kB)\n",
      "Collecting azure-mgmt-storage>=1.5.0\n",
      "  Downloading azure_mgmt_storage-11.1.0-py2.py3-none-any.whl (547 kB)\n",
      "Collecting azure-mgmt-resource>=1.2.1\n",
      "  Downloading azure_mgmt_resource-10.1.0-py2.py3-none-any.whl (892 kB)\n",
      "Collecting pathspec\n",
      "  Downloading pathspec-0.8.0-py2.py3-none-any.whl (28 kB)\n",
      "Collecting azure-mgmt-keyvault>=0.40.0\n",
      "  Downloading azure_mgmt_keyvault-2.2.0-py2.py3-none-any.whl (89 kB)\n",
      "Collecting contextlib2\n",
      "  Downloading contextlib2-0.6.0.post1-py2.py3-none-any.whl (9.8 kB)\n",
      "Collecting jmespath\n",
      "  Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\n",
      "Collecting pyarrow>=0.17.0\n",
      "  Downloading pyarrow-1.0.0-cp36-cp36m-manylinux2014_x86_64.whl (17.2 MB)\n",
      "Collecting azureml-dataprep~=1.9\n",
      "  Downloading azureml_dataprep-1.10.1-py3-none-any.whl (27.7 MB)\n",
      "Collecting fusepy>=3.0.1; extra == \"fuse\"\n",
      "  Downloading fusepy-3.0.1.tar.gz (11 kB)\n",
      "Collecting pandas>=0.23.4; extra == \"pandas\"\n",
      "  Downloading pandas-1.1.0-cp36-cp36m-manylinux1_x86_64.whl (10.5 MB)\n",
      "Requirement already satisfied, skipping upgrade: setuptools>=41.0.0 in /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e/lib/python3.6/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.2->-r /azureml-environment-setup/condaenv.8jaujdn9.requirements.txt (line 1)) (49.2.0.post20200714)\n",
      "Collecting markdown>=2.6.8\n",
      "  Downloading Markdown-3.2.2-py3-none-any.whl (88 kB)\n",
      "Collecting werkzeug>=0.11.15\n",
      "  Downloading Werkzeug-1.0.1-py2.py3-none-any.whl (298 kB)\n",
      "Collecting h5py\n",
      "  Downloading h5py-2.10.0-cp36-cp36m-manylinux1_x86_64.whl (2.9 MB)\n",
      "Collecting jeepney>=0.4.2\n",
      "  Downloading jeepney-0.4.3-py3-none-any.whl (21 kB)\n",
      "Collecting importlib-metadata\n",
      "  Downloading importlib_metadata-1.7.0-py2.py3-none-any.whl (31 kB)\n",
      "Collecting ruamel.yaml.clib>=0.1.2; platform_python_implementation == \"CPython\" and python_version < \"3.9\"\n",
      "  Downloading ruamel.yaml.clib-0.2.0-cp36-cp36m-manylinux1_x86_64.whl (548 kB)\n",
      "Collecting backports.weakref\n",
      "  Downloading backports.weakref-1.0.post1-py2.py3-none-any.whl (5.2 kB)\n",
      "Collecting websocket-client>=0.32.0\n",
      "  Downloading websocket_client-0.57.0-py2.py3-none-any.whl (200 kB)\n",
      "Collecting cffi!=1.11.3,>=1.8\n",
      "  Downloading cffi-1.14.1-cp36-cp36m-manylinux1_x86_64.whl (400 kB)\n",
      "Collecting pyasn1>=0.1.1\n",
      "  Downloading pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)\n",
      "Collecting isodate>=0.6.0\n",
      "  Downloading isodate-0.6.0-py2.py3-none-any.whl (45 kB)\n",
      "Collecting requests-oauthlib>=0.5.0\n",
      "  Downloading requests_oauthlib-1.3.0-py2.py3-none-any.whl (23 kB)\n",
      "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e/lib/python3.6/site-packages (from msrest>=0.5.1->azureml-core~=1.10.0->-r /azureml-environment-setup/condaenv.8jaujdn9.requirements.txt (line 3)) (2020.6.20)\n",
      "Collecting idna<3,>=2.5\n",
      "  Downloading idna-2.10-py2.py3-none-any.whl (58 kB)\n",
      "Collecting chardet<4,>=3.0.2\n",
      "  Downloading chardet-3.0.4-py2.py3-none-any.whl (133 kB)\n",
      "Collecting azureml-dataprep-native<15.0.0,>=14.2.1\n",
      "  Downloading azureml_dataprep_native-14.2.1-cp36-cp36m-manylinux1_x86_64.whl (1.3 MB)\n",
      "Collecting azure-identity<1.3.0,>=1.2.0\n",
      "  Downloading azure_identity-1.2.0-py2.py3-none-any.whl (58 kB)\n",
      "Collecting cloudpickle>=1.1.0\n",
      "  Downloading cloudpickle-1.5.0-py3-none-any.whl (22 kB)\n",
      "Collecting dotnetcore2>=2.1.14\n",
      "  Downloading dotnetcore2-2.1.14-py3-none-manylinux1_x86_64.whl (29.3 MB)\n",
      "Collecting zipp>=0.5\n",
      "  Downloading zipp-3.1.0-py3-none-any.whl (4.9 kB)\n",
      "Collecting pycparser\n",
      "  Downloading pycparser-2.20-py2.py3-none-any.whl (112 kB)\n",
      "Collecting oauthlib>=3.0.0\n",
      "  Downloading oauthlib-3.1.0-py2.py3-none-any.whl (147 kB)\n",
      "Collecting msal<2.0.0,>=1.0.0\n",
      "  Downloading msal-1.4.3-py2.py3-none-any.whl (49 kB)\n",
      "Collecting msal-extensions~=0.1.3\n",
      "  Downloading msal_extensions-0.1.3-py2.py3-none-any.whl (9.0 kB)\n",
      "Collecting azure-core<2.0.0,>=1.0.0\n",
      "  Downloading azure_core-1.7.0-py2.py3-none-any.whl (121 kB)\n",
      "Collecting distro>=1.2.0\n",
      "  Downloading distro-1.5.0-py2.py3-none-any.whl (18 kB)\n",
      "Collecting portalocker~=1.0\n",
      "  Downloading portalocker-1.7.1-py2.py3-none-any.whl (10 kB)\n",
      "Building wheels for collected packages: gast, wrapt, absl-py, termcolor, fusepy\n",
      "  Building wheel for gast (setup.py): started\n",
      "  Building wheel for gast (setup.py): finished with status 'done'\n",
      "  Created wheel for gast: filename=gast-0.2.2-py3-none-any.whl size=7539 sha256=61b99097f1ff947219a29dbb2ccb8b7059d282b1403d3462172b939aefb53db3\n",
      "  Stored in directory: /root/.cache/pip/wheels/19/a7/b9/0740c7a3a7d1d348f04823339274b90de25fbcd217b2ee1fbe\n",
      "  Building wheel for wrapt (setup.py): started\n",
      "  Building wheel for wrapt (setup.py): finished with status 'done'\n",
      "  Created wheel for wrapt: filename=wrapt-1.12.1-cp36-cp36m-linux_x86_64.whl size=66298 sha256=dcff3701a601372362fe59fbdeceed0a3a22400eaebe79f363f8f5d46b9e1a32\n",
      "  Stored in directory: /root/.cache/pip/wheels/32/42/7f/23cae9ff6ef66798d00dc5d659088e57dbba01566f6c60db63\n",
      "  Building wheel for absl-py (setup.py): started\n",
      "  Building wheel for absl-py (setup.py): finished with status 'done'\n",
      "  Created wheel for absl-py: filename=absl_py-0.9.0-py3-none-any.whl size=121931 sha256=d282c9b8d86e64a18c5e0b4b6fc5c3bafda1c46fe0d00cc8824cdade1519aa34\n",
      "  Stored in directory: /root/.cache/pip/wheels/c3/af/84/3962a6af7b4ab336e951b7877dcfb758cf94548bb1771e0679\n",
      "  Building wheel for termcolor (setup.py): started\n",
      "  Building wheel for termcolor (setup.py): finished with status 'done'\n",
      "  Created wheel for termcolor: filename=termcolor-1.1.0-py3-none-any.whl size=4830 sha256=aece9408ecee6b6bda07f4d3f84623331051592a0de7bd758302d8d174ffba48\n",
      "  Stored in directory: /root/.cache/pip/wheels/93/2a/eb/e58dbcbc963549ee4f065ff80a59f274cc7210b6eab962acdc\n",
      "  Building wheel for fusepy (setup.py): started\n",
      "  Building wheel for fusepy (setup.py): finished with status 'done'\n",
      "  Created wheel for fusepy: filename=fusepy-3.0.1-py3-none-any.whl size=10503 sha256=ad63a9f96ec4deb8381b188bbc5080de5f0d84942a31857ab433af0b9a30f4c5\n",
      "  Stored in directory: /root/.cache/pip/wheels/21/5c/83/1dd7e8a232d12227e5410120f4374b33adeb4037473105b079\n",
      "Successfully built gast wrapt absl-py termcolor fusepy\n",
      "Installing collected packages: gast, wrapt, six, numpy, keras-preprocessing, tensorflow-estimator, grpcio, google-pasta, absl-py, protobuf, zipp, importlib-metadata, markdown, werkzeug, tensorboard, opt-einsum, termcolor, astor, h5py, keras-applications, tensorflow, pillow, idna, chardet, urllib3, requests, PyJWT, python-dateutil, pycparser, cffi, cryptography, adal, isodate, oauthlib, requests-oauthlib, msrest, msrestazure, jeepney, SecretStorage, jsonpickle, ruamel.yaml.clib, ruamel.yaml, pyopenssl, backports.weakref, backports.tempfile, websocket-client, docker, azure-common, azure-graphrbac, pytz, azure-mgmt-containerregistry, azure-mgmt-authorization, pyasn1, ndg-httpsclient, azure-mgmt-storage, azure-mgmt-resource, pathspec, azure-mgmt-keyvault, contextlib2, jmespath, azureml-core, pyarrow, azureml-dataprep-native, msal, portalocker, msal-extensions, azure-core, azure-identity, cloudpickle, distro, dotnetcore2, azureml-dataprep, fusepy, pandas, azureml-dataset-runtime\n",
      "Successfully installed PyJWT-1.7.1 SecretStorage-3.1.2 absl-py-0.9.0 adal-1.2.4 astor-0.8.1 azure-common-1.1.25 azure-core-1.7.0 azure-graphrbac-0.61.1 azure-identity-1.2.0 azure-mgmt-authorization-0.60.0 azure-mgmt-containerregistry-2.8.0 azure-mgmt-keyvault-2.2.0 azure-mgmt-resource-10.1.0 azure-mgmt-storage-11.1.0 azureml-core-1.10.0.post1 azureml-dataprep-1.10.1 azureml-dataprep-native-14.2.1 azureml-dataset-runtime-1.10.0 backports.tempfile-1.0 backports.weakref-1.0.post1 cffi-1.14.1 chardet-3.0.4 cloudpickle-1.5.0 contextlib2-0.6.0.post1 cryptography-3.0 distro-1.5.0 docker-4.2.2 dotnetcore2-2.1.14 fusepy-3.0.1 gast-0.2.2 google-pasta-0.2.0 grpcio-1.30.0 h5py-2.10.0 idna-2.10 importlib-metadata-1.7.0 isodate-0.6.0 jeepney-0.4.3 jmespath-0.10.0 jsonpickle-1.4.1 keras-applications-1.0.8 keras-preprocessing-1.1.2 markdown-3.2.2 msal-1.4.3 msal-extensions-0.1.3 msrest-0.6.18 msrestazure-0.6.4 ndg-httpsclient-0.5.1 numpy-1.19.1 oauthlib-3.1.0 opt-einsum-3.3.0 pandas-1.1.0 pathspec-0.8.0 pillow-7.2.0 portalocker-1.7.1 protobuf-3.12.4 pyarrow-1.0.0 pyasn1-0.4.8 pycparser-2.20 pyopenssl-19.1.0 python-dateutil-2.8.1 pytz-2020.1 requests-2.24.0 requests-oauthlib-1.3.0 ruamel.yaml-0.16.10 ruamel.yaml.clib-0.2.0 six-1.15.0 tensorboard-1.15.0 tensorflow-1.15.2 tensorflow-estimator-1.15.1 termcolor-1.1.0 urllib3-1.25.10 websocket-client-0.57.0 werkzeug-1.0.1 wrapt-1.12.1 zipp-3.1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "#\n",
      "# To activate this environment, use\n",
      "#\n",
      "#     $ conda activate /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e\n",
      "#\n",
      "# To deactivate an active environment, use\n",
      "#\n",
      "#     $ conda deactivate\n",
      "\n",
      "\u001b[91m\n",
      "\n",
      "==> WARNING: A newer version of conda exists. <==\n",
      "  current version: 4.7.12\n",
      "  latest version: 4.8.3\n",
      "\n",
      "Please update conda by running\n",
      "\n",
      "    $ conda update -n base -c defaults conda\n",
      "\n",
      "\n",
      "\u001b[0mWARNING: /root/.conda/pkgs does not exist\n",
      "Removing intermediate container f5fbd106f4c6\n",
      " ---> 10d36a8966ad\n",
      "Step 9/15 : ENV PATH /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e/bin:$PATH\n",
      " ---> Running in cb31623b7605\n",
      "Removing intermediate container cb31623b7605\n",
      " ---> 6fe0cd66471a\n",
      "Step 10/15 : ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e\n",
      " ---> Running in 0c5d2269c593\n",
      "Removing intermediate container 0c5d2269c593\n",
      " ---> 05ee0f65f702\n",
      "Step 11/15 : ENV LD_LIBRARY_PATH /azureml-envs/azureml_40740abd0db39f33e22f8442e072813e/lib:$LD_LIBRARY_PATH\n",
      " ---> Running in 8a60cb2dfccd\n",
      "Removing intermediate container 8a60cb2dfccd\n",
      " ---> a23cf063f2f1\n",
      "Step 12/15 : COPY azureml-environment-setup/spark_cache.py azureml-environment-setup/log4j.properties /azureml-environment-setup/\n",
      " ---> 464b770fd1c4\n",
      "Step 13/15 : RUN if [ $SPARK_HOME ]; then /bin/bash -c '$SPARK_HOME/bin/spark-submit  /azureml-environment-setup/spark_cache.py'; fi\n",
      " ---> Running in 6b98986009b9\n",
      "Removing intermediate container 6b98986009b9\n",
      " ---> 75962fc70c09\n",
      "Step 14/15 : ENV AZUREML_ENVIRONMENT_IMAGE True\n",
      " ---> Running in 1997aa702fb9\n",
      "Removing intermediate container 1997aa702fb9\n",
      " ---> 6f932ca5706e\n",
      "Step 15/15 : CMD [\"bash\"]\n",
      " ---> Running in 2bf9828ec89f\n",
      "Removing intermediate container 2bf9828ec89f\n",
      " ---> 801df69c930c\n",
      "Successfully built 801df69c930c\n",
      "Successfully tagged lab01a91a38f72.azurecr.io/azureml/azureml_0d02b0cd070a04992634d620a6f8b3ce:latest\n",
      "2020/08/02 09:14:31 Successfully executed container: acb_step_0\n",
      "2020/08/02 09:14:31 Executing step ID: acb_step_1. Timeout(sec): 5400, Working directory: '', Network: 'acb_default_network'\n",
      "2020/08/02 09:14:31 Pushing image: lab01a91a38f72.azurecr.io/azureml/azureml_0d02b0cd070a04992634d620a6f8b3ce:latest, attempt 1\n",
      "The push refers to repository [lab01a91a38f72.azurecr.io/azureml/azureml_0d02b0cd070a04992634d620a6f8b3ce]\n",
      "5b4247307a8d: Preparing\n",
      "80b9122b3d47: Preparing\n",
      "7bcdeb7d22f4: Preparing\n",
      "bfaaa5f16753: Preparing\n",
      "0bb769d26255: Preparing\n",
      "7926deeb8000: Preparing\n",
      "f5aa53607bc1: Preparing\n",
      "07eae5ad8c0b: Preparing\n",
      "facf43cddd83: Preparing\n",
      "69a9bdc813b0: Preparing\n",
      "7e2b9752143f: Preparing\n",
      "63a67842a4c7: Preparing\n",
      "ae3a847dbd6b: Preparing\n",
      "4ae3adcb66cb: Preparing\n",
      "aa6685385151: Preparing\n",
      "0040d8f00d7e: Preparing\n",
      "9e6f810a2aab: Preparing\n",
      "7926deeb8000: Waiting\n",
      "f5aa53607bc1: Waiting\n",
      "07eae5ad8c0b: Waiting\n",
      "facf43cddd83: Waiting\n",
      "69a9bdc813b0: Waiting\n",
      "7e2b9752143f: Waiting\n",
      "63a67842a4c7: Waiting\n",
      "ae3a847dbd6b: Waiting\n",
      "4ae3adcb66cb: Waiting\n",
      "aa6685385151: Waiting\n",
      "0040d8f00d7e: Waiting\n",
      "9e6f810a2aab: Waiting\n",
      "bfaaa5f16753: Pushed\n",
      "0bb769d26255: Pushed\n",
      "7bcdeb7d22f4: Pushed\n",
      "5b4247307a8d: Pushed\n",
      "7926deeb8000: Pushed\n",
      "f5aa53607bc1: Pushed\n",
      "07eae5ad8c0b: Pushed\n",
      "63a67842a4c7: Pushed\n",
      "facf43cddd83: Pushed\n",
      "4ae3adcb66cb: Pushed\n",
      "aa6685385151: Pushed\n",
      "0040d8f00d7e: Pushed\n",
      "7e2b9752143f: Pushed\n",
      "69a9bdc813b0: Pushed\n",
      "\n",
      "ae3a847dbd6b: Pushed\n",
      "9e6f810a2aab: Pushed\n",
      "\n",
      "80b9122b3d47: Pushed\n",
      "latest: digest: sha256:d2dd5257b1932490992d7a0ae681d353053d3669c71e4a9627b7e938682b0c0f size: 3883\n",
      "2020/08/02 09:16:30 Successfully pushed image: lab01a91a38f72.azurecr.io/azureml/azureml_0d02b0cd070a04992634d620a6f8b3ce:latest\n",
      "2020/08/02 09:16:30 Step ID: acb_step_0 marked as successful (elapsed time in seconds: 193.258789)\n",
      "2020/08/02 09:16:30 Populating digests for step ID: acb_step_0...\n",
      "2020/08/02 09:16:32 Successfully populated digests for step ID: acb_step_0\n",
      "2020/08/02 09:16:32 Step ID: acb_step_1 marked as successful (elapsed time in seconds: 118.884577)\n",
      "2020/08/02 09:16:32 The following dependencies were found:\n",
      "2020/08/02 09:16:32 \n",
      "- image:\n",
      "    registry: lab01a91a38f72.azurecr.io\n",
      "    repository: azureml/azureml_0d02b0cd070a04992634d620a6f8b3ce\n",
      "    tag: latest\n",
      "    digest: sha256:d2dd5257b1932490992d7a0ae681d353053d3669c71e4a9627b7e938682b0c0f\n",
      "  runtime-dependency:\n",
      "    registry: mcr.microsoft.com\n",
      "    repository: azureml/intelmpi2018.3-ubuntu16.04\n",
      "    tag: 20200423.v1\n",
      "    digest: sha256:a8f6491296cbc183e95e9de29b59098e97d50ea87b4f1afa93e2afd43afeaf6d\n",
      "  git: {}\n",
      "\n",
      "Run ID: ca3 was successful after 5m19s\n"
     ]
    },
    {
     "ename": "ExperimentExecutionException",
     "evalue": "ExperimentExecutionException:\n\tMessage: The output streaming for the run interrupted.\nBut the run is still executing on the compute target. \nDetails for canceling the run can be found here: https://aka.ms/aml-docs-cancel-run\n\tInnerException None\n\tErrorResponse \n{\n    \"error\": {\n        \"message\": \"The output streaming for the run interrupted.\\nBut the run is still executing on the compute target. \\nDetails for canceling the run can be found here: https://aka.ms/aml-docs-cancel-run\"\n    }\n}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/run.py\u001b[0m in \u001b[0;36mwait_for_completion\u001b[0;34m(self, show_output, timeout_seconds, raise_on_error)\u001b[0m\n\u001b[1;32m    736\u001b[0m                 return self._stream_run_output(timeout_seconds=timeout_seconds,\n\u001b[0;32m--> 737\u001b[0;31m                                                raise_on_error=raise_on_error)\n\u001b[0m\u001b[1;32m    738\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/run.py\u001b[0m in \u001b[0;36m_stream_run_output\u001b[0;34m(self, timeout_seconds, raise_on_error)\u001b[0m\n\u001b[1;32m    781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 782\u001b[0;31m             \u001b[0mcurrent_details\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_details\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    783\u001b[0m             available_logs = [x for x in current_details[\"logFiles\"]\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/core/run.py\u001b[0m in \u001b[0;36mget_details\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1060\u001b[0m         \"\"\"\n\u001b[0;32m-> 1061\u001b[0;31m         \u001b[0mdetails\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_runstatus\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey_transformer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcamel_case_transformer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1062\u001b[0m         \u001b[0minput_datasets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdetails\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'inputDatasets'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_run_impl/run_history_facade.py\u001b[0m in \u001b[0;36mget_runstatus\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    161\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_runstatus\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m         \u001b[0mrun_status\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_runstatus\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_cached_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_status\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/run_client.py\u001b[0m in \u001b[0;36mget_runstatus\u001b[0;34m(self, caller, custom_headers, is_async)\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 105\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_execute_with_run_arguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_details\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/run_client.py\u001b[0m in \u001b[0;36m_execute_with_run_arguments\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m    552\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_execute_with_run_arguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 553\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_execute_with_arguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_arguments\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/workspace_client.py\u001b[0m in \u001b[0;36m_execute_with_arguments\u001b[0;34m(self, func, args_list, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_api\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mErrorResponseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/clientbase.py\u001b[0m in \u001b[0;36m_call_api\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m    231\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_execute_with_base_arguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/clientbase.py\u001b[0m in \u001b[0;36m_execute_with_base_arguments\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m    320\u001b[0m         return ClientBase._execute_func_internal(\n\u001b[0;32m--> 321\u001b[0;31m             back_off, total_retry, self._logger, func, _noop_reset, *args, **kwargs)\n\u001b[0m\u001b[1;32m    322\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/clientbase.py\u001b[0m in \u001b[0;36m_execute_func_internal\u001b[0;34m(cls, back_off, total_retry, logger, func, reset_func, *args, **kwargs)\u001b[0m\n\u001b[1;32m    333\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 334\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    335\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/operations/run_operations.py\u001b[0m in \u001b[0;36mget_details\u001b[0;34m(self, subscription_id, resource_group_name, workspace_name, experiment_name, run_id, custom_headers, raw, **operation_config)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquery_parameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader_parameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moperation_config\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/service_client.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, headers, content, **kwargs)\u001b[0m\n\u001b[1;32m    335\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m             \u001b[0mpipeline_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    337\u001b[0m             \u001b[0;31m# There is too much thing that expects this method to return a \"requests.Response\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/pipeline/__init__.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0mfirst_node\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_impl_policies\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_impl_policies\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sender\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfirst_node\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpipeline_request\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/pipeline/__init__.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    149\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/pipeline/requests.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m             except (oauth2.rfc6749.errors.InvalidGrantError,\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/pipeline/requests.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 137\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    138\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/pipeline/__init__.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    149\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/pipeline/requests.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    192\u001b[0m             \u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhttp_request\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    194\u001b[0m         )\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/universal_http/requests.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0mrequests_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_configure_send\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRequestsHTTPSender\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mrequests_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    334\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/msrest/universal_http/requests.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    141\u001b[0m                 \u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m                 **kwargs)\n\u001b[0m\u001b[1;32m    143\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRequestException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    529\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    642\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 643\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    644\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    448\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m                 )\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    676\u001b[0m                 \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m                 \u001b[0mchunked\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunked\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    678\u001b[0m             )\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    425\u001b[0m                     \u001b[0;31m# Otherwise it looks like a bug in the code.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 426\u001b[0;31m                     \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    427\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mSocketTimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBaseSSLError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSocketError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    420\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 421\u001b[0;31m                     \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    422\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1345\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1346\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1347\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    306\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    267\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 268\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    269\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1011\u001b[0m                   self.__class__)\n\u001b[0;32m-> 1012\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    873\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 874\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    875\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mSSLError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    630\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 631\u001b[0;31m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    632\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mExperimentExecutionException\u001b[0m              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-40271434b370>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mRunDetails\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpipeline_run\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mpipeline_run\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_completion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshow_output\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/run.py\u001b[0m in \u001b[0;36mwait_for_completion\u001b[0;34m(self, show_output, timeout_seconds, raise_on_error)\u001b[0m\n\u001b[1;32m    293\u001b[0m                             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m                                 step_run.wait_for_completion(timeout_seconds=timeout_seconds - time_elapsed,\n\u001b[0;32m--> 295\u001b[0;31m                                                              raise_on_error=raise_on_error)\n\u001b[0m\u001b[1;32m    296\u001b[0m                             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m                                 \u001b[0;31m# If there are package conflicts in the user's environment, the run rehydration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/pipeline/core/run.py\u001b[0m in \u001b[0;36mwait_for_completion\u001b[0;34m(self, show_output, timeout_seconds, raise_on_error)\u001b[0m\n\u001b[1;32m    742\u001b[0m                                 \u001b[0;34m\"https://aka.ms/aml-docs-cancel-run\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    743\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 744\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mExperimentExecutionException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_message\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    745\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    746\u001b[0m             \u001b[0mstatus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mExperimentExecutionException\u001b[0m: ExperimentExecutionException:\n\tMessage: The output streaming for the run interrupted.\nBut the run is still executing on the compute target. \nDetails for canceling the run can be found here: https://aka.ms/aml-docs-cancel-run\n\tInnerException None\n\tErrorResponse \n{\n    \"error\": {\n        \"message\": \"The output streaming for the run interrupted.\\nBut the run is still executing on the compute target. \\nDetails for canceling the run can be found here: https://aka.ms/aml-docs-cancel-run\"\n    }\n}"
     ]
    }
   ],
   "source": [
    "from azureml.widgets import RunDetails\n",
    "RunDetails(pipeline_run).show()\n",
    "\n",
    "pipeline_run.wait_for_completion(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_on_datastore = mnist_blob.path('mnist/0.png')\n",
    "single_image_ds = Dataset.File.from_files(path=path_on_datastore, validate=False)\n",
    "\n",
    "pipeline_run_2 = experiment.submit(pipeline, \n",
    "                                   pipeline_parameters={\"mnist_param\": single_image_ds, \n",
    "                                                        \"batch_size_param\": \"1\",\n",
    "                                                        \"process_count_param\": 1}\n",
    "                                  )\n",
    "\n",
    "pipeline_run_2.wait_for_completion(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tempfile\n",
    "\n",
    "batch_run = pipeline_run.find_step_run(parallelrun_step.name)[0]\n",
    "batch_output = batch_run.get_output_data(output_dir.name)\n",
    "\n",
    "target_dir = tempfile.mkdtemp()\n",
    "batch_output.download(local_path=target_dir)\n",
    "result_file = os.path.join(target_dir, batch_output.path_on_datastore, parallel_run_config.append_row_file_name)\n",
    "\n",
    "df = pd.read_csv(result_file, delimiter=\":\", header=None)\n",
    "df.columns = [\"Filename\", \"Prediction\"]\n",
    "print(\"Prediction has \", df.shape[0], \" rows\")\n",
    "df.head(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}