Merge pull request #520 from GoogleCloudPlatform/kfp-datadependency-fix

Remove data dependencies in the KFP modules

notebooks/kubeflow_pipelines/cicd/labs/kfp_cicd_vertex.ipynb

@@ -41,7 +41,8 @@
     "PROJECT_ID = PROJECT_ID[0]\n",
     "REGION = \"us-central1\"\n",
     "ARTIFACT_STORE = f\"gs://{PROJECT_ID}-kfp-artifact-store\"\n",
-    "os.environ[\"REGION\"] = REGION"
+    "os.environ[\"REGION\"] = REGION\n",
+    "os.environ[\"ARTIFACT_STORE\"] = ARTIFACT_STORE"
    ]
   },
   {
@@ -60,6 +61,26 @@
     "!gsutil ls | grep ^{ARTIFACT_STORE}/$ || gsutil mb -l {REGION} {ARTIFACT_STORE}"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Also, this notebook assumes the dataset is already created and stored in Google Cloud Storage following the instructions covered in the [walkthrough notebook](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/notebooks/kubeflow_pipelines/walkthrough/solutions/kfp_walkthrough_vertex.ipynb).\n",
+    "\n",
+    "If you haven't run it, please run the cell below and create the dataset before running the pipeline."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "gsutil cp gs://asl-public/data/covertype/training/dataset.csv $ARTIFACT_STORE/data/training/dataset.csv\n",
+    "gsutil cp gs://asl-public/data/covertype/validation/dataset.csv $ARTIFACT_STORE/data/validation/dataset.csv"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

notebooks/kubeflow_pipelines/cicd/solutions/kfp_cicd_vertex.ipynb

@@ -54,7 +54,8 @@
     "PROJECT_ID = PROJECT_ID[0]\n",
     "REGION = \"us-central1\"\n",
     "ARTIFACT_STORE = f\"gs://{PROJECT_ID}-kfp-artifact-store\"\n",
-    "os.environ[\"REGION\"] = REGION"
+    "os.environ[\"REGION\"] = REGION\n",
+    "os.environ[\"ARTIFACT_STORE\"] = ARTIFACT_STORE"
    ]
   },
   {
@@ -67,12 +68,38 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "!gsutil ls | grep ^{ARTIFACT_STORE}/$ || gsutil mb -l {REGION} {ARTIFACT_STORE}"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "Also, this notebook assumes the dataset is already created and stored in Google Cloud Storage following the instructions covered in the [walkthrough notebook](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/notebooks/kubeflow_pipelines/walkthrough/solutions/kfp_walkthrough_vertex.ipynb).\n",
+    "\n",
+    "If you haven't run it, please run the cell below and create the dataset before running the pipeline."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "gsutil cp gs://asl-public/data/covertype/training/dataset.csv $ARTIFACT_STORE/data/training/dataset.csv\n",
+    "gsutil cp gs://asl-public/data/covertype/validation/dataset.csv $ARTIFACT_STORE/data/validation/dataset.csv"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

notebooks/kubeflow_pipelines/pipelines/labs/kfp_pipeline_vertex_lightweight.ipynb

@@ -302,6 +302,26 @@
     "!gsutil ls | grep ^{ARTIFACT_STORE}/$ || gsutil mb -l {REGION} {ARTIFACT_STORE}"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Also, this notebook assumes the dataset is already created and stored in Google Cloud Storage following the instructions covered in the [walkthrough notebook](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/notebooks/kubeflow_pipelines/walkthrough/solutions/kfp_walkthrough_vertex.ipynb).\n",
+    "\n",
+    "If you haven't run it, please run the cell below and create the dataset before running the pipeline."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "gsutil cp gs://asl-public/data/covertype/training/dataset.csv $TRAINING_FILE_PATH\n",
+    "gsutil cp gs://asl-public/data/covertype/validation/dataset.csv $VALIDATION_FILE_PATH"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

notebooks/kubeflow_pipelines/pipelines/labs/kfp_pipeline_vertex_prebuilt.ipynb

@@ -376,6 +376,26 @@
     "!gsutil ls | grep ^{ARTIFACT_STORE}/$ || gsutil mb -l {REGION} {ARTIFACT_STORE}"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Also, this notebook assumes the dataset is already created and stored in Google Cloud Storage following the instructions covered in the [walkthrough notebook](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/notebooks/kubeflow_pipelines/walkthrough/solutions/kfp_walkthrough_vertex.ipynb).\n",
+    "\n",
+    "If you haven't run it, please run the cell below and create the dataset before running the pipeline."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "gsutil cp gs://asl-public/data/covertype/training/dataset.csv $TRAINING_FILE_PATH\n",
+    "gsutil cp gs://asl-public/data/covertype/validation/dataset.csv $VALIDATION_FILE_PATH"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

notebooks/kubeflow_pipelines/pipelines/solutions/kfp_pipeline_vertex_lightweight.ipynb

@@ -312,6 +312,26 @@
     "!gsutil ls | grep ^{ARTIFACT_STORE}/$ || gsutil mb -l {REGION} {ARTIFACT_STORE}"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Also, this notebook assumes the dataset is already created and stored in Google Cloud Storage following the instructions covered in the [walkthrough notebook](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/notebooks/kubeflow_pipelines/walkthrough/solutions/kfp_walkthrough_vertex.ipynb).\n",
+    "\n",
+    "If you haven't run it, please run the cell below and create the dataset before running the pipeline."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "gsutil cp gs://asl-public/data/covertype/training/dataset.csv $TRAINING_FILE_PATH\n",
+    "gsutil cp gs://asl-public/data/covertype/validation/dataset.csv $VALIDATION_FILE_PATH"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

notebooks/kubeflow_pipelines/pipelines/solutions/kfp_pipeline_vertex_prebuilt.ipynb

@@ -384,6 +384,28 @@
     "!gsutil ls | grep ^{ARTIFACT_STORE}/$ || gsutil mb -l {REGION} {ARTIFACT_STORE}"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Also, this notebook assumes the dataset is already created and stored in Google Cloud Storage following the instructions covered in the [walkthrough notebook](https://github.com/GoogleCloudPlatform/asl-ml-immersion/blob/master/notebooks/kubeflow_pipelines/walkthrough/solutions/kfp_walkthrough_vertex.ipynb).\n",
+    "\n",
+    "If you haven't run it, please run the cell below and create the dataset before running the pipeline."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "gsutil cp gs://asl-public/data/covertype/training/dataset.csv $TRAINING_FILE_PATH\n",
+    "gsutil cp gs://asl-public/data/covertype/validation/dataset.csv $VALIDATION_FILE_PATH"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},