diff --git a/Reference cells and variables/Reference cells and variables.ipynb b/Reference cells and variables/Reference cells and variables.ipynb
index 404c54f..4957bbe 100644
--- a/Reference cells and variables/Reference cells and variables.ipynb
+++ b/Reference cells and variables/Reference cells and variables.ipynb
@@ -1,121 +1,160 @@
{
- "metadata": {
- "kernelspec": {
- "display_name": "Streamlit Notebook",
- "name": "streamlit"
- }
- },
- "nbformat_minor": 5,
- "nbformat": 4,
"cells": [
{
"cell_type": "markdown",
"id": "d40f15d5-0f06-4c81-b4e6-a760771d44c2",
"metadata": {
- "name": "cell1",
- "collapsed": false
+ "collapsed": false,
+ "name": "cell1"
},
- "source": "# Reference cells and variables in Snowflake Notebooks"
+ "source": [
+ "# Reference cells and variables in Snowflake Notebooks"
+ ]
},
{
"cell_type": "markdown",
"id": "884f6e12-725b-4ae2-b9c9-5eaa4f4f964f",
"metadata": {
- "name": "cell2",
- "collapsed": false
+ "collapsed": false,
+ "name": "cell2"
},
- "source": "You can reference the results of previous cells in a cell in your notebook. This allows you to seamless switch between working in Python and SQL and reuse the results and variables.\n\n"
+ "source": [
+ "You can reference the results of previous cells in a cell in your notebook. This allows you to seamless switch between working in Python and SQL and reuse the results and variables.\n",
+ "\n"
+ ]
},
{
"cell_type": "markdown",
"id": "1ad40569-c979-461e-a2a0-98449785ba2f",
"metadata": {
- "name": "cell3",
- "collapsed": false
+ "collapsed": false,
+ "name": "cell3"
},
- "source": "## Referencing SQL output in Python cells\n\nWe can access the SQL results directly in Python and convert the results to a Snowpark or pandas dataframe.\n\nThe cell reference is based on the cell name. Note that if you change the cell name, you will also need to update the subsequent cell reference accordingly.\n\n\n### Example 1: Access SQL results as Snowpark or Pandas Dataframes"
+ "source": [
+ "## Referencing SQL output in Python cells\n",
+ "\n",
+ "We can access the SQL results directly in Python and convert the results to a Snowpark or pandas dataframe.\n",
+ "\n",
+ "The cell reference is based on the cell name. Note that if you change the cell name, you will also need to update the subsequent cell reference accordingly.\n",
+ "\n",
+ "\n",
+ "### Example 1: Access SQL results as Snowpark or Pandas Dataframes"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "3775908f-ca36-4846-8f38-5adca39217f2",
"metadata": {
+ "codeCollapsed": false,
"language": "sql",
- "name": "cell4",
- "codeCollapsed": false
+ "name": "cell4"
},
- "source": "SELECT 'FRIDAY' as SNOWDAY, 0.2 as CHANCE_OF_SNOW\nUNION ALL\nSELECT 'SATURDAY',0.5\nUNION ALL \nSELECT 'SUNDAY', 0.9;",
- "execution_count": null,
- "outputs": []
+ "outputs": [],
+ "source": [
+ "-- assign Query Tag to Session. This helps with performance monitoring and troubleshooting\n",
+ "ALTER SESSION SET query_tag = '{\"origin\":\"sf_sit-is\",\"name\":\"notebook_demo_pack\",\"version\":{\"major\":1, \"minor\":0},\"attributes\":{\"is_quickstart\":0, \"source\":\"sql\", \"vignette\":\"reference_cells\"}}';\n",
+ "\n",
+ "SELECT 'FRIDAY' as SNOWDAY, 0.2 as CHANCE_OF_SNOW\n",
+ "UNION ALL\n",
+ "SELECT 'SATURDAY',0.5\n",
+ "UNION ALL \n",
+ "SELECT 'SUNDAY', 0.9;"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "8d50cbf4-0c8d-4950-86cb-114990437ac9",
"metadata": {
+ "codeCollapsed": false,
"language": "python",
- "name": "cell5",
- "codeCollapsed": false
+ "name": "cell5"
},
- "source": "snowpark_df = cell4.to_df()",
- "execution_count": null,
- "outputs": []
+ "outputs": [],
+ "source": [
+ "snowpark_df = cell4.to_df()"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "c695373e-ac74-4b62-a1f1-08206cbd5c81",
"metadata": {
+ "codeCollapsed": false,
"language": "python",
- "name": "cell6",
- "codeCollapsed": false
+ "name": "cell6"
},
- "source": "pandas_df = cell4.to_pandas()",
- "execution_count": null,
- "outputs": []
+ "outputs": [],
+ "source": [
+ "pandas_df = cell4.to_pandas()"
+ ]
},
{
"cell_type": "markdown",
"id": "585a54f7-5dd4-412a-9c42-89d5c5d5978c",
"metadata": {
- "name": "cell7",
- "collapsed": false
+ "collapsed": false,
+ "name": "cell7"
},
- "source": "## Referencing variables in SQL code\n\nYou can use the Jinja syntax `{{..}}` to reference Python variables within your SQL queries as follows.\n\n### Example 2: Using Python variable value in a SQL query\n"
+ "source": [
+ "## Referencing variables in SQL code\n",
+ "\n",
+ "You can use the Jinja syntax `{{..}}` to reference Python variables within your SQL queries as follows.\n",
+ "\n",
+ "### Example 2: Using Python variable value in a SQL query\n"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "e73b633a-57d4-436c-baae-960c92c9cef6",
"metadata": {
- "language": "sql",
- "name": "cell8",
"codeCollapsed": false,
- "collapsed": false
+ "collapsed": false,
+ "language": "sql",
+ "name": "cell8"
},
"outputs": [],
- "source": "-- Create a dataset of countries\nCREATE OR REPLACE TABLE countries (\n country_name VARCHAR(100)\n);\n\nINSERT INTO countries (country_name) VALUES\n ('USA'),('Canada'),('United Kingdom'),('Germany'),('France'),\n ('Australia'),('Japan'),('China'),('India'),('Brazil');",
- "execution_count": null
+ "source": [
+ "-- Create a dataset of countries\n",
+ "CREATE OR REPLACE TABLE countries (\n",
+ " country_name VARCHAR(100)\n",
+ ");\n",
+ "\n",
+ "INSERT INTO countries (country_name) VALUES\n",
+ " ('USA'),('Canada'),('United Kingdom'),('Germany'),('France'),\n",
+ " ('Australia'),('Japan'),('China'),('India'),('Brazil');"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "e7a6f119-4f67-4ef5-a35f-117a7f502475",
"metadata": {
+ "codeCollapsed": false,
"language": "python",
- "name": "cell9",
- "codeCollapsed": false
+ "name": "cell9"
},
"outputs": [],
- "source": "c = \"'USA'\"",
- "execution_count": null
+ "source": [
+ "c = \"'USA'\""
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "60a59077-a4b1-4699-81a5-645addd8ad6d",
"metadata": {
+ "codeCollapsed": false,
"language": "sql",
- "name": "cell10",
- "codeCollapsed": false
+ "name": "cell10"
},
"outputs": [],
- "source": "-- Filter to record where country is USA\nSELECT * FROM countries WHERE COUNTRY_NAME = {{c}}",
- "execution_count": null
+ "source": [
+ "-- Filter to record where country is USA\n",
+ "SELECT * FROM countries WHERE COUNTRY_NAME = {{c}}"
+ ]
},
{
"cell_type": "markdown",
@@ -123,31 +162,50 @@
"metadata": {
"name": "cell11"
},
- "source": "### Example 3: Using Python dataframe in a SQL query"
+ "source": [
+ "### Example 3: Using Python dataframe in a SQL query"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "9b49d972-3966-4fa6-9457-f028b06484a3",
"metadata": {
+ "codeCollapsed": false,
"language": "sql",
- "name": "cell12",
- "codeCollapsed": false
+ "name": "cell12"
},
"outputs": [],
- "source": "-- Create dataset with columns PRODUCT_ID, RATING, PRICE\nSELECT CONCAT('SNOW-',UNIFORM(1000,9999, RANDOM())) AS PRODUCT_ID, \n ABS(NORMAL(5, 3, RANDOM())) AS RATING, \n ABS(NORMAL(750, 200::FLOAT, RANDOM())) AS PRICE\nFROM TABLE(GENERATOR(ROWCOUNT => 100));",
- "execution_count": null
+ "source": [
+ "-- Create dataset with columns PRODUCT_ID, RATING, PRICE\n",
+ "SELECT CONCAT('SNOW-',UNIFORM(1000,9999, RANDOM())) AS PRODUCT_ID, \n",
+ " ABS(NORMAL(5, 3, RANDOM())) AS RATING, \n",
+ " ABS(NORMAL(750, 200::FLOAT, RANDOM())) AS PRICE\n",
+ "FROM TABLE(GENERATOR(ROWCOUNT => 100));"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"id": "b7040f85-0ab8-4bdb-a36e-33599b79ea54",
"metadata": {
+ "codeCollapsed": false,
"language": "sql",
- "name": "cell13",
- "codeCollapsed": false
+ "name": "cell13"
},
"outputs": [],
- "source": "-- Filter to products where price is greater than 500\nSELECT * FROM {{cell12}} where PRICE > 500",
- "execution_count": null
+ "source": [
+ "-- Filter to products where price is greater than 500\n",
+ "SELECT * FROM {{cell12}} where PRICE > 500"
+ ]
}
- ]
-}
\ No newline at end of file
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/Role_Based_Access_Auditing_with_Streamlit/Role_Based_Access_Auditing_with_Streamlit.ipynb b/Role_Based_Access_Auditing_with_Streamlit/Role_Based_Access_Auditing_with_Streamlit.ipynb
new file mode 100644
index 0000000..4643da6
--- /dev/null
+++ b/Role_Based_Access_Auditing_with_Streamlit/Role_Based_Access_Auditing_with_Streamlit.ipynb
@@ -0,0 +1,198 @@
+{
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "cc4fb15e-f9db-44eb-9f60-1b9589b755cb",
+ "metadata": {
+ "name": "md_title",
+ "collapsed": false,
+ "resultHeight": 551
+ },
+ "source": "# Role-Based Access Auditing in Snowflake Notebooks with Streamlit\n\nA utility notebook to audit and report on user roles and privileges, ensuring adherence to security policies.\n\nHere's what we're implementing:\n1. User Role Analysis\n2. Role Grant Analysis\n\nFor each of these implementation, we're doing the following:\n1. SQL query for retrieving the data\n2. Converting data to a Pandas DataFrame\n3. Preparing and reshaping the data\n4. Creating a dashboard with Streamlit and Altair"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6d90f1b1-315e-4cde-a397-8e8ff8467fe0",
+ "metadata": {
+ "name": "md_user_role",
+ "collapsed": false,
+ "resultHeight": 204
+ },
+ "source": "## 1. User Role Analysis\n\nFirst, we'll start by retrieving user details (name, disabled status, last login, creation date) and their active role assignments (granted roles, who granted them, when granted) by joining the USERS and GRANTS_TO_USERS tables."
+ },
+ {
+ "cell_type": "code",
+ "id": "1e72bf27-b152-40a3-85e9-99e4b67cf8eb",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_user_role",
+ "resultHeight": 439,
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "SELECT \n u.name,\n u.disabled,\n u.last_success_login,\n u.created_on as user_created_on,\n g.role as granted_role,\n g.granted_by,\n g.created_on as grant_created_on\nFROM \n SNOWFLAKE.ACCOUNT_USAGE.USERS u\nLEFT JOIN \n SNOWFLAKE.ACCOUNT_USAGE.GRANTS_TO_USERS g\n ON u.name = g.grantee_name\nWHERE \n g.deleted_on IS NULL\nORDER BY \n u.name, g.role;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9da1de7e-1489-4d48-9634-a2f08a00667b",
+ "metadata": {
+ "name": "md_df_user_role",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Next, we'll convert the above SQL query output to a Pandas DataFrame."
+ },
+ {
+ "cell_type": "code",
+ "id": "3c5d60de-212a-4b7a-a3da-6ed6d15fa7ee",
+ "metadata": {
+ "language": "python",
+ "name": "df_user_role",
+ "codeCollapsed": false,
+ "collapsed": false,
+ "resultHeight": 439
+ },
+ "outputs": [],
+ "source": "sql_user_role.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bb7b86d1-5b36-4b05-a58d-fea830d30ab7",
+ "metadata": {
+ "name": "md_prepare_user_role",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Then, we'll prepare the data for subsequent data visualization."
+ },
+ {
+ "cell_type": "code",
+ "id": "2dc63ec1-0b35-43cb-bcd3-f914cc1525c2",
+ "metadata": {
+ "language": "python",
+ "name": "py_prepare_user_role",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "# Create user activity matrix\nuser_activity = (\n # Group by user and role, count occurrences\n df_user_role.groupby(['NAME', 'GRANTED_ROLE']) \n .size()\n .reset_index()\n .pivot(index='NAME', columns='GRANTED_ROLE', values=0) \n .fillna(0)\n)\n\n# Convert to long format for heatmap\nuser_activity_long = user_activity.reset_index().melt(\n id_vars=['NAME'],\n var_name='ROLE',\n value_name='HAS_ROLE'\n)\n\n# Add user status information \nuser_status = df_user_role[['NAME', 'DISABLED', 'LAST_SUCCESS_LOGIN']].drop_duplicates()\nuser_activity_long = user_activity_long.merge(\n user_status,\n on='NAME', # Changed from left_on/right_on to simple on\n how='left'\n)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9eece958-ec03-4f00-993e-409f5341c10e",
+ "metadata": {
+ "name": "md_st_user_role",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Finally, we'll use Streamlit to create a simple dashboard for user analysis."
+ },
+ {
+ "cell_type": "code",
+ "id": "575f9fe0-e16e-46d2-89c8-cca014a47314",
+ "metadata": {
+ "language": "python",
+ "name": "py_st_user_role",
+ "codeCollapsed": false,
+ "resultHeight": 1277,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "import pandas as pd\nimport altair as alt\nimport streamlit as st\n\nst.title(\"User Analysis Dashboard\")\n\n# Streamlit filters\ncol1, col2 = st.columns(2)\nwith col1:\n selected_users = st.multiselect(\n 'Select Users',\n options=sorted(user_activity_long['NAME'].unique()),\n default=sorted(user_activity_long['NAME'].unique())\n )\nwith col2:\n selected_roles = st.multiselect(\n 'Select Roles',\n options=sorted(user_activity_long['ROLE'].unique()),\n default=sorted(user_activity_long['ROLE'].unique())\n )\n\n# Filter data based on selections\nfiltered_data = user_activity_long[\n user_activity_long['NAME'].isin(selected_users) & \n user_activity_long['ROLE'].isin(selected_roles)\n]\n\n# Display summary metrics\nwith st.expander(\"View Summary Metrics\", expanded=True):\n metric_col1, metric_col2, metric_col3 = st.columns(3)\n with metric_col1:\n st.metric(\"Selected Users\", len(selected_users))\n with metric_col2:\n st.metric(\"Selected Roles\", len(selected_roles))\n with metric_col3:\n st.metric(\"Total Assignments\", len(filtered_data[filtered_data['HAS_ROLE'] > 0]))\n\n# Create styled heatmap\nheatmap = alt.Chart(filtered_data).mark_rect(\n stroke='black',\n strokeWidth=1\n).encode(\n x=alt.X('ROLE:N', \n title='Roles',\n axis=alt.Axis(\n labels=True,\n tickMinStep=1,\n labelOverlap=False,\n labelPadding=10\n )),\n y=alt.Y('NAME:N', \n title='Users',\n axis=alt.Axis(\n labels=True,\n labelLimit=200,\n tickMinStep=1,\n labelOverlap=False,\n labelPadding=10\n )),\n color=alt.Color('HAS_ROLE:Q', \n title='Has Role',\n scale=alt.Scale(scheme='blues')),\n tooltip=[\n alt.Tooltip('NAME:N', title='User'),\n alt.Tooltip('ROLE:N', title='Role'),\n alt.Tooltip('HAS_ROLE:Q', title='Has Role'),\n alt.Tooltip('DISABLED:N', title='Is Disabled'),\n alt.Tooltip('LAST_SUCCESS_LOGIN:T', title='Last Login')\n ]\n).properties(\n title='User Role Assignment Matrix'\n).configure_view(\n stroke=None,\n continuousHeight=400\n).configure_axis(\n labelFontSize=10\n)\n\n# Display the chart\nst.altair_chart(heatmap, use_container_width=True)\n\nwith st.expander(\"View DataFrame\"):\n st.dataframe(filtered_data)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "14473e03-2a00-41b4-873a-0d7e6b810c9a",
+ "metadata": {
+ "name": "md_role_grants",
+ "collapsed": false,
+ "resultHeight": 153
+ },
+ "source": "## 2. Role Grant Analysis\n\nSecondly, we'll craft a SQL query to show all active privileges granted to roles, including what type of privilege was granted, what object it was granted on, the specific object name, who granted it and when it was created."
+ },
+ {
+ "cell_type": "code",
+ "id": "dc1bb3f3-0eb6-4740-8c25-0c3938c9668f",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_role_grants",
+ "codeCollapsed": false,
+ "resultHeight": 511,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "SELECT \n grantee_name,\n privilege,\n granted_on,\n name as object_name,\n granted_by,\n created_on\nFROM SNOWFLAKE.ACCOUNT_USAGE.GRANTS_TO_ROLES\nWHERE deleted_on IS NULL;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bfdf0b7c-e33f-4a85-ac9a-2425535cef86",
+ "metadata": {
+ "name": "md_df_role_grants",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Then, we'll prepare the data for subsequent data visualization."
+ },
+ {
+ "cell_type": "code",
+ "id": "b4a7bf1a-8d77-4428-8054-b8683b5f5af7",
+ "metadata": {
+ "language": "python",
+ "name": "df_role_grants",
+ "collapsed": false,
+ "resultHeight": 439
+ },
+ "outputs": [],
+ "source": "sql_role_grants.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "711f2d6c-8c15-482d-85fb-e4e98031b268",
+ "metadata": {
+ "name": "md_st_role_grants",
+ "collapsed": false,
+ "resultHeight": 83
+ },
+ "source": "Finally, we'll use Streamlit to create a simple dashboard for role grant analysis.\n\nGo ahead and adjust the select box widgets for **privileges** and **object types**."
+ },
+ {
+ "cell_type": "code",
+ "id": "5e047ba1-9976-477b-a2d4-db8ad7f24c45",
+ "metadata": {
+ "language": "python",
+ "name": "py_st_role_grants",
+ "codeCollapsed": false,
+ "resultHeight": 1131,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "import pandas as pd\nimport altair as alt\n\nst.title(\"Role Grant Dashboard\")\n\n# Create selectboxes for filtering\ncol1, col2 = st.columns(2)\nwith col1:\n selected_privilege = st.multiselect(\n 'Select Privileges',\n options=sorted(df_role_grants['PRIVILEGE'].unique()),\n default=sorted(df_role_grants['PRIVILEGE'].unique())[:10]\n )\n\nwith col2:\n selected_granted_on = st.multiselect(\n 'Select Object Types',\n options=sorted(df_role_grants['GRANTED_ON'].unique()),\n default=sorted(df_role_grants['GRANTED_ON'].unique())\n )\n\n# Filter data\nfiltered_df = df_role_grants[\n df_role_grants['PRIVILEGE'].isin(selected_privilege) &\n df_role_grants['GRANTED_ON'].isin(selected_granted_on)\n]\n\n# Show summary metrics\nwith st.expander(\"View Summary Metrics\", expanded=True):\n metric_col1, metric_col2 = st.columns(2)\n \n with metric_col1:\n st.metric(\"Total Role Grants\", len(filtered_df))\n \n with metric_col2:\n st.metric(\"Unique Users\", filtered_df['GRANTEE_NAME'].nunique())\n\n# Create Top N user chart\ntop_N_chart = alt.Chart(filtered_df).mark_bar(\n stroke='black',\n strokeWidth=1\n).encode(\n x=alt.X('count():Q', \n title='Number of Role Grants',\n axis=alt.Axis(\n labels=True,\n tickMinStep=1,\n labelOverlap=False\n )),\n y=alt.Y('GRANTEE_NAME:N', \n title='Users',\n sort='-x',\n axis=alt.Axis(\n labels=True,\n labelLimit=200,\n tickMinStep=1,\n labelOverlap=False,\n labelPadding=10\n )),\n color=alt.Color('PRIVILEGE:N', \n title='Privilege Type'),\n tooltip=[\n alt.Tooltip('GRANTEE_NAME:N', title='Users'),\n alt.Tooltip('count():Q', title='Total Grants'),\n alt.Tooltip('PRIVILEGE:N', title='Privilege Type'),\n alt.Tooltip('GRANTED_ON:N', title='Granted On')\n ]\n).transform_window(\n rank='rank(count())',\n sort=[alt.SortField('count()', order='descending')]\n).transform_filter(\n alt.datum.rank <= 20\n).properties(\n title='Top N Users by Number of Role Grants'\n).configure_view(\n stroke=None,\n continuousHeight=400\n).configure_axis(\n labelFontSize=10\n)\n\n# Display chart\nst.altair_chart(top_N_chart, use_container_width=True)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bb08b81b-6d62-4d11-8ece-6d22fcfe6eb8",
+ "metadata": {
+ "name": "md_resources",
+ "collapsed": false,
+ "resultHeight": 217
+ },
+ "source": "## Want to learn more?\n\n- Snowflake Docs on [Account Usage](https://docs.snowflake.com/en/sql-reference/account-usage), [USERS view](https://docs.snowflake.com/en/sql-reference/account-usage/users) and [GRANTS_TO_USERS](https://docs.snowflake.com/en/sql-reference/account-usage/grants_to_users)\n- More about [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake)\n- For more inspiration on how to use Streamlit widgets in Notebooks, check out [Streamlit Docs](https://docs.streamlit.io/) and this list of what is currently supported inside [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake#label-notebooks-streamlit-support)\n- Check out the [Altair User Guide](https://altair-viz.github.io/user_guide/data.html) for further information on customizing Altair charts"
+ }
+ ]
+}
diff --git a/Role_Based_Access_Auditing_with_Streamlit/environment.yml b/Role_Based_Access_Auditing_with_Streamlit/environment.yml
new file mode 100644
index 0000000..bfe5f22
--- /dev/null
+++ b/Role_Based_Access_Auditing_with_Streamlit/environment.yml
@@ -0,0 +1,6 @@
+name: app_environment
+channels:
+ - snowflake
+dependencies:
+ - altair=*
+ - pandas=*
diff --git a/Scheduled_Query_Execution_Report/Scheduled_Query_Execution_Report.ipynb b/Scheduled_Query_Execution_Report/Scheduled_Query_Execution_Report.ipynb
new file mode 100644
index 0000000..636ea86
--- /dev/null
+++ b/Scheduled_Query_Execution_Report/Scheduled_Query_Execution_Report.ipynb
@@ -0,0 +1,126 @@
+{
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "cc4fb15e-f9db-44eb-9f60-1b9589b755cb",
+ "metadata": {
+ "name": "md_title",
+ "collapsed": false,
+ "resultHeight": 285
+ },
+ "source": "# Scheduled Query Execution Report\n\nA notebook to report on failed or long-running scheduled queries, providing insights into reliability issues.\n\nHere's a breakdown of the steps:\n1. Retrieve Data\n2. Convert Table to a DataFrame\n3. Create an Interactive Slider Widget & Data Preparation\n4. Create a Heatmap for Visualizing Scheduled Query Execution"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "42a7b143-0779-4706-affc-c214213f55c5",
+ "metadata": {
+ "name": "md_retrieve_data",
+ "collapsed": false,
+ "resultHeight": 170
+ },
+ "source": "## 1. Retrieve Data\n\nFirstly, we'll write an SQL query to retrieve the execution history for scheduled queries, along with their status, timing metrics, and execution status. \n\nWe're obtaining this from the `snowflake.account_usage.task_history` table."
+ },
+ {
+ "cell_type": "code",
+ "id": "39f7713b-dd7a-41a2-872e-cc534c6dc4f6",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_data",
+ "resultHeight": 439,
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "SELECT \n name,\n database_name,\n query_id,\n query_text,\n schema_name,\n scheduled_time,\n query_start_time,\n completed_time,\n DATEDIFF('second', query_start_time, completed_time) as execution_time_seconds,\n state,\n error_code,\n error_message,\nFROM snowflake.account_usage.task_history\nWHERE scheduled_time >= DATEADD(days, -1, CURRENT_TIMESTAMP())\nORDER BY scheduled_time DESC;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "870b69dd-aae0-4dd3-93f7-7adce1268159",
+ "metadata": {
+ "name": "md_dataframe",
+ "collapsed": false,
+ "resultHeight": 102
+ },
+ "source": "## 2. Convert Table to a DataFrame\n\nNext, we'll convert the table to a Pandas DataFrame."
+ },
+ {
+ "cell_type": "code",
+ "id": "4a5559a8-ef3a-40c3-a9d5-54602403adab",
+ "metadata": {
+ "language": "python",
+ "name": "py_dataframe",
+ "codeCollapsed": false,
+ "resultHeight": 439,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "sql_data.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "59b04137-ca95-4fb8-b216-133272349a78",
+ "metadata": {
+ "name": "md_data_preparation",
+ "collapsed": false,
+ "resultHeight": 195
+ },
+ "source": "## 3. Create an Interactive Slider Widget & Data Preparation\n\nHere, we'll create an interactive slider for dynamically selecting the number of days to analyze. This would then trigger the filtering of the DataFrame to the specified number of days.\n\nNext, we'll reshape the data by calculating the frequency count by hour and task name, which will subsequently be used for creating the heatmap in the next step."
+ },
+ {
+ "cell_type": "code",
+ "id": "ba8fa564-d7d5-4d1c-9f6b-400f9c05ecae",
+ "metadata": {
+ "language": "python",
+ "name": "py_data_preparation",
+ "codeCollapsed": false,
+ "resultHeight": 216
+ },
+ "outputs": [],
+ "source": "import pandas as pd\nimport streamlit as st\nimport altair as alt\n\n# Create date filter slider\nst.subheader(\"Select time duration\")\ndays = st.slider('Select number of days to analyze', \n min_value=10, \n max_value=90, \n value=30, \n step=10)\n \n# Filter data according to day duration\nlatest_date = pd.to_datetime(df['SCHEDULED_TIME']).max()\ncutoff_date = latest_date - pd.Timedelta(days=days)\nfiltered_df = df[pd.to_datetime(df['SCHEDULED_TIME']) > cutoff_date].copy()\n \n# Prepare data for heatmap\nfiltered_df['HOUR_OF_DAY'] = pd.to_datetime(filtered_df['SCHEDULED_TIME']).dt.hour\nfiltered_df['HOUR_DISPLAY'] = filtered_df['HOUR_OF_DAY'].apply(lambda x: f\"{x:02d}:00\")\n \n# Calculate frequency count by hour and task name\nagg_df = filtered_df.groupby(['NAME', 'HOUR_DISPLAY', 'STATE']).size().reset_index(name='COUNT')\n\nst.warning(f\"Analyzing data for the last {days} days!\")",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "35f31e4e-95d5-4ee5-a146-b9e93dd9d570",
+ "metadata": {
+ "name": "md_heatmap",
+ "collapsed": false,
+ "resultHeight": 128
+ },
+ "source": "## 4. Create a Heatmap for Visualizing Scheduled Query Execution\n\nFinally, a heatmap and summary statistics table are generated that will allow us to gain insights on the task name and state (e.g. `SUCCEEDED`, `FAILED`, `SKIPPED`)."
+ },
+ {
+ "cell_type": "code",
+ "id": "e3049001-f3ba-4b66-ba54-c9f02f551992",
+ "metadata": {
+ "language": "python",
+ "name": "py_heatmap",
+ "codeCollapsed": false,
+ "resultHeight": 791
+ },
+ "outputs": [],
+ "source": "# Create heatmap\nchart = alt.Chart(agg_df).mark_rect(\n stroke='black',\n strokeWidth=1\n).encode(\n x=alt.X('HOUR_DISPLAY:O', \n title='Hour of Day',\n axis=alt.Axis(\n labels=True,\n tickMinStep=1,\n labelOverlap=False\n )),\n y=alt.Y('NAME:N', \n title='',\n axis=alt.Axis(\n labels=True,\n labelLimit=200,\n tickMinStep=1,\n labelOverlap=False,\n labelPadding=10\n )),\n color=alt.Color('COUNT:Q', \n title='Number of Executions'),\n row=alt.Row('STATE:N', \n title='Task State',\n header=alt.Header(labelAlign='left')),\n tooltip=[\n alt.Tooltip('NAME', title='Task Name'),\n alt.Tooltip('HOUR_DISPLAY', title='Hour'),\n alt.Tooltip('STATE', title='State'),\n alt.Tooltip('COUNT', title='Number of Executions')\n ]\n).properties(\n height=100,\n width=450\n).configure_view(\n stroke=None,\n continuousWidth=300\n).configure_axis(\n labelFontSize=10\n)\n\n# Display the chart\nst.subheader(f'Task Execution Frequency by State ({days} Days)')\nst.altair_chart(chart)\n\n# Optional: Display summary statistics\nst.subheader(\"Summary Statistics\")\nsummary_df = filtered_df.groupby('NAME').agg({\n 'STATE': lambda x: pd.Series(x).value_counts().to_dict()\n}).reset_index()\n\n# Format the state counts as separate columns\nstate_counts = pd.json_normalize(summary_df['STATE']).fillna(0).astype(int)\nsummary_df = pd.concat([summary_df['NAME'], state_counts], axis=1)\n\nst.dataframe(summary_df)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "eb3e9b67-6a6e-4218-b17a-3f8564a04d18",
+ "metadata": {
+ "name": "md_resources",
+ "collapsed": false,
+ "resultHeight": 217
+ },
+ "source": "## Want to learn more?\n\n- Snowflake Docs on [Account Usage](https://docs.snowflake.com/en/sql-reference/account-usage) and [TASK_HISTORY view](https://docs.snowflake.com/en/sql-reference/account-usage/task_history)\n- More about [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake)\n- For more inspiration on how to use Streamlit widgets in Notebooks, check out [Streamlit Docs](https://docs.streamlit.io/) and this list of what is currently supported inside [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake#label-notebooks-streamlit-support)\n- Check out the [Altair User Guide](https://altair-viz.github.io/user_guide/data.html) for further information on customizing Altair charts"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/Scheduled_Query_Execution_Report/environment.yml b/Scheduled_Query_Execution_Report/environment.yml
new file mode 100644
index 0000000..bfe5f22
--- /dev/null
+++ b/Scheduled_Query_Execution_Report/environment.yml
@@ -0,0 +1,6 @@
+name: app_environment
+channels:
+ - snowflake
+dependencies:
+ - altair=*
+ - pandas=*
diff --git a/Schema_Change_Tracker/Schema_Change_Tracker.ipynb b/Schema_Change_Tracker/Schema_Change_Tracker.ipynb
new file mode 100644
index 0000000..615598a
--- /dev/null
+++ b/Schema_Change_Tracker/Schema_Change_Tracker.ipynb
@@ -0,0 +1,230 @@
+{
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "cc4fb15e-f9db-44eb-9f60-1b9589b755cb",
+ "metadata": {
+ "name": "md_title",
+ "collapsed": false,
+ "resultHeight": 311
+ },
+ "source": "# Schema Change Tracker\n\nThis utility notebook helps to log and track schema changes (e.g., dropped columns) across databases for better governance.\n\nHere's our 4 step process:\n1. SQL query to retrieve data\n2. Convert SQL table to a Pandas DataFrame\n3. Data preparation and filtering (using user input from Streamlit widgets)\n4. Data visualization and exploration"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "42a7b143-0779-4706-affc-c214213f55c5",
+ "metadata": {
+ "name": "md_retrieve_data",
+ "collapsed": false,
+ "resultHeight": 128
+ },
+ "source": "## 1. Retrieve Data\n\nTo gain insights on query costs, we'll write a SQL query to retrieve data on *dropped columns* from the `snowflake.account_usage.columns` table.\n"
+ },
+ {
+ "cell_type": "code",
+ "id": "d549f7ac-bbbd-41f4-9ee3-98284e587de1",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_columns",
+ "resultHeight": 439,
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "-- Track dropped columns\nSELECT\n COLUMN_ID,\n COLUMN_NAME,\n TABLE_ID,\n TABLE_NAME,\n TABLE_SCHEMA_ID,\n TABLE_SCHEMA,\n TABLE_CATALOG_ID,\n TABLE_CATALOG,\n DATA_TYPE,\n CHARACTER_MAXIMUM_LENGTH,\n DELETED\nFROM \n SNOWFLAKE.ACCOUNT_USAGE.COLUMNS\nWHERE \n DELETED >= DATEADD(days, -90, CURRENT_DATE())",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "a083d5e7-3edd-4f8e-987b-a188d1fe788b",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_tables",
+ "resultHeight": 439,
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "-- Track dropped tables\nSELECT\n id as table_id,\n table_name,\n table_created,\n table_dropped,\n \n table_schema_id,\n table_schema,\n schema_created,\n schema_dropped,\n \n table_catalog_id,\n table_catalog,\n catalog_created,\n catalog_dropped\nFROM\n SNOWFLAKE.ACCOUNT_USAGE.TABLE_STORAGE_METRICS\nWHERE\n table_dropped >= DATEADD(days, -90, CURRENT_DATE())",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "5637961e-8f62-4b9f-954d-f51612761d4b",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_databases",
+ "resultHeight": 439,
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "-- Track dropped databases\nSELECT\n database_id,\n database_name,\n database_owner,\n created,\n last_altered,\n deleted\nFROM\n SNOWFLAKE.ACCOUNT_USAGE.DATABASES\nWHERE\n deleted >= DATEADD(days, -90, CURRENT_DATE())",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "870b69dd-aae0-4dd3-93f7-7adce1268159",
+ "metadata": {
+ "name": "md_dataframe",
+ "collapsed": false,
+ "resultHeight": 102
+ },
+ "source": "## 2. Convert Table to a DataFrame\n\nNext, we'll convert the tables to a Pandas DataFrame.\n"
+ },
+ {
+ "cell_type": "code",
+ "id": "4a5559a8-ef3a-40c3-a9d5-54602403adab",
+ "metadata": {
+ "language": "python",
+ "name": "py_columns",
+ "codeCollapsed": false,
+ "resultHeight": 439,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "sql_columns.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "dbd92f00-caea-4e43-a00a-ef4161271a28",
+ "metadata": {
+ "language": "python",
+ "name": "py_tables",
+ "collapsed": false,
+ "resultHeight": 439
+ },
+ "outputs": [],
+ "source": "sql_tables.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "0b84612f-a8c8-48aa-8061-235219c0a1a9",
+ "metadata": {
+ "language": "python",
+ "name": "py_databases",
+ "collapsed": false,
+ "resultHeight": 439
+ },
+ "outputs": [],
+ "source": "sql_databases.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "59b04137-ca95-4fb8-b216-133272349a78",
+ "metadata": {
+ "name": "md_data_preparation",
+ "collapsed": false,
+ "resultHeight": 267
+ },
+ "source": "## 3. Create an Interactive Widget & Data Preparation\n\nHere, we'll create an interactive widget for dynamically selecting the entity of interest (e.g. Column, Table, Schema, Catalog or Database). This would then trigger the filtering of the DataFrame accordingly.\n\n### 3.1. Create Interactive Widget\nNext, we'll reshape the data by calculating the frequency count by hour and task name, which will subsequently be used for creating the heatmap in the next step.\n"
+ },
+ {
+ "cell_type": "code",
+ "id": "e133dfd8-2f48-4250-9811-2c85b41b2db3",
+ "metadata": {
+ "language": "python",
+ "name": "py_data_preparation",
+ "resultHeight": 609,
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "import streamlit as st\n\nst.header(\"Schema Change Tracker\")\nsnowflake_option = st.selectbox(\"Select an option\", (\"Column\", \n \"Table\", \n \"Schema\", \n \"Catalog\", \n \"Database\"))\nif snowflake_option == \"Column\":\n df = py_columns.copy()\n date_deleted = \"DELETED\"\n col_name = \"COLUMN_NAME\"\nif snowflake_option == \"Table\":\n df = py_tables.copy()\n date_deleted = \"TABLE_DROPPED\"\n col_name = \"TABLE_NAME\"\nif snowflake_option == \"Schema\":\n df = py_tables.copy()\n date_deleted = \"SCHEMA_DROPPED\"\n col_name = \"SCHEMA_NAME\"\nif snowflake_option == \"Catalog\":\n df = py_tables.copy()\n date_deleted = \"CATALOG_DROPPED\"\n col_name = \"CATALOG_NAME\"\nif snowflake_option == \"Database\":\n df = py_databases.copy()\n date_deleted = \"DELETED\"\n col_name = \"DATABASE_NAME\"\n\nst.write(f\"You selected: `{snowflake_option}`\")\nst.dataframe(df)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fc10e390-a5d1-4bb6-9934-68e3c869b477",
+ "metadata": {
+ "name": "md_data_filtering",
+ "collapsed": false,
+ "resultHeight": 164
+ },
+ "source": "### 3.2. Data Filtering\n\nHere, we'll filter the DataFrame by defining the `start_date` variable, add the `WEEK` column to the DataFrame and reshape the data by applying the `groupby()` method to the DataFrame so that the data is now aggregated by `WEEK` and `col_name` (e.g. `COLUMN_NAME`, `TABLE_NAME`, `SCHEMA_NAME`, `CATALOG_NAME`, `DATABASE_NAME`)."
+ },
+ {
+ "cell_type": "code",
+ "id": "aeff0dbb-5a3d-4c15-bcc6-f19e5f2398ac",
+ "metadata": {
+ "language": "python",
+ "name": "py_data_filtering",
+ "resultHeight": 439,
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "# Data filtering\nimport pandas as pd\n\n# Get the minimum date from date column\nstart_date = pd.to_datetime(df[date_deleted]).min()\n\n# Create week numbers for x-axis\ndf['WEEK'] = pd.to_datetime(df[date_deleted]).dt.isocalendar().week\n\n# Create aggregation for heatmap\nagg_df = df.groupby(['WEEK', col_name]).size().reset_index(name='COUNT')\nagg_df",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "88d679a6-feef-4aad-893c-ded57e8467cb",
+ "metadata": {
+ "name": "md_week_legend",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Next, we'll define what the Week numbers correspond to. Particularly, the date range for each week."
+ },
+ {
+ "cell_type": "code",
+ "id": "b38c57ea-a8e8-42b2-b3a5-6e1bb79c22ad",
+ "metadata": {
+ "language": "python",
+ "name": "py_week_legend",
+ "resultHeight": 217,
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "# Week legend\nimport pandas as pd\nfrom datetime import datetime\n\n# Get unique weeks\nweeks = sorted(df['WEEK'].unique())\n\n# Create week ranges\nfor week in weeks:\n monday = datetime.strptime(f'2024-W{week:02d}-1', '%Y-W%W-%w')\n print(f\"Week {week}: {monday.strftime('%b %d')} - {(monday + pd.Timedelta(days=6)).strftime('%b %d')}\")",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1004d933-0fbd-4fc1-982a-b73012665ce6",
+ "metadata": {
+ "name": "md_heatmap",
+ "collapsed": false,
+ "resultHeight": 102
+ },
+ "source": "## Creation of the Heatmap\n\nHere, we're visualizing the data as a heatmap."
+ },
+ {
+ "cell_type": "code",
+ "id": "2c2da67a-cabd-4d11-bb1d-ccd3743faeb7",
+ "metadata": {
+ "language": "python",
+ "name": "py_heatmap",
+ "resultHeight": 423,
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "# Create the heatmap\nimport pandas as pd\nimport altair as alt\nimport numpy as np\n\n\nheatmap = alt.Chart(agg_df).mark_rect(stroke='black', strokeWidth=1).encode(\n x=alt.X('WEEK:O', \n title='Week Number',\n axis=alt.Axis(\n labelAngle=0,\n labelOverlap=False\n )),\n y=alt.Y(f'{col_name}:N', \n title='',\n axis=alt.Axis(\n labels=True,\n labelLimit=250,\n tickMinStep=1,\n labelOverlap=False,\n labelPadding=10\n )),\n color=alt.Color('COUNT:Q',\n title=f'Number of {snowflake_option}'),\n tooltip=['WEEK', col_name, 'COUNT']\n).properties(\n title=f'{snowflake_option} Usage Heatmap by Week and Table (Starting from {start_date.strftime(\"%Y-%m-%d\")})',\n width=800,\n height=df[col_name].nunique()*20 # Multiply the number of unique values by 15 \n)\n\nst.altair_chart(heatmap, use_container_width=True)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "eb3e9b67-6a6e-4218-b17a-3f8564a04d18",
+ "metadata": {
+ "name": "md_resources",
+ "collapsed": false,
+ "resultHeight": 217
+ },
+ "source": "## Want to learn more?\n\n- Snowflake Docs on [Account Usage](https://docs.snowflake.com/en/sql-reference/account-usage), [COLUMNS view](https://docs.snowflake.com/en/sql-reference/account-usage/columns), [TABLES view](https://docs.snowflake.com/en/sql-reference/account-usage/tables) and [DATABASES view](https://docs.snowflake.com/en/sql-reference/account-usage/databases)\n- More about [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake)\n- For more inspiration on how to use Streamlit widgets in Notebooks, check out [Streamlit Docs](https://docs.streamlit.io/) and this list of what is currently supported inside [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake#label-notebooks-streamlit-support)\n- Check out the [Altair User Guide](https://altair-viz.github.io/user_guide/data.html) for further information on customizing Altair charts\n"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/Schema_Change_Tracker/environment.yml b/Schema_Change_Tracker/environment.yml
new file mode 100644
index 0000000..bfe5f22
--- /dev/null
+++ b/Schema_Change_Tracker/environment.yml
@@ -0,0 +1,6 @@
+name: app_environment
+channels:
+ - snowflake
+dependencies:
+ - altair=*
+ - pandas=*
diff --git a/Snowflake_Notebooks_Summit_2024_Demo/aileen_summit_notebook.ipynb b/Snowflake_Notebooks_Summit_2024_Demo/aileen_summit_notebook.ipynb
new file mode 100644
index 0000000..2172e89
--- /dev/null
+++ b/Snowflake_Notebooks_Summit_2024_Demo/aileen_summit_notebook.ipynb
@@ -0,0 +1,192 @@
+{
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "30fcf7ae-e7f3-4a88-8afc-6568831d1c2a",
+ "metadata": {
+ "name": "Title",
+ "collapsed": false,
+ "resultHeight": 333
+ },
+ "source": "# :date: Send :orange[Daily Digest] of Fresh Foods Customer Reviews to :orange[Slack] \n\n## Features\n:gray[In this demo, we'll cover the following features:]\n- :gray[Calling Snowflake Cortex functions]\n- :gray[Integrating with external endpoints, i.e. Slack APIs]\n- :gray[Scheduling the notebook to run daily]\n- :gray[Keeping version control with Git]\n- :green[**BONUS**] :gray[- Run one notebook from another :knot: :knot: :knot:]"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "754480e1-8983-4b6c-8ba7-270e9dc5994f",
+ "metadata": {
+ "name": "Step_1_Get_data",
+ "collapsed": false,
+ "resultHeight": 60
+ },
+ "source": "## Step :one: - Get the customer reviews data :speech_balloon:"
+ },
+ {
+ "cell_type": "code",
+ "id": "465f4adb-3571-483b-90da-cd3e576b9435",
+ "metadata": {
+ "language": "sql",
+ "name": "Get_data",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "USE SCHEMA PUBLIC.PUBLIC;\nSELECT * FROM FRESH_FOODS_REVIEWS;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "89f98a73-ef13-4a4e-a8c6-7ed8bf620930",
+ "metadata": {
+ "language": "python",
+ "name": "Set_review_date",
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "from datetime import date\nimport streamlit as st\n\nreview_date = date(2024, 6, 4) # change to `date.today()` to always grab the current date \nst.write(review_date)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d3530f1e-55dd-43d9-9e09-0c0797116102",
+ "metadata": {
+ "name": "Step_2_Cortex",
+ "collapsed": false,
+ "resultHeight": 377
+ },
+ "source": "## Step :two: - Ask Snowflake Cortex to generate the daily digest :mega:\nSnowflake Cortex is a fully-managed service that enables access to industry-leading large language models (LLMs).\n- COMPLETE: Given a prompt, returns a response that completes the prompt. This function accepts either a single prompt or a conversation with multiple prompts and responses.\n\n- EMBED_TEXT_768: Given a piece of text, returns a vector embedding that represents that text.\n\n- EXTRACT_ANSWER: Given a question and unstructured data, returns the answer to the question if it can be found in the data.\n\n- SENTIMENT: Returns a sentiment score, from -1 to 1, representing the detected positive or negative sentiment of the given text.\n\n- SUMMARIZE: Returns a summary of the given text.\n\n- TRANSLATE: Translates given text from any supported language to any other."
+ },
+ {
+ "cell_type": "code",
+ "id": "58a6bf2f-34df-452d-946f-ba416b07118d",
+ "metadata": {
+ "language": "sql",
+ "name": "Cortex_SUMMARIZE",
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "WITH CUSTOMER_REVIEWS AS(\n SELECT LISTAGG(DISTINCT REVIEW) AS REVIEWS \n FROM {{Get_data}} \n WHERE to_date(DATE) = '{{review_date}}' )\n\nSELECT SNOWFLAKE.CORTEX.SUMMARIZE(REVIEWS) FROM CUSTOMER_REVIEWS;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "eea93bfd-ed59-4478-9931-b145261dab5b",
+ "metadata": {
+ "language": "python",
+ "name": "Summary",
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "summary_text = Cortex_SUMMARIZE.to_pandas().iloc[0]['SNOWFLAKE.CORTEX.SUMMARIZE(REVIEWS)']\nst.write(summary_text)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "4849cc86-d8b4-4b7c-a4b2-f73174798593",
+ "metadata": {
+ "language": "sql",
+ "name": "Daily_avg_score",
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "SELECT AVG(SNOWFLAKE.CORTEX.SENTIMENT(REVIEW)) AS AVERAGE_RATING FROM FRESH_FOODS_REVIEWS WHERE DATE = '{{review_date}}';",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c61883bc-ff05-4627-9558-681383d477f6",
+ "metadata": {
+ "name": "Step_3_Slack",
+ "collapsed": false,
+ "resultHeight": 60
+ },
+ "source": "## Step :three: - Send the summary and sentiment to Slack :tada:\n"
+ },
+ {
+ "cell_type": "code",
+ "id": "f69f5fcf-f470-48a6-a688-259440c95741",
+ "metadata": {
+ "language": "python",
+ "name": "Send_to_Slack",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "import requests\nimport numpy as np\n\n\nheaders = {\n 'Content-Type': 'Content-type: application/json',\n}\n\n# Extract Daily_avg_score contents\nsentiment_score = str(np.round(Daily_avg_score.to_pandas().values[0][0], 2))\n\n\ndata = {\n\t\"blocks\": [\n\t\t{\n\t\t\t\"type\": \"section\",\n\t\t\t\"text\": {\n\t\t\t\t\"type\": \"mrkdwn\",\n\t\t\t\t\"text\": f\":mega: *Daily summary | Sentiment score: {sentiment_score} | {review_date}*\"\n\t\t\t}\n\t\t},\n\t\t{\n\t\t\t\"type\": \"section\",\n\t\t\t\"text\": {\n\t\t\t\t\"type\": \"mrkdwn\",\n\t\t\t\t\"text\": summary_text\n\t\t\t}\n\t\t},\n\t\t{\n\t\t\t\"type\": \"divider\"\n\t\t},\n\t\t{\n\t\t\t\"type\": \"context\",\n\t\t\t\"elements\": [\n\t\t\t\t{\n\t\t\t\t\t\"type\": \"mrkdwn\",\n\t\t\t\t\t\"text\": \"\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t]\n}\n\nresponse = requests.post(\n 'https://hooks.slack.com/services/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', \n headers=headers, \n json=data)\n\nif response.status_code == 200:\n st.write('โ
Daily summary sent to Slack')",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "89b1c2bd-043b-4313-a20c-91a927e4dbd6",
+ "metadata": {
+ "name": "Step_4_Schedule",
+ "collapsed": false,
+ "resultHeight": 60
+ },
+ "source": "## Step :four: - Schedule the notebook to send daily updates automatically"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8780c297-a747-44f9-8f94-ae9a3084814d",
+ "metadata": {
+ "name": "Git_integration",
+ "collapsed": false,
+ "resultHeight": 538
+ },
+ "source": "## Let's keep track of code changes!\n- :rainbow[GitHub], :orange[GitLab], :blue[BitBucket], :violet[Azure DevOps]\n\n"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a1089358-dc72-4c1b-bb20-29d86e6ecdd2",
+ "metadata": {
+ "name": "Bonus_Chain_notebooks",
+ "collapsed": false,
+ "resultHeight": 60
+ },
+ "source": "## Bonus - :chains: Chain multiple notebooks together "
+ },
+ {
+ "cell_type": "code",
+ "id": "440692da-0080-4352-87ee-37e94d24027f",
+ "metadata": {
+ "language": "sql",
+ "name": "Run_2nd_notebook",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "EXECUTE NOTEBOOK PUBLIC.PUBLIC.AILEEN_SUMMIT_DEEP_ANALYSIS_2()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "97229677-6288-414c-906f-9e74ee1d31de",
+ "metadata": {
+ "name": "cell1",
+ "collapsed": false,
+ "resultHeight": 176
+ },
+ "source": "## You can also:\n- ### Wrap EXECUTE NOTEBOOK in business logic and call it from a Python cell :bulb:\n- ### Integrate with other orchestration tools :keyboard:"
+ },
+ {
+ "cell_type": "code",
+ "id": "3157f79a-f841-4be8-9a50-de312a474723",
+ "metadata": {
+ "language": "python",
+ "name": "Run_on_condition",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "from snowflake.snowpark.context import get_active_session\nsession = get_active_session()\n\nsentiment_score_flt = np.round(Daily_avg_score.to_pandas().values[0][0], 2)\n \nif sentiment_score_flt < 0.9:\n st.markdown(\"\"\"\n :rotating_light: Sentiment is below threshold! \n \n Kick off the 2nd notebook Deep Analysis.\"\"\")\n session.sql(\"EXECUTE NOTEBOOK PUBLIC.PUBLIC.AILEEN_SUMMIT_DEEP_ANALYSIS_2()\").collect()\nelse:\n st.write(\":sunflower: Sentiment is good. Do nothing.\")",
+ "execution_count": null
+ }
+ ]
+}
diff --git a/Snowflake_Semantic_View/environment.yml b/Snowflake_Semantic_View/environment.yml
new file mode 100644
index 0000000..04fc14e
--- /dev/null
+++ b/Snowflake_Semantic_View/environment.yml
@@ -0,0 +1,4 @@
+name: app_environment
+channels:
+ - snowflake
+dependencies: []
diff --git a/Snowflake_Semantic_View/getting-started-with-snowflake-semantic-view.ipynb b/Snowflake_Semantic_View/getting-started-with-snowflake-semantic-view.ipynb
new file mode 100644
index 0000000..4737630
--- /dev/null
+++ b/Snowflake_Semantic_View/getting-started-with-snowflake-semantic-view.ipynb
@@ -0,0 +1,304 @@
+{
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ },
+ "lastEditStatus": {
+ "notebookId": "eet2n3pip62drxyvjfdd",
+ "authorId": "7086005961584",
+ "authorName": "DATAPROFESSOR",
+ "authorEmail": "hellodataprofessor@gmail.com",
+ "sessionId": "8739c32f-e74f-4978-98ce-f2abd8a2e44a",
+ "lastEditTime": 1750731644861
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "b2c83e2e-e59c-4ccd-ac51-3cb16515e422",
+ "metadata": {
+ "name": "md_title",
+ "collapsed": false
+ },
+ "source": "# Getting Started with Snowflake Semantic View\n\nThis notebook guides you through setting up and querying a Snowflake Semantic View using TPC-DS sample data. You will learn how to:\n\n1. Create a new database and schema.\n2. Create views from existing sample data tables.\n3. Define a Semantic View to simplify data analysis.\n4. Query the Semantic View.\n5. Explore the Semantic View in Cortex Analyst.\n\nLet's get started!"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1fab659d-7a49-4f0a-a922-c2fd425c251b",
+ "metadata": {
+ "name": "md_step1",
+ "collapsed": false
+ },
+ "source": "## Step 1: Set up your Database and Schema\n\nFirst, we'll create a new database named `SAMPLE_DATA` and a schema named `TPCDS_SF10TCL` to organize our data. We will then set the context to use this new schema."
+ },
+ {
+ "cell_type": "code",
+ "id": "a7a7f7b6-7c04-4404-bd25-e769d04dacd9",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step1",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "-- Create a new test database named SAMPLE_DATA\nCREATE OR REPLACE DATABASE SAMPLE_DATA;\n\n-- Use the newly created database\nUSE DATABASE SAMPLE_DATA;\n\n-- Create a new schema named TPCDS_SF10TCL within SAMPLE_DATA\nCREATE SCHEMA TPCDS_SF10TCL;\n\n-- Set the context to use the new schema\nUSE SCHEMA TPCDS_SF10TCL;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5426bf7f-9c3d-48ad-afea-92c836399ae9",
+ "metadata": {
+ "name": "md_step2",
+ "collapsed": false
+ },
+ "source": "## Step 2: Create Views from Sample Data\n\nNext, we'll create views for the tables we want to analyze. These views will be based on the `SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL` dataset, allowing us to work with a subset of the data without modifying the original tables."
+ },
+ {
+ "cell_type": "code",
+ "id": "64efbf80-99d3-4370-855d-0ded817eece1",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step2"
+ },
+ "outputs": [],
+ "source": "-- Create or replace views for the tables from SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL\n\nCREATE OR REPLACE VIEW CUSTOMER AS\nSELECT * FROM SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL.CUSTOMER;\n\nCREATE OR REPLACE VIEW CUSTOMER_DEMOGRAPHICS AS\nSELECT * FROM SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL.CUSTOMER_DEMOGRAPHICS;\n\nCREATE OR REPLACE VIEW DATE_DIM AS\nSELECT * FROM SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL.DATE_DIM;\n\nCREATE OR REPLACE VIEW ITEM AS\nSELECT * FROM SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL.ITEM;\n\nCREATE OR REPLACE VIEW STORE AS\nSELECT * FROM SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL.STORE;\n\nCREATE OR REPLACE VIEW STORE_SALES AS\nSELECT * FROM SNOWFLAKE_SAMPLE_DATA.TPCDS_SF10TCL.STORE_SALES;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "638fcb07-5d39-413f-a685-cff868a36f87",
+ "metadata": {
+ "name": "md_step3",
+ "collapsed": false
+ },
+ "source": "## Step 3: Verify your Environment Setup\n\nBefore proceeding, let's ensure our warehouse, database, and schema are correctly set, and then list the views we just created."
+ },
+ {
+ "cell_type": "code",
+ "id": "3775908f-ca36-4846-8f38-5adca39217f2",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step3"
+ },
+ "source": "-- Select the warehouse, database, and schema\nUSE WAREHOUSE COMPUTE_WH;\nUSE DATABASE SAMPLE_DATA;\nUSE SCHEMA TPCDS_SF10TCL;\n\n-- Show all views in the current schema to verify creation\nSHOW VIEWS;",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6a42b8b3-352d-4d16-aab7-50a68be1ae3c",
+ "metadata": {
+ "name": "md_step4",
+ "collapsed": false
+ },
+ "source": "## Step 4: Define the Semantic View\n\nNow, we'll define our `TPCDS_SEMANTIC_VIEW_SM` semantic view. This view will establish relationships between our tables, define facts (measures), and dimensions (attributes), making it easier to query and analyze our data without complex joins."
+ },
+ {
+ "cell_type": "code",
+ "id": "8d50cbf4-0c8d-4950-86cb-114990437ac9",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step4"
+ },
+ "source": "-- Switch to ACCOUNTADMIN role to create the semantic view\nUSE ROLE ACCOUNTADMIN;\n\n-- Create or replace the semantic view named TPCDS_SEMANTIC_VIEW_SM\nCREATE OR REPLACE SEMANTIC VIEW TPCDS_SEMANTIC_VIEW_SM\n\ttables (\n\t\tCUSTOMER primary key (C_CUSTOMER_SK),\n\t\tDATE as DATE_DIM primary key (D_DATE_SK),\n\t\tDEMO as CUSTOMER_DEMOGRAPHICS primary key (CD_DEMO_SK),\n\t\tITEM primary key (I_ITEM_SK),\n\t\tSTORE primary key (S_STORE_SK),\n\t\tSTORESALES as STORE_SALES\n primary key (SS_SOLD_DATE_SK,SS_CDEMO_SK,SS_ITEM_SK,SS_STORE_SK,SS_CUSTOMER_SK)\n\t)\n\trelationships (\n\t\tSALESTOCUSTOMER as STORESALES(SS_CUSTOMER_SK) references CUSTOMER(C_CUSTOMER_SK),\n\t\tSALESTODATE as STORESALES(SS_SOLD_DATE_SK) references DATE(D_DATE_SK),\n\t\tSALESTODEMO as STORESALES(SS_CDEMO_SK) references DEMO(CD_DEMO_SK),\n\t\tSALESTOITEM as STORESALES(SS_ITEM_SK) references ITEM(I_ITEM_SK),\n\t\tSALETOSTORE as STORESALES(SS_STORE_SK) references STORE(S_STORE_SK)\n\t)\n\tfacts (\n\t\tITEM.COST as i_wholesale_cost,\n\t\tITEM.PRICE as i_current_price,\n\t\tSTORE.TAX_RATE as S_TAX_PRECENTAGE,\n STORESALES.SALES_QUANTITY as SS_QUANTITY\n\t)\n\tdimensions (\n\t\tCUSTOMER.BIRTHYEAR as C_BIRTH_YEAR,\n\t\tCUSTOMER.COUNTRY as C_BIRTH_COUNTRY,\n\t\tCUSTOMER.C_CUSTOMER_SK as c_customer_sk,\n\t\tDATE.DATE as D_DATE,\n\t\tDATE.D_DATE_SK as d_date_sk,\n\t\tDATE.MONTH as D_MOY,\n\t\tDATE.WEEK as D_WEEK_SEQ,\n\t\tDATE.YEAR as D_YEAR,\n\t\tDEMO.CD_DEMO_SK as cd_demo_sk,\n\t\tDEMO.CREDIT_RATING as CD_CREDIT_RATING,\n\t\tDEMO.MARITAL_STATUS as CD_MARITAL_STATUS,\n\t\tITEM.BRAND as I_BRAND,\n\t\tITEM.CATEGORY as I_CATEGORY,\n\t\tITEM.CLASS as I_CLASS,\n\t\tITEM.I_ITEM_SK as i_item_sk,\n\t\tSTORE.MARKET as S_MARKET_ID,\n\t\tSTORE.SQUAREFOOTAGE as S_FLOOR_SPACE,\n\t\tSTORE.STATE as S_STATE,\n\t\tSTORE.STORECOUNTRY as S_COUNTRY,\n\t\tSTORE.S_STORE_SK as s_store_sk,\n\t\tSTORESALES.SS_CDEMO_SK as ss_cdemo_sk,\n\t\tSTORESALES.SS_CUSTOMER_SK as ss_customer_sk,\n\t\tSTORESALES.SS_ITEM_SK as ss_item_sk,\n\t\tSTORESALES.SS_SOLD_DATE_SK as ss_sold_date_sk,\n\t\tSTORESALES.SS_STORE_SK as ss_store_sk\n\t)\n\tmetrics (\n\t\tSTORESALES.TOTALCOST as SUM(item.cost),\n\t\tSTORESALES.TOTALSALESPRICE as SUM(SS_SALES_PRICE),\n\t\tSTORESALES.TOTALSALESQUANTITY as SUM(SS_QUANTITY)\n WITH SYNONYMS = ( 'total sales quantity', 'total sales amount')\n\t)\n\n;",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d3561721-710e-4a79-905b-194a6a99203e",
+ "metadata": {
+ "name": "md_step5",
+ "collapsed": false
+ },
+ "source": "## Step 5: Verify the Semantic View Creation\n\nLet's confirm that our semantic view has been successfully created by listing all semantic views in the current database."
+ },
+ {
+ "cell_type": "code",
+ "id": "c695373e-ac74-4b62-a1f1-08206cbd5c81",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step5"
+ },
+ "source": "-- Lists semantic views in the database that is currently in use\nSHOW SEMANTIC VIEWS;",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "id": "87a4e121-fa51-4968-b53d-b65aadb13beb",
+ "metadata": {
+ "name": "md_step6",
+ "collapsed": false
+ },
+ "source": "## Step 6: Describe the Semantic View\n\nTo understand the structure and components of our newly created semantic view, we can use the `DESC SEMANTIC VIEW` command. This will provide details about its tables, relationships, facts, and dimensions."
+ },
+ {
+ "cell_type": "code",
+ "id": "08994b45-88a7-4d79-a570-6a37b8db2e25",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step6"
+ },
+ "outputs": [],
+ "source": "-- Describes the semantic view named TPCDS_SEMANTIC_VIEW_SM, and as a special bonus uses our new flow operator to filter and project only the metric and dimension names\nDESC SEMANTIC VIEW TPCDS_SEMANTIC_VIEW_SM\n ->> SELECT \"object_kind\",\"property_value\" as \"parent_object\",\"object_name\" FROM $1\n WHERE \"object_kind\" IN ('METRIC','DIMENSION') AND \"property\" IN ('TABLE')\n;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ea0ccd5b-4dc6-489f-b3e4-2e15fdc2dd7f",
+ "metadata": {
+ "name": "md_step7",
+ "collapsed": false
+ },
+ "source": "## Step 7: \"Talk To\" the Semantic View with Cortex Analyst\n\nSnowflake's Cortex Analyst allows you to interact with your semantic views using natural language. \n\nLet's dynamically generate a link to Cortex Analyst so that you can access the semantic view.\n\nGo to the link in the cell below:"
+ },
+ {
+ "cell_type": "code",
+ "id": "55bb9979-789a-4414-83ba-659fd053ab64",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step7"
+ },
+ "outputs": [],
+ "source": "SELECT 'https://app.snowflake.com/' || CURRENT_ORGANIZATION_NAME() || '/' || CURRENT_ACCOUNT_NAME() || '/#/studio/analyst/databases/SAMPLE_DATA/schemas/TPCDS_SF10TCL/semanticView/TPCDS_SEMANTIC_VIEW_SM/edit' AS RESULT;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "65d22ac9-c5fb-4a29-8588-3dfe5dea66d7",
+ "metadata": {
+ "language": "python",
+ "name": "py_link",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "import streamlit as st\n\nlink = sql_step7.to_pandas()['RESULT'].iloc[0]\n\nst.link_button(\"Go to Cortex Analyst\", link)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "14b68fc8-c9b9-42c4-8362-d0426ce3cfee",
+ "metadata": {
+ "name": "md_question",
+ "collapsed": false
+ },
+ "source": "You can ask in natural language:\n\n*Show me the top selling brands in by total sales quantity in the state 'TX' in the 'Books' category in the year 2003*"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "465be0c3-2ee0-498a-9e35-a48da80ac67c",
+ "metadata": {
+ "name": "md_step8",
+ "collapsed": false
+ },
+ "source": "## Step 8: Query the Semantic View Using SQL\n\nNow that our semantic view is defined, we can easily query it to retrieve aggregated data. The following query demonstrates how to find the top-selling brands in a specific state and category for a given year and month, leveraging the simplified structure provided by the semantic view."
+ },
+ {
+ "cell_type": "code",
+ "id": "6d7e8487-51be-4521-b6a2-489dc69cc647",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_step8"
+ },
+ "outputs": [],
+ "source": "-- Query the semantic view to find top selling brands\nSELECT * FROM SEMANTIC_VIEW\n( \n TPCDS_SEMANTIC_VIEW_SM\n DIMENSIONS \n Item.Brand,\n Item.Category, \n Date.Year,\n Date.Month,\n Store.State\n METRICS \n StoreSales.TotalSalesQuantity\n WHERE\n Date.Year = '2002' AND Date.Month = '12' AND Store.State ='TX' AND Item.Category = 'Books'\n) \nORDER BY TotalSalesQuantity DESC LIMIT 10;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4071ee9d-8b8b-46b6-a029-765ce01c98cb",
+ "metadata": {
+ "name": "step9_streamlit",
+ "collapsed": false
+ },
+ "source": "## Step 9 (Optional): Build an Interactive Data App\n\nIn this step, we'll build 2 simple interactive data apps:\n\n1. Interactive data visualization app\n2. Simple interactive dashboard\n\nFirstly, we'll modify the SQL query to show data for month 12."
+ },
+ {
+ "cell_type": "code",
+ "id": "0ff53394-1dd7-4bd8-b2f2-4b72093dcb4d",
+ "metadata": {
+ "language": "sql",
+ "name": "cell1"
+ },
+ "outputs": [],
+ "source": "-- Query the semantic view for month 12\nSELECT * FROM SEMANTIC_VIEW\n( \n TPCDS_SEMANTIC_VIEW_SM\n DIMENSIONS \n Item.Brand,\n Item.Category, \n Date.Year,\n Date.Month,\n Store.State\n METRICS \n StoreSales.TotalSalesQuantity\n WHERE\n Date.Year = '2002' AND Date.Month = '12' AND Item.Category = 'Books'\n) \nORDER BY TotalSalesQuantity DESC;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1c7f4a26-4ea7-4b25-937d-e19c15d9b55b",
+ "metadata": {
+ "name": "md_df",
+ "collapsed": false
+ },
+ "source": "Next, we'll convert the SQL table to a Pandas DataFrame."
+ },
+ {
+ "cell_type": "code",
+ "id": "b7d4b9f8-f952-4497-93dd-98daf3a2a313",
+ "metadata": {
+ "language": "python",
+ "name": "df",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "cell1.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3697d6f0-203a-444a-9e53-1f899bbd627a",
+ "metadata": {
+ "name": "md_visualization",
+ "collapsed": false
+ },
+ "source": "### App 1. Interactive Data Visualization\n\nHere the user can interactively explore the sales data:"
+ },
+ {
+ "cell_type": "code",
+ "id": "259e5219-0505-472b-bba1-8c0267cae5c6",
+ "metadata": {
+ "language": "python",
+ "name": "py_visualization",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "import streamlit as st\nimport pandas as pd\n\nst.title(\"๐ Sales Data Interactive Visualization\")\n\n# Create selectbox for grouping option\ngroup_by = st.selectbox(\n \"Select grouping option:\",\n options=['BRAND', 'STATE'],\n index=0\n)\n\n# Group the data based on selection\nif group_by == 'BRAND':\n grouped_data = df.groupby('BRAND')['TOTALSALESQUANTITY'].sum().reset_index()\n grouped_data = grouped_data.set_index('BRAND')\n chart_title = \"Total Sales Quantity by Brand\"\nelse: # group_by == 'STATE'\n grouped_data = df.groupby('STATE')['TOTALSALESQUANTITY'].sum().reset_index()\n grouped_data = grouped_data.set_index('STATE')\n chart_title = \"Total Sales Quantity by State\"\n\n# Display the chart\nst.subheader(chart_title)\nst.bar_chart(grouped_data['TOTALSALESQUANTITY'])\n\n# Optional: Display the data table\nif st.checkbox(\"Show data table\"):\n st.subheader(\"Grouped Data\")\n st.dataframe(grouped_data)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "06fa1609-7084-41ee-9646-a8a0a79047c4",
+ "metadata": {
+ "name": "md_dashboard",
+ "collapsed": false
+ },
+ "source": "### App 2. Dashboard\n\nHere's a simple dashboard we're we've included a row of metrics:"
+ },
+ {
+ "cell_type": "code",
+ "id": "54077554-94ee-4a56-b499-02a327c506de",
+ "metadata": {
+ "language": "python",
+ "name": "py_dashboard",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "import streamlit as st\nimport pandas as pd\n\nst.title(\"๐ Sales Data Dashboard\")\n\n# Create selectbox for grouping option\ngroup_by = st.selectbox(\n \"Select grouping option:\",\n options=['BRAND', 'STATE'],\n index=0\n)\n\n# Group the data based on selection\nif group_by == 'BRAND':\n grouped_data = df.groupby('BRAND')['TOTALSALESQUANTITY'].sum().reset_index()\n grouped_data = grouped_data.set_index('BRAND')\n chart_title = \"Total Sales Quantity by Brand\"\nelse: # group_by == 'STATE'\n grouped_data = df.groupby('STATE')['TOTALSALESQUANTITY'].sum().reset_index()\n grouped_data = grouped_data.set_index('STATE')\n chart_title = \"Total Sales Quantity by State\"\n\n# Calculate KPIs based on current grouping\ntotal_sales = df['TOTALSALESQUANTITY'].sum()\navg_sales = df['TOTALSALESQUANTITY'].mean()\ntop_performer = grouped_data['TOTALSALESQUANTITY'].max()\ntop_performer_name = grouped_data['TOTALSALESQUANTITY'].idxmax()\n\n# Display KPI metrics in 3 columns\ncol1, col2, col3 = st.columns(3)\n\nwith col1:\n st.metric(\n label=\"Total Sales Quantity\", \n value=f\"{total_sales:,.0f}\",\n delta=None\n )\n\nwith col2:\n if group_by == 'BRAND':\n st.metric(\n label=\"Average Sales per Brand\", \n value=f\"{avg_sales:,.0f}\",\n delta=f\"{((avg_sales/total_sales)*100):.3f}% of total\"\n )\n else:\n st.metric(\n label=\"Average Sales per State\", \n value=f\"{avg_sales:,.0f}\",\n delta=f\"{len(df['STATE'].unique())} state(s)\"\n )\n\nwith col3:\n st.metric(\n label=f\"Top {group_by.title()}\", \n value=f\"{top_performer:,.0f}\",\n delta=f\"{top_performer_name}\"\n )\n\n# Display the chart\nst.subheader(chart_title)\nst.bar_chart(grouped_data['TOTALSALESQUANTITY'])\n\n# Optional: Display the data table\nif st.checkbox(\"Show data table\"):\n st.subheader(\"Grouped Data\")\n st.dataframe(grouped_data)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a27ed12b-9cbf-47c6-81ef-e8d84307e93f",
+ "metadata": {
+ "name": "md_resources",
+ "collapsed": false
+ },
+ "source": "## Related Resources\n\nArticles:\n\n- [Using SQL commands to create and manage semantic views](https://docs.snowflake.com/user-guide/views-semantic/sql)\n- [Using the Cortex Analyst Semantic View Generator](https://docs.snowflake.com/en/user-guide/views-semantic/ui)\n- [Sample Data: TPC-DS](https://docs.snowflake.com/en/user-guide/sample-data-tpcds)\n- [TPC-DS Benchmark Overview](https://www.tpc.org/tpcds/) - Understanding the sample dataset used in this guide\n\nDocumentation:\n- [Overview of semantic views](https://docs.snowflake.com/en/user-guide/views-semantic/overview)\n- [CREATE SEMANTIC VIEW](https://docs.snowflake.com/en/sql-reference/sql/create-semantic-view)\n- [DROP SEMANTIC VIEW](https://docs.snowflake.com/en/sql-reference/sql/drop-semantic-view)\n- [SHOW SEMANTIC VIEWS](https://docs.snowflake.com/en/sql-reference/sql/show-semantic-views)\n- [DESCRIBE SEMANTIC VIEW](https://docs.snowflake.com/en/sql-reference/sql/desc-semantic-view)"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/Snowflake_Trail_Alerts_Notifications/environment.yml b/Snowflake_Trail_Alerts_Notifications/environment.yml
new file mode 100644
index 0000000..f0682b2
--- /dev/null
+++ b/Snowflake_Trail_Alerts_Notifications/environment.yml
@@ -0,0 +1,7 @@
+name: app_environment
+channels:
+ - snowflake
+dependencies:
+ - snowflake=*
+ - snowflake-ml-python=*
+ - snowflake-snowpark-python=*
diff --git a/Snowflake_Trail_Alerts_Notifications/screenshot.png b/Snowflake_Trail_Alerts_Notifications/screenshot.png
new file mode 100644
index 0000000..38231fe
Binary files /dev/null and b/Snowflake_Trail_Alerts_Notifications/screenshot.png differ
diff --git a/Snowflake_Trail_Alerts_Notifications/truck_sentiment_analysis_with_trail.ipynb b/Snowflake_Trail_Alerts_Notifications/truck_sentiment_analysis_with_trail.ipynb
new file mode 100644
index 0000000..914258b
--- /dev/null
+++ b/Snowflake_Trail_Alerts_Notifications/truck_sentiment_analysis_with_trail.ipynb
@@ -0,0 +1,781 @@
+{
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat_minor": 2,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "name": "md_intro",
+ "resultHeight": 547,
+ "collapsed": false
+ },
+ "source": "# Snowflake Trail for Observability\n[Snowflake Trail](https://www.snowflake.com/en/data-cloud/snowflake-trail/) is a set of Snowflake capabilities that enables developers to better monitor, troubleshoot, debug, and take actions on pipelines, applications, user code, and compute utilization.\n\n## Truck Analysis\nIn this demo, we'll explore how to add observability - traces, logs, and alerts for a simple Truck Reviews sentiment analysis use case. We'll integrate [Slack Webhook](https://api.slack.com/messaging/webhooks) to deliver notifications to a Slack channel.\n\nBy the end of this demo, you will understand:\n- How to enable Telemetry in your Snowflake account\n- What the various object levels are at which Telemetry can be set\n- How to define Serverless Alerts\n- How to integrate Slack notifications via Webhooks\n\n>**IMPORTANT**\n>\n>Before getting started, make sure you have access to a Slack workspace where you can add a webhook integration\n",
+ "id": "ce110000-1111-2222-3333-ffffff000000"
+ },
+ {
+ "cell_type": "code",
+ "id": "7da07c23-e866-4858-be9a-6ac57afc578e",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_currents",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "SELECT current_role() as current_role",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "2ab7d0b2-52e4-4079-a777-6708ada8255a",
+ "metadata": {
+ "language": "python",
+ "name": "imports",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "import streamlit as st",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "044cbada-d1e6-416d-bcaf-e6bf8f116ca4",
+ "metadata": {
+ "name": "cell2",
+ "collapsed": false,
+ "resultHeight": 102
+ },
+ "source": "## Object Names\nLet us define variables that will hold the various object and resource names used throughout this demo."
+ },
+ {
+ "cell_type": "code",
+ "id": "ec1e66cb-72b4-4767-971f-8c0e28cb3ea7",
+ "metadata": {
+ "language": "python",
+ "name": "variables",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "outputs": [],
+ "source": "__current_role=sql_currents.to_pandas().iloc[0]['CURRENT_ROLE']\n__current_role\n__database = \"kamesh_build_24_demos\"\n__analytics_schema = \"analytics\"\n__data_schema = \"data\"\n__stages_schema = \"stages\"\n__src_schema = \"src\"\n__task_schema = \"tasks\"\n__alerts_schema = \"alerts_and_notifications\"\n__telemetry_schema = \"telemetry\"\n__warehouse = \"kamesh_snowpark_demo_wh\"\n__task_name = \"truck_sentiment\"",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "21cd8327-b1c9-4661-8ce2-4c9b79bdf30b",
+ "metadata": {
+ "name": "cell3",
+ "collapsed": false,
+ "resultHeight": 201
+ },
+ "source": "## Database Setup\nIn the following steps, we will:\n- Create necessary Snowflake objects and resources\n- Ingest data required for truck sentiment analysis\n- Set up alert triggers for Slack channel notifications"
+ },
+ {
+ "cell_type": "code",
+ "id": "171f227f-a22f-4a86-9b6f-a3db86658db3",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_context",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "CREATE DATABASE IF NOT EXISTS {{__database}};\nUSE DATABASE {{__database}};",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d76b3281-8186-4433-bb4c-bda4f4c227a5",
+ "metadata": {
+ "name": "cell4",
+ "collapsed": false,
+ "resultHeight": 311
+ },
+ "source": "Let us create schemas to group our various objects.\n\n| Schema Name | Purpose |\n| :----: | :---- |\n| analytics | Holds the analytical data |\n| stages | Holds all internal and external stages |\n| src | Holds the sources of the UDFs and Stored Procedures |\n| task | Holds all Tasks used in this demo |\n| alerts | Holds all Alert definitions |\n| telemetry | Holds database-level event table |\n"
+ },
+ {
+ "cell_type": "code",
+ "id": "e629dabf-6b9c-4f06-aab0-e42b47a8e092",
+ "metadata": {
+ "language": "sql",
+ "name": "create_schemas",
+ "collapsed": false,
+ "resultHeight": 438
+ },
+ "outputs": [],
+ "source": "CREATE SCHEMA IF NOT EXISTS {{__analytics_schema}};\nCREATE SCHEMA IF NOT EXISTS {{__data_schema}};\nCREATE SCHEMA IF NOT EXISTS {{__stages_schema}};\nCREATE SCHEMA IF NOT EXISTS {{__src_schema}};\nCREATE SCHEMA IF NOT EXISTS {{__task_schema}};\nCREATE SCHEMA IF NOT EXISTS {{__alerts_schema}};\nCREATE SCHEMA IF NOT EXISTS {{__telemetry_schema}};\n\nSHOW SCHEMAS in database {{__database}};",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "83574809-7ffb-4549-86d5-2148c05b6f27",
+ "metadata": {
+ "name": "md_data_load",
+ "collapsed": false,
+ "resultHeight": 128
+ },
+ "source": "## Load Truck Data\nThe demo uses truck data from Tasty Bytes. Please ensure that you load the data from `data_load.sql` script into your `__database`. The SQL objects and other related data definitions are available [here](https://github.com/Snowflake-Labs/build24-trail-demo/tree/main/scripts)."
+ },
+ {
+ "cell_type": "code",
+ "id": "6bed04c3-22e5-40eb-9c8a-dcab48b819be",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_load_truck_data",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "CREATE API INTEGRATION if not exists git_api_integration\n API_PROVIDER = git_https_api\n API_ALLOWED_PREFIXES = ('https://github.com/snowflake-labs')\n ENABLED = TRUE;\n\nCREATE OR REPLACE GIT REPOSITORY {{__database}}.{{__data_schema}}.build24_trail_demo\n API_INTEGRATION = git_api_integration\n ORIGIN = 'https://github.com/snowflake-labs/build24-trail-demo';",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "73bc1cb3-3aae-4a00-b587-2a4e392903d8",
+ "metadata": {
+ "name": "cell6",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Let's refresh the repository and pull the latest changes."
+ },
+ {
+ "cell_type": "code",
+ "id": "4dfb5ee5-d482-4d0f-a53e-a703fee0568e",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_list_main_branch_scripts_files",
+ "collapsed": false,
+ "resultHeight": 251,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "ALTER git repository {{__database}}.{{__data_schema}}.build24_trail_demo fetch;\nls @{{__database}}.{{__data_schema}}.build24_trail_demo/branches/main/scripts;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "898c1aac-6615-4f7c-a6a0-c264609c91e6",
+ "metadata": {
+ "name": "cell7",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Let's run the script to create the database objects and ingest the data."
+ },
+ {
+ "cell_type": "code",
+ "id": "609b16ff-03f2-48f4-960d-6f9b5844a6bf",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_load_data",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "EXECUTE IMMEDIATE FROM @{{__database}}.{{__data_schema}}.build24_trail_demo/branches/main/scripts/data_setup.j2.sql \nUSING ( demo_role => '{{__current_role}}', demo_database => '{{__database}}' );",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "00c11a4f-4455-4ff0-9b82-77c2b62922b1",
+ "metadata": {
+ "language": "python",
+ "name": "py_snowpark_session",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "from snowflake.snowpark.context import get_active_session\nfrom snowflake.core import CreateMode, Root\nfrom snowflake.core.schema import Schema\nfrom snowflake.core.database import Database\n\nsession = get_active_session()\nroot = Root(session)\n",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b7c1897c-8a03-4861-a287-059db25af975",
+ "metadata": {
+ "name": "md_udf_classify_sentiment",
+ "collapsed": false,
+ "resultHeight": 128
+ },
+ "source": "## UDF Sentiment Class\nA Python UDF that converts Snowflake Cortex sentiment scores into textual sentiment classifications: `negative`, `neutral`, or `positive`."
+ },
+ {
+ "cell_type": "code",
+ "id": "8ee3f3d7-9134-4cc2-aff4-750cbeb903db",
+ "metadata": {
+ "language": "python",
+ "name": "udf_src_stage",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "from snowflake.core.stage import Stage\n\n__udf_stage_name = \"udfs\"\n__udf_stage = Stage(name=__udf_stage_name)\n_ = root.databases[__database].schemas[__src_schema].stages.create(\n __udf_stage,\n mode=CreateMode.if_not_exists,\n)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "68218028-9fa1-4ff5-8883-c73ffb53bdcf",
+ "metadata": {
+ "language": "python",
+ "name": "udf_classify_sentiment",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "from snowflake.snowpark.functions import udf\n\n@udf(\n name=f\"{__database}.{__data_schema}.classify_sentiment\",\n is_permanent=True,\n packages=[\"snowflake-telemetry-python\"],\n stage_location=f\"{__database}.{__src_schema}.{__udf_stage_name}\",\n replace=True,\n)\ndef classify_sentiment(sentiment_score: float) -> str:\n \"\"\"Classify sentiment as positive,neutral or negative based on the score.\"\"\"\n import logging\n\n import snowflake.telemetry as telemetry\n\n logging.info(\"Classifying sentiment score\")\n\n telemetry.set_span_attribute(\"processing\", \"classify_sentiment\")\n logging.debug(f\"Classifying sentiment score {sentiment_score:.2f}\")\n\n if sentiment_score < -0.5:\n logging.debug(f\"Sentiment {sentiment_score:.2f} is negative\")\n return \"negative\"\n elif sentiment_score >= 0.5 and sentiment_score <= 1.0:\n logging.debug(f\"Sentiment {sentiment_score:.2f} is positive\")\n return \"positive\"\n else:\n logging.debug(f\"Sentiment {sentiment_score:.2f} is neutral\")\n return \"netural\"",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7725e882-cbdf-453a-8d5d-910683c2495d",
+ "metadata": {
+ "name": "md_stored_proc_truck_review_sentiments",
+ "collapsed": false,
+ "resultHeight": 128
+ },
+ "source": "## Stored Procedure `truck_review_sentiments`\nThe stored procedure builds the `truck_review_sentiments` table and uses the `sentiment_class` UDF to categorize sentiments into text classifications."
+ },
+ {
+ "cell_type": "code",
+ "id": "30d66652-1671-415b-bd42-176c98f99f32",
+ "metadata": {
+ "language": "python",
+ "name": "sp_build_truck_review_sentiments",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "# stage to hold the stored procedure sources\nfrom snowflake.snowpark.functions import sproc\nfrom snowflake.snowpark.session import Session\nfrom snowflake.core.stage import Stage\n\n__pros_stage_name = \"procs\"\n__procs_stage = Stage(name=__pros_stage_name)\n_ = (\n root.databases[__database]\n .schemas[__src_schema]\n .stages.create(\n __procs_stage,\n mode=CreateMode.if_not_exists,\n )\n)\n\n@sproc(\n name=f\"{__database}.{__data_schema}.build_truck_review_sentiments\",\n replace=True,\n is_permanent=True,\n packages=[\n \"snowflake-telemetry-python\",\n \"snowflake-ml-python\",\n ],\n stage_location=f\"{__database}.{__src_schema}.{__procs_stage.name}\",\n source_code_display=True,\n comment=\"Build the build_truck_review_sentiments table. This procedure will be called from a Task.\",\n)\ndef build_truck_review_sentiments(session: Session) -> None:\n \"\"\"Build the Truck Review Sentiments table.\"\"\"\n import logging\n\n import snowflake.cortex as cortex\n import snowflake.snowpark.functions as F\n import snowflake.telemetry as telemetry\n from snowflake.snowpark.types import DecimalType\n\n logging.debug(\"START::Truck Review Sentiments\")\n telemetry.set_span_attribute(\"executing\", \"build_truck_review_sentiments\")\n\n try:\n telemetry.set_span_attribute(\"building\", \"truck_reviews\")\n review_df = (\n session.table(f\"{__database}.{__analytics_schema}.truck_reviews_v\")\n .select(\n F.col(\"TRUCK_ID\"),\n F.col(\"REVIEW\"),\n )\n .filter(F.year(F.col(\"DATE\")) == 2024)\n )\n telemetry.set_span_attribute(\"building\", \"add_sentiment_score\")\n review_sentiment_score_df = review_df.withColumn(\n \"SENTIMENT_SCORE\",\n cortex.Sentiment(F.col(\"REVIEW\")).cast(DecimalType(2, 2)),\n )\n telemetry.set_span_attribute(\"building\", \"add_sentiment_class\")\n review_sentiment_class_df = review_sentiment_score_df.withColumn(\n \"SENTIMENT_CLASS\",\n classify_sentiment(\n F.col(\"SENTIMENT_SCORE\"),\n ),\n )\n logging.debug(review_sentiment_score_df.show(5))\n __table = f\"{__database}.{__data_schema}.truck_review_sentiments\"\n telemetry.set_span_attribute(\"save\", f\"save_to_{__table}\")\n review_sentiment_class_df.write.mode(\"overwrite\").save_as_table(__table)\n except Exception as e:\n logging.error(f\"Error building truck_review_sentiments,{e}\", exc_info=True)\n\n logging.debug(\"END::Truck Review Sentiments Complete\")\n",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7c331022-6549-4328-97b6-b4e9b6edb977",
+ "metadata": {
+ "name": "md_telemetry_settings",
+ "collapsed": false,
+ "resultHeight": 154
+ },
+ "source": "## Telemetry Settings\nIn the following steps, we will set up Telemetry Events (logs/traces) at the database level. While Snowflake defaults to storing events in `SNOWFLAKE.TELEMETRY.EVENTS`, for this demo we will configure event collection at the database level."
+ },
+ {
+ "cell_type": "code",
+ "id": "fda3d506-3439-4447-94c5-be7f0d43aaf2",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_check_current_telementry_settings",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "-- check the current event_table\nSHOW PARAMETERS LIKE 'event_table' IN DATABASE {{__database}};",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cc595e6e-6045-4100-af79-a292909c37f6",
+ "metadata": {
+ "name": "cell8",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Create the event table at the database level and set it as the default Events table for the database."
+ },
+ {
+ "cell_type": "code",
+ "id": "be2b0ff5-3148-4219-adf8-e499b3eb5d52",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_setup_telemetry",
+ "collapsed": false,
+ "resultHeight": 111,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "-- create event table \nCREATE EVENT TABLE IF NOT EXISTS {{__database}}.{{__telemetry_schema}}.events;\n-- set to new event table\nALTER DATABASE {{__database}} SET EVENT_TABLE = {{__database}}.{{__telemetry_schema}}.events;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "70dcee7b-af96-4498-b26b-d30fb29d6a49",
+ "metadata": {
+ "name": "cell9",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "In the following cells, we will examine the parameters for logs, traces, and metrics in the demo database."
+ },
+ {
+ "cell_type": "code",
+ "id": "126ccb24-978c-4859-badc-c2bb99a2d54e",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_current_log_level",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "SHOW PARAMETERS LIKE 'LOG_LEVEL' IN DATABASE {{__database}};",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "208d9160-e64e-4604-88d7-2a0dd981ecaa",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_current_trace_level",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "SHOW PARAMETERS LIKE 'TRACE_LEVEL' IN DATABASE {{__database}};",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "43699fa7-9a2e-4a7e-85c1-51c7b9f84940",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_current_metric_level",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "SHOW PARAMETERS LIKE 'METRIC_LEVEL' IN DATABASE {{__database}};",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3ac7ae23-5db5-4b27-b1ec-b2eda710641a",
+ "metadata": {
+ "name": "cell10",
+ "collapsed": false,
+ "resultHeight": 67
+ },
+ "source": "Alter the demo database to set the logging level to DEBUG, trace level to ALWAYS, and metrics collection level to ALL"
+ },
+ {
+ "cell_type": "code",
+ "id": "c3a07e2d-430e-4a08-942e-a335fc3e0918",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_set_telemetry_levels",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "-- set log, trace and metrtic levels\nALTER DATABASE {{__database}} SET LOG_LEVEL = DEBUG;\nALTER DATABASE {{__database}} SET TRACE_LEVEL = ALWAYS;\nALTER DATABASE {{__database}} SET METRIC_LEVEL = ALL;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7c0e73f5-8636-468d-aa68-d50fb5cf99ef",
+ "metadata": {
+ "name": "md_view_data",
+ "collapsed": false,
+ "resultHeight": 102
+ },
+ "source": "## Truck Reviews\nLet's ensure we have the data ingested and ready to use."
+ },
+ {
+ "cell_type": "code",
+ "id": "1190b9de-b97c-4c61-8e32-7a8c0abf0447",
+ "metadata": {
+ "language": "sql",
+ "name": "py_sql_truck_reviews",
+ "collapsed": false,
+ "resultHeight": 251
+ },
+ "outputs": [],
+ "source": "select * \nfrom {{__database}}.analytics.truck_reviews_v\nlimit 5;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8f513c50-8e1c-467d-b155-0273d5ed1956",
+ "metadata": {
+ "name": "md_tasks",
+ "collapsed": false,
+ "resultHeight": 102
+ },
+ "source": "## Tasks\nLet's create a few tasks to execute the stored procedure and build our truck_review_sentiments table."
+ },
+ {
+ "cell_type": "code",
+ "id": "314a5999-c26a-4974-9ede-53e08710ac94",
+ "metadata": {
+ "language": "python",
+ "name": "task_truck_sentiments",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "from datetime import timedelta\n\nfrom snowflake.core.task import StoredProcedureCall, Task\n\ntruck_sentiment_task = Task(\n name=__task_name,\n warehouse=__warehouse,\n definition=StoredProcedureCall(build_truck_review_sentiments),\n schedule=timedelta(minutes=1),\n)\n\ntask_truck_sentiment = (\n root.databases[__database].schemas[__task_schema].tasks[__task_name]\n)\n\ntask_truck_sentiment.create_or_alter(truck_sentiment_task)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "94fba846-827a-44ba-8a96-902065ddeaff",
+ "metadata": {
+ "language": "python",
+ "name": "task_status",
+ "collapsed": false,
+ "resultHeight": 42
+ },
+ "outputs": [],
+ "source": "tasks = root.databases[__database].schemas[__task_schema].tasks\n__task_truck_sentiment = tasks[__task_name]\ntask_detials = __task_truck_sentiment.fetch()\nst.write(f\"Current Task Status:`{task_detials.state}`\")",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b6375f4e-6904-4793-9fb9-db5005a0505c",
+ "metadata": {
+ "name": "cell11",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Resume the task."
+ },
+ {
+ "cell_type": "code",
+ "id": "41c03646-2289-4d77-b18d-d336339caad8",
+ "metadata": {
+ "language": "python",
+ "name": "resume_task",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "__task_truck_sentiment.resume()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ad7923a5-c985-4188-a650-0cbbc6b1d89e",
+ "metadata": {
+ "name": "cell12",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Suspend the task."
+ },
+ {
+ "cell_type": "code",
+ "id": "b5e73f5e-026d-4186-9d4b-1d5c1ff56d81",
+ "metadata": {
+ "language": "python",
+ "name": "suspend_task",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "__task_truck_sentiment.suspend()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c54f601b-29b6-434c-a247-dc3b2cde8f7d",
+ "metadata": {
+ "name": "cell13",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Execute the task immediately."
+ },
+ {
+ "cell_type": "code",
+ "id": "044a67f7-854e-4b91-b4ac-ba78b112e9b9",
+ "metadata": {
+ "language": "python",
+ "name": "run_task_immediately",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "__task_truck_sentiment.execute()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6d3871ca-b779-46d9-8db4-fed18101cc51",
+ "metadata": {
+ "name": "cell1",
+ "collapsed": false,
+ "resultHeight": 74
+ },
+ "source": "# Alerts and Notifications"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1989ae1b-25dd-4389-b435-3f05da32c3c2",
+ "metadata": {
+ "name": "md_serverless_alerts",
+ "collapsed": false,
+ "resultHeight": 179
+ },
+ "source": "## Serverless Alerts\nAlerts that use the serverless compute model are called serverless alerts. When using the serverless compute model, Snowflake automatically resizes and scales the required compute resources for the alert. Snowflake determines the ideal compute resource size for each run based on a dynamic analysis of statistics from the alert's most recent previous executions."
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3010ee4b-fb63-405d-b12d-5db5388121ba",
+ "metadata": {
+ "name": "cell5",
+ "collapsed": false,
+ "resultHeight": 321
+ },
+ "source": "## Slack Notifications\nTo create a Slack Webhook notification, we need to complete the following steps:\n\n1. Create a Slack Webhook using the [Slack API](https://api.slack.com/apps) to enable posting to a channel. For detailed instructions, refer to the [Slack Webhooks documentation](https://api.slack.com/messaging/webhooks).\n\n2. Obtain the Slack Webhook URL for channel posting. The URL format follows this pattern:\n `https://hooks.slack.com/services/`\n\n3. Create a string-type secret containing the `` value.\n\n4. Create a `NOTIFICATION INTEGRATION` using both the `secret` and the `Slack Webhook URL`."
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4780713e-4b60-4d09-970d-ba0ae9d7096f",
+ "metadata": {
+ "name": "md_slack_webhook_secret",
+ "collapsed": false,
+ "resultHeight": 140
+ },
+ "source": "### Create Slack Webhook Secret\nThe Slack webhook secret can be extracted from the Webhook URL. For example, if your URL is `https://hooks.slack.com/services/Txxxxxxx/B000000000/xxxxxxxxxx`, use the string `Txxxxxxx/B000000000/xxxxxxxxxx` as the `SECRET_STRING`."
+ },
+ {
+ "cell_type": "code",
+ "id": "3af9d926-8784-40b9-a647-e727e54d888a",
+ "metadata": {
+ "language": "python",
+ "name": "get_slack_secret",
+ "collapsed": false,
+ "resultHeight": 84
+ },
+ "outputs": [],
+ "source": "slack_webhook_secret = st.text_input(\"Enter Slack Webhook Secret:\",type=\"password\")\nif slack_webhook_secret == \"\":\n raise Exception(\"Slack webhook secret is required.\")",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8ca2e9ca-650d-40d9-8675-2fd75349254d",
+ "metadata": {
+ "name": "cell14",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Let's define variables to hold the names of the alert and notification objects."
+ },
+ {
+ "cell_type": "code",
+ "id": "dcee0962-2136-4936-bdb2-a1ec71ad8fd0",
+ "metadata": {
+ "language": "python",
+ "name": "alert_variables",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "__slack_webhook_secret_name='slack_alerts_notifications_webhook'\n__slack_notification='slack_channel_alerts_notify'\n__truck_negatives_alert='truck_review_alert'",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "31f03170-08ea-40ce-8a1b-dabba703ba22",
+ "metadata": {
+ "name": "cell15",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Let's create a secret to hold the Slack webhook secret."
+ },
+ {
+ "cell_type": "code",
+ "id": "e17233e5-4180-4712-86fd-414a71c44570",
+ "metadata": {
+ "language": "sql",
+ "name": "slack_webhook_secret",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "CREATE OR REPLACE SECRET {{__database}}.{{__alerts_schema}}.{{__slack_webhook_secret_name}}\n TYPE = GENERIC_STRING\n SECRET_STRING = '{{slack_webhook_secret}}';",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ec05b6b4-5509-43b8-936f-c7bd73f2d17d",
+ "metadata": {
+ "name": "cell16",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "[Notification Integration](https://docs.snowflake.com/en/sql-reference/commands-integration) enables us to trigger a notification on an alert."
+ },
+ {
+ "cell_type": "code",
+ "id": "972cb37d-c610-403a-9ceb-4d1322801849",
+ "metadata": {
+ "language": "sql",
+ "name": "slack_notification",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "-- send to channel \nCREATE OR REPLACE NOTIFICATION INTEGRATION {{__slack_notification}}\n TYPE = WEBHOOK\n ENABLED = true\n WEBHOOK_URL = 'https://hooks.slack.com/services/SNOWFLAKE_WEBHOOK_SECRET'\n WEBHOOK_SECRET = {{__database}}.{{__alerts_schema}}.{{__slack_webhook_secret_name}}\n WEBHOOK_BODY_TEMPLATE='SNOWFLAKE_WEBHOOK_MESSAGE'\n WEBHOOK_HEADERS=('Content-Type'='application/json');",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dec6abbe-cce8-4a54-a722-80591aa990de",
+ "metadata": {
+ "name": "cell17",
+ "collapsed": false,
+ "resultHeight": 305
+ },
+ "source": "## Serverless Alert\nLet's define a serverless alert that triggers when data in `truck_review_sentiments` has the class `negative` and a sentiment score less than `-0.8`. For simplicity in this demo, we will retrieve only the top three negative records.\n\nOnce we have the negative records, we will use [Cortex Complete](https://docs.snowflake.com/en/sql-reference/functions/complete-snowflake-cortex) to construct a Slack message that will be sent as part of the notification.\n\n> *NOTE*:\n>\n> To convert a normal alert to a serverless alert, omit the `WAREHOUSE` property."
+ },
+ {
+ "cell_type": "code",
+ "id": "804ef84e-8eee-4268-aa5e-ac1caf4e822d",
+ "metadata": {
+ "language": "sql",
+ "name": "truck_review_alert",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "-- Alert - alerts when there is stronger negative feedback\n-- Truck Review Alert\nCREATE OR REPLACE ALERT {{__database}}.{{__alerts_schema}}.{{__truck_negatives_alert}}\n SCHEDULE = '1 minute'\n IF(\n EXISTS(\n WITH negative_reviews AS (\n SELECT \n truck_id,\n review,\n sentiment_score,\n ROW_NUMBER() OVER (PARTITION BY truck_id ORDER BY sentiment_score ASC) as worst_review_rank\n FROM data.truck_review_sentiments\n WHERE sentiment_class = 'negative'\n AND sentiment_score < -0.8\n )\n SELECT \n truck_id,\n review,\n sentiment_score\n FROM negative_reviews\n WHERE worst_review_rank = 1\n ORDER BY sentiment_score ASC\n LIMIT 3 -- top 3 only\n )\n )\n THEN\n BEGIN\n -- TODO add event\n LET rs RESULTSET := (\n WITH REVIEW_DATA AS (\n SELECT truck_id, review\n FROM TABLE(RESULT_SCAN(SNOWFLAKE.ALERT.GET_CONDITION_QUERY_UUID()))\n ),\n SUMMARIZED_CONTENT AS (\n SELECT \n SNOWFLAKE.CORTEX.COMPLETE(\n 'llama3.1-405b',\n CONCAT(\n 'Summarize the review as bullets formatted for slack notification blocks with right and consistent emojis and always add truck id to the Review Alert header along with truck emoji and stay consistent with Header like Review Truck ID - :',\n '', \n REVIEW, \n '',\n 'Quote the truck id.', \n TRUCK_ID,\n '.Generate only Slack blocks and strictly ignore other text.'\n )) AS SUMMARY\n FROM REVIEW_DATA\n ),\n FORMATTED_BLOCKS AS (\n SELECT SNOWFLAKE.NOTIFICATION.SANITIZE_WEBHOOK_CONTENT(SUMMARY) AS CLEAN_BLOCKS\n FROM SUMMARIZED_CONTENT\n ),\n JSON_BLOCKS AS (\n SELECT SNOWFLAKE.NOTIFICATION.APPLICATION_JSON(CONCAT('{\"blocks\":',CLEAN_BLOCKS,'}')) AS BLOCKS\n FROM FORMATTED_BLOCKS\n )\n -- slack message content blocks\n SELECT BLOCKS FROM JSON_BLOCKS\n );\n \n FOR record IN rs DO\n let slack_message varchar := record.BLOCKS;\n SYSTEM$LOG_INFO('SLACK MESSAGE:',OBJECT_CONSTRUCT('slack_message', slack_message));\n CALL SYSTEM$SEND_SNOWFLAKE_NOTIFICATION(\n :slack_message,\n SNOWFLAKE.NOTIFICATION.INTEGRATION('{{__slack_notification}}')\n );\n END FOR;\n END;\n",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ea5dc107-2515-4f32-be3f-4059b26c8cdb",
+ "metadata": {
+ "name": "cell18",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Let's trigger the alert immediately."
+ },
+ {
+ "cell_type": "code",
+ "id": "69e4a26c-65c5-4c5b-acbe-33c2446e9e15",
+ "metadata": {
+ "language": "sql",
+ "name": "execute_alert",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "EXECUTE ALERT {{__database}}.{{__alerts_schema}}.{{__truck_negatives_alert}};",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "16d0a845-fbb5-4e25-b4bd-2fa293f7fdb0",
+ "metadata": {
+ "name": "cell19",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Suspend the alert if needed."
+ },
+ {
+ "cell_type": "code",
+ "id": "52a95ce6-e3f1-4919-8cc9-3285f7cf663d",
+ "metadata": {
+ "language": "sql",
+ "name": "suspend_alert",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "ALTER ALERT {{__database}}.{{__alerts_schema}}.{{__truck_negatives_alert}} SUSPEND;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8a73600c-570a-4b0d-97c8-35592ac8a6b8",
+ "metadata": {
+ "name": "cell20",
+ "collapsed": false,
+ "resultHeight": 451
+ },
+ "source": "## Alert and Notification History\n\nSnowflake provides dedicated stored procedures to view the execution history of alerts and notifications. These procedures allow you to monitor and audit your alert and notification activities.\n\nTo retrieve historical data, use these stored procedures:\n\n### Alert History\n```sql\nINFORMATION_SCHEMA.ALERT_HISTORY\n```\nThis procedure returns detailed records of past alert executions.\n\n### Notification History\n```sql\nINFORMATION_SCHEMA.NOTIFICATION_HISTORY\n```"
+ },
+ {
+ "cell_type": "code",
+ "id": "3890fbdf-894c-49d6-998f-07a8b7dc84a2",
+ "metadata": {
+ "language": "python",
+ "name": "st_view_alert_history",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "st.header(\"Alert History\")\nscheduled_time_range_start = st.slider(\"Schedule Time Range Start(mins):\",min_value=5,max_value=60)\n#alert_history_tf=session.table_function(information_schema.alert_history)\n",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "0c5a98c2-ce2f-4293-aaf4-fed3c5b5d000",
+ "metadata": {
+ "language": "python",
+ "name": "view_alert_history",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "df=session.sql(f\"\"\"\nSelect name,database_name,schema_name,action,state,sql_error_message\nfrom\n table(information_schema.alert_history(\n scheduled_time_range_start\n =>dateadd('minutes',-{scheduled_time_range_start},current_timestamp())))\norder by scheduled_time desc\n\"\"\")\nst.dataframe(df)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "fec79688-f59a-4b4d-b2d9-12e478e23462",
+ "metadata": {
+ "language": "python",
+ "name": "st_notification_history",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "st.header(\"Notification History\")\n_start_time = st.slider(\"Start time(mins):\",min_value=5,max_value=60)\n#alert_history_tf=session.table_function(information_schema.alert_history)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "c81fc3a4-dbc8-4097-ac01-d0e0a215e3e7",
+ "metadata": {
+ "language": "python",
+ "name": "notification_history",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "notify_df=session.sql(f\"\"\"\nSELECT INTEGRATION_NAME,STATUS,ERROR_MESSAGE \nFROM TABLE(INFORMATION_SCHEMA.NOTIFICATION_HISTORY(\n START_TIME => dateadd('minutes',-{_start_time},current_timestamp()),\n INTEGRATION_NAME => '{__slack_notification}'\n))\n\"\"\")\nst.dataframe(notify_df)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f92b0393-4165-4fd9-a77d-a30f481c6f1b",
+ "metadata": {
+ "name": "md_cleanup",
+ "collapsed": false,
+ "resultHeight": 102
+ },
+ "source": "## Resource Cleanup\n\nTo prevent unnecessary resource consumption and cost."
+ },
+ {
+ "cell_type": "code",
+ "id": "6f507949-f5c5-4193-9b57-339334993d20",
+ "metadata": {
+ "language": "sql",
+ "name": "cleanup",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "DROP NOTIFICATION INTEGRATION {{__slack_notification}};\nDROP DATABASE {{__database}}",
+ "execution_count": null
+ }
+ ]
+}
\ No newline at end of file
diff --git a/Streamlit_Zero_To_Hero_Machine_Learning_App/Streamlit_Machine_Learning_App.ipynb b/Streamlit_Zero_To_Hero_Machine_Learning_App/Streamlit_Machine_Learning_App.ipynb
new file mode 100644
index 0000000..6cf8814
--- /dev/null
+++ b/Streamlit_Zero_To_Hero_Machine_Learning_App/Streamlit_Machine_Learning_App.ipynb
@@ -0,0 +1,327 @@
+{
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat_minor": 2,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "name": "md_intro",
+ "resultHeight": 599,
+ "collapsed": false
+ },
+ "source": "\n# Building an Interactive Machine Learning Demo with Streamlit in Snowflake\n\nIn this notebook, we'll create and deploy an interactive Machine Learning application using Streamlit, running it entirely within a Snowflake Notebook environment. This hands-on exercise will demonstrate how to combine the power of Streamlit's user interface capabilities with Snowflake Notebook in quickly building an interactive Machine Learning application.\n\n## Learning Objectives\n\nBy completing this exercise, you will:\n\n- Master the usage of Streamlit widgets to create interactive data applications\n- Deploy and run a Streamlit application within a Snowflake Notebook\n- Implement a practical classification model using scikit-learn\n- Create interactive ML predictions using Streamlit's dynamic interface capabilities\n\nThe unique aspect of this tutorial is that everything runs directly within your Snowflake Notebook environment, providing a seamless development experience.\n\n## Resources\n\n- Reference Implementation: [Streamlit Machine Learning Demo](https://github.com/Snowflake-Labs/st-ml-app)\n- Detailed Tutorial: [Bootstrapping Your Transition from Streamlit OSS to Streamlit in Snowflake (SiS)](https://snowflake-labs.github.io/streamlit-oss-to-sis-bootstrap/) - A comprehensive guide by Snowflake Developers on building Streamlit applications\n",
+ "id": "ce110000-1111-2222-3333-ffffff000000"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cea10b02-7b79-4fb4-8f08-5d58f6398ee8",
+ "metadata": {
+ "name": "md_pre_req",
+ "collapsed": false,
+ "resultHeight": 623
+ },
+ "source": "## Pre-requisite\n\nBefore we dive into building our Machine Learning application, this notebook will guide you through the essential setup steps required to prepare your Snowflake account. These preparations are crucial for deploying and running the Streamlit ML App successfully.\n\n## Setup Steps\n\nWe will complete the following configuration tasks:\n\n1. Database Structure Setup\n\n - Create necessary schemas\n - Set up required tables for our ML application\n\n\n2. External Storage Configuration\n\n - Create and configure an external stage connected to Amazon S3 \n - Establish secure data access pathways\n\n3. Data Preparation\n\n - Load the Penguins dataset into Snowflake\n - Prepare the data structure for ML operations\n\nThis foundational setup will ensure smooth execution of our Machine Learning application within the Snowflake environment. \n\nLet's proceed with these prerequisites step by step."
+ },
+ {
+ "cell_type": "markdown",
+ "id": "34c3b93e-b674-4603-a68e-8f0fd3c2e2f7",
+ "metadata": {
+ "name": "md_env_schemas",
+ "collapsed": false,
+ "resultHeight": 638
+ },
+ "source": "\n## Environment Setup: Schemas and Stages\n\nIn this section, we'll establish the foundational database structures needed for our Streamlit ML application. We'll create dedicated schemas to ensure proper organization and separation of concerns.\n\n> *NOTE*: The schemas will default to the database where the Notebook is located.\n\n## Schema Organization\n\n| Schema | Purpose |\n|--------|----------|\n| `apps` | Houses all application components, specifically our Streamlit application |\n| `data` | Stores all data tables, including our Penguins dataset |\n| `stages` | Contains all staging areas for data loading and file management |\n| `file_formats` | Defines the file formats used for data ingestion |\n\nEach schema serves a specific purpose in our application architecture:\n- The `apps` schema keeps our application code isolated\n- The `data` schema maintains our datasets in an organized manner\n- The `stages` schema manages our external connections\n- The `file_formats` schema ensures consistent data loading formats\n\nLet's proceed with creating these schemas in our Snowflake environment."
+ },
+ {
+ "cell_type": "code",
+ "id": "0019ab10-21cf-493d-b328-ab8f836d7844",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_schemas",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "-- data schema\nCREATE SCHEMA IF NOT EXISTS DATA;\n-- create schema to hold all stages\nCREATE SCHEMA IF NOT EXISTS STAGES;\n-- create schema to hold all file formats\nCREATE SCHEMA IF NOT EXISTS FILE_FORMATS;\n-- apps to hold all streamlit apps\nCREATE SCHEMA IF NOT EXISTS APPS;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "11bdb7e7-20da-42de-a27e-f074ea90962f",
+ "metadata": {
+ "name": "md_env_stages",
+ "collapsed": false,
+ "resultHeight": 398
+ },
+ "source": "## Stage and File Format Configuration\n\nIn this section, we'll set up the necessary staging area and file format for our data loading process. Specifically, we will:\n\n1. Create a stage named `stages.st_ml_app_penguins` that will:\n - Connect to the S3 bucket `s3://sfquickstarts/misc`\n - Serve as our data loading pipeline\n\n2. Configure a file format `file_formats.csv` that will:\n - Define how we parse and load CSV files\n - Be associated with our stage for data processing\n\nThis setup will establish the foundation for loading our Penguins dataset into Snowflake.\n\nLet's proceed with creating these configurations...\n"
+ },
+ {
+ "cell_type": "code",
+ "id": "8c4b0e50-0df8-42bc-a512-a8be6155020e",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_stages",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "-- add an external stage to a s3 bucket\nCREATE STAGE IF NOT EXISTS STAGES.ST_ML_APP_PENGUINS\n URL='s3://sfquickstarts/misc';\n\n-- default CSV file format and allow values to quoted by \"\nCREATE FILE FORMAT IF NOT EXISTS FILE_FORMATS.CSV\n TYPE='CSV'\n SKIP_HEADER=1\n FIELD_OPTIONALLY_ENCLOSED_BY = '\"';",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8fb9e105-5d23-48dc-b66d-2a807a50a03d",
+ "metadata": {
+ "name": "md_load_penguins_dataset",
+ "collapsed": false,
+ "resultHeight": 513
+ },
+ "source": "## Loading the Penguins Dataset\n\nAs our next step, we'll load the penguins dataset that will serve as the foundation for our ML demo application. The dataset contains various measurements of different penguin species, making it perfect for our classification tasks.\n\n## Data Loading Process\n\nWe will:\n- Create a table `data.penguins` to store our penguin details\n- Load data from the file `penguins_cleaned.csv` located in our external stage\n- Use the previously configured stage path: `@stages.st_ml_app_penguins/penguins_cleaned.csv`\n\nThis dataset will be used throughout our demo to:\n- Train our machine learning model\n- Make predictions on penguin species\n- Demonstrate interactive data visualization\n\nLet's proceed with the data loading commands..."
+ },
+ {
+ "cell_type": "code",
+ "id": "c0d35b0f-638c-45b9-a026-4cead0159f8e",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_tables",
+ "collapsed": false,
+ "resultHeight": 111
+ },
+ "outputs": [],
+ "source": "-- Create table to hold penguins data\nCREATE OR ALTER TABLE DATA.PENGUINS(\n SPECIES STRING NOT NULL,\n ISLAND STRING NOT NULL,\n BILL_LENGTH_MM NUMBER NOT NULL,\n BILL_DEPTH_MM NUMBER NOT NULL,\n FLIPPER_LENGTH_MM NUMBER NOT NULL,\n BODY_MASS_G NUMBER NOT NULL,\n SEX STRING NOT NULL\n);\n\n-- Load the data from penguins_cleaned.csv\nCOPY INTO DATA.PENGUINS\nFROM @STAGES.ST_ML_APP_PENGUINS/PENGUINS_CLEANED.CSV\nFILE_FORMAT=(FORMAT_NAME='FILE_FORMATS.CSV');",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fa430827-c3b5-4be8-b90b-79d9443a1ab4",
+ "metadata": {
+ "name": "md_app_intro",
+ "collapsed": false,
+ "resultHeight": 513
+ },
+ "source": "## Building Our Streamlit ML Application\n\nNow that we have our environment set up and the penguins dataset loaded, let's start building our interactive Machine Learning application using Streamlit. We'll create a user-friendly interface that allows users to:\n\n- Visualize the penguins dataset\n- Input penguin measurements through interactive widgets\n- Make real-time predictions using our trained ML model\n- Display the results in an engaging way\n\n### Getting Started\nWe'll begin by importing the necessary libraries and setting up our Streamlit application structure. Our app will leverage:\n- Streamlit for the interactive web interface\n- scikit-learn for our ML model\n- Snowflake for data access\n- Pandas for data manipulation\n\nLet's dive into the code and build our application step by step..."
+ },
+ {
+ "cell_type": "code",
+ "id": "384139f1-3cfd-44bc-ae55-c2c4ffde00fa",
+ "metadata": {
+ "language": "python",
+ "name": "py_imports",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "import streamlit as st\nimport os\nimport pandas as pd\nimport numpy as np\nfrom sklearn.ensemble import RandomForestClassifier\nfrom snowflake.snowpark.session import Session\nfrom snowflake.snowpark.functions import col\nfrom snowflake.snowpark.types import StringType, DecimalType",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "28fb1802-478d-4b4b-99fd-a387a34bbbbc",
+ "metadata": {
+ "name": "md_select_penguins_data",
+ "collapsed": false,
+ "resultHeight": 41
+ },
+ "source": "Let us select the penguins data for further use,"
+ },
+ {
+ "cell_type": "code",
+ "id": "8f4baab3-acf5-419c-b6c2-633bb8971be4",
+ "metadata": {
+ "language": "sql",
+ "name": "penguins_data",
+ "collapsed": false,
+ "resultHeight": 438
+ },
+ "outputs": [],
+ "source": "SELECT * FROM DATA.PENGUINS;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3e3ee447-4c08-477a-900b-ec8c29f6ad3a",
+ "metadata": {
+ "name": "md_dp_details",
+ "collapsed": false,
+ "resultHeight": 248
+ },
+ "source": "### Data Preprocessing Steps\n1. Import SQL output to pandas DataFrame, you can refer to the cell name in Snowflake Notebooks in this case `penguins_data`\n2. Standardize column names to lowercase for consistency and easier reference\n3. Set appropriate data types for each column:\n - Numeric columns: Convert to float64\n - Text columns: Convert to string\n\nThe text is clear, concise, and properly structured with the correct heading level (##), numbered list, and nested bullet points. No changes are needed."
+ },
+ {
+ "cell_type": "code",
+ "id": "efbd843d-0ffb-4785-8be0-1bb2d47fd05c",
+ "metadata": {
+ "language": "python",
+ "name": "py_prep_data",
+ "collapsed": false,
+ "resultHeight": 0
+ },
+ "outputs": [],
+ "source": "df = penguins_data.to_pandas()\n\n# for consistency and easiness let us change the column names to be of lower case\ndf.columns=df.columns.str.lower()\n\n## Set the columns to right data type\ndf['island'] = df['island'].astype('str')\ndf['species'] = df['species'].astype('str')\ndf['bill_length_mm'] = df['bill_length_mm'].astype('float64')\ndf['bill_depth_mm'] = df['bill_depth_mm'].astype('float64')\ndf['flipper_length_mm'] = df['flipper_length_mm'].astype('float64')\ndf['body_mass_g'] = df['body_mass_g'].astype('float64')\ndf['sex'] = df['sex'].astype('str')\n",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "17147ef3-62d4-45dc-aec9-adaa838c2056",
+ "metadata": {
+ "name": "md_st_exapander",
+ "collapsed": false,
+ "resultHeight": 436
+ },
+ "source": "\n### Streamlit Expander Widget ๐\n\nAn `st.expander` creates a collapsible section in your app that can be expanded/collapsed by clicking. It's useful for:\n- Hiding optional details or settings\n- Organizing long-form content\n- Creating FAQ-style interfaces\n- Showing additional visualizations on demand\n\n#### Key Features\n- Maintains a clean UI by hiding secondary content\n- Can contain any Streamlit elements (text, charts, inputs, etc.)\n- Default state can be set (expanded/collapsed)\n- Customizable label text\n\n๐ Documentation: https://docs.streamlit.io/library/api-reference/layout/st.expander"
+ },
+ {
+ "cell_type": "code",
+ "id": "8382cb6f-d738-4794-a20d-ee443d819510",
+ "metadata": {
+ "language": "python",
+ "name": "st_show_raw_data",
+ "collapsed": false,
+ "resultHeight": 64
+ },
+ "outputs": [],
+ "source": "with st.expander(\"**Raw Data**\"):\n df.columns = df.columns.str.lower()\n \n st.write(\"**X**\")\n st.write(\"The input features that will use to build the model.\")\n X_raw = df.drop(\"species\", axis=1)\n X_raw\n\n st.write(\"**y**\")\n st.write(\"The target of our predicted model.\")\n y_raw = df.species\n y_raw",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ed1ea63b-fca5-4704-8447-33af38515948",
+ "metadata": {
+ "name": "md_st_visualization",
+ "collapsed": false,
+ "resultHeight": 577
+ },
+ "source": "### Scatter Plot Visualization using Altair in Streamlit ๐\n\nAltair (powered by Vega-Lite) provides more customizable scatter plots than Streamlit's built-in charts. Perfect for the penguins dataset with features like:\n- Interactive tooltips with custom formatting\n- Layered visualizations\n- Color encoding by categorical variables\n- Dynamic filtering and zooming\n- Configurable axis and legend properties\n\n#### Key Advantages\n- Declarative grammar of graphics\n- Seamless integration with pandas DataFrames\n- Publication-quality aesthetics\n- Compositional layering system\n\n๐ Documentation:\n- Altair: https://altair-viz.github.io/user_guide/marks/scatter.html\n- Streamlit-Altair Integration: https://docs.streamlit.io/library/api-reference/charts/st.altair_chart\n\n*Note: Altair works natively with Streamlit using `st.altair_chart()`. No additional configuration needed.*"
+ },
+ {
+ "cell_type": "code",
+ "id": "884c6956-caef-4486-8d51-980abcd6fb67",
+ "metadata": {
+ "language": "python",
+ "name": "st_data_visualization",
+ "collapsed": false,
+ "resultHeight": 437
+ },
+ "outputs": [],
+ "source": "import altair as alt\n\nwith st.expander(\"Data Visualization\",expanded=True):\n sp=alt.Chart(df).mark_circle().encode(\n alt.X('bill_length_mm').scale(zero=False),\n alt.Y('body_mass_g').scale(zero=False, padding=1),\n color='species',\n )\n\n st.altair_chart(sp)\n",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "37c935a6-bc55-4359-bcac-451a72bf806d",
+ "metadata": {
+ "name": "md_input_widgets",
+ "collapsed": false,
+ "resultHeight": 826
+ },
+ "source": "\n### Interactive Widgets for Data Filtering ๐๏ธ\n\nStreamlit provides several widgets to create dynamic, interactive filters for your data:\n\n#### Select Box (`st.selectbox`)\n- Dropdown menu for single selection\n- Perfect for categorical filters (e.g., penguin species)\n- Clean interface for limited options\n๐ [Select Box Documentation](https://docs.streamlit.io/library/api-reference/widgets/st.selectbox)\n\n#### Radio Button (`st.radio`)\n- Visual selection for mutually exclusive options\n- Great for 2-5 choices\n- More visible than dropdown menus\n๐ [Radio Button Documentation](https://docs.streamlit.io/library/api-reference/widgets/st.radio)\n\n#### Slider (`st.slider`)\n- Interactive range selection\n- Works with numbers, dates, and times\n- Supports single value or range selection\n- Ideal for numerical filters (e.g., bill length range)\n๐ [Slider Documentation](https://docs.streamlit.io/library/api-reference/widgets/st.slider)\n\n#### Sidebar Organization (`st.sidebar`)\nAll these widgets can be neatly organized in a collapsible sidebar using `st.sidebar`:\n- Keeps main content area clean\n- Creates intuitive filter panel\n- Automatically responsive\n- Perfect for filter controls and app navigation\n๐ [Sidebar Documentation](https://docs.streamlit.io/library/api-reference/layout/st.sidebar)\n\n*๐ก Pro Tip: Using `with st.sidebar:` context manager keeps your sidebar code organized and readable. Very useful for standalone apps.*"
+ },
+ {
+ "cell_type": "code",
+ "id": "b07a2a93-2dc4-4b9a-a4b1-9adf0af9c574",
+ "metadata": {
+ "language": "python",
+ "name": "st_input_features",
+ "collapsed": false,
+ "resultHeight": 633
+ },
+ "outputs": [],
+ "source": "st.header(\"Input Features\")\n# Islands\nislands = df.island.unique().astype(str)\nisland = st.selectbox(\n \"Island\",\n islands,\n)\n# Bill Length\nmin, max, mean = (\n df.bill_length_mm.min(),\n df.bill_length_mm.max(),\n df.bill_length_mm.mean().round(2),\n)\nbill_length_mm = st.slider(\n \"Bill Length(mm)\",\n min_value=min,\n max_value=max,\n value=mean,\n)\n# Bill Depth\nmin, max, mean = (\n df.bill_depth_mm.min(),\n df.bill_depth_mm.max(),\n df.bill_depth_mm.mean().round(2),\n)\nbill_depth_mm = st.slider(\n \"Bill Depth(mm)\",\n min_value=min,\n max_value=max,\n value=mean,\n)\n# Filpper Length\nmin, max, mean = (\n df.flipper_length_mm.min(),\n df.flipper_length_mm.max(),\n df.flipper_length_mm.mean().round(2),\n)\nflipper_length_mm = st.slider(\n \"Flipper Length(mm)\",\n min_value=min,\n max_value=max,\n value=mean,\n)\n# Body Mass\nmin, max, mean = (\n df.body_mass_g.min(),\n df.body_mass_g.max(),\n df.body_mass_g.mean().round(2),\n)\nbody_mass_g = st.slider(\n \"Body Mass(g)\",\n min_value=min,\n max_value=max,\n value=mean,\n)\n# Gender\ngender = st.radio(\n \"Gender\",\n (\"male\", \"female\"),\n)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "63c01a8f-ccfe-41d2-8404-7055561a6615",
+ "metadata": {
+ "name": "md_dataframe",
+ "collapsed": false,
+ "resultHeight": 114
+ },
+ "source": "### Display Input Features\nWe will use Streamlit's [data display elements](https://docs.streamlit.io/library/api-reference/data/st.dataframe) to showcase our input features. The `st.dataframe()` function provides an interactive table with sorting and filtering capabilities."
+ },
+ {
+ "cell_type": "code",
+ "id": "9d97ac23-af20-480d-8053-f01a4b448ca9",
+ "metadata": {
+ "language": "python",
+ "name": "st_input_features_df",
+ "collapsed": false,
+ "resultHeight": 666
+ },
+ "outputs": [],
+ "source": "data = {\n \"island\": island,\n \"bill_length_mm\": bill_length_mm,\n \"bill_depth_mm\": bill_depth_mm,\n \"flipper_length_mm\": flipper_length_mm,\n \"body_mass_g\": body_mass_g,\n \"sex\": gender,\n}\ninput_df = pd.DataFrame(data, index=[0])\ninput_penguins = pd.concat([input_df, X_raw], axis=0)\n\nwith st.expander(\"Input Features\"):\n st.write(\"**Input Penguins**\")\n input_df\n st.write(\"**Combined Penguins Data**\")\n input_penguins",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5ffd1344-e95f-4b19-88ab-c523067cbc7f",
+ "metadata": {
+ "name": "md_dp_encode",
+ "collapsed": false,
+ "resultHeight": 200
+ },
+ "source": "### Data Preparation\n\nFor the data preparation step in this demo, we'll keep things straightforward and focus on:\n1. Encoding string features - converting text values into numbers that our ML model can understand\n2. Preparing the target variable - ensuring our prediction target is properly encoded\n\nThis will be a minimal demonstration without additional preprocessing steps like feature scaling, handling missing values, or feature engineering. "
+ },
+ {
+ "cell_type": "code",
+ "id": "201896f7-0d16-4314-afc1-85c4cc5e880e",
+ "metadata": {
+ "language": "python",
+ "name": "py_model_data_prep",
+ "collapsed": false,
+ "resultHeight": 666
+ },
+ "outputs": [],
+ "source": "X_encode = [\"island\", \"sex\"]\ndf_penguins = pd.get_dummies(input_penguins, prefix=X_encode)\nX = df_penguins[1:]\ninput_row = df_penguins[:1]\n\n## Encode Y\ntarget_mapper = {\n \"Adelie\": 0,\n \"Chinstrap\": 1,\n \"Gentoo\": 2,\n}\n\ny = y_raw.apply(lambda v: target_mapper[v])\n\nwith st.expander(\"Data Preparation\"):\n st.write(\"**Encoded X (input penguins)**\")\n input_row\n st.write(\"**Encoded y**\")\n y",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f17782f0-8c05-4908-851c-4acf6e6fcede",
+ "metadata": {
+ "name": "md_train_predict",
+ "collapsed": false,
+ "resultHeight": 535
+ },
+ "source": "### Model Training and Prediction\n\nFor this final step, we'll use RandomForestClassifier - an ensemble learning method that operates by constructing multiple decision trees during training and outputs the class that is the mode of the classes predicted by individual trees. We'll display the progress and results using Streamlit's container and progress components for a better user experience, followed by a success message showing the prediction results.\n\nRandomForest is a good choice for our demonstration as it:\n- Handles both numerical and categorical features well\n- Provides feature importance rankings\n- Is less prone to overfitting compared to single decision trees\n- Requires minimal hyperparameter tuning to get reasonable results\n\n**References:**\n* [Scikit-learn RandomForestClassifier Documentation](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html)\n* [Scikit-learn Ensemble Methods Guide](https://scikit-learn.org/stable/modules/ensemble.html#forest)\n* [User Guide: Forest of randomized trees](https://scikit-learn.org/stable/modules/forest.html)\n* [Streamlit Container API](https://docs.streamlit.io/library/api-reference/layout/st.container)\n* [Streamlit Progress and Status API](https://docs.streamlit.io/library/api-reference/status/st.progress)\n* [Streamlit Success Message](https://docs.streamlit.io/library/api-reference/status/st.success)"
+ },
+ {
+ "cell_type": "code",
+ "id": "a610ccd1-d748-44cf-b9c9-e03e11baac1d",
+ "metadata": {
+ "language": "python",
+ "name": "st_train_predict",
+ "collapsed": false,
+ "resultHeight": 260
+ },
+ "outputs": [],
+ "source": "with st.container():\n st.subheader(\"**Prediction Probability**\")\n ## Model Training\n rf_classifier = RandomForestClassifier()\n # Fit the model\n rf_classifier.fit(X, y)\n # predict using the model\n prediction = rf_classifier.predict(input_row)\n prediction_prob = rf_classifier.predict_proba(input_row)\n\n # reverse the target_mapper\n p_cols = dict((v, k) for k, v in target_mapper.items())\n df_prediction_prob = pd.DataFrame(prediction_prob)\n # set the column names\n df_prediction_prob.columns = p_cols.values()\n # set the Penguin name\n df_prediction_prob.rename(columns=p_cols)\n\n st.dataframe(\n df_prediction_prob,\n column_config={\n \"Adelie\": st.column_config.ProgressColumn(\n \"Adelie\",\n help=\"Adelie\",\n format=\"%f\",\n width=\"medium\",\n min_value=0,\n max_value=1,\n ),\n \"Chinstrap\": st.column_config.ProgressColumn(\n \"Chinstrap\",\n help=\"Chinstrap\",\n format=\"%f\",\n width=\"medium\",\n min_value=0,\n max_value=1,\n ),\n \"Gentoo\": st.column_config.ProgressColumn(\n \"Gentoo\",\n help=\"Gentoo\",\n format=\"%f\",\n width=\"medium\",\n min_value=0,\n max_value=1,\n ),\n },\n hide_index=True,\n )\n\n# display the prediction\nst.subheader(\"Predicted Species\")\nst.success(p_cols[prediction[0]])\n",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "05b25160-ff9c-4d04-a1ae-939dc28c30b6",
+ "metadata": {
+ "name": "md_note",
+ "collapsed": false,
+ "resultHeight": 309
+ },
+ "source": "\nโ ๏ธ **Important Note:**\n* When changing input features, cells don't automatically re-run\n* After modifying `st_input_features`, you need to manually run these cells in sequence:\n 1. `st_input_features_df` - Updates the features DataFrame\n 2. `py_model_data_prep` - Prepares data for model training\n 3. `st_train_predict` - Trains model and shows prediction\n\nHere is execution of cells flow:\n\n`Change inputs[st_input_features]` โ `Update DataFrame[st_input_features_df]` โ `Prepare ML data[py_model_data_prep]` โ `Train & predict[st_train_predict]`\n "
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3dbd7c05-603d-4a91-9a4e-87271ca6aad9",
+ "metadata": {
+ "name": "md_summary",
+ "collapsed": false,
+ "resultHeight": 607
+ },
+ "source": "## Summary and Further Reading\n\nThroughout this course, we've seen how Snowflake Notebooks and Streamlit work together to create powerful, interactive machine learning applications. This combination offers several advantages:\n\n1. **Unified Development Environment**: Snowflake Notebooks provide a seamless environment for data preparation, model development, and testing, all within the Snowflake ecosystem.\n\n2. **Interactive User Interfaces**: Streamlit enables us to transform our machine learning models into user-friendly applications, making complex analytics accessible to non-technical users.\n\n3. **Scalable Processing**: By leveraging Snowflake's computational power, our applications can handle large-scale data processing without compromising performance.\n\n4. **Real-time Analytics**: The integration allows for real-time data updates and model predictions, making our applications more dynamic and valuable for business decisions.\n\n## Further Reading\n\n- [Streamlit in Snowflake](https://docs.snowflake.com/en/developer-guide/streamlit/about-streamlit) - Learn more about building interactive data applications\n- [Snowpark Python DataFrames](https://docs.snowflake.com/en/developer-guide/snowpark/python/working-with-dataframes) - Deep dive into data manipulation techniques\n- [Snowflake ML](https://docs.snowflake.com/en/developer-guide/snowflake-ml/snowpark-ml) - Explore advanced machine learning capabilities\n- [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks) - Master the notebook environment for development\n- [Snowflake Quickstarts](https://quickstarts.snowflake.com/) - Get hands-on experience with guided tutorials and examples\n\nHappy building!"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/Streamlit_Zero_To_Hero_Machine_Learning_App/environment.yml b/Streamlit_Zero_To_Hero_Machine_Learning_App/environment.yml
new file mode 100644
index 0000000..65cfb04
--- /dev/null
+++ b/Streamlit_Zero_To_Hero_Machine_Learning_App/environment.yml
@@ -0,0 +1,9 @@
+name: app_environment
+channels:
+ - snowflake
+dependencies:
+ - streamlit=1.35.0
+ - snowflake-snowpark-python
+ - scikit-learn=1.3.0
+ - pandas=2.0.3
+ - numpy=1.24.3
diff --git a/Visual Data Stories with Snowflake Notebooks/Visual Data Stories with Snowflake Notebooks.ipynb b/Visual Data Stories with Snowflake Notebooks/Visual Data Stories with Snowflake Notebooks.ipynb
index 5fb7832..a893382 100644
--- a/Visual Data Stories with Snowflake Notebooks/Visual Data Stories with Snowflake Notebooks.ipynb
+++ b/Visual Data Stories with Snowflake Notebooks/Visual Data Stories with Snowflake Notebooks.ipynb
@@ -1,180 +1,91 @@
{
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ },
+ "lastEditStatus": {
+ "notebookId": "ogkrlqqvlepeplsa5vrg",
+ "authorId": "61119818470",
+ "authorName": "",
+ "authorEmail": "",
+ "sessionId": "f70b7346-b7d5-4a6a-9bbf-87296f4a63d1",
+ "lastEditTime": 1744835079409
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
"cells": [
{
"cell_type": "markdown",
- "id": "60aa4826-ef81-4217-9dbd-336397e056c0",
+ "id": "19dd2287-3024-443b-979f-386c8ba4e8ce",
"metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell1"
+ "name": "try-now"
},
"source": [
- "# Visual Data Stories with Snowflake Notebooks\n",
- "\n",
- "In this tutorial, we will walk you through the different ways you can enrich your data narrative through engaging visuals in Snowflake Notebooks. We will demonstrate how you can develop visualizations, work with Markdown text, embed images, and build awesome data apps all within your notebook, alongside your code and data.\n",
- "\n",
- "**Requirements:** Please add the `matplotlib` and `plotly` package from the package picker on the top right. We will be using these packages in the notebook."
+ "# Try this demo out!\n",
+ "```python\n",
+ "# Install snowflake.demos and run this in your local development environment or in Snowflake Notebooks\n",
+ "import snowflake.demos\n",
+ "snowflake.demos.load_demo('visual-data-stories')\n",
+ "```"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "60aa4826-ef81-4217-9dbd-336397e056c0",
+ "metadata": {
+ "collapsed": false,
+ "name": "intro_md",
+ "resultHeight": 167
+ },
+ "source": "# Visual Data Stories with Snowflake Notebooks\n\nIn this tutorial, we will walk you through the different ways you can enrich your data narrative through engaging visuals in Snowflake Notebooks. We will demonstrate how you can develop visualizations, work with Markdown text, embed images, and build awesome data apps all within your notebook, alongside your code and data.\n\n**Note**: Before we start, please make sure that you've installed Plotly from the `Packages` drop down in the top right corner. "
+ },
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"id": "7e1072e0-562d-4584-92f9-988e3d0e7465",
"metadata": {
"codeCollapsed": false,
"language": "python",
- "name": "cell2"
+ "name": "import_packages",
+ "resultHeight": 0
},
"outputs": [],
- "source": [
- "# Import python packages\n",
- "import streamlit as st\n",
- "import pandas as pd"
- ]
+ "source": "# Import python packages\nimport streamlit as st\nimport pandas as pd\nimport numpy as np"
},
{
"cell_type": "markdown",
"id": "27269da4-823f-4a23-8324-5ccd5db61720",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell3"
+ "name": "visualizations_md",
+ "resultHeight": 143
},
- "source": [
- "## Data visualizations ๐\n",
- "\n",
- "With Snowflake Notebook, you can use your favorite Python visualization library, including matplotlib and plotly, to develop your visualization.\n",
- "\n",
- "First, let's generate some toy data for the Iris dataset."
- ]
+ "source": "## Data visualizations ๐\n\nWith Snowflake Notebooks, you can use your favorite Python visualization library, including plotly, altair, and matplotlib to develop your visualization.\n\nFirst, let's generate a toy snowfall dataset."
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"id": "3775908f-ca36-4846-8f38-5adca39217f2",
"metadata": {
"codeCollapsed": false,
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
"language": "python",
- "name": "cell4"
+ "name": "visualizations",
+ "resultHeight": 391
},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "\n",
- " \n",
- " \n",
- " \n",
- " | \n",
- " species | \n",
- " measurement | \n",
- " value | \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " | 0 | \n",
- " setosa | \n",
- " sepal_length | \n",
- " 5.1 | \n",
- " \n",
- " \n",
- " | 1 | \n",
- " setosa | \n",
- " sepal_width | \n",
- " 3.5 | \n",
- " \n",
- " \n",
- " | 2 | \n",
- " setosa | \n",
- " petal_length | \n",
- " 1.4 | \n",
- " \n",
- " \n",
- " | 3 | \n",
- " versicolor | \n",
- " sepal_length | \n",
- " 6.2 | \n",
- " \n",
- " \n",
- " | 4 | \n",
- " versicolor | \n",
- " sepal_width | \n",
- " 2.9 | \n",
- " \n",
- " \n",
- " | 5 | \n",
- " versicolor | \n",
- " petal_length | \n",
- " 4.3 | \n",
- " \n",
- " \n",
- " | 6 | \n",
- " virginica | \n",
- " sepal_length | \n",
- " 7.3 | \n",
- " \n",
- " \n",
- " | 7 | \n",
- " virginica | \n",
- " sepal_width | \n",
- " 3.0 | \n",
- " \n",
- " \n",
- " | 8 | \n",
- " virginica | \n",
- " petal_length | \n",
- " 6.3 | \n",
- " \n",
- " \n",
- " \n",
- "
measurement=%{x}
value=%{y}",
- "legendgroup": "setosa",
- "marker": {
- "color": "#000001",
- "pattern": {
- "shape": ""
- }
- },
- "name": "setosa",
- "offsetgroup": "setosa",
- "orientation": "v",
- "showlegend": true,
- "textposition": "auto",
- "type": "bar",
- "x": [
- "sepal_length",
- "sepal_width",
- "petal_length"
- ],
- "xaxis": "x",
- "y": [
- 5.1,
- 3.5,
- 1.4
- ],
- "yaxis": "y"
- },
- {
- "alignmentgroup": "True",
- "hovertemplate": "species=versicolor
measurement=%{x}
value=%{y}",
- "legendgroup": "versicolor",
- "marker": {
- "color": "#000002",
- "pattern": {
- "shape": ""
- }
- },
- "name": "versicolor",
- "offsetgroup": "versicolor",
- "orientation": "v",
- "showlegend": true,
- "textposition": "auto",
- "type": "bar",
- "x": [
- "sepal_length",
- "sepal_width",
- "petal_length"
- ],
- "xaxis": "x",
- "y": [
- 6.2,
- 2.9,
- 4.3
- ],
- "yaxis": "y"
- },
- {
- "alignmentgroup": "True",
- "hovertemplate": "species=virginica
measurement=%{x}
value=%{y}",
- "legendgroup": "virginica",
- "marker": {
- "color": "#000003",
- "pattern": {
- "shape": ""
- }
- },
- "name": "virginica",
- "offsetgroup": "virginica",
- "orientation": "v",
- "showlegend": true,
- "textposition": "auto",
- "type": "bar",
- "x": [
- "sepal_length",
- "sepal_width",
- "petal_length"
- ],
- "xaxis": "x",
- "y": [
- 7.3,
- 3,
- 6.3
- ],
- "yaxis": "y"
- }
- ],
- "layout": {
- "autosize": true,
- "barmode": "relative",
- "legend": {
- "title": {
- "text": "species"
- },
- "tracegroupgap": 0
- },
- "margin": {
- "t": 60
- },
- "template": {
- "data": {
- "candlestick": [
- {
- "decreasing": {
- "line": {
- "color": "#000033"
- }
- },
- "increasing": {
- "line": {
- "color": "#000032"
- }
- },
- "type": "candlestick"
- }
- ],
- "contour": [
- {
- "colorscale": [
- [
- 0,
- "#000011"
- ],
- [
- 0.1111111111111111,
- "#000012"
- ],
- [
- 0.2222222222222222,
- "#000013"
- ],
- [
- 0.3333333333333333,
- "#000014"
- ],
- [
- 0.4444444444444444,
- "#000015"
- ],
- [
- 0.5555555555555556,
- "#000016"
- ],
- [
- 0.6666666666666666,
- "#000017"
- ],
- [
- 0.7777777777777778,
- "#000018"
- ],
- [
- 0.8888888888888888,
- "#000019"
- ],
- [
- 1,
- "#000020"
- ]
- ],
- "type": "contour"
- }
- ],
- "contourcarpet": [
- {
- "colorscale": [
- [
- 0,
- "#000011"
- ],
- [
- 0.1111111111111111,
- "#000012"
- ],
- [
- 0.2222222222222222,
- "#000013"
- ],
- [
- 0.3333333333333333,
- "#000014"
- ],
- [
- 0.4444444444444444,
- "#000015"
- ],
- [
- 0.5555555555555556,
- "#000016"
- ],
- [
- 0.6666666666666666,
- "#000017"
- ],
- [
- 0.7777777777777778,
- "#000018"
- ],
- [
- 0.8888888888888888,
- "#000019"
- ],
- [
- 1,
- "#000020"
- ]
- ],
- "type": "contourcarpet"
- }
- ],
- "heatmap": [
- {
- "colorscale": [
- [
- 0,
- "#000011"
- ],
- [
- 0.1111111111111111,
- "#000012"
- ],
- [
- 0.2222222222222222,
- "#000013"
- ],
- [
- 0.3333333333333333,
- "#000014"
- ],
- [
- 0.4444444444444444,
- "#000015"
- ],
- [
- 0.5555555555555556,
- "#000016"
- ],
- [
- 0.6666666666666666,
- "#000017"
- ],
- [
- 0.7777777777777778,
- "#000018"
- ],
- [
- 0.8888888888888888,
- "#000019"
- ],
- [
- 1,
- "#000020"
- ]
- ],
- "type": "heatmap"
- }
- ],
- "histogram2d": [
- {
- "colorscale": [
- [
- 0,
- "#000011"
- ],
- [
- 0.1111111111111111,
- "#000012"
- ],
- [
- 0.2222222222222222,
- "#000013"
- ],
- [
- 0.3333333333333333,
- "#000014"
- ],
- [
- 0.4444444444444444,
- "#000015"
- ],
- [
- 0.5555555555555556,
- "#000016"
- ],
- [
- 0.6666666666666666,
- "#000017"
- ],
- [
- 0.7777777777777778,
- "#000018"
- ],
- [
- 0.8888888888888888,
- "#000019"
- ],
- [
- 1,
- "#000020"
- ]
- ],
- "type": "histogram2d"
- }
- ],
- "icicle": [
- {
- "textfont": {
- "color": "white"
- },
- "type": "icicle"
- }
- ],
- "sankey": [
- {
- "textfont": {
- "color": "#000036"
- },
- "type": "sankey"
- }
- ],
- "scatter": [
- {
- "marker": {
- "line": {
- "width": 0
- }
- },
- "type": "scatter"
- }
- ],
- "table": [
- {
- "cells": {
- "fill": {
- "color": "#000038"
- },
- "font": {
- "color": "#000037"
- },
- "line": {
- "color": "#000039"
- }
- },
- "header": {
- "fill": {
- "color": "#000040"
- },
- "font": {
- "color": "#000036"
- },
- "line": {
- "color": "#000039"
- }
- },
- "type": "table"
- }
- ],
- "waterfall": [
- {
- "connector": {
- "line": {
- "color": "#000036",
- "width": 2
- }
- },
- "decreasing": {
- "marker": {
- "color": "#000033"
- }
- },
- "increasing": {
- "marker": {
- "color": "#000032"
- }
- },
- "totals": {
- "marker": {
- "color": "#000034"
- }
- },
- "type": "waterfall"
- }
- ]
- },
- "layout": {
- "coloraxis": {
- "colorscale": [
- [
- 0,
- "#000011"
- ],
- [
- 0.1111111111111111,
- "#000012"
- ],
- [
- 0.2222222222222222,
- "#000013"
- ],
- [
- 0.3333333333333333,
- "#000014"
- ],
- [
- 0.4444444444444444,
- "#000015"
- ],
- [
- 0.5555555555555556,
- "#000016"
- ],
- [
- 0.6666666666666666,
- "#000017"
- ],
- [
- 0.7777777777777778,
- "#000018"
- ],
- [
- 0.8888888888888888,
- "#000019"
- ],
- [
- 1,
- "#000020"
- ]
- ]
- },
- "colorscale": {
- "diverging": [
- [
- 0,
- "#000021"
- ],
- [
- 0.1,
- "#000022"
- ],
- [
- 0.2,
- "#000023"
- ],
- [
- 0.3,
- "#000024"
- ],
- [
- 0.4,
- "#000025"
- ],
- [
- 0.5,
- "#000026"
- ],
- [
- 0.6,
- "#000027"
- ],
- [
- 0.7,
- "#000028"
- ],
- [
- 0.8,
- "#000029"
- ],
- [
- 0.9,
- "#000030"
- ],
- [
- 1,
- "#000031"
- ]
- ],
- "sequential": [
- [
- 0,
- "#000011"
- ],
- [
- 0.1111111111111111,
- "#000012"
- ],
- [
- 0.2222222222222222,
- "#000013"
- ],
- [
- 0.3333333333333333,
- "#000014"
- ],
- [
- 0.4444444444444444,
- "#000015"
- ],
- [
- 0.5555555555555556,
- "#000016"
- ],
- [
- 0.6666666666666666,
- "#000017"
- ],
- [
- 0.7777777777777778,
- "#000018"
- ],
- [
- 0.8888888888888888,
- "#000019"
- ],
- [
- 1,
- "#000020"
- ]
- ],
- "sequentialminus": [
- [
- 0,
- "#000011"
- ],
- [
- 0.1111111111111111,
- "#000012"
- ],
- [
- 0.2222222222222222,
- "#000013"
- ],
- [
- 0.3333333333333333,
- "#000014"
- ],
- [
- 0.4444444444444444,
- "#000015"
- ],
- [
- 0.5555555555555556,
- "#000016"
- ],
- [
- 0.6666666666666666,
- "#000017"
- ],
- [
- 0.7777777777777778,
- "#000018"
- ],
- [
- 0.8888888888888888,
- "#000019"
- ],
- [
- 1,
- "#000020"
- ]
- ]
- },
- "colorway": [
- "#000001",
- "#000002",
- "#000003",
- "#000004",
- "#000005",
- "#000006",
- "#000007",
- "#000008",
- "#000009",
- "#000010"
- ]
- }
- },
- "xaxis": {
- "anchor": "y",
- "autorange": true,
- "domain": [
- 0,
- 1
- ],
- "range": [
- -0.5,
- 2.5
- ],
- "title": {
- "text": "measurement"
- },
- "type": "category"
- },
- "yaxis": {
- "anchor": "x",
- "autorange": true,
- "domain": [
- 0,
- 1
- ],
- "range": [
- 0,
- 19.578947368421055
- ],
- "title": {
- "text": "value"
- },
- "type": "linear"
- }
- }
- },
- "image/png": "iVBORw0KGgoAAAANSUhEUgAABbAAAAFoCAYAAACG+t3qAAAAAXNSR0IArs4c6QAAIABJREFUeF7s3Qm8XtO9P/6VRDTm2TUWVdQ8VV1DTb3GioZyUWMNQYnW2BgTBKWmas1aVLU1lpi52prSKjVGDKWGkpaKS0UNEfm/vuv+T34RiZOzs59z9rP3e79eeUXO2Xvttd7f7ezzfJ71rN1rwoQJE5KNAAECBAgQIECAAAECBAgQIECAAAECBAhUTKCXALtiFdEdAgQIECBAgAABAgQIECBAgAABAgQIEMgCAmwXAgECBAgQIECAAAECBAgQIECAAAECBAhUUkCAXcmy6BQBAgQIECBAgAABAgQIECBAgAABAgQICLBdAwQIECBAgAABAgQIECBAgAABAgQIECBQSQEBdiXLolMECBAgQIAAAQIECBAgQIAAAQIECBAgIMB2DRAgQIAAAQIECBAgQIAAAQIECBAgQIBAJQUE2JUsi04RIECAAAECBAgQIECAAAECBAgQIECAgADbNUCAAAECBAgQIECAAAECBAgQIECAAAEClRQQYFeyLDpFgAABAgQIECBAgAABAgQIECBAgAABAgJs1wABAgQIECBAgAABAgQIECBAgAABAgQIVFJAgF3JsugUAQIECBAgQIAAAQIECBAgQIAAAQIECAiwXQMECBAgQIAAAQIECBAgQIAAAQIECBAgUEkBAXYly6JTBAgQIECAAAECBAgQIECAAAECBAgQICDAdg0QIECAAAECBAgQIECAAAECBAgQIECAQCUFBNiVLItOESBAgAABAgQIECBAgAABAgQIECBAgIAA2zVAgAABAgQIECBAgAABAgQIECBAgAABApUUEGBXsiw6RYAAAQIECBAgQIAAAQIECBAgQIAAAQICbNcAAQIECBAgQIAAAQIECBAgQIAAAQIECFRSQIBdybLoFAECBAgQIECAAAECBAgQIECAAAECBAgIsF0DBAgQIECAAAECBAgQIECAAAECBAgQIFBJAQF2JcuiUwQIECBAgAABAgQIECBAgAABAgQIECAgwHYNECBAgAABAgQIECBAgAABAgQIECBAgEAlBQTYlSyLThEgQIAAAQIECBAgQIAAAQIECBAgQICAANs1QIAAAQIECBAgQIAAAQIECBAgQIAAAQKVFBBgV7IsOkWAAAECBAgQIECAAAECBAgQIECAAAECAmzXAAECBAgQIECAAAECBAgQIECAAAECBAhUUkCAXcmy6BQBAgQIECBAgAABAgQIECBAgAABAgQICLBdAwQIECBAgAABAgQIECBAgAABAgQIECBQSQEBdiXLolMECBAgQIAAAQIECBAgQIAAAQIECBAgIMB2DRAgQIAAAQIECBAgQIAAAQIECBAgQIBAJQUE2JUsi04RIECAAAECBAgQIECAAAECBAgQIECAgADbNUCAAAECBAgQIECAAAECBAgQIECAAAEClRQQYFeyLDpFgAABAgQIECBAgAABAgQIECBAgAABAgJs1wABAgQIECBAgAABAgQIECBAgAABAgQIVFJAgF3JsugUAQIECBAgQIAAAQIECBAgQIAAAQIECAiwXQMECBAgQIAAAQIECBAgQIAAAQIECBAgUEkBAXYly6JTBAgQIECAAAECBAgQIECAAAECBAgQICDAdg0QIECAAAECBAgQIECAAAECBAgQIECAQCUFBNiVLItOESBAgAABAgQIECBAgAABAgQIECBAgIAA2zVAgAABAgQIECBAgAABAgQIECBAgAABApUUEGBXsiw6RYAAAQIECBAgQIAAAQIECBAgQIAAAQICbNcAAQIECBAgQIAAAQIECBAgQIAAAQIECFRSQIBdybLoFAECBAgQIECAAAECBAgQIECAAAECBAgIsF0DBAgQIECAAAECBAgQIECAAAECBAgQIFBJAQF2JcuiUwQIECBAgAABAgQIECBAgAABAgQIECAgwHYNECBAgAABAgQIECBAgAABAgQIECBAgEAlBQTYlSyLThEgQIAAAQIECBAgQIAAAQIECBAgQICAANs1QIAAAQIECBAgQIAAAQIECBAgQIAAAQKVFBBgV7IsOkWAAAECBAgQIECAAAECBAgQIECAAAECAmzXAAECBAgQIECAAAECBAgQIECAAAECBAhUUkCAXcmy6BQBAgQIECBAgAABAgQIECBAgAABAgQICLBdAwQIECBAgAABAgQIECBAgAABAgQIECBQSQEBdiXLolMECBAgQIAAAQIECBAgQIAAAQIECBAgIMB2DRAgQIAAAQIECBAgQIAAAQIECBAgQIBAJQUE2JUsi04RIECAAAECBAgQIECAAAECBAgQIECAgADbNUCAAAECBAgQIECAAAECBAgQIECAAAEClRQQYFeyLDpFgAABAgQIECBAgAABAgQIECBAgAABAgJs1wABAgQIECBAgAABAgQIECBAgAABAgQIVFJAgF3JsugUAQIECBAgQIAAAQIECBAgQIAAAQIECAiwXQMECBAgQIAAAQIECBAgQIAAAQIECBAgUEkBAXYly6JTBAgQIECAAAECBAgQIECAAAECBAgQICDAdg0QIECAAAECBAgQIECAAAECBAgQIECAQCUFBNiVLItOESBAgAABAgQIECBAgAABAgQIECBAgIAA2zVAgAABAgQIECBAgAABAgQIECBAgAABApUUEGBXsiw6RYAAAQIECBAgQIAAAQIECBAgQIAAAQICbNcAAQIECBAgQIAAAQIECBAgQIAAAQIECFRSQIBdybLoFAECBAgQIECAAAECBAgQIECAAAEC7Szw8MMPp5dffjkNGDCgnYfR430XYPd4CXSAAAECBAgQIECAAAECBAgQIECAAIG6CQwZMiSNGDEi3XnnnXUbWreOR4DdrdxORoAAAQIECBAgQIAAAQIECBAgQIBAEwT+/e9/pw8//DDNOeecTRhuy8YowG4ZrYYJECBAgAABAgQIECBAgAABAgQIEOhJgeuvvz5deeWV6c0330wzzTRTWnbZZdNBBx2U5p9//nT00UfncPmDDz5If/zjH3PYvNFGG6WDDz449e3bN3f7nXfeST/60Y/SAw88kN5///202GKLpUMOOSS3E9vHH3+cfvWrX6VbbrklvfHGG2nuuedO6623Xtpvv/3yeaPdM888cyLBPffck37605+m0aNHp1lmmSVtvPHGaeDAgfl8cf5zzz03xT7vvvtumn322dOaa66ZDj300J4k7PFzC7B7vASt78AKK6yUnnxyVOtP5AwECBBoQ4EhQ45JQ4cOacOe6zIBAgQIECBAgAABAgQIfJZALN8Ry3iss846+U+ExjfccEM65phj0qqrrpp23HHHHDovscQSaa211kp33313evXVV9O2226b9tlnnxxO77LLLjlMjnWsI+y++uqr8zHXXHNNmm222dJpp52Wbr/99rT00kvnMPrZZ5/NS4bEn7POOivddddd6cYbb8zdjPaHDRuWVl999RyUP/744/nY7bffPu21117p7LPPzvtGvxZZZJH0zDPPpJtuuinv0+RNgN2A6guwG1BkQyRAoLCAALswnQMJECBAgAABAgQIECBQaYGLL744z4K+4oor8ozr2CKUHj9+fJ7xHEHxwgsvnEPoji0C67feeisHyb/97W/TySefnE499dQceMcWAfX++++fDjvssPy1b33rWzkcHzp06MQ2Xn/99Xy+yQPsCKojBL/gggsm7huzwV955ZUcjO+7774pjo1wvHfv3nmf9957L88cb/ImwG5A9QXYDSiyIRIgUFhAgF2YzoEECBAgQIAAAQIECBCotMDTTz+dBg0alMPgNdZYI6288sppiy22yEt3xBYB9jLLLPOJ8LljRvWtt96aLrroonTdddeleeedd+I4IwCP5Uh22mmn9KUvfSnP5h48eHD62te+9imLSQPscePG5XNHcD7HHHNM3DfaijZjxvavf/3rvLxI9C+WDok+x0ztjjC70tgt7JwAu4W4VWlagF2VSugHAQJVFBBgV7Eq+kSAAAECBAgQIECAAIFyBP7617/mNapHjhyZl/6IMPiSSy5JCy200BQD7JhtHWFyLNsRa1/H2taxVvbkWyw78sILL+QlQeJPBM6Tb5MG2GPHjk1bb711WnvttXMoPenWq1evvG52bA899FC69tprc39jze2Yyf2LX/wixT5N3QTYDai8ALsBRTZEAgQKCwiwC9M5kAABAgQIECBAgAABApUWiJnNk85e7lj+Y+edd0677bbbFAPsbbbZJs+SjqVH4gGQ55xzTn6w4lJLLfWJsU6YMCG99NJLae+9984zq2MpkI6t47yTLyES+8XDH08//fRPtRUB9aT9jfZjRvbPfvaz/BDIFVZYodLWreycALuVuhVpW4BdkULoBgEClRQQYFeyLDpFgAABAgQIECBAgACB6RaIoDjWkO7fv39eBiQeohizr4844og8CzqWEIntgAMOSH369MlrTz/22GN52ZGtttoqxazp2CcC7QMPPDAttthiObSOB0FGm9FGrFsdM7Fj/0033TTFjO9YBiQC8MkD7JhJfdlll+X9Yv8PP/wwPfLII+m+++7L62LHgxw7HjgZ57z00ktTPIgyjvuP//iP6fZo1wYE2O1auS70W4DdBSy7EiDQOAEBduNKbsAECBAgQIAAAQIECDRE4Oabb84zqGP96djiAYobb7xxGjhwYP53hNMda1B3kMSDFvfcc8+JS3Y8//zz+UGOEVx3bBGGDxkyJK+BHcuSnHjiiXnJj44tZlmfffbZeQmS//mf/8kPhIwtZlj//Oc/z0uaxH93bBFoH3rooflhkrF0SccWD5iMh0pOaX3thpQwD1OA3YBqC7AbUGRDJECgsIAAuzCdAwkQIECAAAECBAgQIFB5gViKI0Lqjz766FOzmDse4njUUUelMWPGpPnmmy/PxJ7SFutR//Of/0xzzTVXmnXWWT+1S8f3o41+/fp9pkv0KdqKv2P/SZc5GT9+fHrttdfyOWafffbK+3ZHBwXY3aHcw+cQYPdwAZyeAIFKCwiwK10enSNAgAABAgQIECBAgEDLBDoC7KFDh7bsHBqefgEB9vQbVr4FAXblS6SDBAj0oIAAuwfxnZoAAQIECBAgQIAAAQI9KBBrYce61rGOta26AgLs6tamtJ4JsEuj1BABAjUUEGDXsKiGRIAAAQIECBAgQIAAAQK1ERBg16aUUx+IALsBRTZEAgQKCwiwC9M5kAABAgQIECBAgAABAgQItFxAgN1y4p4/gQC752ugBwQIVFdAgF3d2ugZAQIECBAgQIAAAQIECBAQYDfgGhBgN6DIhkiAQGEBAXZhOgcSIECAAAECBAgQIECAAIGWCwiwW07c8ycQYPd8DfSAAIHqCgiwq1sbPSNAgAABAgQIECBAgAABAgLsBlwDAuwGFNkQCRAoLCDALkznQAIECBAgQIAAAQIECNRGYMyYMemJJ56Y5vHMM888acUVV5zm/e1YXECAXdyubY4UYLdNqXSUAIEeEBBg9wC6UxIgQIAAAQIECBAgQKBiAjfeeGPaaqtvTHOv+vffMg0fPnya97djcQEBdnG7tjlSgN02pdJRAgR6QECA3QPoTkmAAAECBAgQIECAAIGKCbRDgH344YenvfbaKy299NIV02ttdwTYrfWtROsC7EqUQScIEKiogAC7ooXRLQIECBAgQIAAAQIECHSjQDsE2BtvvHEaNmxYWnPNNbtRpudPJcDu+Rq0vAcC7JYTOwEBAm0sIMBu4+LpOgECBAgQIECAAAECBEoS6M4A+/3330+nnXZaGjFiRBo/fnxabLHF0hFHHJGWWGKJ9Oqrr6Yf/OAH6dlnn81f32abbdJmm22W97/99tvTLLPMkv98/etfT9/61rfSb3/723Teeeelt956Ky288MLpyCOPnDhD+4orrkjXXnttevfdd9Pcc8+d9t5777TRRhulyy67LF1zzTUp+tG3b9+0xRZbpP333z/16tWrJM1ymxFgl+tZydYE2JUsi04RIFARAQF2RQqhGwQIECBAgAABAgQIEOhBge4MsC+88MJ0/fXXpxNPPDHNMMMMOYT+z//8z7Taaqul7bffPi2zzDJp5513Ti+++GI666yz0iWXXJL+/e9/55B5p512Sssuu2xacMEF0wcffJC+853v5AD6q1/9aorA+plnnsnhdPwdS44cfPDBackll0wPP/xw+uijj3K7EYTHeRdddNH08ssvp1NOOSUdddRRaYMNNujBCkz91ALsSpal3E4JsMv11BoBAvUSEGDXq55GQ4AAAQIECBAgQIAAgSIC3Rlgn3322Tm0jpnWEVZ3zHz+wx/+kI499tgcbM8666x5GPHfm2++eQ6eJ19CJMLt3//+9zkMj23MmDFphx12yLO5+/Xrl4YMGZKOOeaYtPbaa+fAetLtueeeSyNHjkxvvPFGuu6669J///d/p913370IXcuPEWC3nLjnTyDA7vka6AEBAtUVEGBXtzZ6RoAAAQIECBAgQIAAge4S6M4Ae/To0Tmofumll1Lv3r1zMB0zqW+55ZZ0wQUXpIUWWugTw1533XXz8h+TB9iDBw/O+0UQ3rH1798/h9gRSJ900knpvvvuy99aaaWV0ve+97086/rkk0/OAfqXvvSl/O977703DRgwIO25557dxd2l8wiwu8TVnjsLsNuzbnpNgED3CAiwu8fZWQgQIECAAAECBAgQIFBlge4MsDscIsj+05/+lNewjoB6nnnmyWH0zTff/KkZ03FMBNjHH398WmuttXITp556avrzn/+crrzyyvzvsWPHpq233jovGxKztmOLtbEfe+yxdO655+Y1tg877LAccJ9wwgl52ZLY9t1337TGGmsIsKt8gda9bwLsulfY+AgQmB4BAfb06DmWAAECBAgQIECAAAEC9RDozgD78ssvz+tYr7rqqjl0juVB9ttvv7zUR6yBvd566+XZ0rFFwD1u3Li0ySabpL322isfM3DgwPxgxlGjRuVlQiKwjlnal156aRo+fHheC/upp55K//rXv9J//dd/pT59+qSjjz46P/zxoIMOSt/85jfz+aLNhx56KM/IjhnbZmDX41puy1EIsNuybDpNgEA3CQiwuwnaaQgQIECAAAECBAgQIFBhge4MsC+++OKJs6ZjCZHVV189z4iOoDketjhs2LD0zjvvZK34/qGHHppnX995553pzDPPzIF2LBVy4IEHpnPOOWfiGtiT7jtixIh03HHHpY8//ji384UvfCEvW7Lwwguniy66KF111VX563POOWdub6uttkp77LFHJStkCZFKlqXcTgmwy/XUGgEC9RIQYNernkZDgAABAgQIECBAgACBIgLdGWBH/z766KP8AMX55psvB9eTb2+//XYOlmNZkY6HPMY+EUi/+eabn/j6+++/n/75z3/mtbMnbWvChAn5wY4zzzxz/jPpFjO/Y4b25OttF7Fr9TEC7FYLV6B9AXYFiqALBAhUVkCAXdnS6BgBAgQIECBAgAABAgS6TSBmLB911FHTfL5YhzoekmhrvYAAu/XGPX4GAXaPl0AHCBCosIAAu8LF0TUCBAgQIECAAAECBAgQaLyAALsBl4AAuwFFNkQCBAoLCLAL0zmQAAECBAgQIECAAAECBAi0XECA3XLinj+BALvna6AHBAhUV0CAXd3a6BkBAgQIECBAgAABAgQIEBBgN+AaEGA3oMiGSIBAYQEBdmE6BxIgQIAAAQIECBAgQIAAgZYLCLBbTtzzJxBg93wN9IAAgeoKCLCrWxs9I0CAAAECBAgQIECAAAECAuwGXAMC7AYU2RAJECgsIMAuTOdAAgQIECBAgAABAgQIECDQcgEBdsuJe/4EAuyer4EeECBQXQEBdnVro2cECBAgQIAAAQIECBDoLoExY8akJ554YppPN88886QVV1xxmve3Y3EBAfYkduPHj0+9evVKvXv3Li5awSMF2BUsii4RIFAZAQF2ZUqhIwQIECBAgAABAgQIEOgxgRtvvDFttdWAaT5///5bpuHDb5jm/Xtix/fffz/16dMn9e3bt/Dpb7vttrTGGmukCOx7ahNg///y7733Xtp1113TLrvskrbaaquJ9bjzzjvTqaee+qn6xEXdr1+/nqpbl84rwO4Sl50JEGiYgAC7YQU3XAIECBAgQIAAAQIECExBoI4B9m677ZaWW2659P3vf79wzTfeeON00kkn5RC7pzYBdko5oI6gOrZBgwZ9IsC+44470umnn57OP//8T9Ro8cUXz7O122ETYLdDlfSRAIGeEhBg95S88xIgQIAAAQIECBAgQKA6AnUMsP/617+mmWeeOS2wwAKFoQXYhenKPfB///d/0wcffJD23nvv/GfSGdgRYJ911lnplltuKfek3diaALsbsZ2KAIG2ExBgt13JdJgAAQIECBAgQIAAAQKlC3RngH3IIYekr3zlK2n77befOI6YVLvFFlukzTffPP3pT39KP/rRj9Ibb7yRVltttbTjjjumlVZaKf3lL39JQ4cOTXH85Zdfnt5888102WWXpSuuuCJde+216d13301zzz13zjc32mijdMopp6Qll1wybbvttmnChAnpmmuuyX/eeuuttOCCC6bvfve7adVVV02//e1v03nnnZe/vvDCC6cjjzwyLb300rlvkwbYb7/9dm7zz3/+c5pxxhnTZpttlvbdd9+8TMkNN9yQHnvssdzPW2+9NZ/38MMPL6VOZmBPwjhgwIC0xx57fCrA/uEPf5i+/OUvp8997nNp9dVXzxfSDDPMkI+M4ld9W3HFldOTT46qejf1jwABAj0iEAH2kCHH9si5nZQAgdYJxCfl2uH3tNYJaJkAgToJ+JlWp2oaCwECVV3RoDsD7IsuuijdfPPN6Te/+U1e4WHkyJHpoIMOykF0rFu955575nB7vfXWS7EGdawcMXz48PyQyQivY1njTTbZJP8dmWUExQcffHAOjR9++OH00UcfpZ133jntv//+aYUVVkj77bdfDph/8pOfpG222SatvfbaacSIEWn22WfPQfp3vvOdHJ5/9atfzX145plnctAds7cnDbDjPC+88EJuL8LzGEecJ5Zk/tnPfpZ+9atf5dne0e8IyLfccstSLngBdicB9uOPP54vlDnnnDONHj063X///Wn99ddPRx99dD4y3pmo+rb22uump556uurd1D8CBAj0iMD3v39YGjy4+HpgPdJpJyVAoFOBmGwQv7jbCBAgUAeBmNk2fvz4OgzFGAgQIJAztipu3Rlgv/7662mnnXbKyxrHDOgTTzwxz7Y+88wzc8gcM6KHDRuWmeJ32gitzz777DRu3Lj83xF8zzrrrPn7EUQPGTIkHXPMMTmY7ph0G9+bNMCO9bAjVP7BD37wCf5YeeL3v/99uv766/PXx4wZk3bYYYd0xBFH5FncHQF2rKUdk39j1nZHMH3yySenRx99NF155ZU5wP7d736XZ4T37t271BILsCfhnNIM7Mm1492HCy64IE+Fn/SCKLUqJTdmCZGSQTVHgECtBCwhUqtyGgwBAgQIECBAgAABAgQKCXRngB0dPPDAA9Ncc82Vg+JvfOMbOYBed9110+DBg9MjjzzyqXWrY1mQmDEdAXYsedwxkz1C7XjI4n333ZfHHUt4fO9730uLLrroJwLsTTfdNO21115pu+22+4RPnC+2SYPt/v375xA7QvaOAHveeedNAwcOTBdeeGFaYokl8jExK/ycc85Jt99+ew6w//jHP+bvl70JsCcRnZYA+5577kknnHBCios6pum3wybAbocq6SMBAj0lIMDuKXnnJUCAAAECBAgQIECAQHUEujvAvvvuu/Ms61iO45JLLslLfMTM5dNOOy299NJL6cc//vGncGKliMkD7I6dYpWIWIP63HPPzQFzBNKTzsCOJUlWWWWVHJhPusUs8FjTOmZRxzZ27Ni09dZb5yVJYhnljgB7qaWWyuF3x8zs2Ddmhcfs7euuu06A3epLOT6K9fHHH+ci7LrrrineZejbt28+7S9+8Yu0zDLL5PViYqHyeFciZl5ffPHFre5Wae0LsEuj1BABAjUUEGDXsKiGRIAAAQIECBAgQIAAgS4KdHeAHXlkZJAxgzoesrjPPvvkHj/00EM5JD7ggAPS17/+9ZxHxgznNdZYI7333nufCrAjCP/Xv/6V/uu//is/TDGWPZ5lllnysiKTBtgRiMeKEt///vfTOuuskx8UGWF1LEUS+0ZgHTPAL7300jyzOtbCnn/++T+xBnYsQzLTTDOlY489Nq+BHeeKYw499FABdhevty7vHu9cxDsYk24d0+HjXY+4SDq2WIg8puXHNPx22QTY7VIp/SRAoCcEBNg9oe6cBAgQIECAAAECBAgQqJZAdwfYMfpYfiPWnr788ss/sWRILGEcD0iMCbexzTbbbOmMM87IQfXkM7BjDezjjjtu4r5f+MIXcsC88MILp0GDBqXll18+7bvvvjn8jlUlHnzwwdxmzPaOhzJ+7Wtfm9iPjq9HIB0zr2OLv2Ot63hYZDzAMY7peCbg0ksvnWd6R/9iFnksIRJLL5e9WUJkGkTj6Z+xuHoUI9amabdNgN1uFdNfAgS6U0CA3Z3azkWAAAECBAgQIECAAIFqCvREgP1ZEhMmTMgPdpxxxhnTHHPM8ZlosW88fHHmmWfOfz5r+/DDD3MAHWtaT/qwxcg///nPf6aFFlooz+T+rO21115Ln/vc57rtgZwC7Gr+P1NqrwTYpXJqjACBmgkIsGtWUMMhQIAAAQIECBAgQIBAAYGYyXzUUUdP85FrrbVWOumkE6d5fzsWFxBgF7drmyMF2G1TKh0lQKAHBATYPYDulAQIECBAgAABAgQIECBAYBoFBNjTCNXOuwmw27l6+k6AQKsFBNitFtY+AQIECBAgQIAAAQIECBAoLiDALm7XNkcKsNumVDpKgEAPCAiwewDdKQkQIECAAAECBAgQIECAwDQKCLBQCNj8AAAgAElEQVSnEaqddxNgt3P19J0AgVYLCLBbLax9AgQIECBAgAABAgQIECBQXECAXdyubY4UYLdNqXSUAIEeEBBg9wC6UxIgQIAAAQIECBAgQIAAgWkUEGBPI1Q77ybAbufq6TsBAq0WEGC3Wlj7BAgQIECAAAECBAgQIECguIAAu7hd2xwpwG6bUukoAQI9ICDA7gF0pyRAgAABAgQIECBAgEDFBMaMGZOeeOKJae7VPPPMk1ZcccVp3t+OxQUE2MXt2uZIAXbblEpHCRDoAQEBdg+gOyUBAgQIECBAgAABAgQqJnDjjTemrbbaepp71b//lmn48Ounef9p3fHjjz9O77//fpp55pmn9ZBP7NeV48eNG5fGjx+f+vXrV+hc3XWQALu7pHvwPALsHsR3agIEKi8gwK58iXSQAAECBAgQIECAAAECLReoSoA9YsSINGTIkHTVVVelueaaq8vj7srxZ599drr//vvTlVde2eXzdOcBAuzu1O6hcwmwewjeaQkQaAsBAXZblEknCRAgQIAAAQIECBAg0FKBqgTY77zzTnrppZfSsssum/r06dPlMXfl+Ndeey3F/l/84he7fJ7uPECA3Z3aPXQuAXYPwTstAQJtISDAbosy6SQBAgQIECBAgAABAgRaKtCdAfYhhxySvvKVr6Ttt99+4pgGDRqUtthii7T00kunY489Nl1++eWpd+/eaZdddknf/va307333puee+65NHTo0DT//POnU045JT344IP5+MUXXzwtueSS6fDDD0/PP//8p45fZ5110t13353efPPNtPHGG6f9998/zTTTTCnG/PDDD+cZ37E99NBD6Sc/+Un6+9//nuacc8603XbbpW233Ta3O3LkyBRLjsTX99xzz7TZZpu1tB6TNi7A7jbqnjuRALvn7J2ZAIHqCwiwq18jPSRAgAABAgQIECBAgECrBbozwL7ooovSzTffnH7zm9+kXr165XD4oIMOSldccUV6/fXX83/fdttteQZ2BM6xbbDBBmmhhRZKm266afrxj3+cnnrqqRxsR3h93nnnpRlnnDHFkiAdbU16fITOu+++ew6tI/g+8sgj0/rrr58uvfTSdN9996WLL744vfjii2nvvfdOa665Ztpmm23yvx999NF0/PHH5zB9qaWWSvHgyt/97nfp6quvzn+i3e7YBNjdodzD5xBg93ABnJ4AgUoLCLArXR6dI0CAAAECBAgQIECAQLcIdGeAHSH1TjvtlE499dS06qqrphNPPDG98cYb6cwzz5xiAH3MMcek9dZbLzvEAx779++fZ1EPGDAgf+2cc85JzzzzzFQD7AitV1tttbxvzKaed95589+TBtgRfkfoHcF6hOqTbvFgyFGjRqWnn346/fOf/0zXXXddOuOMM9KKK67YLbURYHcLc8+eRIDds/7OToBAtQUE2NWuj94RIECAAAECBAgQIECgOwS6M8CO8Rx44IH5IY1HHHFE+sY3vpEipF533XWnGGBPGkC//PLLeQmPCy64IH3hC1/ocoA9bNiw9NFHH+WlSCYNsCPQji1C9Um3sWPHpgMOOCAvK7LyyiunBRdcMN1yyy3ptNNOy//ujk2A3R3KPXwOAXYPF8DpCRCotIAAu9Ll0TkCBAgQIECAAAECBAh0i0B3B9ixJnWEyfvtt1+65JJL0g033JDXvJ7SEiCTBtgxG3rzzTdPgwcPThtuuGFpAXYE17GmdiwNMul255135lD7mmuuSXPMMUf+VixrIsDulsuyOScRYDen1kZKgEDXBQTYXTdzBAECBAgQIECAAAECBOom0N0B9vjx4/NSIPFgxHhQ4j777JNJOwuwY59YwzqW89h5553Tu+++m371q1+lL37xi9O0hMjUZmCPGDEiP8xxjz32yDPCY8Z1rI8da1/H12Pd7QUWWCAvMRIztwXYdfs/oIfHI8Du4QI4PQEClRYQYFe6PDpHgAABAgQIECBAgACBbhHo7gA7BhVrV19//fX5IYkRDk8twO5YK7sD4h//+Ed+GONLL72UPv/5z6eYld2vX788U3pKAfikx0eAHeF5hNKXXXZZuvfee/NDHGOLmeC//OUvJ3pvueWWadCgQenggw9OTz75ZP76l770pRyen3766WmllVbqltpYQqRbmHv2JALsnvV3dgIEqi0gwK52ffSOAAECBAgQIECAAAEC3SHQEwF20XFFAN2nT598eITXsQxJhMnxYMfp3aK9eFBjrM8944wzTmwuHjIZS5zMPffc03uKLh8vwO4yWfsdIMBuv5rpMQEC3ScgwO4+a2ciQIAAAQIECBAgQIBAVQViCY2jjjpmmru31lr/mU466cRp3r/MHS+88MJ01113pYUWWiiNHj06vfPOO3lZj/nnn7/M01SmLQF2ZUrRuo4IsFtnq2UCBNpfQIDd/jU0AgIECBAgQIAAAQIECDRJIJYQeeCBB3JwHaH1WmutlWabbbbaEgiwa1va/zcwAXYDimyIBAgUFhBgF6ZzIAECBAgQIECAAAECBAgQaLmAALvlxD1/AgF2z9dADwgQqK6AALu6tdEzAgQIECBAgAABAgQIECAgwG7ANSDAbkCRDZEAgcICAuzCdA4kQIAAAQIECBAgQIAAAQItFxBgt5y4508gwO75GugBAQLVFRBgV7c2ekaAAAECBAgQIECAAAECBATYDbgGBNgNKLIhEiBQWECAXZjOgQQIECBAgAABAgQIECBAoOUCAuyWE/f8CQTYPV8DPSBAoLoCAuzq1kbPCBAgQIAAAQIECBAgQICAALsB14AAuwFFNkQCBAoLCLAL0zmQAAECBAgQIECAAAECBAi0XECA3XLinj+BALvna6AHBAhUV0CAXd3a6BkBAgQIEJhc4NJLL0uXXfZzMAQIECAwBYElllg8/exnP2VDoHYCAuzalfTTAxJgN6DIhkiAQGEBAXZhOgcSIECAAIFuFxg69Lh03HEndPt5nZAAAQLtILDCCsunJ554rB26qo8EuiQgwO4SV3vuLMBuz7rpNQEC3SMgwO4eZ2chQIAAAQJlCAiwy1DUBgECdRUQYNe1ssYlwG7ANSDAbkCRDZEAgcICAuzCdA4kQIAAAQLdLiDA7nZyJyRAoI0EBNhtVCxd7ZKAALtLXO25swC7Peum1wQIdI+AALt7nJ2FAAECBAiUISDALkNRGwQI1FVAgF3XyhqXALsB14AAuwFFNkQCBAoLCLAL0zmQAAECBAh0u4AAu9vJnZAAgTYSEGC3UbF0tUsClQqwJ0yYkF588cU0evTotOSSS6YFFlggvfTSS2nmmWdO8803X5cGZuf/JyDAdjUQIEBg6gICbFcHAQIECBBoHwEBdvvUSk8JEOh+AQF295s7Y/cIVCbAHjt2bDrggAPSq6++mke+//77pwEDBqQDDzww/f3vf09XX31194jU8CwC7BoW1ZAIEChNQIBdGqWGCBAgQIBAywUE2C0ndgICBNpYQIDdxsXT9c8UqEyA/etf/zr9/Oc/T/vss0+66qqr0nbbbZcD7AceeCAdffTR6Yorrkjzzz+/chYQWGGFFdOTT44qcKRDCBAgUH+BIUOOTUOHDqn/QI2QAAECBAjUQECAXYMiGgIBAi0TEGC3jFbDPSxQmQB7++23TxtssEHab7/9coi9+eab5wB7zJgxaYcddkinn356WmmllXqYqz1PL8Buz7rpNQEC3SMgwO4eZ2chQIAAAQJlCAiwy1DUBgECdRUQYNe1ssZVmQA7Zlxvuummaa+99vpEgP3cc8/lUPuSSy5JiyyyiIoVEBBgF0BzCAECjREQYDem1AZKgAABAjUQEGDXoIiGQIBAywQE2C2j1XAPC1QmwD7++OPTww8/nM4///w0ZMiQPAN7k002SYcddlh64YUX0k033ZR69+7dw1zteXoBdnvWTa8JEOgeAQF29zg7CwECBAgQKENAgF2GojYIEKirgAC7rpU1rsoE2G+++Wbabbfd0vvvv5+rMssss6T33nsvffzxx+mII45IG220kWoVFBBgF4RzGAECjRAQYDeizAZJgAABAjUREGDXpJCGQYBASwQE2C1h1WgFBCoTYIdFBNZXXnllGjVqVBo7dmxabLHF0tZbb52WXnrpClC1bxcE2O1bOz0nQKD1AgLs1hs7AwECBAgQKEtAgF2WpHYIEKijgAC7jlU1phCoVICtJK0REGC3xlWrBAjUQ0CAXY86GgUBAgQINENAgN2MOhslAQLFBATYxdwcVX2BygTYzz//fHr77benKrbyyiunPn36VF+0gj0UYFewKLpEgEBlBATYlSmFjhAgQIAAgU4FBNidEtmBAIEGCwiwG1z8mg+9MgH2oEGD0tNPPz1V7quvvjrNOeecNS9Ha4YnwG6Nq1YJEKiHgAC7HnU0CgIECBBohoAAuxl1NkoCBIoJCLCLuTmq+gKVCbBHjx6d3n333U+JDR06NC244ILp1FNPTb17926p6Pjx41OvXr2meJ533nknjRs3Ls0999wt7UMrGhdgt0JVmwQI1EVAgF2XShoHAQIECDRBQIDdhCobIwECRQUE2EXlHFd1gcoE2FODuvvuu9OwYcPSddddl2abbbaWecYDJHfddde0yy67pK222mrieSJUHzx48MTZ4QsssEA644wz0nzzzdeyvpTdsAC7bFHtESBQJwEBdp2qaSwECBAgUHcBAXbdK2x8BAhMj4AAe3r0HFtlgcoH2C+99FLaa6+90imnnJJWW221lljG7O4777wztx1LmUwaYF944YXplltuSRdccEGaeeaZ0wEHHJAWXXTRHKq3yybAbpdK6ScBAj0hIMDuCXXnJECAAAECxQQE2MXcHEWAQDMEBNjNqHMTR1mZAPv1119PMQu6Y5swYUIaO3Zs+uUvf5kefPDB1Mo1sP/3f/83ffDBB2nvvffOfyYNsHfccce04YYbpoEDB+au3XrrrXkG9h133JGXG2mHTYDdDlXSRwIEekpAgN1T8s5LgAABAgS6LiDA7rqZIwgQaI6AALs5tW7aSCsTYE/tIY6x7nWEyLvvvnvLazNgwIC0xx57fCLA3nTTTdNBBx2UNttss3z+kSNH5n9fc801aY455mh5n8o4gQC7DEVtECBQVwEBdl0ra1wECBAgUEcBAXYdq2pMBAiUJSDALktSO1UTqEyA/fTTT6c333zzEz6zzjprWn755VOfPn26xW3yADtmgW+yySbpyCOPzLOwY3vuuefSfvvtly677LK00EILpbfffrtb+jY9J1lrrbXTU089PT1NOJYAAQK1FRg8+Psp/tgIEKiXwAwzzJA++uijeg3KaAgQSD/4wSnpBz84lQQBAgQITEFgueWWTSNG3Fd5m3aZEFp5yAZ1sDIBdhXMpzYD++CDD04xEzu2yWdgf/zxx1Xo+mf2YaWVVk5PPjmq8v3UQQIECPSEwLHHHpNiFraNAAECBAgQqL7Acccdn44/vn2eR1R9UT0kQKBOAjED+7HHHqn8kGK1BRuBrgj0aIB91113pb/97W/T1N8ddtgh9evXb5r2LbrTlALsydfAjgc6nnnmmdbALorsOAIECFRMwBIiFSuI7hAgQIAAgc8QsISIy4MAAQJTF7CEiKujrgI9GmAfcsgh6fHHH58m21auOT1+/PgUM6m32267tOuuu6b+/funvn375n5deOGFKULr+HummWZKBxxwQFp00UXTsGHt866/NbCn6RKzEwECDRUQYDe08IZNgAABAm0pIMBuy7LpNAEC3SQgwO4maKfpdoEeDbC7fbRTOeGUgvQIrJdYYok0duzYdPjhh6e//OUv+ej5558/z8COv9tlE2C3S6X0kwCBnhAQYPeEunMSIECAAIFiAgLsYm6OIkCgGQIC7GbUuYmjFGBPY9XfeuutNG7cuDTffPNN4xHV2U2AXZ1a6AkBAtUTEGBXryZ6RIAAAQIEpiYgwHZtECBAYOoCAmxXR10FKhVg33fffenBBx9M77777qe8Y5Z0LOFh67qAALvrZo4gQKA5AgLs5tTaSAkQIECg/QUE2O1fQyMgQKB1AgLs1tlquWcFKhNg33jjjenss89O8STSWI96ttlmy+tQv/nmm/nvK6+8Mn/N1nUBAXbXzRxBgEBzBATY5dT69ddfT+eee145jWmFAAECNRTYf//vtOWnOatWCgF21SqiPwQIVElAgF2lauhLmQKVCbD32GOPNMccc6QhQ4bkhyleeumlaeGFF87rTT/88MPp8ssvL3PcjWpLgN2ochssAQJdFBBgdxFsKruPHDkyrbjiKuU0phUCBAjUUGDkyMfS8ssvX8ORde+QBNjd6+1sBAi0l4AAu73qpbfTLlCZAHvAgAFpp512St/85jfTpptumk499dS06qqrpmeffTbtv//+qeOhitM+NHt2CAiwXQsECBCYuoAAu5yrQ4BdjqNWCBCor4AAu5zaCrDLcdQKAQL1FBBg17OuRpVSZQLsmHW92WabpT333DPtuOOOaaONNkp77713ihfEBx100MRAW9G6LiDA7rqZIwgQaI6AALucWguwy3HUCgEC9RUQYJdTWwF2OY5aIUCgngIC7HrW1agqFGDHLOvYzjnnnPzn+uuvzyH2o48+mt555500fPjwNMMMM6hZAQEBdgE0hxAg0BgBAXY5pRZgl+OoFQIE6isgwC6ntgLschy1QoBAPQUE2PWsq1FVKMAeNWpUeu2119KGG26YPvzww3T00UenRx55JK+DPXDgwLT22murV0EBAXZBOIcRINAIAQF2OWUWYJfjqBUCBOorIMAup7YC7HIctUKAQD0FBNj1rKtRVSjAHj16dFpwwQVTr169Jtbl448/Tr1791an6RQQYE8noMMJEKi1gAC7nPIKsMtx1AoBAvUVEGCXU1sBdjmOWiFAoJ4CAux61tWoKhRgDxo0KP3jH/9I8TDHLbfcMs0xxxzqU5KAALskSM0QIFBLAQF2OWUVYJfjqBUCBOorIMAup7YC7HIctUKAQD0FBNj1rKtRVSjAfuihh9IVV1yRH9oY21prrZW23XbbtNJKK6nTdAoIsKcT0OEECNRaQIBdTnkF2OU4aoUAgfoKCLDLqa0AuxxHrRAgUE8BAXY962pUFQqwO4oxZsyYdNNNN6UbbrghP7xx7rnnzkH2Nttsk/r06aNmBQQE2AXQHEKAQGMEBNjllFqAXY6jVggQqK+AALuc2gqwy3HUCgEC9RQQYNezrkZVwQC7oyix/vXPfvazdOWVV+YvXX311WnOOedUswICAuwCaA4hQKAxAgLsckotwC7HUSsECNRXQIBdTm0F2OU4aoUAgXoKCLDrWVejqmCA/cYbb+QZ2MOHD88zsPv165e22mqrtPvuu6e+ffuqWQEBAXYBNIcQINAYAQF2OaUWYJfjqBUCBOorIMAup7YC7HIctUKAQD0FBNj1rKtRVSjAnnwN7OWXXz4vHbL22mun3r17q9V0CAiwpwPPoQQI1F5AgF1OiQXY5ThqhQCB+goIsMuprQC7HEetECBQTwEBdj3ralQVCrAHDRqUXnzxxdS/f/80YMCANP/886tPSQIC7JIgNUOAQC0FBNjllFWAXY6jVggQqK+AALuc2gqwy3HUCgEC9RQQYNezrkZVoQD7hRdeSIsttpjZ1i24KgXYLUDVJAECtREQYJdTSgF2OY5aIUCgvgIC7HJqK8Aux1ErBAjUU0CAXc+6GlWFAmzFaJ2AALt1tlomQKD9BQTY5dRQgF2Oo1YIEKivgAC7nNoKsMtx1AoBAvUUEGDXs65GJcBuxDUgwG5EmQ2SAIGCAgLsgnCTHSbALsdRKwQI1FdAgF1ObQXY5ThqhQCBegoIsOtZV6MSYDfiGhBgN6LMBkmAQEEBAXZBOAF2OXBaIUCgMQIC7HJKLcAux1ErBAjUU0CAXc+6GpUAuxHXgAC7EWU2SAIECgoIsAvCCbDLgdMKAQKNERBgl1NqAXY5jlohQKCeAgLsetbVqATYjbgGBNiNKLNBEiBQUECAXRBOgF0OnFYIEGiMgAC7nFILsMtx1AoBAvUUEGDXs65GJcBuxDUgwG5EmQ2SAIGCAgLsgnAC7HLgtEKAQGMEBNjllFqAXY6jVggQqKeAALuedTUqAXYjrgEBdiPKbJAECBQUEGAXhBNglwOnFQIEGiMgwC6n1ALschy1QoBAPQUE2PWsq1EJsBtxDQiwG1FmgyRAoKCAALsgnAC7HDitECDQGAEBdjmlFmCX46gVAgTqKSDArmddjUqA3YhrQIDdiDIbJAECBQUE2AXhBNjlwGmFAIHGCAiwyym1ALscR60QIFBPAQF2PetqVALsRlwDAuxGlNkgCRAoKCDALggnwC4HTisECDRGQIBdTqkF2OU4aoUAgXoKCLDrWVejEmA34hoQYDeizAZJgEBBAQF2QTgBdjlwWiFAoDECAuxySi3ALsdRKwQI1FNAgF3PuhqVALsR14AAuxFlNkgCBAoKCLALwgmwy4HTCgECjREQYJdTagF2OY5aIUCgngIC7HrW1agE2I24BgTYjSizQRIgUFBAgF0QToBdDpxWCBBojIAAu5xSC7DLcdQKAQL1FBBg17OuRiXAbsQ1IMBuRJkNkgCBggIC7IJwAuxy4LRCgEBjBATY5ZRagF2Oo1YIEKingAC7nnU1KgF2I64BAXYjymyQBAgUFBBgF4QTYJcDpxUCBBojIMAup9QC7HIctUKAQD0FBNj1rKtRCbAbcQ0IsBtRZoMkQKCggAC7IJwAuxw4rRAg0BgBAXY5pRZgl+OoFQIE6ikgwK5nXY1KgN2Ia0CA3YgyGyQBAgUFBNgF4QTY5cBphQCBxggIsMsptQC7HEetECBQTwEBdj3ralQC7EZcAwLsRpTZIAkQKCggwC4IJ8AuB04rBAg0RkCAXU6pBdjlOGqFAIF6Cgiw61lXoxJgN+IaEGA3oswGSYBAQQEBdkE4AXY5cFohQKAxAgLsckotwC7HUSsECNRTQIBdz7oalQC7EdeAALsRZTZIAgQKCgiwC8JNMcBeuZzGtEKAAIEaCowc+Xhafvnlaziy7h2SALt7vZ2NAIH2EhBgt1e99HbaBXpNmDBhwrTvbs92FBBgt2PV9JkAge4SEGCXIz1y5Mi04ooC7HI0tUKAQB0FBNjlVFWAXY6jVggQqKeAALuedTUqM7AbcQ0IsBtRZoMkQKCggAC7INxkhwmwy3HUCgEC9RUQYJdTWwF2OY5aIUCgngIC7HrW1agE2I24BgTYjSizQRIgUFBAgF0QToBdDpxWCBBojIAAu5xSC7DLcdQKAQL1FBBg17OuRiXAbsQ1IMBuRJkNkgCBggIC7IJwAuxy4LRCgEBjBATY5ZRagF2Oo1YIEKingAC7nnU1KgF2I64BAXYjymyQBAgUFBBgF4QTYJcDpxUCBBojIMAup9QC7HIctUKAQD0FBNj1rKtRCbAbcQ0IsBtRZoMkQKCggAC7IJwAuxw4rRAg0BgBAXY5pRZgl+OoFQIE6ikgwK5nXY1KgN2Ia0CA3YgyGyQBAgUFBNgF4QTY5cBphQCBxggIsMsptQC7HEetECBQTwEBdj3ralQC7EZcAwLsRpTZIAkQKCggwC4IJ8AuB04rBAg0RkCAXU6p/y/APr6cxrRCgACBmgn8X4D9eM1GZTgEBNiNuAYE2I0os0ESIFBQQIBdEE6AXQ6cVggQaIyAALucUguwy3HUCgEC9RQQYNezrkYlwG7ENSDAbkSZDZIAgYICAuyCcALscuC0QoBAYwQE2OWUWoBdjqNWCBCop4AAu551NSoBdqfXwJ133plOPfXUT+134403pn79+nV6fBV2EGBXoQr6QIBAVQUE2OVUZuTIkWnFFVcupzGtECBAoIYCAuxyiirALsdRKwQI1FNAgF3PuhqVALvTa+COO+5Ip59+ejr//PM/se/iiy+eevXq1enxVdhBgF2FKugDAQJVFRBgl1MZAXY5jlohQKC+AgLscmorwC7HUSsECNRTQIBdz7oalQC702sgAuyzzjor3XLLLZ3uW9UdBNhVrYx+ESBQBQEBdjlVEGCX46gVAgTqKyDALqe2AuxyHLVCgEA9BQTY9ayrUQmwO70GIsD+4Q9/mL785S+nz33uc2n11VdPm2++eZphhhk6PbYqO/Tvv1V65ZVXqtId/SBAgEClBHba6Vvp0EMPrVSf2rEzAux2rJo+EyDQnQIC7HK0BdjlOGqFAIF6Cgiw61lXoxJgd3oNPP744+m2225Lc845Zxo9enS6//770/rrr5+OPvrofOzYsWM7baOnd/jKV76Snnrq6Z7uhvMTIECgkgJHHDE4HXnkkZXsWzt1atSoUWnNNddqpy7rKwECBLpV4E9/+mNadtllu/WcdTzZSSednE4++Qd1HJoxESBAYLoFlltu2fTAA3+c7nZa3cCss87a6lNov2YCvSZMmDChZmNq6XCuueaadMEFF6Rbb701z8IeN25cS89XRuOrrLJKGjXqqTKa0gYBAgRqJ3DMMUenY445pnbj6u4BPfnkk2nVVVfv7tM6HwECBNpG4NFHH07LLbdc2/S3qh094YRhKf7YCBAgQODTAssvv1x65JGHK0/Tt2/fyvdRB6slIMDuYj3uueeedMIJJ6Qbb7wx9evXr4tH98zuK6ywQnryyVE9c3JnJUCAQMUF/m8N7KEV72X1u2cJkerXSA8JEOhZAUuIlONvCZFyHLVCgEA9BSwhUs+6GpUlRDq9Bn7xi1+kZZZZJkUI/Pbbb6fBgwfnmdcXX3xxp8dWZQcBdlUqoR8ECFRRQIBdTlUE2OU4aoUAgfoKCLDLqa0AuxxHrRAgUE8BAXY962pUAuxOr4HTTjst3X777RP3W2CBBdJJJ52UFl100U6PrcoOAuyqVEI/CBCoooAAu5yqCLDLcdQKAQL1FRBgl1NbAXY5jlohQKCeAgLsetbVqATY03QNvAMzVcYAACAASURBVP/+++n1119Ps802W5prrrmm6Zgq7STArlI19IUAgaoJCLDLqYgAuxxHrRAgUF8BAXY5tRVgl+OoFQIE6ikgwK5nXY1KgN2Ia0CA3YgyGyQBAgUFBNgF4SY7TIBdjqNWCBCor4AAu5zaCrDLcdQKAQL1FBBg17OuRiXAbsQ1IMBuRJkNkgCBggIC7IJwAuxy4LRCgEBjBATY5ZRagF2Oo1YIEKingAC7nnU1KgF2I64BAXYjymyQBAgUFBBgF4QTYJcDpxUCBBojIMAup9QC7HIctUKAQD0FBNj1rKtRCbAbcQ0IsBtRZoMkQKCggAC7IJwAuxw4rRAg0BgBAXY5pRZgl+OoFQIE6ikgwK5nXY1KgN2Ia0CA3YgyGyQBAgUFBNgF4QTY5cBphQCBxggIsMsptQC7HEetECBQTwEBdj3ralQC7EZcAwLsRpTZIAkQKCggwC4IJ8AuB04rBAg0RkCAXU6pBdjlOGqFAIF6Cgiw61lXoxJgN+IaEGA3oswGSYBAQQEBdkE4AXY5cFohQKAxAgLsckotwC7HUSsECNRTQIBdz7oalQC7EdeAALsRZTZIAgQKCgiwC8IJsMuB0woBAo0REGCXU2oBdjmOWiFAoJ4CAux61tWoBNiNuAYE2I0os0ESIFBQQIBdEE6AXQ6cVggQaIyAALucUguwy3HUCgEC9RQQYNezrkYlwG7ENSDAbkSZDZIAgYICAuyCcALscuC0QoBAYwQE2OWUWoBdjqNWCBCop4AAu551NSoBdiOuAQF2I8pskAQIFBQQYBeEE2CXA6cVAgQaIyDALqfUAuxyHLVCgEA9BQTY9ayrUQmwG3ENCLAbUWaDJECgoIAAuyDcZIeNGjUq7bTTzuU0phUCBAjUUOCXv7wiLbvssjUcWfcOSYDdvd7ORoBAewkIsNurXno77QK9JkyYMGHad7dnOwoIsNuxavpMgEB3CQiwy5EeOXJkWnHFlcppTCsECBCoocDIkU+k5ZdfvoYj694hCbC719vZCBBoLwEBdnvVS2+nXUCAPe1WbbunALttS6fjBAh0g4AAuxxkAXY5jlohQKC+AgLscmorwC7HUSsECNRTQIBdz7oalSVEGnENCLAbUWaDJECgoIAAuyDcZIcJsMtx1AoBAvUVEGCXU1sBdjmOWiFAoJ4CAux61tWoBNiNuAYE2I0os0ESIFBQQIBdEE6AXQ6cVggQaIyAALucUguwy3HUCgEC9RQQYNezrkYlwG7ENSDAbkSZDZIAgYICAuyCcALscuC0QoBAYwQE2OWUWoBdjqNWCBCop4AAu551NSoBdiOuAQF2I8pskAQIFBQQYBeEE2CXA6cVAgQaIyDALqfUAuxyHLVCgEA9BQTY9ayrUQmwG3ENCLAbUWaDJECgoIAAuyCcALscOK0QINAYAQF2OaUWYJfjqBUCBOopIMCuZ12NSoDdiGtAgN2IMhskAQIFBQTYBeEE2OXAaYUAgcYICLDLKbUAuxxHrRAgUE8BAXY962pUAuxGXAMC7EaU2SAJECgoIMAuCCfALgdOKwQINEZAgF1OqQXY5ThqhQCBegoIsOtZV6MSYDfiGhBgN6LMBkmAQEEBAXZBOAF2OXBaIUCgMQIC7HJKLcAux1ErBAjUU0CAXc+6GpUAuxHXgAC7EWU2SAIECgoIsAvCCbDLgdMKAQKNERBgl1NqAXY5jlohQKCeAgLsetbVqATYjbgGBNiNKLNBEiBQUECAXRBOgF0OnFYIEGiMgAC7nFILsMtx1AoBAvUUEGDXs65GJcBuxDUgwG5EmQ2SAIGCAgLsgnAC7HLgtEKAQGMEBNjllFqAXY6jVggQqKeAALuedTUqAXYjrgEBdiPKbJAECBQUEGAXhBNglwOnFQIEGiMgwC6n1ALschy1QoBAPQUE2PWsq1EJsBtxDQiwG1FmgyRAoKCAALsgnAC7HDitECDQGAEBdjmlFmCX46gVAgTqKSDArmddjUqA3YhrQIDdiDIbJAECBQUE2AXhBNjlwGmFAIHGCAiwyym1ALscR60QIFBPAQF2PetqVALsRlwDAuxGlNkgCRAoKCDALggnwC4HTisECDRGQIBdTqkF2OU4aoUAgXoKCLDrWVejEmA34hoQYDeizAZJgEBBAQF2QTgBdjlwWiFAoDECAuxySi3ALsdRKwQI1FNAgF3PuhqVALsR14AAuxFlNkgCBAoKCLALwgmwy4HTCgECjREQYJdTagF2OY5aIUCgngIC7HrW1agE2I24BgTYjSizQRIgUFBAgF0QToBdDpxWCBBojIAAu5xSC7DLcdQKAQL1FBBg17OuRiXAbsQ1IMBuRJkNkgCBggIC7IJwAuxy4LRCgEBjBATY5ZRagF2Oo1YIEKingAC7nnU1KgF2I64BAXYjymyQBAgUFBBgF4QTYJcDpxUCBBojIMAup9QC7HIctUKAQD0FBNj1rKtRCbAbcQ0IsBtRZoMkQKCggAC7IJwAuxw4rRAg0BgBAXY5pRZgl+OoFQIE6ikgwK5nXY1KgN2Ia0CA3YgyGyQBAgUFBNgF4QTY5cBphQCBxggIsMsp9WmnnZauuOKKchrTCgECBGomsOiii6bhw4fXbFSGQ0CA3YhrQIDdiDIbJAECBQUE2AXhBNjlwGmFAIHGCAiwyyn10KFD03HHHV9OY1ohQIBAzQT+bwb2EzUbleEQEGA34hoQYDeizAZJgEBBAQF2QTgBdjlwWiFAoDECAuxySi3ALsdRKwQI1FNAgF3PuhqVALsR14AAuxFlNkgCBAoKCLALwgmwy4HTCgECjREQYJdTagF2OY5aIUCgngIC7HrW1agE2I24BgTYjSizQRIgUFBAgF0QToBdDpxWCBBojIAAu5xSC7DLcdQKAQL1FBBg17OuRiXAbsQ1IMBuRJkNkgCBggIC7IJwAuxy4LRCgEBjBATY5ZRagF2Oo1YIEKingAC7nnU1KgF2I64BAXYjymyQBAgUFBBgF4QTYJcDpxUCBBojIMAup9QC7HIctUKAQD0FBNj1rKtRCbAbcQ0IsBtRZoMkQKCggAC7IJwAuxw4rRAg0BgBAXY5pRZgl+OoFQIE6ikgwK5nXY1KgN2Ia0CA3YgyGyQBAgUFBNgF4QTY5cBphQCBxggIsMsptQC7HEetECBQTwEBdj3ralQC7EZcAwLsRpTZIAkQKCggwC4IJ8AuB04rBAg0RkCAXU6pBdjlOGqFAIF6Cgiw61lXoxJgT/M18M4776Rx48alueeee5qPqcqOAuyqVEI/CBCoooAAu5yqjBw5Mq244krlNKYVAgQI1FBAgF1OUQXY5ThqhQCBegoIsOtZV6MSYHd6Dbz77rtp8ODB6emnn877LrDAAumMM85I8803X6fHVmUHAXZVKqEfBAhUUUCAXU5VBNjlOGqFAIH6Cgiwy6mtALscR60QIFBPAQF2PetqVALsTq+BCy+8MN1yyy3pggsuSDPPPHM64IAD0qKLLpqGDRvW6bFV2UGAXZVK6AcBAlUUEGCXUxUBdjmOWiFAoL4CAuxyaivALsdRKwQI1FNAgF3PuhqVALvTa2DHHXdMG264YRo4cGDe99Zbb80zsO+4447Uq1evTo+vwg4C7CpUQR8IEKiqgAC7nMoIsMtx1AoBAvUVEGCXU1sBdjmOWiFAoJ4CAux61tWoBNidXgObbrppOuigg9Jmm22W940X6PHva665Js0xxxydHl+FHQTYVaiCPhAgUFUBAXY5lRFgl+OoFQIE6isgwC6ntgLschy1QoBAPQUE2PWsq1EJsD/zGpgwYULaZJNN0pFHHplnYcf23HPPpf322y9ddtllaaGFFkqXX3555a+jI444Ir366ujK91MHCRAg0BMCW289IG299dY9cepanfOVV15JRx55VK3GZDAECBAoU+Ckk05MiyyySJlNNrKt3/zmN+k3v7m+kWM3aAIECHQmsMgiC6eTTjqps916/Pu77LJLj/dBB9pLoNeESGltUxWIGdgHH3xwir9jm3wG9s9//nN6BAgQqIVAv3790vvvv1+LsRgEAQIEZp111jR27FgQBAgQqIWA39NqUUaDIEDg/xfYddddWRDokoAAuxOuydfAjgc6nnnmmW21BnaXrgg7EyBAgAABAgQIECBAgAABAgQIECBAoCICAuxOCnHhhRemCK3j75lmmikdcMABadFFF03Dhg2rSAl1gwABAgQIECBAgAABAgQIECBAgAABAvUUEGB3Utf46Onhhx+e/vKXv+Q9559//jwDO/62ESDwSYG//vWv6fXXX0//+Z//2SlNV/bttLEu7PDWW2+lESNGpC222KILR9mVAAECBKoo0PEzffPNN0+9evUqpYuxlFKfPn1S3759P9Xexx9/nG677ba0zjrrTPFh3u4xpZRAIwQIEKiEQPy8X2ONNdI888zTaX+6sm+njXVhhz/96U9p7rnnTl/84he7cJRdCRAg0H4CAuxprFm8IBk3blyab775pvEIuxFonsCll16a7rvvvnTxxRd3Oviu7NtpY5+xQ7wBtddee6Wll1467/X444+nQw45xDJA04PqWAIECFREoOPZJBEcROhcxrbbbrul5ZZbLn3/+9//VHPxu2C8AfqjH/0o73PRRRelhRdeeOKbou4xZVRAGwQIEGiNwOQ/szs7y8Ybb5wfhhchdmdbV/btrK2pff/3v/99fi1z4IEHTtzlO9/5TlpllVXSwIEDizbrOAIECLSFgAC7LcqkkwTaQ6AroXRX9p2e0ccvk7Hkz5prrpmbES5Mj6ZjCRAgUC2BVgTY8QmhmWeeOS2wwAKdBtgRIiyzzDJp//33d4+p1qWhNwQIEPiUwOQ/szsj6koo3ZV9Ozvv1L5/+eWXp7vuuivF66iOTYBdVNNxBAi0m4AAu90qpr8EuiDw8ssvp1NOOSU999xzeWbaCiuskMPcGWecMcXHzWIG2RtvvJFWW221FA8sXWmllfJyOccee2xab7318vrvH374Ydpmm23SPvvsk8982WWXpWuuuSbFR6zj49UxEy1euMdHt7sSSk++b2f92WSTTdINN9yQ+7DDDjvkP7HFx7l/9rOfpeuuuy5/SmKJJZbIfY72TzvttHT77benWWaZJf/5+te/ng1iBva3vvWtKbbXBV67EiBAgMBkAkXuO9HELrvskme43X///enNN99Mq6++er4XRZAc94cf/vCH6V//+lc+28orr5yOOuqovITHtAbYca/49re/nQYNGpS+/OUvpxdeeCEdffTR6Zxzzklzzjlnvt/FJ4hipl3cN5dccsm07bbb5vNFn84666wUn8aLUPsf//hHvn+++OKL+e+4v84111z5/vPf//3f7jH+ryBAgEALBeL1wD333JOfT/Xggw/mvw866KC0/vrr57O++uqr6Qc/+EF69tln02KLLZZfx2y22Wb55/zkP7PjNUGE2rEEYmzxiZr49OaXvvSl/O+uhNKT7vvee+/l+8a9996bX4NstdVW+Z4SfY3+x9dnn332fH9ZcMEF07777jtxCca4P5188sn5PjXbbLPle1Qcu+yyy+bXXPF6p2M505/+9Kfp4IMPTrPOOmuKpU+ff/75fI+M53Z9/vOfb2EVNE2AAIHuFxBgd7+5MxLoNoEIneOFdfxi9vbbb6drr702BwIRDuy5555p++23z0F1fPT6zjvvTMOHD09PPPFEfvG96qqr5sD3gQceyN+LZUHil8AIhGeYYYb8MNOOoCKChA022KBwgB3tdNaf+EUyfnn729/+lkP0q6++Ov9CF8H1eeedl0ODtddeO9166625j9Hn+MU1ftHbaaed8i998QtiBBAxvqm1123FcSICBAjUUKDIfSfeAI0X/vGCfNddd81Bwi9+8Yv8xmkED3FfijdX4+d4hALxwj4+VXPooYdOc4Ad1LGcVNzb4r4QHyO/6qqr8ov8b3zjGznMjqAglg2J78ebnfvtt1++z8X9KULvb37zmxPD6whBYs3R2H+hhRZKAwYMyAHC+PHj3WNqeF0bEgEC1RGIiSu/+tWv0pZbbpnD2nhNEGFvBNQR7sbrm/hkzM4775zfaIwg+ZJLLsmvXyb/mb3IIovk1xIxmScm+MR+8Voj2p+eAHvIkCF5AlHcY+Ied+qpp6ZYniruNx39j/veV7/61fzaJe5xcc6YIBSTdOadd978pmscG/e8eJ0T96AzzjgjPfbYY/n+F1u88RvniOA6xrvUUkulH//4x/le17FPdSqnJwQIEJg+AQH29Pk5mkClBeIXpZgVdsQRR6T/+I//mNjXn/zkJ+m3v/1tno0d20cffZRfcJ999tn5F7/J14iOACFC4PjFKbb4hSxmvcXs7filL36p2n333QsH2F3tT8z6jl/KNtpoo3zeCA9i1lxsEbhHEBEBdscvnp0tITJpe5UuqM4RIECg4gJF7jsRTMcL+Zj5HCFCbPFzO2aTxSy62CLUfvjhh9Nrr72Wfve73+UZbTF7elpnYEcbF154YXrooYfy33FPizdB403euPdFAB1v9sZ9ZdIAO94gvfHGG9PNN9+cg4TJ18CeliVE3GMqftHqHgECbSUQAfAf//jH/LO84/4QP9Pj3/EJmZisc+KJJ+Y3FWOL/44H/UbAO6UlROKN0QiFIwR/8sknJ07e6Xgd0dU1sOMN0JhxHffDjntavF6K+1jcbybvfyxbFW/YRhD/9NNPp2OOOSbFzOqOGdTxWudrX/ta/qTStCwhEu3Ep2WvvPLKtqqrzhIgQKAzAQF2Z0K+T6CNBSLMjRAg3s2PF/uxTEjMShg8eHB65JFHPrW+5957750/zjZ5gB0fy4sgPH4hjFkAEX7HDOaYhR0fgYsX/jFDregSIl3tTwTqcb6YIR7BQMxQ2G677XKligTYk7bXxuXWdQIECPS4QJH7zrrrrvupAPuXv/xlniF9/fXX51l1Z555Zv5o9xe/+MUUL/Zjptz555/fpQA77nvx0fAIBuJ+F3/H/SP+jntKxyd7Jg2w4w3RWJYqZs/FViTAdo/p8ctSBwgQqJHA5AFwDG3TTTdN3/3ud9O///3vdMEFF+TJLZNucZ+Jn/uTB9gx8zmO69evX/7kTfy8//Of//yJiTBdDbBj9nQ8UDH+jntVxxaf2ol72dQC+AinI5iPN07j06QdW1cD7JjRHQZx/7QRIECgTgIC7DpV01gITEEg1v2Mj5Xdcccd+ReZmJ0QS4m89NJL+SNmk29Teshh//7989pxHWtPn3DCCRPXaYs12+Lja9MTYMda1V3pz6RhQMzEjjVJOz4mN6UA+/jjj09rrbVWHuqUxidc8L8OAQIEyhPo6n0nzjz5DOwIjkePHp1f6McblLFMVceDEmNJq5hJ3dUAuyN8juVAIqyIj3jHDLlYXiruQR0fGZ80wI43ZmPGdyxdFduUAuyll146f4TbPaa8a0hLBAgQmJrA5AHwK6+8kt+MjEk7EWDHJ3fiUzOxZMjkWwTYk/7Mjok5EWLHfaV3797p0UcfTYcddth0BdjRfix7GGF1hOKTb58VYMcnj2L5qo43VOPYSQPsWF4rPmXacU+K70/+EEcBtv93CBCoq4AAu66VNS4CKeWHGMayH4svvngaNWpU+t73vpfXgYuPysWyIvGCO2Yxx/rY8U5/BNHxvZiBHR/Nnm+++XLYHR9Bi7A7ZjNEe/GLVTxUMQKEjnXZpifAjnY6608E8PHx7dgmDZwjlI+ZChFwRH9//etf56VNOpYQ6VjzNGZCvPvuu3k90yktkdIxo9uFQ4AAAQLFBYrcd2LNzgiw4+d1rGkan+yJF/4RLscDd+PveEBi/OyOn+ExGy7Wq+5qgN3xQj/CiqFDh6Z11lknhwARCMSbtNF+bJMG2E899VSesRf3z6985St5OZEIumMN7OWWWy73IWZ2x7/jHhMPD3OPKX79OJIAAQKdCUQAHK9bYjmODz74IJ177rl5gkq8Zol/dzzjJ35uxxYPAo43H+O1y+Q/s+PTPiNGjMg/w2NJxWir4/k/cWzRhzjG/Szai/tVvD6JpUFiZneE0Z8VYMeSj/HaLJYPiU+4xjJZMdZ4PkQsIRKvmeLZQ1dccUUO3OMTsnHPWmWVVfKs79gE2J1dQb5PgEC7Cgiw27Vy+k1gGgQGDRqUf2GKLWabxUzqjl9uYm20eIhVzJSLLcKAeDDIv/71r/ziu2/fvvmXvdhiXbaYSRBbx4Ov4r9j/dDYJ9Z522OPPXIQEMFDzGLobJt83876M3mAHb8YxvIhsTxK/AIbH7mL/iy55JL5yeQdH72LIDuCkOhnjD9m8U0pXOhor7N++z4BAgQITF2gyH0n3mSNkGDS+87666+fl7uKGXT33XdfDgHi53i8YI9P3cSyWBE0dGUN7Oh13HsisLjpppvy+Tpm7sWao/FQ49hiDMsvv3yKTxjFPTKWHYn1UWOLmXXxgOC478Ta3bFmavQzHo4cb/LG/cU9xv8hBAgQaJ1Ax0MQO84Q94PjjjsuP9AxtnheQszGfuedd/K/474Rn9SM+8zkP7NjIk78zI7JL7HF64j45Oqkz9KJfeKTO51t0X7HvrEWd3wCNN4w7dgimI5QPR4UGa9bYpmP2GJt7FjDO5YQiftbhNRxf4sHz8ca2jGeWAYyJutEKN7x0MY4dvjw4bn/kwbYt912Ww7qLSHSWcV8nwCBdhMQYLdbxfSXQBcFIuCNGdaTPsSxo4kJEybkX9hifbY55pgjf7ljiY345Sd+cYp39uMXv0m3+HhbBN2Try/XsU+EDDETbWpbhOnxZ/JtSv3pbLgRLsTM7I7Z2fHL4B/+8Ie8HnfHFvtEuDDPPPNM3K+zdn2fAAECBIoJdPW+E2fpWELkC1/4Qr4/TH6PGD9+fJ7dHMt9RPA8pftH3OumtkUQ3vFAryKjintltBFvlE5pGzNmTH6GxJT6VuR8jiFAgACBKQt0zGCOT4vG65H4/X5KW9wT4jXJlH7/n/xndtxf4jXPzDPPPMW2IgyP+9DUtqndG+J+GK+nYj3sKS1pMqX24jzxgOHY4rwxiSjC8DXXXHPi7jG2eP0200wzuUwIECDQGAEBdmNKbaAEpk1gSmtET9uR/2+vmCkQ7/xPbYtfxOJjfGVs8dHteLhkzOCLjw3GzIqYhbH22muX0bw2CBAgQKAbBCZfA7urp4wX+R1LgEzp2FiHNJYCsREgQIBAewtM6SGOrR5RPDPh73//+xRPE8F0zJgua4s1rWMiUDz08ZlnnkmLLLJIfl01+YSiss6nHQIECLSLgAC7XSqlnwS6SSBmJNx99915nel22OIXvPvvvz9//C5mv8VH7eIXPRsBAgQItI9AfNR53XXXzbPUbAQIECBAYGoCsYxTLNHRsexT3aTi2QtPPvlk+vDDD/Na2DEpR3hdtyobDwECRQQE2EXUHEOAAAECBAgQIECAAAECBAgQIECAAAECLRcQYLec2AkIECBAgAABAgQIECBAgAABAgQIECBAoIiAALuImmMIECBAgAABAgQIECBAgAABAgQIECBAoOUCAuyWEzsBAQIECBAgQIAAAQIECBAgQIAAAQIECBQREGAXUXMMAQIECBAgQIAAAQIECBAgQIAAAQIECLRcQIDdcmInIECAAAECBAgQIECAAAECBAgQIECAAIEiAgLsImqOIUCAAAECBAgQIECAAAECBAgQIECAAIGWCwiwW07sBAQIECBAgAABAgQIECBAgAABAgQIECBQRECAXUTNMQQIECBAgAABAgQIECBAgAABAgQIECDQcgEBdsuJnYAAAQIECBAgQIAAAQIECBAgQIAAAQIEiggIsIuoOYYAAQIECBAgQIAAAQIECBAgQIAAAQIEWi4gwG45sRMQIECAAAECBAgQIECAAAECBAgQIECAQBEBAXYRNccQIECAAAECBAgQIECAAAECBAgQIECAQMsFBNgtJ3YCAgQIECBAgAABAgQIECBAgAABAgQIECgiIMAuouYYAgQIECBAgAABAgQIECBAgAABAgQIEGi5gAC75cROQIAAAQIECBAgQIAAAQIECBAgQIAAAQJFBATYRdQcQ4AAAQIECBAgQIAAAQIECBAgQIAAAQItFxBgt5zYCQgQIECAAAECBAgQIECAAAECBAgQIECgiIAAu4iaYwgQIECAAAECBAgQIECAAAECBAgQIECg5QIC7JYTOwEBAgQIECBAgAABAgQIECBAgAABAgQIFBEQYBdRcwwBAgQIECBAgAABAgQIECBAgAABAgQItFxAgN1yYicgQIAAAQIECBAgQIAAAQIECBAgQIAAgSICAuwiao4hQIAAAQIECBAgQIAAAQIECBAgQIAAgZYLCLBbTuwEBAgQIECAAAECBAgQIECAAAECBAgQIFBEQIBdRM0xBAgQIECAAAECBAgQIECAAAECBAgQINByAQF2y4mdgAABAgQIECBAgAABAgQIECBAgAABAgSKCAiwi6g5hgABAgQIECBAgECJAn/84x/TMccck37605+mz3/+8yW2rCkCBAgQIECAAAEC7S0gwG7v+uk9AQIECBAgQIBADQRGjBiRhgwZki6++OK02GKL1WBEhkCAAAECBAgQIECgHAEBdjmOWiFAgAABAgQI1E7g6KOPTvPMM0/68MMP0913353Ht9VWW6VvfvOb6cwzz0yPPvpommuuudIOO+yQ+vfvP3H8L7zwQjrjjDPSs88+m/r06ZNWXnnldOihh+a2YjvkkEPS888/n959993Ut2/ftNJKK6Xvfe97aYEFFsjf/8c//pF+9KMfpZEjR+Zzx9e33nrrNGDAgBRB7wUXXJDOPffcNMsss+T9R40alU455ZR08sknp4UWWig98sgj6ayzzkrHHntsuummm9KTTz6Z1ltvvbTzzjunz+rbe++9l/bdd9+0ySabpIceeii3O/vss6fvfOc7ab755ks//vGP04svvpiWGvkY2gAACdFJREFUWWaZtMcee6RVVlll4pjvueeePHt69OjRuV8bb7xxGjhwYB5fR7ubbrppevjhh9MTTzyRFlxwwbTjjjum+FqMN87xzjvvpHnnnTfNOOOMafHFF0/HHXdc7a4pAyJAgAABAgQIECDQVQEBdlfF7E+AAAECBAgQaIhABKxvvPFGWnTRRdOGG26Yg+A///nPefRLLrlkWmedddKDDz6YnnrqqfTrX/86B9Svv/562mmnndLCCy+cA+e33norXXXVVXlZjPPPPz8fe+CBB+bQOtp988030+WXX54WWWSRdOGFF+bv77bbbvnr3/72t9PnPve5fI5oN0LrW265JYfn11xzTZpjjjny/n/605/SUUcdlduPfkXYPmzYsPy9OeecMy2xxBI5RI9Q+bP6FgHyNttsk49bffXV04orrphuu+22HDDHttZaa+Xwevjw4enjjz9OV199df56x/nimI022ig9/vjj6fbbb0/bb7992muvvXIw3dHuqquumvty77335hD/2muvTb169cqBfbQTbxDMPffc+c/mm2/ekCvNMAkQIECAAAECBAhMXUCA7eogQIAAAQIECBCYokAE2DH7OWZTR8g6bty4tMUWW+TgeujQofmYjnA2ZlB//etfT6eeemq666670nXXXTdxhvQvf/nLdMkll0wMuTtO9v777+eA++c//3m68847c+gbW8xKXn/99VPMAO/YYhbzTDPN1KUAe//9988hesfWWd9i5nMEzbvvvnsOumOLWeaHHXZYniEe44utI7CO4D18IqiOoDxmhndsBx10UHrllVdyyN1hFMF8zAKPbcyYMXnmesxG32yzzfLMckuI+B+RAAECBAgQIECAwKcFBNiuCgIECBAgQIAAgSkKRIAdM447wurYKQLsmMkcAW3HFv+OMHbPPffMS2v87W9/y0thdGwRPsdyITFzeoUVVsgh9GWXXZZnWU+6xddjyY0IdWMWcywHErOaN9hggzxjO7auzMCOc0QbHVtnfYu1pyPA/u53v5u23HLLfNjLL7+cxxUPWIxlSGKLpU1i/Keddlpabrnlskn0u2NGeOwTY4tZ2hHMdwTYk7Yb+4Tbt771rTzTXIDtf0ICBAgQIECAAAECUxYQYLsyCBAgQIAAAQIEpigwpQA71rqOZTKmFmDHzOXevXvnpTMm32L5jAimY23nNdZYI89cjqVD/ud//ic/vLAjwP7oo4/Sb37zmxTrSsc62hEExyzlCLY7AuyY2RyznmOb2hIikwfYnfUtZplPHmC/+uqreUb2pAF2LJkSy6BEgB1LlsT63GuvvXZ2mXSL9iL0FmD7H4wAAQIECBAgQIBAcQEBdnE7RxIgQIAAAQIEai1QJMA+8sgj80MKY93rWPKjY5swYUJehiRmYUcI3RFWx/djuZHzzjtv4tcisI4QPLZYtiTC41h7O5YY+cMf/pBnhEfgHTOmY3vggQfyciOTr4E9eYDdWd+mFDR3FmDHetYxA3vZZZdNp59++ieuh44xT0uAHQ+ePPzww9M555yTll566VpfVwZHgAABAgQIECBAoCsCAuyuaNmXAAECBAgQINAggSIBdjzoMdaLXmqppdLAgQPTbLPNlkaNGpViHex4CGOsKX3SSSelaDtmYcf+sQZ2BNURasdSI7HUR/yJZUMi/I39I9SONj744IP0/7V3xyiNRlEYQP/CLWhnE1yCYOM63EU6G3eTIpCNuIh0dgqxMVZBCAzfg0gYHR0HBi7c8zdKYuK9573q43FfToEfTnDnhHbma+fz3wXY39V2cnLy4xPYCbCXy+UYiZLZ3bmE8e3tbUogfX9/P+Zi/02Anc9kxnZmf9/c3Eyvr6/T5eVlo92mVQIECBAgQIAAAQKfCwiw7QwCBAgQIECAAIFPBTJyI0H08QzsP40QSSCd0DlPLjnMSeuE0YcnozbyWi5KvLu7GwFvnsyOznvr9XoE2BkfklnRDw8P75/N6eaM7Li4uBiv5fLE1Wo1Quuc1L66uhonsxMWz2azD5csHjf3VW353xkhcnxh4+Pj45TLF3PB4vX19fiq1Dqfz8eJ64TsCdcTwqem/H54Emjf3t5+uOjy8H5mYMc4I0ryLBaLcXI9feVyyPTpIUCAAAECBAgQINBdQIDdfQfonwABAgQIECDwnwS22+0Ib8/OzkZwffy8vLxMef/8/Px9XMjx+7vdbtpsNiPI/f2z+buEvE9PT2OG9mHcyE/a+Kq2n3zP8d9mZMjz8/OUn6enp/9U136/H9+RSzBzItxDgAABAgQIECBAoLuAALv7DtA/AQIECBAgQIAAAQIECBAgQIAAAQIEigoIsIsujLIIECBAgAABAgQIECBAgAABAgQIECDQXUCA3X0H6J8AAQIECBAgQIAAAQIECBAgQIAAAQJFBQTYRRdGWQQIECBAgAABAgQIECBAgAABAgQIEOguIMDuvgP0T4AAAQIECBAgQIAAAQIECBAgQIAAgaICAuyiC6MsAgQIECBAgAABAgQIECBAgAABAgQIdBcQYHffAfonQIAAAQIECBAgQIAAAQIECBAgQIBAUQEBdtGFURYBAgQIECBAgAABAgQIECBAgAABAgS6Cwiwu+8A/RMgQIAAAQIECBAgQIAAAQIECBAgQKCogAC76MIoiwABAgQIECBAgAABAgQIECBAgAABAt0FBNjdd4D+CRAgQIAAAQIECBAgQIAAAQIECBAgUFRAgF10YZRFgAABAgQIECBAgAABAgQIECBAgACB7gIC7O47QP8ECBAgQIAAAQIECBAgQIAAAQIECBAoKiDALrowyiJAgAABAgQIECBAgAABAgQIECBAgEB3AQF29x2gfwIECBAgQIAAAQIECBAgQIAAAQIECBQVEGAXXRhlESBAgAABAgQIECBAgAABAgQIECBAoLuAALv7DtA/AQIECBAgQIAAAQIECBAgQIAAAQIEigoIsIsujLIIECBAgAABAgQIECBAgAABAgQIECDQXUCA3X0H6J8AAQIECBAgQIAAAQIECBAgQIAAAQJFBQTYRRdGWQQIECBAgAABAgQIECBAgAABAgQIEOguIMDuvgP0T4AAAQIECBAgQIAAAQIECBAgQIAAgaICAuyiC6MsAgQIECBAgAABAgQIECBAgAABAgQIdBcQYHffAfonQIAAAQIECBAgQIAAAQIECBAgQIBAUQEBdtGFURYBAgQIECBAgAABAgQIECBAgAABAgS6Cwiwu+8A/RMgQIAAAQIECBAgQIAAAQIECBAgQKCogAC76MIoiwABAgQIECBAgAABAgQIECBAgAABAt0FBNjdd4D+CRAgQIAAAQIECBAgQIAAAQIECBAgUFTgF/KP4SGdh9qKAAAAAElFTkSuQmCC",
- "text/html": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
+ "outputs": [],
"source": [
"import plotly.express as px\n",
- "px.bar(df, x='measurement', y='value', color='species')\n",
- "px."
+ "\n",
+ "# Create a faceted bar chart using Plotly\n",
+ "fig = px.bar(\n",
+ " df,\n",
+ " x=\"month\",\n",
+ " y=\"snowfall_inches\",\n",
+ " color=\"month\",\n",
+ " facet_col=\"region\",\n",
+ " title=\"Snowfall by Region and Month\"\n",
+ ")\n",
+ "fig"
]
},
{
@@ -1219,10 +159,8 @@
"id": "8105c206-febd-48c6-a2e2-88881894a8d6",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell16"
+ "name": "markdowns_md",
+ "resultHeight": 170
},
"source": [
"## Develop your narrative with Markdown cells\n",
@@ -1237,10 +175,8 @@
"id": "629ff770-3d1c-46a2-ac4b-299b0e4663c9",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell17"
+ "name": "markdowns2_md",
+ "resultHeight": 308
},
"source": [
"# Top-level Header \n",
@@ -1258,10 +194,8 @@
"id": "640931e5-d7c7-44d4-a311-155abf60af4e",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell18"
+ "name": "markdowns3_md",
+ "resultHeight": 214
},
"source": [
"### Inline Text Formatting\n",
@@ -1280,10 +214,8 @@
"id": "7bc5008a-aebb-45a2-9852-aec6142e335a",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell19"
+ "name": "markdowns4_md",
+ "resultHeight": 239
},
"source": [
"From here on, you can double click onto each Markdown cell to take a look at the underlying Markdown content.\n",
@@ -1302,10 +234,8 @@
"id": "8e969fb2-d46a-4ce6-9fa5-4032c5fa7de5",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell20"
+ "name": "markdowns5_md",
+ "resultHeight": 400
},
"source": [
"## Formatting code\n",
@@ -1331,10 +261,8 @@
"id": "ca69a287-6866-4fa9-b610-ec9b8e28b9ba",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell21"
+ "name": "image_embedding_md",
+ "resultHeight": 856
},
"source": [
"## Embedding Images ๐ผ๏ธ\n",
@@ -1360,10 +288,8 @@
"id": "37bbb377-515d-4559-beb0-7450d9c33828",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell22"
+ "name": "image_embedding2_md",
+ "resultHeight": 278
},
"source": [
"## Bring your Notebook alive with Streamlit\n",
@@ -1384,11 +310,9 @@
"metadata": {
"codeCollapsed": false,
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
"language": "python",
- "name": "cell23"
+ "name": "image_embedding",
+ "resultHeight": 225
},
"outputs": [],
"source": [
@@ -1402,11 +326,9 @@
"id": "08bf80ac-bc12-4e41-8079-cfff2ce29e7d",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
"language": "python",
- "name": "cell24"
+ "name": "image_embedding2",
+ "resultHeight": 448
},
"outputs": [],
"source": [
@@ -1419,10 +341,8 @@
"id": "c3bd5c15-eca9-4ba5-a4b4-06a280b2f992",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell25"
+ "name": "image_embedding_stage_md",
+ "resultHeight": 41
},
"source": [
"Let's say you have some images in your Snowflake stage, you can stream in the image file and display it with Streamlit."
@@ -1431,95 +351,104 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "317e0475-7e55-449b-89dc-a2057f1bf90a",
+ "id": "c4af080a-3939-42da-b504-1de31ee8cc97",
"metadata": {
- "codeCollapsed": false,
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
"language": "sql",
- "name": "cell26"
+ "name": "create_stage",
+ "resultHeight": 111
},
"outputs": [],
"source": [
- "LS @IMAGE_STAGE;"
+ "create or replace stage NOTEBOOK;"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "57bc8d6a-c5d3-48f2-a835-8ca8f15602be",
+ "id": "8c8ed0ed-1e74-40cc-8dc3-31c33dfbeeb3",
"metadata": {
- "codeCollapsed": false,
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
"language": "python",
- "name": "cell27"
+ "name": "put_file",
+ "resultHeight": 354
},
"outputs": [],
"source": [
"from snowflake.snowpark.context import get_active_session\n",
"session = get_active_session()\n",
- "image=session.file.get_stream(\"@IMAGE_STAGE/snowflake-logo.png\", decompress=False).read() \n",
- "st.image(image)"
+ "session.file.put('snowflake_logo.png',\n",
+ " '@NOTEBOOK/Visual_Data_Stories_with_Snowflake_Notebooks',\n",
+ " auto_compress=False,\n",
+ " overwrite=True)"
]
},
{
- "cell_type": "markdown",
- "id": "941238e2-3632-49c3-a76c-d1d22345688c",
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "317e0475-7e55-449b-89dc-a2057f1bf90a",
"metadata": {
+ "codeCollapsed": false,
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell28"
+ "language": "sql",
+ "name": "image_embedding_stage",
+ "resultHeight": 111
},
+ "outputs": [],
"source": [
- "## Interactive data apps ๐น๏ธ\n",
- "\n",
- "Think of each cell in your Snowflake Notebook as a mini Streamlit app. As you interact with your data app, the relevant cells will get re-executed and the results in your app updates.\n"
+ "LS @NOTEBOOK;"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "aca7e5b1-78a5-4799-bc9a-af44c777a333",
+ "id": "57bc8d6a-c5d3-48f2-a835-8ca8f15602be",
"metadata": {
"codeCollapsed": false,
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
"language": "python",
- "name": "cell29"
+ "name": "image_embedding_stage2",
+ "resultHeight": 120
},
"outputs": [],
"source": [
- "st.markdown(\"\"\"# Interactive Filtering with Streamlit! :balloon: \n",
- "Values will automatically cascade down the notebook cells\"\"\")\n",
- "value = st.slider(\"Move the slider to change the filter value ๐\", df.value.min(), df.value.max(), df.value.mean(), step = 0.3 )"
+ "# Add a query tag to the session. This helps with debugging and performance monitoring.\n",
+ "session.query_tag = {\"origin\":\"sf_sit-is\", \"name\":\"aiml_notebooks_fs_with_dbt\", \"version\":{\"major\":1, \"minor\":0}, \"attributes\":{\"is_quickstart\":0, \"source\":\"notebook\"}}\n",
+ "\n",
+ "image=session.file.get_stream(\"@NOTEBOOK/Visual_Data_Stories_with_Snowflake_Notebooks/snowflake_logo.png\", decompress=False).read()\n",
+ "st.image(image)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "941238e2-3632-49c3-a76c-d1d22345688c",
+ "metadata": {
+ "collapsed": false,
+ "name": "filtering_md",
+ "resultHeight": 127
+ },
+ "source": [
+ "## Interactive data apps ๐น๏ธ\n",
+ "\n",
+ "Think of each cell in your Snowflake Notebook as a mini Streamlit app. As you interact with your data app, the relevant cells will get re-executed and the results in your app updates.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "1f39f7e7-f7ac-4da4-afea-609b2f3e30af",
+ "id": "aca7e5b1-78a5-4799-bc9a-af44c777a333",
"metadata": {
"codeCollapsed": false,
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
"language": "python",
- "name": "cell30"
+ "name": "filtering",
+ "resultHeight": 214
},
"outputs": [],
"source": [
- "# Filter the table from above using the Streamlit slider\n",
- "df[df[\"value\"]>value].sort_values(\"value\")"
+ "st.markdown(\"\"\"# Interactive Filtering with Streamlit! :balloon:\n",
+ "Values will automatically cascade down the notebook cells\"\"\")\n",
+ "value = st.slider(\"Move the slider to change the filter value ๐\", df.snowfall_inches.min(), df.snowfall_inches.max(), df.snowfall_inches.mean(), step = 0.3 )"
]
},
{
@@ -1529,14 +458,19 @@
"metadata": {
"codeCollapsed": false,
"language": "python",
- "name": "cell31"
+ "name": "plotting",
+ "resultHeight": 338
},
"outputs": [],
"source": [
- "alt.Chart(df).mark_bar().encode(\n",
- " x= alt.X(\"measurement\", axis = alt.Axis(labelAngle=0)),\n",
- " y=\"value\",\n",
- " color=\"species\"\n",
+ "# Filter the table from above using the Streamlit slider\n",
+ "filtered_df = df[df[\"snowfall_inches\"]>value].sort_values(\"snowfall_inches\")\n",
+ "count_df = filtered_df.groupby('region').count()['month'].reset_index()\n",
+ "\n",
+ "# Chart the number of months above the average\n",
+ "alt.Chart(count_df, title = f\"Months above {np.round(value,2)}\\\" snowfall by region\").mark_bar().encode(\n",
+ " x= alt.X(\"region\", axis = alt.Axis(labelAngle=0)),\n",
+ " y=alt.Y(\"month\", title = 'Number of months'),\n",
").properties(width=500,height=300)"
]
},
@@ -1545,10 +479,8 @@
"id": "4c568906-1bfe-4f9c-9d39-eed4c80ccb9d",
"metadata": {
"collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- },
- "name": "cell32"
+ "name": "next_steps_md",
+ "resultHeight": 115
},
"source": [
"# Now it's your turn! ๐ \n",
@@ -1556,26 +488,5 @@
"Try out Notebooks yourself to build your own data narrative!"
]
}
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.5"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
+ ]
}
diff --git a/Warehouse_Utilization_with_Streamlit/Warehouse_Utilization_with_Streamlit.ipynb b/Warehouse_Utilization_with_Streamlit/Warehouse_Utilization_with_Streamlit.ipynb
new file mode 100644
index 0000000..e732bf4
--- /dev/null
+++ b/Warehouse_Utilization_with_Streamlit/Warehouse_Utilization_with_Streamlit.ipynb
@@ -0,0 +1,118 @@
+{
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "cc4fb15e-f9db-44eb-9f60-1b9589b755cb",
+ "metadata": {
+ "name": "md_title",
+ "collapsed": false
+ },
+ "source": "# Analyze Warehouse Utilization in Snowflake Notebooks with Streamlit\n\nA notebook that generates a heatmap of warehouse usage patterns to identify peak hours that can help with cost optimization.\n\nHere's what we're implementing to investigate the tables:\n1. Retrieve warehouse utilization data\n2. Convert table to a DataFrame\n3. Create an interactive slider widget\n4. Create a Heatmap for visualizing warehouse usage patterns"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "42a7b143-0779-4706-affc-c214213f55c5",
+ "metadata": {
+ "name": "md_retrieve_data",
+ "collapsed": false
+ },
+ "source": "## 1. Retrieve warehouse utilization data\n\nFirstly, we'll write a SQL query to retrieve warehouse utilization data."
+ },
+ {
+ "cell_type": "code",
+ "id": "e17f14a5-ea50-4a1d-bc15-c64a6447d0a8",
+ "metadata": {
+ "language": "sql",
+ "name": "sql_warehouse_data",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "SELECT \n DATE(start_time) AS usage_date,\n HOUR(start_time) AS hour_of_day,\n warehouse_name,\n avg_running,\n avg_queued_load,\n start_time,\n end_time\nFROM snowflake.account_usage.warehouse_load_history\nWHERE start_time >= DATEADD(month, -1, CURRENT_TIMESTAMP())\nORDER BY warehouse_name, start_time;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b2ef4485-566e-4b11-bb5a-8085c9bc0c97",
+ "metadata": {
+ "name": "md_dataframe",
+ "collapsed": false
+ },
+ "source": "## 2. Convert table to a DataFrame\n\nNext, we'll convert the table to a Pandas DataFrame."
+ },
+ {
+ "cell_type": "code",
+ "id": "014ceccb-9447-43c9-ad8f-a91a80722de1",
+ "metadata": {
+ "language": "python",
+ "name": "py_dataframe",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "sql_warehouse_data.to_pandas()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d4027f90-ae2a-41e7-8a09-5c088b3ab3bf",
+ "metadata": {
+ "name": "md_",
+ "collapsed": false
+ },
+ "source": "## 3. Create an Interactive slider widget\n\nLet's create an interactive slider using Streamlit. This would allow users to select the number of days to analyze, which would filter the DataFrame. \n\nFinally, we'll calculate the total warehouse load (`TOTAL_LOAD`) and format the hour display (`HOUR_DISPLAY`) for each record."
+ },
+ {
+ "cell_type": "code",
+ "id": "137f2fc5-c5df-4dd4-b223-0e0690b6f8a6",
+ "metadata": {
+ "language": "python",
+ "name": "py_data_preparation",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "import pandas as pd\nimport streamlit as st\n\n# Get data\ndf = py_dataframe.copy()\n\n# Create date filter slider\ndays = st.slider('Select number of days to analyze', \n min_value=10, \n max_value=90, \n value=30, \n step=10)\n\n# Filter data based on selected days and create a copy\nlatest_date = pd.to_datetime(df['USAGE_DATE']).max()\ncutoff_date = latest_date - pd.Timedelta(days=days)\nfiltered_df = df[pd.to_datetime(df['USAGE_DATE']) > cutoff_date].copy()\n\n# Prepare data and create heatmap\n#filtered_df.loc[:, 'TOTAL_LOAD'] = filtered_df['AVG_RUNNING'] + filtered_df['AVG_QUEUED_LOAD']\n#filtered_df.loc[:, 'HOUR_DISPLAY'] = filtered_df['HOUR_OF_DAY'].apply(lambda x: f\"{x:02d}:00\")\nfiltered_df['TOTAL_LOAD'] = filtered_df['AVG_RUNNING'] + filtered_df['AVG_QUEUED_LOAD']\nfiltered_df['HOUR_DISPLAY'] = filtered_df['HOUR_OF_DAY'].apply(lambda x: f\"{x:02d}:00\")\n\nst.warning(f\"You've selected {days} days to analyze!\")\nfiltered_df",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "84929a0b-de27-4655-93dc-fd15bac9f3e5",
+ "metadata": {
+ "name": "md_heatmap",
+ "collapsed": false
+ },
+ "source": "## 4. Create a Heatmap for visualizing warehouse usage patterns\n\nFinally, we're create a heatmap using Altair. The heatmap shows the warehouse usage pattern across different hours of the day. Color intensity represents the total load and interactive tooltips showing detailed metrics for each cell."
+ },
+ {
+ "cell_type": "code",
+ "id": "f84a45e7-288f-400c-8a99-badb37a13707",
+ "metadata": {
+ "language": "python",
+ "name": "py_heatmap",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "import altair as alt\nimport streamlit as st\n\nchart = alt.Chart(filtered_df).mark_rect(\n stroke='black',\n strokeWidth=1\n).encode(\n x=alt.X('HOUR_DISPLAY:O', \n title='Hour of Day',\n axis=alt.Axis(\n labels=True,\n tickMinStep=1,\n labelOverlap=False\n )),\n y=alt.Y('WAREHOUSE_NAME:N', \n title='Warehouse Name',\n axis=alt.Axis(\n labels=True,\n labelLimit=200,\n tickMinStep=1,\n labelOverlap=False,\n labelPadding=10\n )),\n color=alt.Color('TOTAL_LOAD:Q', title='Total Load'),\n tooltip=['WAREHOUSE_NAME', 'HOUR_DISPLAY', 'TOTAL_LOAD', \n 'AVG_RUNNING', 'AVG_QUEUED_LOAD']\n).properties(\n #width=700,\n #height=450,\n title=f'Warehouse Usage Patterns ({days} Days)'\n).configure_view(\n stroke=None,\n continuousHeight=400\n).configure_axis(\n labelFontSize=10\n)\n\n# Display the chart\nst.altair_chart(chart, use_container_width=True)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f6e54924-57e2-4dfb-8bf1-bad9b7fb635d",
+ "metadata": {
+ "name": "md_resources",
+ "collapsed": false
+ },
+ "source": "## Want to learn more?\n\n- Snowflake Docs on [Account Usage](https://docs.snowflake.com/en/sql-reference/account-usage) and [WAREHOUSE_LOAD_HISTORY view](https://docs.snowflake.com/en/sql-reference/account-usage/warehouse_load_history)\n- More about [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake)\n- For more inspiration on how to use Streamlit widgets in Notebooks, check out [Streamlit Docs](https://docs.streamlit.io/) and this list of what is currently supported inside [Snowflake Notebooks](https://docs.snowflake.com/en/user-guide/ui-snowsight/notebooks-use-with-snowflake#label-notebooks-streamlit-support)\n- Check out the [Altair User Guide](https://altair-viz.github.io/user_guide/data.html) for further information on customizing Altair charts"
+ }
+ ]
+}
diff --git a/Warehouse_Utilization_with_Streamlit/environment.yml b/Warehouse_Utilization_with_Streamlit/environment.yml
new file mode 100644
index 0000000..bfe5f22
--- /dev/null
+++ b/Warehouse_Utilization_with_Streamlit/environment.yml
@@ -0,0 +1,6 @@
+name: app_environment
+channels:
+ - snowflake
+dependencies:
+ - altair=*
+ - pandas=*
diff --git a/Working with Files/Working with Files.ipynb b/Working with Files/Working with Files.ipynb
index b980b77..0021f9d 100644
--- a/Working with Files/Working with Files.ipynb
+++ b/Working with Files/Working with Files.ipynb
@@ -1,386 +1,407 @@
{
- "metadata": {
- "kernelspec": {
- "display_name": "Streamlit Notebook",
- "name": "streamlit"
- }
- },
- "nbformat_minor": 5,
- "nbformat": 4,
- "cells": [
- {
- "cell_type": "markdown",
- "id": "dfa83513-f551-4576-a9b1-ba72fea7a3f8",
- "metadata": {
- "name": "cell1",
- "collapsed": false
- },
- "source": "# How to work with files in Snowflake Notebooks \ud83d\uddc4\ufe0f\n\nIn this example, we will show you how you can work with files in notebooks and how to save them permanently to a stage."
- },
- {
- "cell_type": "markdown",
- "id": "60bb7c26-7567-4da9-994c-7d45bbeaefbe",
- "metadata": {
- "name": "cell2",
- "collapsed": false
- },
- "source": "## Working with Temporary Files\n\nAny files you write from the notebook are temporarily stored in the local stage associated with your notebook.\n\n**Note that you will no longer have access to these files as soon as you exit out of the notebook session.**\n\nLet's take a look at an example of how this works by creating a simple file."
- },
- {
- "cell_type": "code",
- "id": "3775908f-ca36-4846-8f38-5adca39217f2",
- "metadata": {
- "language": "python",
- "name": "cell3",
- "codeCollapsed": false,
- "collapsed": false
- },
- "source": "with open(\"myfile.txt\",'w') as f:\n f.write(\"abc\")\nf.close()",
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "markdown",
- "id": "1d0f320f-bb0b-49c5-8afe-f79f67ba61d3",
- "metadata": {
- "name": "cell4"
- },
- "source": "Taking a look at what's the files on my stage. Note that `notebook_app.ipynb` and `environment.yml` are files automatically created as part of Snowflake notebook. You can see the new file we created `myfile.txt`."
- },
- {
- "cell_type": "code",
- "id": "8d50cbf4-0c8d-4950-86cb-114990437ac9",
- "metadata": {
- "language": "python",
- "name": "cell5",
- "codeCollapsed": false,
- "collapsed": false
- },
- "source": "import os\nos.listdir()",
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "markdown",
- "id": "c695373e-ac74-4b62-a1f1-08206cbd5c81",
- "metadata": {
- "name": "cell6",
- "collapsed": false
- },
- "source": "Now let's disconnect the notebook from the session. You can do this by closing/refreshing the browser page or clicking on the `Active` button on the top right corner and press `End session`.\n\nNow if you rerun the notebook starting from this cell, the file you created during your previous notebook session `myfile.txt` will be lost. "
- },
- {
- "cell_type": "code",
- "id": "9c22bca7-1787-400d-ae28-482987817906",
- "metadata": {
- "language": "python",
- "name": "cell7",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "import os\nos.listdir()",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "65556fd5-8be3-4084-87e4-81e7263489ef",
- "metadata": {
- "name": "cell8",
- "collapsed": false
- },
- "source": "## Working with Permanent Files\n\nWhat if you want to save the file to a permanent location that you can access again when you come back to the session? For example, you may trained a model and want to save your model for use later, or you may want to store the results of your analysis. Since files created during the notebook session is temporary by default, we show you how you can do save files permanently by moving your files to a permanent Snowflake stage.\n\nFirst, let's create a stage called `PERMANENT_STAGE`:"
- },
- {
- "cell_type": "code",
- "id": "6646015e-f40b-4ff4-affe-b6f98f1158dd",
- "metadata": {
- "language": "sql",
- "name": "cell9",
- "codeCollapsed": false,
- "collapsed": false
- },
- "outputs": [],
- "source": "CREATE OR REPLACE STAGE PERMANENT_STAGE;",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "2c197f0c-0500-407a-ad41-3cd241fc3320",
- "metadata": {
- "name": "cell10",
- "collapsed": false
- },
- "source": "Now let's write `myfile.txt` to the temporary local stage again"
- },
- {
- "cell_type": "code",
- "id": "20c5df62-c520-4776-b74f-5c6fbc398e47",
- "metadata": {
- "language": "python",
- "name": "cell11",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "with open(\"myfile.txt\",'w') as f:\n f.write(\"abc\")\nf.close()",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "4cd337ae-4a68-4d5d-afe8-ce6606d48324",
- "metadata": {
- "name": "cell12",
- "collapsed": false
- },
- "source": "Now let's use Snowpark to upload the local file we created to the stage location. In Notebooks, we can use `get_active_session` method to get the [session](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.Session#snowflake.snowpark.Session) context variable to work with Snowpark as follows:"
- },
- {
- "cell_type": "code",
- "id": "deb5f941-d916-4bb3-b0be-d4c3cbc9bced",
- "metadata": {
- "language": "python",
- "name": "cell13",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "from snowflake.snowpark.context import get_active_session\nsession = get_active_session()",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "ef94acc8-a486-4441-a647-25422542314a",
- "metadata": {
- "name": "cell14",
- "collapsed": false
- },
- "source": "Let's use the [session.file.put](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.put) command in Snowpark to move `myfile.txt` to the stage location `@PERMANENT_STAGE`"
- },
- {
- "cell_type": "code",
- "id": "4f626f09-809f-4c6e-b6ed-bf7521041544",
- "metadata": {
- "language": "python",
- "name": "cell15",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "put_result = session.file.put(\"myfile.txt\",\"@PERMANENT_STAGE\", auto_compress= False)\nput_result[0].status",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "b9e31ad7-aec0-4431-a907-167291fca0e2",
- "metadata": {
- "name": "cell16",
- "collapsed": false
- },
- "source": "The file has now been uploaded to the permanent stage. "
- },
- {
- "cell_type": "code",
- "id": "b8557a5f-bb17-42d4-96fe-4875fee51d91",
- "metadata": {
- "language": "sql",
- "name": "cell17",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "LS @PERMANENT_STAGE;",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "319aa72b-8356-4fba-a260-655cc1786b85",
- "metadata": {
- "name": "cell18",
- "collapsed": false
- },
- "source": "Now if you disconnect the notebook session, you will see that the file still persist in the permanent stage."
- },
- {
- "cell_type": "code",
- "id": "1e61830a-c637-47f4-9ceb-705464262210",
- "metadata": {
- "language": "sql",
- "name": "cell19",
- "codeCollapsed": false,
- "collapsed": false
- },
- "outputs": [],
- "source": "LS @PERMANENT_STAGE;",
- "execution_count": null
- },
- {
- "cell_type": "code",
- "id": "aa745d07-4ebf-4c94-a017-e6131c24cd2b",
- "metadata": {
- "language": "python",
- "name": "cell20",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "from snowflake.snowpark.context import get_active_session\nsession = get_active_session()\n\nf = session.file.get_stream(\"@PERMANENT_STAGE/myfile.txt\")\nprint(f.readline())\nf.close()",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "4fcdca0d-9860-4178-8013-b2a6135e789d",
- "metadata": {
- "name": "cell21",
- "collapsed": false
- },
- "source": "Alternatively, if you prefer to download the file locally first before reading it, you can using the [session.file.get](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.get) command: "
- },
- {
- "cell_type": "code",
- "id": "4637b83b-4171-4545-ac3b-2f3878ae21ed",
- "metadata": {
- "language": "python",
- "name": "cell22",
- "codeCollapsed": false,
- "collapsed": false
- },
- "outputs": [],
- "source": "# Download the file from stage to current local path\nget_status = session.file.get(\"@PERMANENT_STAGE/myfile.txt\",\"./\")\nget_status[0].status",
- "execution_count": null
- },
- {
- "cell_type": "code",
- "id": "2b6a333c-143a-477b-9760-046748c9fd2e",
- "metadata": {
- "language": "python",
- "name": "cell23",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "import os\nos.listdir()",
- "execution_count": null
- },
- {
- "cell_type": "code",
- "id": "a514fbb3-af35-40ed-afba-485600492d3f",
- "metadata": {
- "language": "python",
- "name": "cell24",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "# Open the file locally\nwith open(\"myfile.txt\",'r') as f:\n print(f.readline())\nf.close()",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "39ba2226-35b0-4cc8-91c9-1392debeef6a",
- "metadata": {
- "name": "cell25",
- "collapsed": false
- },
- "source": "## Bonus: Working with data files from stage\n\nStage is common location for storing data file before it is loaded into Snowflake. In the previous section, we saw how you can read and write a generic file to a Snowflake stage. Here, we show a few common examples of how you can work with tabular data files stored in stage.\n"
- },
- {
- "cell_type": "code",
- "id": "47e912a8-fa21-42ec-ab8b-31289cd14970",
- "metadata": {
- "language": "python",
- "name": "cell26"
- },
- "outputs": [],
- "source": "from snowflake.snowpark.context import get_active_session\nsession = get_active_session()",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "bca5c08e-bd46-4cbf-a2da-20b73905e60b",
- "metadata": {
- "name": "cell27",
- "collapsed": false
- },
- "source": "We have an example dataset recording the amount of snowfall at different ski resort locations across different days."
- },
- {
- "cell_type": "code",
- "id": "c6905253-fb4a-4e6e-b563-cc481c608b9d",
- "metadata": {
- "language": "python",
- "name": "cell28",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "# Create a Snowpark DataFrame with sample data\ndf = session.create_dataframe([[1, 'Big Bear', 8],[2, 'Big Bear', 10],[3, 'Big Bear', 5],\n [1, 'Tahoe', 3],[2, 'Tahoe', 20],[3, 'Tahoe', 13]], \n schema=[\"DAY\", \"LOCATION\", \"SNOWFALL\"])\ndf",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "035e576d-0428-45b0-a23b-7ded6df3dfb1",
- "metadata": {
- "name": "cell29"
- },
- "source": "This is how we can write a Snowpark dataframe to a CSV file on stage:"
- },
- {
- "cell_type": "code",
- "id": "bdd17871-bc46-439c-a29e-c06fa663524e",
- "metadata": {
- "language": "python",
- "name": "cell30",
- "codeCollapsed": false,
- "collapsed": false
- },
- "outputs": [],
- "source": "df.write.copy_into_location(\"@PERMANENT_STAGE/snowfall.csv\",file_format_type=\"csv\",header=True)",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "ffd05f5d-3a12-4cbb-8db3-7c7207d12b96",
- "metadata": {
- "name": "cell31"
- },
- "source": "To access the file on stage, read a CSV file from stage location back to a Snowpark dataframe:"
- },
- {
- "cell_type": "code",
- "id": "aa177f0c-69a6-44a3-b0db-554078108add",
- "metadata": {
- "language": "python",
- "name": "cell32",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "df = session.read.options({\"infer_schema\":True}).csv('@PERMANENT_STAGE/snowfall.csv')",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "f903d26c-0323-4ccf-848b-b65c020b07d6",
- "metadata": {
- "name": "cell33",
- "collapsed": false
- },
- "source": "To learn more about how you can work with data files in notebooks, check out our tutorial on how to [work with CSV files from an external S3 stage](https://github.com/Snowflake-Labs/snowflake-demo-notebooks/blob/main/Load%20CSV%20from%20S3/Load%20CSV%20from%20S3.ipynb) and [load data from a public endpoint to a Snowflake table](https://github.com/Snowflake-Labs/snowflake-demo-notebooks/blob/main/Ingest%20Public%20JSON/Ingest%20Public%20JSON.ipynb). "
- },
- {
- "cell_type": "code",
- "id": "58de86d9-778e-4e61-841c-c2f4fda0a13a",
- "metadata": {
- "language": "sql",
- "name": "cell34",
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "-- Teardown stage created as part of this tutorial\nDROP STAGE PERMANENT_STAGE;",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "f9f1fb29-b4d8-45bf-9918-1133d1132c60",
- "metadata": {
- "name": "cell35",
- "collapsed": false
- },
- "source": "### Conclusion\n\nIn this tutorial, we showed how you can upload local files from your notebook to a permanent Snowflake stage to persist results across notebook sessions. We used Snowpark's file operation commands (e.g., [file.get](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.get), [file.put](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.put)) to move files between your local file path and the stage location. You can learn more about working with files with Snowpark [here](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/io)."
- }
- ]
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ },
+ "lastEditStatus": {
+ "notebookId": "3vdbrpduryiypkn325mi",
+ "authorId": "56160401252",
+ "authorName": "DOLEE",
+ "authorEmail": "doris.lee@snowflake.com",
+ "sessionId": "b582237b-3399-4305-b81d-3887b327cb44",
+ "lastEditTime": 1738223021808
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "dfa83513-f551-4576-a9b1-ba72fea7a3f8",
+ "metadata": {
+ "name": "intro_md",
+ "collapsed": false
+ },
+ "source": "# How to work with files in Snowflake Notebooks ๐๏ธ\n\nIn this example, we will show you how you can work with files in notebooks and how to save them permanently to a stage."
+ },
+ {
+ "cell_type": "markdown",
+ "id": "60bb7c26-7567-4da9-994c-7d45bbeaefbe",
+ "metadata": {
+ "name": "temp_files_md",
+ "collapsed": false
+ },
+ "source": "## Working with Temporary Files\n\nAny files you write from the notebook are temporarily stored in the local stage associated with your notebook.\n\n**Note that you will no longer have access to these files as soon as you exit out of the notebook session.**\n\nLet's take a look at an example of how this works by creating a simple file."
+ },
+ {
+ "cell_type": "code",
+ "id": "d5fad36d-60b2-4e06-bff9-9d399dd1dd5e",
+ "metadata": {
+ "language": "python",
+ "name": "create_working_folder",
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "import os\nos.mkdir(\"myfolder/\")\nos.chdir(\"myfolder/\")",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "3775908f-ca36-4846-8f38-5adca39217f2",
+ "metadata": {
+ "language": "python",
+ "name": "temp_file",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "source": "with open(\"myfile.txt\",'w') as f:\n f.write(\"abc\")\nf.close()",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1d0f320f-bb0b-49c5-8afe-f79f67ba61d3",
+ "metadata": {
+ "name": "temp_files2_md",
+ "collapsed": false
+ },
+ "source": "Taking a look at what's the files on my stage. Note that `notebook_app.ipynb` and `environment.yml` are files automatically created as part of Snowflake notebook. You can see the new file we created `myfile.txt`."
+ },
+ {
+ "cell_type": "code",
+ "id": "8d50cbf4-0c8d-4950-86cb-114990437ac9",
+ "metadata": {
+ "language": "python",
+ "name": "temp_files2",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "source": "import os\nos.listdir()",
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c695373e-ac74-4b62-a1f1-08206cbd5c81",
+ "metadata": {
+ "name": "temp_files3_md",
+ "collapsed": false
+ },
+ "source": "Now let's disconnect the notebook from the session. You can do this by closing/refreshing the browser page or clicking on the `Active` button on the top right corner and press `End session`.\n\nNow if you rerun the notebook starting from this cell, the file you created during your previous notebook session `myfile.txt` will be lost. "
+ },
+ {
+ "cell_type": "code",
+ "id": "9c22bca7-1787-400d-ae28-482987817906",
+ "metadata": {
+ "language": "python",
+ "name": "temp_files3",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "import os\nos.listdir()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "65556fd5-8be3-4084-87e4-81e7263489ef",
+ "metadata": {
+ "name": "perm_files_md",
+ "collapsed": false
+ },
+ "source": "## Working with Permanent Files\n\nWhat if you want to save the file to a permanent location that you can access again when you come back to the session? For example, you may trained a model and want to save your model for use later, or you may want to store the results of your analysis. Since files created during the notebook session is temporary by default, we show you how you can do save files permanently by moving your files to a permanent Snowflake stage.\n\nFirst, let's create a stage called `PERMANENT_STAGE`:"
+ },
+ {
+ "cell_type": "code",
+ "id": "6646015e-f40b-4ff4-affe-b6f98f1158dd",
+ "metadata": {
+ "language": "sql",
+ "name": "perm_files",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "CREATE OR REPLACE STAGE PERMANENT_STAGE;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2c197f0c-0500-407a-ad41-3cd241fc3320",
+ "metadata": {
+ "name": "perm_files2_md",
+ "collapsed": false
+ },
+ "source": "Now let's write `myfile.txt` to the temporary local stage again"
+ },
+ {
+ "cell_type": "code",
+ "id": "20c5df62-c520-4776-b74f-5c6fbc398e47",
+ "metadata": {
+ "language": "python",
+ "name": "perm_files2",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "with open(\"myfile.txt\",'w') as f:\n f.write(\"abc\")\nf.close()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4cd337ae-4a68-4d5d-afe8-ce6606d48324",
+ "metadata": {
+ "name": "perm_files3_md",
+ "collapsed": false
+ },
+ "source": "Now let's use Snowpark to upload the local file we created to the stage location. In Notebooks, we can use `get_active_session` method to get the [session](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.Session#snowflake.snowpark.Session) context variable to work with Snowpark as follows:"
+ },
+ {
+ "cell_type": "code",
+ "id": "deb5f941-d916-4bb3-b0be-d4c3cbc9bced",
+ "metadata": {
+ "language": "python",
+ "name": "perm_files3",
+ "collapsed": false,
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "from snowflake.snowpark.context import get_active_session\nsession = get_active_session()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ef94acc8-a486-4441-a647-25422542314a",
+ "metadata": {
+ "name": "upload_file_md",
+ "collapsed": false
+ },
+ "source": "Let's use the [session.file.put](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.put) command in Snowpark to move `myfile.txt` to the stage location `@PERMANENT_STAGE`"
+ },
+ {
+ "cell_type": "code",
+ "id": "4f626f09-809f-4c6e-b6ed-bf7521041544",
+ "metadata": {
+ "language": "python",
+ "name": "upload_file",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "put_result = session.file.put(\"myfile.txt\",\"@PERMANENT_STAGE\", auto_compress= False)\nput_result[0].status",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b9e31ad7-aec0-4431-a907-167291fca0e2",
+ "metadata": {
+ "name": "upload_file2_md",
+ "collapsed": false
+ },
+ "source": "The file has now been uploaded to the permanent stage. "
+ },
+ {
+ "cell_type": "code",
+ "id": "b8557a5f-bb17-42d4-96fe-4875fee51d91",
+ "metadata": {
+ "language": "sql",
+ "name": "upload_file2",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "LS @PERMANENT_STAGE;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "319aa72b-8356-4fba-a260-655cc1786b85",
+ "metadata": {
+ "name": "file_on_stage_md",
+ "collapsed": false
+ },
+ "source": "Now if you disconnect the notebook session, you will see that the file still persist in the permanent stage."
+ },
+ {
+ "cell_type": "code",
+ "id": "1e61830a-c637-47f4-9ceb-705464262210",
+ "metadata": {
+ "language": "sql",
+ "name": "file_on_stage",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "LS @PERMANENT_STAGE;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "aa745d07-4ebf-4c94-a017-e6131c24cd2b",
+ "metadata": {
+ "language": "python",
+ "name": "read_file",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "from snowflake.snowpark.context import get_active_session\nsession = get_active_session()\n\nf = session.file.get_stream(\"@PERMANENT_STAGE/myfile.txt\")\nprint(f.readline())\nf.close()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4fcdca0d-9860-4178-8013-b2a6135e789d",
+ "metadata": {
+ "name": "download_file_md",
+ "collapsed": false
+ },
+ "source": "Alternatively, if you prefer to download the file locally first before reading it, you can using the [session.file.get](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.get) command: "
+ },
+ {
+ "cell_type": "code",
+ "id": "4637b83b-4171-4545-ac3b-2f3878ae21ed",
+ "metadata": {
+ "language": "python",
+ "name": "download_file",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "# Download the file from stage to current local path\nget_status = session.file.get(\"@PERMANENT_STAGE/myfile.txt\",\"./\")\nget_status[0].status",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "2b6a333c-143a-477b-9760-046748c9fd2e",
+ "metadata": {
+ "language": "python",
+ "name": "list_files",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "import os\nos.listdir()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "code",
+ "id": "a514fbb3-af35-40ed-afba-485600492d3f",
+ "metadata": {
+ "language": "python",
+ "name": "read_file2",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "# Open the file locally\nwith open(\"myfile.txt\",'r') as f:\n print(f.readline())\nf.close()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "39ba2226-35b0-4cc8-91c9-1392debeef6a",
+ "metadata": {
+ "name": "stage_files_md",
+ "collapsed": false
+ },
+ "source": "## Bonus: Working with data files from stage\n\nStage is common location for storing data file before it is loaded into Snowflake. In the previous section, we saw how you can read and write a generic file to a Snowflake stage. Here, we show a few common examples of how you can work with tabular data files stored in stage.\n"
+ },
+ {
+ "cell_type": "code",
+ "id": "47e912a8-fa21-42ec-ab8b-31289cd14970",
+ "metadata": {
+ "language": "python",
+ "name": "stage_files"
+ },
+ "outputs": [],
+ "source": "from snowflake.snowpark.context import get_active_session\nsession = get_active_session()",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bca5c08e-bd46-4cbf-a2da-20b73905e60b",
+ "metadata": {
+ "name": "stage_files2_md",
+ "collapsed": false
+ },
+ "source": "We have an example dataset recording the amount of snowfall at different ski resort locations across different days."
+ },
+ {
+ "cell_type": "code",
+ "id": "c6905253-fb4a-4e6e-b563-cc481c608b9d",
+ "metadata": {
+ "language": "python",
+ "name": "stage_files2",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "# Create a Snowpark DataFrame with sample data\ndf = session.create_dataframe([[1, 'Big Bear', 8],[2, 'Big Bear', 10],[3, 'Big Bear', 5],\n [1, 'Tahoe', 3],[2, 'Tahoe', 20],[3, 'Tahoe', 13]], \n schema=[\"DAY\", \"LOCATION\", \"SNOWFALL\"])\ndf",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "035e576d-0428-45b0-a23b-7ded6df3dfb1",
+ "metadata": {
+ "name": "df_to_csv_md"
+ },
+ "source": "This is how we can write a Snowpark dataframe to a CSV file on stage:"
+ },
+ {
+ "cell_type": "code",
+ "id": "bdd17871-bc46-439c-a29e-c06fa663524e",
+ "metadata": {
+ "language": "python",
+ "name": "df_to_csv",
+ "codeCollapsed": false,
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": "df.write.copy_into_location(\"@PERMANENT_STAGE/snowfall.csv\",file_format_type=\"csv\",header=True)",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ffd05f5d-3a12-4cbb-8db3-7c7207d12b96",
+ "metadata": {
+ "name": "csv_to_df_md"
+ },
+ "source": "To access the file on stage, read a CSV file from stage location back to a Snowpark dataframe:"
+ },
+ {
+ "cell_type": "code",
+ "id": "aa177f0c-69a6-44a3-b0db-554078108add",
+ "metadata": {
+ "language": "python",
+ "name": "csv_to_df",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "df = session.read.options({\"infer_schema\":True}).csv('@PERMANENT_STAGE/snowfall.csv')",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f903d26c-0323-4ccf-848b-b65c020b07d6",
+ "metadata": {
+ "name": "next_steps_md",
+ "collapsed": false
+ },
+ "source": "To learn more about how you can work with data files in notebooks, check out our tutorial on how to [work with CSV files from an external S3 stage](https://github.com/Snowflake-Labs/snowflake-demo-notebooks/blob/main/Load%20CSV%20from%20S3/Load%20CSV%20from%20S3.ipynb) and [load data from a public endpoint to a Snowflake table](https://github.com/Snowflake-Labs/snowflake-demo-notebooks/blob/main/Ingest%20Public%20JSON/Ingest%20Public%20JSON.ipynb). "
+ },
+ {
+ "cell_type": "code",
+ "id": "58de86d9-778e-4e61-841c-c2f4fda0a13a",
+ "metadata": {
+ "language": "sql",
+ "name": "clean_up",
+ "codeCollapsed": false
+ },
+ "outputs": [],
+ "source": "-- Teardown stage created as part of this tutorial\nDROP STAGE PERMANENT_STAGE;",
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f9f1fb29-b4d8-45bf-9918-1133d1132c60",
+ "metadata": {
+ "name": "conclusion_md",
+ "collapsed": false
+ },
+ "source": "### Conclusion\n\nIn this tutorial, we showed how you can upload local files from your notebook to a permanent Snowflake stage to persist results across notebook sessions. We used Snowpark's file operation commands (e.g., [file.get](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.get), [file.put](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/api/snowflake.snowpark.FileOperation.put)) to move files between your local file path and the stage location. You can learn more about working with files with Snowpark [here](https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/io)."
+ }
+ ]
}
\ No newline at end of file
diff --git a/Working with Git/Working with Git.ipynb b/Working with Git/Working with Git.ipynb
index 72d07ae..6ff86f3 100644
--- a/Working with Git/Working with Git.ipynb
+++ b/Working with Git/Working with Git.ipynb
@@ -1,189 +1,274 @@
{
- "metadata": {
- "kernelspec": {
- "display_name": "Streamlit Notebook",
- "name": "streamlit"
- }
- },
- "nbformat_minor": 5,
- "nbformat": 4,
- "cells": [
- {
- "cell_type": "markdown",
- "id": "38d31fbc-6666-4495-a2b1-d716ffe24329",
- "metadata": {
- "name": "cell1",
- "collapsed": false
- },
- "source": "In this example, we will demonstrate how you can easily go from prototyping for development purposes to production with Git integration.\n\nWe will show an example of a simple data pipeline with one query. By changing the `MODE` variable to `DEV` or `PROD` with different warehouse and schema configurations.\n\nFor `DEV`, we will be using an extra small warehouse on a sample of the TPCH data.\nFor `PROD`, we will be using a large warehouse on a sample of the TPCH data that is 100X the size of the DEV sample."
- },
- {
- "cell_type": "code",
- "id": "3775908f-ca36-4846-8f38-5adca39217f2",
- "metadata": {
- "language": "python",
- "name": "cell2",
- "collapsed": false,
- "codeCollapsed": false
- },
- "source": "MODE = \"DEV\" # Parameter to control whether to run in DEV or PROD mode\n\nif MODE == \"DEV\":\n # For development, use XSMALL warehouse on TPCH data with scale factor of 1\n warehouse_name = \"GIT_EXAMPLE_DEV_WH\"\n schema_name = \"TPCH_SF1\"\n size = 'XSMALL'\nelif MODE == \"PROD\": \n # For production, use LARGE warehouse on TPCH data with scale factor of 100\n warehouse_name = \"GIT_EXAMPLE_PROD_WH\"\n schema_name = \"TPCH_SF100\"\n size = 'LARGE'",
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "markdown",
- "id": "01bd1a4d-1715-4c10-8fdc-08be7b115be5",
- "metadata": {
- "name": "cell3"
- },
- "source": "Let's create and use a warehouse with the specified name and size."
- },
- {
- "cell_type": "code",
- "id": "55bb9c45-e1e4-49ba-a7db-e5eb671ad13a",
- "metadata": {
- "language": "sql",
- "name": "cell4",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "-- Create warehouse with specified name and size\nCREATE OR REPLACE WAREHOUSE {{warehouse_name}} WITH WAREHOUSE_SIZE= {{size}};",
- "execution_count": null
- },
- {
- "cell_type": "code",
- "id": "2b1f4b91-7988-432b-afe1-cb599eea5cc6",
- "metadata": {
- "language": "sql",
- "name": "cell5",
- "collapsed": false
- },
- "outputs": [],
- "source": "-- Use specified warehouse for subsequent query\nUSE WAREHOUSE {{warehouse_name}};",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "f330162f-b59e-467d-bc4e-5c297993c4ee",
- "metadata": {
- "name": "cell6",
- "collapsed": false
- },
- "source": "Use the TPC-H Sample dataset with differing scale factor. \n- Note: Sample data sets are provided in a database named SNOWFLAKE_SAMPLE_DATA that has been shared with your account from the Snowflake SFC_SAMPLES account. If you do not see the database, you can create it yourself. Refer to [Using the Sample Database](https://docs.snowflake.com/en/user-guide/sample-data-using)."
- },
- {
- "cell_type": "code",
- "id": "edb15abf-6061-4e29-9d45-85b0cc806e71",
- "metadata": {
- "language": "sql",
- "name": "cell7",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "USE SCHEMA SNOWFLAKE_SAMPLE_DATA.{{schema_name}}; ",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "024892ff-b2df-4a4d-9308-1760751b4dae",
- "metadata": {
- "name": "cell8",
- "collapsed": false
- },
- "source": "Check out the number of rows in the `LINEITEM` table."
- },
- {
- "cell_type": "code",
- "id": "e73a5b30-fdcc-4dd6-9619-f19a5c31e769",
- "metadata": {
- "language": "sql",
- "name": "cell9",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "SELECT COUNT(*) FROM LINEITEM;",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "115c9b33-f508-4385-806d-20bada66fe18",
- "metadata": {
- "name": "cell10",
- "collapsed": false
- },
- "source": "Now let's run a query on this dataset:\n- The query lists totals for extended price, discounted extended price, discounted extended price plus tax, average quantity, average extended price, and average discount. These aggregates are grouped by RETURNFLAG and LINESTATUS, and listed in ascending order of RETURNFLAG and LINESTATUS. A count of the number of line items in each group is included."
- },
- {
- "cell_type": "code",
- "id": "8d50cbf4-0c8d-4950-86cb-114990437ac9",
- "metadata": {
- "language": "sql",
- "name": "cell11",
- "codeCollapsed": false,
- "collapsed": false
- },
- "source": "select\n l_returnflag,\n l_linestatus,\n sum(l_quantity) as sum_qty,\n sum(l_extendedprice) as sum_base_price,\n sum(l_extendedprice * (1-l_discount)) as sum_disc_price,\n sum(l_extendedprice * (1-l_discount) * (1+l_tax)) as sum_charge,\n avg(l_quantity) as avg_qty,\n avg(l_extendedprice) as avg_price,\n avg(l_discount) as avg_disc,\n count(*) as count_order\n from\n lineitem\n where\n l_shipdate <= dateadd(day, -90, to_date('1998-12-01'))\n group by\n l_returnflag,\n l_linestatus\n order by\n l_returnflag,\n l_linestatus;",
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "markdown",
- "id": "170637df-6e8b-498a-8f2a-fda1a41c21ca",
- "metadata": {
- "name": "cell12",
- "collapsed": false
- },
- "source": "Using the cell referencing, we get the query ID and history of the query we just ran."
- },
- {
- "cell_type": "code",
- "id": "c49eb85b-6956-4da6-949f-1939c6a1dcc4",
- "metadata": {
- "language": "python",
- "name": "cell13",
- "codeCollapsed": false,
- "collapsed": false
- },
- "outputs": [],
- "source": "# Get query ID of the referenced cell\nquery_id = cell11.result_scan_sql().split(\"'\")[1]",
- "execution_count": null
- },
- {
- "cell_type": "code",
- "id": "dfd22f9f-44ef-4a3f-99e6-7c774b02eea7",
- "metadata": {
- "language": "sql",
- "name": "cell14",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "select * from table(information_schema.query_history_by_warehouse('{{warehouse_name}}')) \nwhere query_id = '{{query_id}}';",
- "execution_count": null
- },
- {
- "cell_type": "markdown",
- "id": "ef4d7fcb-9729-4409-8bce-7a7081b98e87",
- "metadata": {
- "name": "cell15"
- },
- "source": "Finally, we compile all of this information into a report to document the run information."
- },
- {
- "cell_type": "code",
- "id": "9b718981-9577-4996-b212-0cf7ffb4f23b",
- "metadata": {
- "language": "python",
- "name": "cell16",
- "collapsed": false,
- "codeCollapsed": false
- },
- "outputs": [],
- "source": "import streamlit as st\nfrom datetime import datetime\nst.header(f\"[{MODE}] Run Report\")\nst.markdown(f\"Generated on: {datetime.now()}\")\n\nst.markdown(f\"### System Information\")\n# Print session information\nfrom snowflake.snowpark.context import get_active_session\nsession = get_active_session()\nst.markdown(f\"**Database:** {session.get_current_database()[1:-1]}\")\nst.markdown(f\"**Schema:** {session.get_current_schema()[1:-1]}\")\nst.markdown(f\"**Warehouse:** {session.get_current_warehouse()[1:-1]}\")\n\nst.markdown(f\"### Query Information\")\n# Print session information\nst.markdown(f\"**Query ID:** {query_id}\")\nresult_info = cell14.to_pandas()\nst.markdown(\"**Query Text:**\")\nst.code(result_info[\"QUERY_TEXT\"].values[0],language='sql',line_numbers=True)\nst.markdown(\"**Runtime information:**\")\nst.dataframe(result_info[['START_TIME','END_TIME','TOTAL_ELAPSED_TIME']])",
- "execution_count": null
- }
- ]
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Streamlit Notebook",
+ "name": "streamlit"
+ },
+ "lastEditStatus": {
+ "notebookId": "sm7bybhs736mosu4apsn",
+ "authorId": "691502871989",
+ "authorName": "KRISHNAN1234",
+ "authorEmail": "sriram.krishnan.work@gmail.com",
+ "sessionId": "1944aea4-1b8b-4250-8ce6-c8879c306273",
+ "lastEditTime": 1755494736120
+ }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "38d31fbc-6666-4495-a2b1-d716ffe24329",
+ "metadata": {
+ "collapsed": false,
+ "name": "cell1"
+ },
+ "source": [
+ "In this example, we will demonstrate how you can easily go from prototyping for development purposes to production with Git integration.\n",
+ "\n",
+ "We will show an example of a simple data pipeline with one query. By changing the `MODE` variable to `DEV` or `PROD` with different warehouse and schema configurations.\n",
+ "\n",
+ "For `DEV`, we will be using an extra small warehouse on a sample of the TPCH data.\n",
+ "For `PROD`, we will be using a large warehouse on a sample of the TPCH data that is 100X the size of the DEV sample."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3775908f-ca36-4846-8f38-5adca39217f2",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "python",
+ "name": "cell2"
+ },
+ "outputs": [],
+ "source": [
+ "MODE = \"DEV\" # Parameter to control whether to run in DEV or PROD mode\n",
+ "\n",
+ "if MODE == \"DEV\":\n",
+ " # For development, use XSMALL warehouse on TPCH data with scale factor of 1\n",
+ " warehouse_name = \"GIT_EXAMPLE_DEV_WH\"\n",
+ " schema_name = \"TPCH_SF1\"\n",
+ " size = 'XSMALL'\n",
+ "elif MODE == \"PROD\": \n",
+ " # For production, use LARGE warehouse on TPCH data with scale factor of 100\n",
+ " warehouse_name = \"GIT_EXAMPLE_PROD_WH\"\n",
+ " schema_name = \"TPCH_SF100\"\n",
+ " size = 'LARGE'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "01bd1a4d-1715-4c10-8fdc-08be7b115be5",
+ "metadata": {
+ "name": "cell3"
+ },
+ "source": [
+ "Let's create and use a warehouse with the specified name and size."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "55bb9c45-e1e4-49ba-a7db-e5eb671ad13a",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "sql",
+ "name": "cell4"
+ },
+ "outputs": [],
+ "source": [
+ "-- Create warehouse with specified name and size\n",
+ "CREATE OR REPLACE WAREHOUSE {{warehouse_name}} WITH WAREHOUSE_SIZE= {{size}};"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2b1f4b91-7988-432b-afe1-cb599eea5cc6",
+ "metadata": {
+ "collapsed": false,
+ "language": "sql",
+ "name": "cell5"
+ },
+ "outputs": [],
+ "source": [
+ "-- Use specified warehouse for subsequent query\n",
+ "USE WAREHOUSE {{warehouse_name}};"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f330162f-b59e-467d-bc4e-5c297993c4ee",
+ "metadata": {
+ "collapsed": false,
+ "name": "cell6"
+ },
+ "source": [
+ "Use the TPC-H Sample dataset with differing scale factor. \n",
+ "- Note: Sample data sets are provided in a database named SNOWFLAKE_SAMPLE_DATA that has been shared with your account from the Snowflake SFC_SAMPLES account. If you do not see the database, you can create it yourself. Refer to [Using the Sample Database](https://docs.snowflake.com/en/user-guide/sample-data-using)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "edb15abf-6061-4e29-9d45-85b0cc806e71",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "sql",
+ "name": "cell7"
+ },
+ "outputs": [],
+ "source": [
+ "USE SCHEMA SNOWFLAKE_SAMPLE_DATA.{{schema_name}}; "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "024892ff-b2df-4a4d-9308-1760751b4dae",
+ "metadata": {
+ "collapsed": false,
+ "name": "cell8"
+ },
+ "source": [
+ "Check out the number of rows in the `LINEITEM` table."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e73a5b30-fdcc-4dd6-9619-f19a5c31e769",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "sql",
+ "name": "cell9"
+ },
+ "outputs": [],
+ "source": [
+ "SELECT COUNT(*) FROM LINEITEM;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "115c9b33-f508-4385-806d-20bada66fe18",
+ "metadata": {
+ "collapsed": false,
+ "name": "cell10"
+ },
+ "source": [
+ "Now let's run a query on this dataset:\n",
+ "- The query lists totals for extended price, discounted extended price, discounted extended price plus tax, average quantity, average extended price, and average discount. These aggregates are grouped by RETURNFLAG and LINESTATUS, and listed in ascending order of RETURNFLAG and LINESTATUS. A count of the number of line items in each group is included."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8d50cbf4-0c8d-4950-86cb-114990437ac9",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "sql",
+ "name": "cell11"
+ },
+ "outputs": [],
+ "source": "select\n l_returnflag,\n l_linestatus,\n sum(l_quantity) as sum_qty,\n sum(l_extendedprice) as sum_base_price,\n sum(l_extendedprice * (1-l_discount)) as sum_disc_price,\n sum(l_extendedprice * (1-l_discount) * (1+l_tax)) as sum_charge,\n avg(l_quantity) as avg_qty,\n avg(l_extendedprice) as avg_price,\n avg(l_discount) as avg_disc,\n count(*) as count_order\n from\n lineitem\n group by\n l_returnflag,\n l_linestatus\n order by\n l_returnflag,\n l_linestatus;"
+ },
+ {
+ "cell_type": "markdown",
+ "id": "170637df-6e8b-498a-8f2a-fda1a41c21ca",
+ "metadata": {
+ "collapsed": false,
+ "name": "cell12"
+ },
+ "source": [
+ "Using the cell referencing, we get the query ID and history of the query we just ran."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c49eb85b-6956-4da6-949f-1939c6a1dcc4",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "python",
+ "name": "cell13"
+ },
+ "outputs": [],
+ "source": [
+ "# Get query ID of the referenced cell\n",
+ "query_id = cell11.result_scan_sql().split(\"'\")[1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dfd22f9f-44ef-4a3f-99e6-7c774b02eea7",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "sql",
+ "name": "cell14"
+ },
+ "outputs": [],
+ "source": [
+ "select * from table(information_schema.query_history_by_warehouse('{{warehouse_name}}')) \n",
+ "where query_id = '{{query_id}}';"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ef4d7fcb-9729-4409-8bce-7a7081b98e87",
+ "metadata": {
+ "name": "cell15"
+ },
+ "source": [
+ "Finally, we compile all of this information into a report to document the run information."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9b718981-9577-4996-b212-0cf7ffb4f23b",
+ "metadata": {
+ "codeCollapsed": false,
+ "collapsed": false,
+ "language": "python",
+ "name": "cell16"
+ },
+ "outputs": [],
+ "source": [
+ "import streamlit as st\n",
+ "from datetime import datetime\n",
+ "st.header(f\"[{MODE}] Run Report\")\n",
+ "st.markdown(f\"Generated on: {datetime.now()}\")\n",
+ "\n",
+ "st.markdown(f\"### System Information\")\n",
+ "# Print session information\n",
+ "from snowflake.snowpark.context import get_active_session\n",
+ "session = get_active_session()\n",
+ "# Add a query tag to the session. This helps with troubleshooting and performance monitoring.\n",
+ "session.query_tag = {\"origin\":\"sf_sit-is\", \n",
+ " \"name\":\"notebook_demo_pack\", \n",
+ " \"version\":{\"major\":1, \"minor\":0},\n",
+ " \"attributes\":{\"is_quickstart\":1, \"source\":\"notebook\", \"vignette\":\"working_with_git\"}}\n",
+ "st.markdown(f\"**Database:** {session.get_current_database()[1:-1]}\")\n",
+ "st.markdown(f\"**Schema:** {session.get_current_schema()[1:-1]}\")\n",
+ "st.markdown(f\"**Warehouse:** {session.get_current_warehouse()[1:-1]}\")\n",
+ "\n",
+ "st.markdown(f\"### Query Information\")\n",
+ "# Print session information\n",
+ "st.markdown(f\"**Query ID:** {query_id}\")\n",
+ "result_info = cell14.to_pandas()\n",
+ "st.markdown(\"**Query Text:**\")\n",
+ "st.code(result_info[\"QUERY_TEXT\"].values[0],language='sql',line_numbers=True)\n",
+ "st.markdown(\"**Runtime information:**\")\n",
+ "st.dataframe(result_info[['START_TIME','END_TIME','TOTAL_ELAPSED_TIME']])"
+ ]
+ }
+ ]
}
\ No newline at end of file
|
![\"Snowflake](\"https://s26.q4cdn.com/463892824/files/doc_multimedia/HI_RES-_Snowflake_Logo_Blue_1800x550.jpg\"\n){kind=logo}
\n \n ![\"Snowflake](\"https://s26.q4cdn.com/463892824/files/doc_multimedia/HI_RES-_Snowflake_Logo_Blue_1800x550.jpg\"){kind=logo}
\n 
+