-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a Storm to Beam conversion utility (#1322)
- Loading branch information
1 parent
8d5e511
commit 65af11e
Showing
1 changed file
with
158 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,158 @@ | ||
{ | ||
"nbformat": 4, | ||
"nbformat_minor": 0, | ||
"metadata": { | ||
"colab": { | ||
"provenance": [], | ||
"authorship_tag": "ABX9TyOddl3p3PRsCszWtVsAJOO4", | ||
"include_colab_link": true | ||
}, | ||
"kernelspec": { | ||
"name": "python3", | ||
"display_name": "Python 3" | ||
}, | ||
"language_info": { | ||
"name": "python" | ||
} | ||
}, | ||
"cells": [ | ||
{ | ||
"cell_type": "markdown", | ||
"metadata": { | ||
"id": "view-in-github", | ||
"colab_type": "text" | ||
}, | ||
"source": [ | ||
"<a href=\"https://colab.research.google.com/github/svetakvsundhar/professional-services/blob/main/colab_notebooks/beam_to_storm.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | ||
] | ||
}, | ||
{ | ||
"cell_type": "markdown", | ||
"source": [ | ||
"**Welcome to the Beam-to-Storm Colab notebook!**\n", | ||
"\n", | ||
"This notebook will allow you to get an initial glimpse of how Apache Beam Code may be represented in terms of Apache Storm constructs. Note that this tool simply provides a template, and no guarantees that the Gemini output is runnable/production ready." | ||
], | ||
"metadata": { | ||
"id": "vCfGpZzbiJ03" | ||
} | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 1, | ||
"metadata": { | ||
"colab": { | ||
"base_uri": "https://localhost:8080/" | ||
}, | ||
"id": "BDD1hnKxgZMH", | ||
"outputId": "e71af5f8-dd7c-4c95-f4f6-277fa2189dc6" | ||
}, | ||
"outputs": [ | ||
{ | ||
"output_type": "stream", | ||
"name": "stdout", | ||
"text": [ | ||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m164.2/164.2 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", | ||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m718.3/718.3 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", | ||
"\u001b[?25h" | ||
] | ||
} | ||
], | ||
"source": [ | ||
"!pip install -q -U google-generativeai" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"source": [ | ||
"import google.generativeai as genai" | ||
], | ||
"metadata": { | ||
"id": "bXbUF00Ogdio" | ||
}, | ||
"execution_count": 2, | ||
"outputs": [] | ||
}, | ||
{ | ||
"cell_type": "markdown", | ||
"source": [ | ||
"**Configure your API key**\n", | ||
"\n", | ||
"To run the following cell, your API key must be stored it in a Colab Secret named GOOGLE_API_KEY. If you don't already have an API key, or you're not sure how to create a Colab Secret, see Authentication for an example." | ||
], | ||
"metadata": { | ||
"id": "VRty7oWwiZgR" | ||
} | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"source": [ | ||
"from google.colab import userdata\n", | ||
"GOOGLE_API_KEY=userdata.get('GOOGLE_API_KEY')\n", | ||
"\n", | ||
"genai.configure(api_key=GOOGLE_API_KEY)" | ||
], | ||
"metadata": { | ||
"id": "u-CE4-Aiglc0" | ||
}, | ||
"execution_count": 3, | ||
"outputs": [] | ||
}, | ||
{ | ||
"cell_type": "markdown", | ||
"source": [ | ||
"Upload your file onto this Colab Notebook (note, please use a .txt file). Then, replace **path_to_your_file.txt** with your actual path." | ||
], | ||
"metadata": { | ||
"id": "JKjVyiSjieX1" | ||
} | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"source": [ | ||
"your_file = genai.upload_file(path='#path_to_your_file.txt')" | ||
], | ||
"metadata": { | ||
"id": "CnKJTDyygpAH" | ||
}, | ||
"execution_count": 5, | ||
"outputs": [] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"source": [ | ||
"your_architecture_description=\"This is an Apache Beam pipeline being executed on Dataflow Runner\"" | ||
], | ||
"metadata": { | ||
"id": "iAiHKFK3gw4Z" | ||
}, | ||
"execution_count": 6, | ||
"outputs": [] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"source": [ | ||
"design_img = \"Gemini, your role is to convert this Apache Beam code into Apache Storm code,from the code provided and the description. Please return Apache Storm code.\"" | ||
], | ||
"metadata": { | ||
"id": "85xu5QWZgzRZ" | ||
}, | ||
"execution_count": 9, | ||
"outputs": [] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"source": [ | ||
"model = genai.GenerativeModel('models/gemini-1.5-pro-latest')\n", | ||
"prompt = model.generate_content(your_architecture_description + design_img)\n", | ||
"response = model.generate_content([prompt.text, your_file])\n", | ||
"print(response.text)" | ||
], | ||
"metadata": { | ||
"id": "8YkEn_rKg8eb" | ||
}, | ||
"execution_count": null, | ||
"outputs": [] | ||
} | ||
] | ||
} |