nlp_pipeline_openai.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z7pa2CrEnk-M",
        "outputId": "438ac6d6-ae83-40a4-afa4-3ee4fd88d814"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: spacy in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 1)) (3.7.5)\n",
            "Requirement already satisfied: wordcloud in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 2)) (1.9.4)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 3)) (3.4.2)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 4)) (3.9.1)\n",
            "Requirement already satisfied: textblob in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 5)) (0.17.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 6)) (1.6.0)\n",
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 7)) (0.13.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 8)) (3.8.0)\n",
            "Collecting rake-nltk (from -r /content/requirements.txt (line 9))\n",
            "  Downloading rake_nltk-1.0.6-py3-none-any.whl.metadata (6.4 kB)\n",
            "Collecting textstat (from -r /content/requirements.txt (line 10))\n",
            "  Downloading textstat-0.7.4-py3-none-any.whl.metadata (14 kB)\n",
            "Collecting gradio (from -r /content/requirements.txt (line 11))\n",
            "  Downloading gradio-5.9.1-py3-none-any.whl.metadata (16 kB)\n",
            "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (from -r /content/requirements.txt (line 12)) (1.57.4)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (1.0.11)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (2.0.10)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (8.2.5)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (1.1.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (2.5.0)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.5.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (0.4.1)\n",
            "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (0.15.1)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (4.67.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (2.32.3)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (2.10.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (3.1.4)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (75.1.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (24.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (3.5.0)\n",
            "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.10/dist-packages (from spacy->-r /content/requirements.txt (line 1)) (1.26.4)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from wordcloud->-r /content/requirements.txt (line 2)) (11.0.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->-r /content/requirements.txt (line 4)) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->-r /content/requirements.txt (line 4)) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->-r /content/requirements.txt (line 4)) (2024.11.6)\n",
            "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->-r /content/requirements.txt (line 6)) (1.13.1)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->-r /content/requirements.txt (line 6)) (3.5.0)\n",
            "Requirement already satisfied: pandas>=1.2 in /usr/local/lib/python3.10/dist-packages (from seaborn->-r /content/requirements.txt (line 7)) (2.2.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r /content/requirements.txt (line 8)) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r /content/requirements.txt (line 8)) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r /content/requirements.txt (line 8)) (4.55.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r /content/requirements.txt (line 8)) (1.4.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r /content/requirements.txt (line 8)) (3.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->-r /content/requirements.txt (line 8)) (2.8.2)\n",
            "Collecting pyphen (from textstat->-r /content/requirements.txt (line 10))\n",
            "  Downloading pyphen-0.17.0-py3-none-any.whl.metadata (3.2 kB)\n",
            "Collecting aiofiles<24.0,>=22.0 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading aiofiles-23.2.1-py3-none-any.whl.metadata (9.7 kB)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r /content/requirements.txt (line 11)) (3.7.1)\n",
            "Collecting fastapi<1.0,>=0.115.2 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading fastapi-0.115.6-py3-none-any.whl.metadata (27 kB)\n",
            "Collecting ffmpy (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading ffmpy-0.5.0-py3-none-any.whl.metadata (3.0 kB)\n",
            "Collecting gradio-client==1.5.2 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading gradio_client-1.5.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Requirement already satisfied: httpx>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from gradio->-r /content/requirements.txt (line 11)) (0.28.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.25.1 in /usr/local/lib/python3.10/dist-packages (from gradio->-r /content/requirements.txt (line 11)) (0.27.0)\n",
            "Collecting markupsafe~=2.0 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.0 kB)\n",
            "Requirement already satisfied: orjson~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r /content/requirements.txt (line 11)) (3.10.12)\n",
            "Collecting pydub (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
            "Collecting python-multipart>=0.0.18 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading python_multipart-0.0.20-py3-none-any.whl.metadata (1.8 kB)\n",
            "Requirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r /content/requirements.txt (line 11)) (6.0.2)\n",
            "Collecting ruff>=0.2.2 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading ruff-0.8.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (25 kB)\n",
            "Collecting safehttpx<0.2.0,>=0.1.6 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading safehttpx-0.1.6-py3-none-any.whl.metadata (4.2 kB)\n",
            "Collecting semantic-version~=2.0 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl.metadata (9.7 kB)\n",
            "Collecting starlette<1.0,>=0.40.0 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading starlette-0.43.0-py3-none-any.whl.metadata (6.3 kB)\n",
            "Collecting tomlkit<0.14.0,>=0.12.0 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading tomlkit-0.13.2-py3-none-any.whl.metadata (2.7 kB)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r /content/requirements.txt (line 11)) (4.12.2)\n",
            "Collecting uvicorn>=0.14.0 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading uvicorn-0.34.0-py3-none-any.whl.metadata (6.5 kB)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==1.5.2->gradio->-r /content/requirements.txt (line 11)) (2024.10.0)\n",
            "Requirement already satisfied: websockets<15.0,>=10.0 in /usr/local/lib/python3.10/dist-packages (from gradio-client==1.5.2->gradio->-r /content/requirements.txt (line 11)) (14.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r /content/requirements.txt (line 12)) (1.9.0)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r /content/requirements.txt (line 12)) (0.8.2)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai->-r /content/requirements.txt (line 12)) (1.3.1)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5.0,>=3.0->gradio->-r /content/requirements.txt (line 11)) (3.10)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5.0,>=3.0->gradio->-r /content/requirements.txt (line 11)) (1.2.2)\n",
            "Collecting starlette<1.0,>=0.40.0 (from gradio->-r /content/requirements.txt (line 11))\n",
            "  Downloading starlette-0.41.3-py3-none-any.whl.metadata (6.0 kB)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio->-r /content/requirements.txt (line 11)) (2024.12.14)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio->-r /content/requirements.txt (line 11)) (1.0.7)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.1->gradio->-r /content/requirements.txt (line 11)) (0.14.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.25.1->gradio->-r /content/requirements.txt (line 11)) (3.16.1)\n",
            "Requirement already satisfied: language-data>=1.2 in /usr/local/lib/python3.10/dist-packages (from langcodes<4.0.0,>=3.2.0->spacy->-r /content/requirements.txt (line 1)) (1.3.0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.2->seaborn->-r /content/requirements.txt (line 7)) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.2->seaborn->-r /content/requirements.txt (line 7)) (2024.2)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy->-r /content/requirements.txt (line 1)) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.27.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy->-r /content/requirements.txt (line 1)) (2.27.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->-r /content/requirements.txt (line 8)) (1.17.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy->-r /content/requirements.txt (line 1)) (3.4.0)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy->-r /content/requirements.txt (line 1)) (2.2.3)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy->-r /content/requirements.txt (line 1)) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy->-r /content/requirements.txt (line 1)) (0.1.5)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r /content/requirements.txt (line 1)) (1.5.4)\n",
            "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy->-r /content/requirements.txt (line 1)) (13.9.4)\n",
            "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<0.5.0,>=0.1.0->spacy->-r /content/requirements.txt (line 1)) (0.20.0)\n",
            "Requirement already satisfied: smart-open<8.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from weasel<0.5.0,>=0.1.0->spacy->-r /content/requirements.txt (line 1)) (7.1.0)\n",
            "Requirement already satisfied: marisa-trie>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy->-r /content/requirements.txt (line 1)) (1.2.1)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy->-r /content/requirements.txt (line 1)) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy->-r /content/requirements.txt (line 1)) (2.18.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from smart-open<8.0.0,>=5.2.1->weasel<0.5.0,>=0.1.0->spacy->-r /content/requirements.txt (line 1)) (1.17.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy->-r /content/requirements.txt (line 1)) (0.1.2)\n",
            "Downloading rake_nltk-1.0.6-py3-none-any.whl (9.1 kB)\n",
            "Downloading textstat-0.7.4-py3-none-any.whl (105 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.1/105.1 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gradio-5.9.1-py3-none-any.whl (57.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.2/57.2 MB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gradio_client-1.5.2-py3-none-any.whl (320 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.4/320.4 kB\u001b[0m \u001b[31m23.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
            "Downloading fastapi-0.115.6-py3-none-any.whl (94 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.8/94.8 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\n",
            "Downloading python_multipart-0.0.20-py3-none-any.whl (24 kB)\n",
            "Downloading ruff-0.8.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.2/11.2 MB\u001b[0m \u001b[31m101.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading safehttpx-0.1.6-py3-none-any.whl (8.7 kB)\n",
            "Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Downloading starlette-0.41.3-py3-none-any.whl (73 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.2/73.2 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tomlkit-0.13.2-py3-none-any.whl (37 kB)\n",
            "Downloading uvicorn-0.34.0-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.3/62.3 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ffmpy-0.5.0-py3-none-any.whl (6.0 kB)\n",
            "Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Downloading pyphen-0.17.0-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m73.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pydub, uvicorn, tomlkit, semantic-version, ruff, python-multipart, pyphen, markupsafe, ffmpy, aiofiles, textstat, starlette, rake-nltk, safehttpx, gradio-client, fastapi, gradio\n",
            "  Attempting uninstall: markupsafe\n",
            "    Found existing installation: MarkupSafe 3.0.2\n",
            "    Uninstalling MarkupSafe-3.0.2:\n",
            "      Successfully uninstalled MarkupSafe-3.0.2\n",
            "Successfully installed aiofiles-23.2.1 fastapi-0.115.6 ffmpy-0.5.0 gradio-5.9.1 gradio-client-1.5.2 markupsafe-2.1.5 pydub-0.25.1 pyphen-0.17.0 python-multipart-0.0.20 rake-nltk-1.0.6 ruff-0.8.4 safehttpx-0.1.6 semantic-version-2.10.0 starlette-0.41.3 textstat-0.7.4 tomlkit-0.13.2 uvicorn-0.34.0\n"
          ]
        }
      ],
      "source": [
        "!pip install -r /content/requirements.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Enter \"your-openai-api-key\" in client section"
      ],
      "metadata": {
        "id": "sBwz52T4pz2l"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "############################################\n",
        "# -------------------- Developed by Partha Pratim Ray -------------------- #\n",
        "# Contact: parthapratimray1986@gmail.com\n",
        "# GitHub: https://github.com/ParthaPRay/\n",
        "##################### GRADIO INTERFACE #####################\n",
        "\n",
        "##### Sample Inputs\n",
        "\n",
        "# text_input[]\n",
        "\n",
        "# OpenAI, based in San Francisco, has developed the GPT model, which is widely used for natural language processing tasks.  The company aims to make artificial intelligence accessible and useful to people worldwide. In 2023, they released GPT-4.\n",
        "\n",
        "########\n",
        "\n",
        "# docs_input[]\n",
        "\n",
        "# OpenAI is an artificial intelligence research lab that focuses on developing safe AI. The lab is well-known for the GPT series of models. GPT-4 is the latest release by OpenAI, showcasing advanced natural language processing capabilities. Artificial intelligence tools like GPT have become integral for tasks like summarization, translation, and content generation.\n",
        "\n",
        "#######\n",
        "\n",
        "import spacy\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from textblob import TextBlob\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.decomposition import LatentDirichletAllocation\n",
        "from sklearn.cluster import KMeans\n",
        "from openai import OpenAI\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from collections import Counter\n",
        "import networkx as nx\n",
        "from wordcloud import WordCloud\n",
        "from rake_nltk import Rake\n",
        "import numpy as np\n",
        "from textstat import textstat\n",
        "import gradio as gr\n",
        "\n",
        "# If you run into issues with different backends, uncomment the next line:\n",
        "# matplotlib.use('Agg')\n",
        "\n",
        "# Download NLTK resources (ensure these are downloaded at least once)\n",
        "nltk.download(\"punkt\")\n",
        "nltk.download(\"punkt_tab\")\n",
        "nltk.download(\"stopwords\")\n",
        "\n",
        "# Load SpaCy model\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "\n",
        "# Instantiate OpenAI client (replace with your actual API key if needed)\n",
        "client = OpenAI(api_key=\"your-openai-api-key\")\n",
        "\n",
        "# -------------------- Utility Functions -------------------- #\n",
        "\n",
        "def compute_tfidf(documents, top_n=5):\n",
        "    vectorizer = TfidfVectorizer(stop_words=\"english\")\n",
        "    tfidf_matrix = vectorizer.fit_transform(documents)\n",
        "    feature_names = vectorizer.get_feature_names_out()\n",
        "    dense = tfidf_matrix.todense()\n",
        "    scores = dense[0].tolist()[0]\n",
        "    tfidf_scores = [(feature_names[i], scores[i]) for i in range(len(scores))]\n",
        "    sorted_scores = sorted(tfidf_scores, key=lambda x: x[1], reverse=True)\n",
        "    return sorted_scores[:top_n]\n",
        "\n",
        "def topic_modeling(documents, n_topics=3):\n",
        "    vectorizer = TfidfVectorizer(stop_words=\"english\")\n",
        "    doc_term_matrix = vectorizer.fit_transform(documents)\n",
        "    lda = LatentDirichletAllocation(n_components=n_topics, random_state=42)\n",
        "    lda.fit(doc_term_matrix)\n",
        "    topics = {}\n",
        "    for idx, topic in enumerate(lda.components_):\n",
        "        topics[f\"Topic {idx + 1}\"] = [\n",
        "            vectorizer.get_feature_names_out()[i] for i in topic.argsort()[-5:]\n",
        "        ]\n",
        "    return topics\n",
        "\n",
        "def summarize_text(text, length=\"short\"):\n",
        "    length_prompt = {\n",
        "        \"short\": \"Summarize in one sentence.\",\n",
        "        \"medium\": \"Summarize in a short paragraph.\",\n",
        "        \"long\": \"Summarize in detail.\",\n",
        "    }\n",
        "    completion = client.chat.completions.create(\n",
        "        model=\"gpt-4o-mini\",\n",
        "        messages=[\n",
        "            {\"role\": \"user\", \"content\": f\"{length_prompt[length]} The text is:\\n{text}\"}\n",
        "        ],\n",
        "    )\n",
        "    return completion.choices[0].message.content.strip()\n",
        "\n",
        "def classify_sentiment(sentiment):\n",
        "    if sentiment[\"polarity\"] > 0.1:\n",
        "        return \"Positive\"\n",
        "    elif sentiment[\"polarity\"] < -0.1:\n",
        "        return \"Negative\"\n",
        "    else:\n",
        "        return \"Neutral\"\n",
        "\n",
        "# -------------------- Visualization Functions -------------------- #\n",
        "\n",
        "def visualize_tfidf_figure(tfidf_scores):\n",
        "    fig, ax = plt.subplots()\n",
        "    words, scores = zip(*tfidf_scores) if tfidf_scores else ([], [])\n",
        "    ax.barh(words, scores)\n",
        "    ax.set_xlabel(\"TF-IDF Score\")\n",
        "    ax.set_title(\"Top TF-IDF Keywords\")\n",
        "    plt.tight_layout()\n",
        "    return fig\n",
        "\n",
        "def generate_wordcloud_figure(text):\n",
        "    wordcloud = WordCloud(width=800, height=400, background_color=\"white\").generate(text)\n",
        "    fig, ax = plt.subplots(figsize=(10, 5))\n",
        "    ax.imshow(wordcloud, interpolation=\"bilinear\")\n",
        "    ax.axis(\"off\")\n",
        "    ax.set_title(\"Word Cloud\")\n",
        "    plt.tight_layout()\n",
        "    return fig\n",
        "\n",
        "def create_cooccurrence_network_figure(tokens):\n",
        "    cooccurrence_graph = nx.Graph()\n",
        "    for i, token1 in enumerate(tokens):\n",
        "        for token2 in tokens[i + 1 : i + 5]:\n",
        "            if token1 != token2:\n",
        "                if cooccurrence_graph.has_edge(token1, token2):\n",
        "                    cooccurrence_graph[token1][token2][\"weight\"] += 1\n",
        "                else:\n",
        "                    cooccurrence_graph.add_edge(token1, token2, weight=1)\n",
        "    pos = nx.spring_layout(cooccurrence_graph, seed=42)\n",
        "    fig, ax = plt.subplots(figsize=(12, 8))\n",
        "    nx.draw(\n",
        "        cooccurrence_graph, pos, with_labels=True,\n",
        "        node_color=\"lightblue\", edge_color=\"gray\", font_size=10, ax=ax\n",
        "    )\n",
        "    ax.set_title(\"Co-occurrence Network\")\n",
        "    plt.tight_layout()\n",
        "    return fig\n",
        "\n",
        "def generate_polarity_heatmap_figure(text):\n",
        "    sentences = text.split(\". \")\n",
        "    polarities = [\n",
        "        TextBlob(sentence).sentiment.polarity for sentence in sentences if sentence\n",
        "    ]\n",
        "    if not polarities:\n",
        "        polarities = [0.0]\n",
        "    fig, ax = plt.subplots(figsize=(10, 2))\n",
        "    data = np.array(polarities).reshape(1, -1)\n",
        "    sns.heatmap(\n",
        "        data, annot=True, fmt=\".2f\", cmap=\"coolwarm\", cbar=True,\n",
        "        xticklabels=range(1, len(polarities) + 1), yticklabels=[\"Polarity\"], ax=ax\n",
        "    )\n",
        "    ax.set_title(\"Sentence Polarity Heatmap\")\n",
        "    ax.set_xlabel(\"Sentence Index\")\n",
        "    plt.tight_layout()\n",
        "    return fig\n",
        "\n",
        "# -------------------- Other NLP Functions -------------------- #\n",
        "\n",
        "def dependency_parsing(text):\n",
        "    doc = nlp(text)\n",
        "    for token in doc:\n",
        "        print(f\"{token.text} -> {token.dep_} -> {token.head.text}\")\n",
        "\n",
        "def compute_semantic_similarity(text, documents):\n",
        "    base_doc = nlp(text)\n",
        "    # This will show a warning if you're using a small model that doesn't have word vectors.\n",
        "    similarities = [(doc, base_doc.similarity(nlp(doc))) for doc in documents]\n",
        "    return sorted(similarities, key=lambda x: x[1], reverse=True)\n",
        "\n",
        "def extract_keywords_rake(text):\n",
        "    rake = Rake()\n",
        "    rake.extract_keywords_from_text(text)\n",
        "    return rake.get_ranked_phrases_with_scores()\n",
        "\n",
        "# -------------------- FIX: Graceful KMeans Clustering -------------------- #\n",
        "def cluster_documents(documents, n_clusters=3):\n",
        "    # If the user doesn't provide enough documents, lower the cluster count or skip\n",
        "    if len(documents) < n_clusters:\n",
        "        # If there's only 0 or 1 document, skip clustering\n",
        "        if len(documents) <= 1:\n",
        "            return [0] * len(documents)  # Return 0 if there's exactly 1 doc\n",
        "        else:\n",
        "            n_clusters = len(documents)\n",
        "\n",
        "    vectorizer = TfidfVectorizer(stop_words=\"english\")\n",
        "    tfidf_matrix = vectorizer.fit_transform(documents)\n",
        "    km = KMeans(n_clusters=n_clusters, random_state=42)\n",
        "    km.fit(tfidf_matrix)\n",
        "    return km.labels_\n",
        "\n",
        "def calculate_readability(text):\n",
        "    readability_scores = {\n",
        "        \"flesch_reading_ease\": textstat.flesch_reading_ease(text),\n",
        "        \"gunning_fog_index\": textstat.gunning_fog(text),\n",
        "        \"smog_index\": textstat.smog_index(text),\n",
        "        \"automated_readability_index\": textstat.automated_readability_index(text),\n",
        "    }\n",
        "    return readability_scores\n",
        "\n",
        "def pos_tagging_analysis(text):\n",
        "    doc = nlp(text)\n",
        "    pos_counts = Counter([token.pos_ for token in doc])\n",
        "    return dict(pos_counts)\n",
        "\n",
        "# -------------------- Main Pipeline -------------------- #\n",
        "\n",
        "def process_text_with_pipeline(text, documents):\n",
        "    # Step 1: Named Entity Recognition (NER)\n",
        "    doc = nlp(text)\n",
        "    entities = [(ent.text, ent.label_) for ent in doc.ents]\n",
        "\n",
        "    # Step 2: Tokenization and Stopword Removal\n",
        "    tokens = word_tokenize(text)\n",
        "    stop_words = set(stopwords.words(\"english\"))\n",
        "    clean_tokens = [\n",
        "        word for word in tokens if word.isalnum() and word.lower() not in stop_words\n",
        "    ]\n",
        "\n",
        "    # Step 3: Word Frequencies\n",
        "    word_freq = Counter(clean_tokens)\n",
        "\n",
        "    # Step 4: Sentiment Analysis\n",
        "    blob = TextBlob(text)\n",
        "    sentiment = {\n",
        "        \"polarity\": blob.sentiment.polarity,\n",
        "        \"subjectivity\": blob.sentiment.subjectivity,\n",
        "        \"classification\": classify_sentiment(blob.sentiment._asdict()),\n",
        "    }\n",
        "\n",
        "    # Step 5: TF-IDF\n",
        "    tfidf_keywords = compute_tfidf(documents)\n",
        "\n",
        "    # Step 6: Topic Modeling\n",
        "    topics = topic_modeling(documents)\n",
        "\n",
        "    # Step 7: Summarization\n",
        "    summary = summarize_text(text)\n",
        "\n",
        "    # Step 8: Dependency Parsing (print in console)\n",
        "    dependency_parsing(text)\n",
        "\n",
        "    # Step 9: Semantic Similarity\n",
        "    similarities = compute_semantic_similarity(text, documents)\n",
        "\n",
        "    # Step 10: RAKE Keywords\n",
        "    keywords = extract_keywords_rake(text)\n",
        "\n",
        "    # Step 11: Clustering\n",
        "    clusters = cluster_documents(documents)\n",
        "\n",
        "    # Figures\n",
        "    polarity_heatmap_fig = generate_polarity_heatmap_figure(text)\n",
        "    wordcloud_fig = generate_wordcloud_figure(text)\n",
        "    cooccurrence_fig = create_cooccurrence_network_figure(clean_tokens)\n",
        "\n",
        "    # Step 16: POS Tagging\n",
        "    pos_counts = pos_tagging_analysis(text)\n",
        "\n",
        "    # Step 17: Readability\n",
        "    readability_scores = calculate_readability(text)\n",
        "\n",
        "    results = {\n",
        "        \"entities\": entities,\n",
        "        \"clean_tokens\": clean_tokens,\n",
        "        \"word_frequencies\": word_freq.most_common(10),\n",
        "        \"sentiment\": sentiment,\n",
        "        \"tfidf_keywords\": tfidf_keywords,\n",
        "        \"topics\": topics,\n",
        "        \"summary\": summary,\n",
        "        \"semantic_similarities\": similarities,\n",
        "        \"rake_keywords\": keywords,\n",
        "        \"clusters\": clusters,\n",
        "        \"pos_counts\": pos_counts,\n",
        "        \"readability_scores\": readability_scores,\n",
        "    }\n",
        "\n",
        "    # Final TF-IDF Figure\n",
        "    tfidf_fig = visualize_tfidf_figure(tfidf_keywords)\n",
        "\n",
        "    return results, wordcloud_fig, cooccurrence_fig, polarity_heatmap_fig, tfidf_fig\n",
        "\n",
        "# -------------------- Gradio Interface -------------------- #\n",
        "\n",
        "def gradio_pipeline(text, documents):\n",
        "    # Convert multiline box into a list of documents\n",
        "    if isinstance(documents, str):\n",
        "        docs_list = [doc.strip() for doc in documents.split(\"\\n\") if doc.strip()]\n",
        "    else:\n",
        "        docs_list = documents\n",
        "\n",
        "    results, wordcloud_fig, cooccurrence_fig, polarity_heatmap_fig, tfidf_fig = process_text_with_pipeline(\n",
        "        text, docs_list\n",
        "    )\n",
        "\n",
        "    return (\n",
        "        results[\"entities\"],\n",
        "        results[\"clean_tokens\"],\n",
        "        results[\"word_frequencies\"],\n",
        "        results[\"sentiment\"],\n",
        "        results[\"tfidf_keywords\"],\n",
        "        results[\"topics\"],\n",
        "        results[\"summary\"],\n",
        "        results[\"semantic_similarities\"],\n",
        "        results[\"rake_keywords\"],\n",
        "        results[\"clusters\"],\n",
        "        results[\"pos_counts\"],\n",
        "        results[\"readability_scores\"],\n",
        "        wordcloud_fig,\n",
        "        cooccurrence_fig,\n",
        "        polarity_heatmap_fig,\n",
        "        tfidf_fig,\n",
        "    )\n",
        "\n",
        "with gr.Blocks() as demo:\n",
        "    gr.Markdown(\n",
        "        \"## NLP Pipeline with Multiple Results Panels\\n\"\n",
        "        \"### Developed by Partha Pratim Ray\\n\"\n",
        "        \"Contact: [parthapratimray1986@gmail.com](mailto:parthapratimray1986@gmail.com)\\n\"\n",
        "        \"GitHub: [https://github.com/ParthaPRay/](https://github.com/ParthaPRay/)\"\n",
        "    )\n",
        "    with gr.Row():\n",
        "        text_input = gr.Textbox(\n",
        "            label=\"Enter your text here\",\n",
        "            lines=5,\n",
        "            placeholder=\"Type or paste the text to analyze...\",\n",
        "        )\n",
        "        docs_input = gr.Textbox(\n",
        "            label=\"Enter your documents\",\n",
        "            lines=5,\n",
        "            placeholder=\"Doc1...\\nDoc2...\\nDoc3...\",\n",
        "        )\n",
        "\n",
        "    submit_button = gr.Button(\"Submit\")\n",
        "\n",
        "    # Panels (outputs)\n",
        "    named_entities = gr.JSON(label=\"Named Entities\")\n",
        "    clean_tokens = gr.JSON(label=\"Clean Tokens\")\n",
        "    word_frequencies = gr.JSON(label=\"Word Frequencies\")\n",
        "    sentiment_analysis = gr.JSON(label=\"Sentiment Analysis\")\n",
        "    tfidf_keywords = gr.JSON(label=\"Top TF-IDF Keywords\")\n",
        "    topics = gr.JSON(label=\"Topics\")\n",
        "    summary = gr.Textbox(label=\"Summary\")\n",
        "    semantic_similarities = gr.JSON(label=\"Semantic Similarities\")\n",
        "    rake_keywords = gr.JSON(label=\"RAKE Keywords\")\n",
        "    clusters = gr.JSON(label=\"Document Clusters\")\n",
        "    pos_counts = gr.JSON(label=\"POS Tagging Counts\")\n",
        "    readability_scores = gr.JSON(label=\"Readability Scores\")\n",
        "\n",
        "    # Plots\n",
        "    wordcloud_plot = gr.Plot(label=\"Word Cloud\")\n",
        "    cooccurrence_plot = gr.Plot(label=\"Co-occurrence Network\")\n",
        "    polarity_heatmap_plot = gr.Plot(label=\"Polarity Heatmap\")\n",
        "    tfidf_plot = gr.Plot(label=\"TF-IDF Chart\")\n",
        "\n",
        "    submit_button.click(\n",
        "        fn=gradio_pipeline,\n",
        "        inputs=[text_input, docs_input],\n",
        "        outputs=[\n",
        "            named_entities,\n",
        "            clean_tokens,\n",
        "            word_frequencies,\n",
        "            sentiment_analysis,\n",
        "            tfidf_keywords,\n",
        "            topics,\n",
        "            summary,\n",
        "            semantic_similarities,\n",
        "            rake_keywords,\n",
        "            clusters,\n",
        "            pos_counts,\n",
        "            readability_scores,\n",
        "            wordcloud_plot,\n",
        "            cooccurrence_plot,\n",
        "            polarity_heatmap_plot,\n",
        "            tfidf_plot,\n",
        "        ],\n",
        "    )\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    demo.launch()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 750
        },
        "id": "XA4hSI6nn2YP",
        "outputId": "adc3e4cd-9a51-49e4-e699-f85a4be8ff86"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package punkt_tab to /root/nltk_data...\n",
            "[nltk_data]   Package punkt_tab is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running Gradio in a Colab notebook requires sharing enabled. Automatically setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
            "\n",
            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
            "* Running on public URL: https://d6eeecb06b9b8fb11e.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from the terminal in the working directory to deploy to Hugging Face Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://d6eeecb06b9b8fb11e.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}