speech_to_diffusion.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMreyvnRR+5DwFrtePuUVMN",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f4e6dc169dfe42f4a9cbc1f83e6b3154": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4192ee41ac4d4567b90ef5237862cf86",
              "IPY_MODEL_c494819b6ef541f4a19d76dbf2d08058",
              "IPY_MODEL_720e7a51ef1345ab9277200568984f40"
            ],
            "layout": "IPY_MODEL_3fce5084f6e24955bd3c62f39ffdf76b"
          }
        },
        "4192ee41ac4d4567b90ef5237862cf86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a6e996e5b1341f3bbdd0b92f7b113d6",
            "placeholder": "​",
            "style": "IPY_MODEL_f2a0135b44c342db8d7abe30d338e0e7",
            "value": "100%"
          }
        },
        "c494819b6ef541f4a19d76dbf2d08058": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e86560c2438a43a8a364a0cc49b1e5a6",
            "max": 47,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6ae4c022d9914c5792210ffb9d4aaefe",
            "value": 47
          }
        },
        "720e7a51ef1345ab9277200568984f40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84a958f9231840a5be242635c45fdc4b",
            "placeholder": "​",
            "style": "IPY_MODEL_bb7bfb1920dc40ee8b12211ca6e85df9",
            "value": " 47/47 [00:14&lt;00:00,  3.23it/s]"
          }
        },
        "3fce5084f6e24955bd3c62f39ffdf76b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a6e996e5b1341f3bbdd0b92f7b113d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2a0135b44c342db8d7abe30d338e0e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e86560c2438a43a8a364a0cc49b1e5a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ae4c022d9914c5792210ffb9d4aaefe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "84a958f9231840a5be242635c45fdc4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb7bfb1920dc40ee8b12211ca6e85df9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/speech_to_diffusion.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xe8wOozrvwyp",
        "outputId": "6b372d2c-8783-4978-c604-3f68eb414448"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l\r\u001b[K     |██████▏                         | 10 kB 24.5 MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 20 kB 9.7 MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 30 kB 8.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 40 kB 3.7 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 51 kB 4.2 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 53 kB 1.2 MB/s \n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "! pip install -q transformers gradio diffusers ftfy"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "header_html = \"\"\"\n",
        "            <div style=\"text-align: center; max-width: 650px; margin: 0 auto;\">\n",
        "              <div\n",
        "                style=\"\n",
        "                  display: inline-flex;\n",
        "                  align-items: center;\n",
        "                  gap: 0.8rem;\n",
        "                  font-size: 1.75rem;\n",
        "                \"\n",
        "              >\n",
        "                <svg\n",
        "                  width=\"0.65em\"\n",
        "                  height=\"0.65em\"\n",
        "                  viewBox=\"0 0 115 115\"\n",
        "                  fill=\"none\"\n",
        "                  xmlns=\"http://www.w3.org/2000/svg\"\n",
        "                >\n",
        "                  <rect width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect y=\"69\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"23\" width=\"23\" height=\"23\" fill=\"#AEAEAE\"></rect>\n",
        "                  <rect x=\"23\" y=\"69\" width=\"23\" height=\"23\" fill=\"#AEAEAE\"></rect>\n",
        "                  <rect x=\"46\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"46\" y=\"69\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"69\" width=\"23\" height=\"23\" fill=\"black\"></rect>\n",
        "                  <rect x=\"69\" y=\"69\" width=\"23\" height=\"23\" fill=\"black\"></rect>\n",
        "                  <rect x=\"92\" width=\"23\" height=\"23\" fill=\"#D9D9D9\"></rect>\n",
        "                  <rect x=\"92\" y=\"69\" width=\"23\" height=\"23\" fill=\"#AEAEAE\"></rect>\n",
        "                  <rect x=\"115\" y=\"46\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"115\" y=\"115\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"115\" y=\"69\" width=\"23\" height=\"23\" fill=\"#D9D9D9\"></rect>\n",
        "                  <rect x=\"92\" y=\"46\" width=\"23\" height=\"23\" fill=\"#AEAEAE\"></rect>\n",
        "                  <rect x=\"92\" y=\"115\" width=\"23\" height=\"23\" fill=\"#AEAEAE\"></rect>\n",
        "                  <rect x=\"92\" y=\"69\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"69\" y=\"46\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"69\" y=\"115\" width=\"23\" height=\"23\" fill=\"white\"></rect>\n",
        "                  <rect x=\"69\" y=\"69\" width=\"23\" height=\"23\" fill=\"#D9D9D9\"></rect>\n",
        "                  <rect x=\"46\" y=\"46\" width=\"23\" height=\"23\" fill=\"black\"></rect>\n",
        "                  <rect x=\"46\" y=\"115\" width=\"23\" height=\"23\" fill=\"black\"></rect>\n",
        "                  <rect x=\"46\" y=\"69\" width=\"23\" height=\"23\" fill=\"black\"></rect>\n",
        "                  <rect x=\"23\" y=\"46\" width=\"23\" height=\"23\" fill=\"#D9D9D9\"></rect>\n",
        "                  <rect x=\"23\" y=\"115\" width=\"23\" height=\"23\" fill=\"#AEAEAE\"></rect>\n",
        "                  <rect x=\"23\" y=\"69\" width=\"23\" height=\"23\" fill=\"black\"></rect>\n",
        "                </svg>\n",
        "                <h1 style=\"font-weight: 900; margin-bottom: 7px;\">\n",
        "                  Speech to Stable Diffusion Demo\n",
        "                </h1>\n",
        "              </div>\n",
        "              <p style=\"margin-bottom: 10px; font-size: 94%\">\n",
        "                Stable Diffusion is a state of the art text-to-image model that generates\n",
        "                images from text.<br>For faster generation and forthcoming API\n",
        "                access you can try\n",
        "                <a\n",
        "                  href=\"http://beta.dreamstudio.ai/\"\n",
        "                  style=\"text-decoration: underline;\"\n",
        "                  target=\"_blank\"\n",
        "                  >DreamStudio Beta</a\n",
        "                >\n",
        "              </p>\n",
        "            </div>\n",
        "        \"\"\"\n",
        "\n",
        "\n",
        "footer_html = \"\"\"\n",
        "                <div class=\"footer\">\n",
        "                    <p>Model by <a href=\"https://huggingface.co/CompVis\" style=\"text-decoration: underline;\" target=\"_blank\">CompVis</a> and <a href=\"https://huggingface.co/stabilityai\" style=\"text-decoration: underline;\" target=\"_blank\">Stability AI</a> - Gradio Demo by 🤗 Hugging Face\n",
        "                    </p>\n",
        "                </div>\n",
        "                <div class=\"acknowledgments\">\n",
        "                    <p><h4>LICENSE</h4>\n",
        "The model is licensed with a <a href=\"https://huggingface.co/spaces/CompVis/stable-diffusion-license\" style=\"text-decoration: underline;\" target=\"_blank\">CreativeML Open RAIL-M</a> license. The authors claim no rights on the outputs you generate, you are free to use them and are accountable for their use which must not go against the provisions set in this license. The license forbids you from sharing any content that violates any laws, produce any harm to a person, disseminate any personal information that would be meant for harm, spread misinformation and target vulnerable groups. For the full list of restrictions please <a href=\"https://huggingface.co/spaces/CompVis/stable-diffusion-license\" target=\"_blank\" style=\"text-decoration: underline;\" target=\"_blank\">read the license</a></p>\n",
        "                    <p><h4>Biases and content acknowledgment</h4>\n",
        "Despite how impressive being able to turn text into image is, beware to the fact that this model may output content that reinforces or exacerbates societal biases, as well as realistic faces, pornography and violence. The model was trained on the <a href=\"https://laion.ai/blog/laion-5b/\" style=\"text-decoration: underline;\" target=\"_blank\">LAION-5B dataset</a>, which scraped non-curated image-text-pairs from the internet (the exception being the removal of illegal content) and is meant for research purposes. You can read more in the <a href=\"https://huggingface.co/CompVis/stable-diffusion-v1-4\" style=\"text-decoration: underline;\" target=\"_blank\">model card</a></p>\n",
        "               </div>\n",
        "           \"\"\"\n",
        "\n",
        "css = \"\"\"\n",
        "        .gradio-container {\n",
        "            font-family: 'IBM Plex Sans', sans-serif;\n",
        "        }\n",
        "        .gr-button {\n",
        "            color: white;\n",
        "            border-color: black;\n",
        "            background: black;\n",
        "        }\n",
        "        input[type='range'] {\n",
        "            accent-color: black;\n",
        "        }\n",
        "        .dark input[type='range'] {\n",
        "            accent-color: #dfdfdf;\n",
        "        }\n",
        "        .container {\n",
        "            max-width: 730px;\n",
        "            margin: auto;\n",
        "            padding-top: 1.5rem;\n",
        "        }\n",
        "        #gallery {\n",
        "            min-height: 22rem;\n",
        "            margin-bottom: 15px;\n",
        "            margin-left: auto;\n",
        "            margin-right: auto;\n",
        "            border-bottom-right-radius: .5rem !important;\n",
        "            border-bottom-left-radius: .5rem !important;\n",
        "        }\n",
        "        #gallery>div>.h-full {\n",
        "            min-height: 20rem;\n",
        "        }\n",
        "        .details:hover {\n",
        "            text-decoration: underline;\n",
        "        }\n",
        "        .gr-button {\n",
        "            white-space: nowrap;\n",
        "        }\n",
        "        .gr-button:focus {\n",
        "            border-color: rgb(147 197 253 / var(--tw-border-opacity));\n",
        "            outline: none;\n",
        "            box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);\n",
        "            --tw-border-opacity: 1;\n",
        "            --tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);\n",
        "            --tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color);\n",
        "            --tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity));\n",
        "            --tw-ring-opacity: .5;\n",
        "        }\n",
        "        #advanced-btn {\n",
        "            font-size: .7rem !important;\n",
        "            line-height: 19px;\n",
        "            margin-top: 12px;\n",
        "            margin-bottom: 12px;\n",
        "            padding: 2px 8px;\n",
        "            border-radius: 14px !important;\n",
        "        }\n",
        "        #advanced-options {\n",
        "            display: none;\n",
        "            margin-bottom: 20px;\n",
        "        }\n",
        "        .footer {\n",
        "            margin-bottom: 45px;\n",
        "            margin-top: 35px;\n",
        "            text-align: center;\n",
        "            border-bottom: 1px solid #e5e5e5;\n",
        "        }\n",
        "        .footer>p {\n",
        "            font-size: .8rem;\n",
        "            display: inline-block;\n",
        "            padding: 0 10px;\n",
        "            transform: translateY(10px);\n",
        "            background: white;\n",
        "        }\n",
        "        .dark .footer {\n",
        "            border-color: #303030;\n",
        "        }\n",
        "        .dark .footer>p {\n",
        "            background: #0b0f19;\n",
        "        }\n",
        "        .acknowledgments h4{\n",
        "            margin: 1.25em 0 .25em 0;\n",
        "            font-weight: bold;\n",
        "            font-size: 115%;\n",
        "        }\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "zhiuDLYNwMjV"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! wget https://huggingface.co/spaces/stabilityai/stable-diffusion/resolve/main/unsafe.png"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d7qrvawCyoWt",
        "outputId": "28936ff1-dfb5-4ce6-bfa3-bfc12a9e2d41"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-09-14 16:17:37--  https://huggingface.co/spaces/stabilityai/stable-diffusion/resolve/main/unsafe.png\n",
            "Resolving huggingface.co (huggingface.co)... 52.6.16.131, 52.202.207.64, 2600:1f18:147f:e800:afa4:a769:1b42:e343, ...\n",
            "Connecting to huggingface.co (huggingface.co)|52.6.16.131|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 29601 (29K) [image/png]\n",
            "Saving to: ‘unsafe.png’\n",
            "\n",
            "unsafe.png          100%[===================>]  28.91K   158KB/s    in 0.2s    \n",
            "\n",
            "2022-09-14 16:17:38 (158 KB/s) - ‘unsafe.png’ saved [29601/29601]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gradio as gr\n",
        "import os\n",
        "import torch\n",
        "from torch import autocast\n",
        "from diffusers import StableDiffusionPipeline\n",
        "from PIL import Image\n",
        "from transformers import pipeline\n",
        "\n",
        "ars_model = pipeline(\"automatic-speech-recognition\")\n",
        "\n",
        "\n",
        "model_id = \"CompVis/stable-diffusion-v1-4\"\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "pipe = StableDiffusionPipeline.from_pretrained(\n",
        "    model_id, use_auth_token='<your_hf_token>', revision=\"fp16\", torch_dtype=torch.float16)\n",
        "pipe = pipe.to(device)\n",
        "\n",
        "\n",
        "def transcribe(audio):\n",
        "    text = ars_model(audio)[\"text\"]\n",
        "    return text\n",
        "\n",
        "\n",
        "def infer(audio, samples, steps, scale, seed):\n",
        "\n",
        "    prompt = transcribe(audio)\n",
        "    generator = torch.Generator(device=device).manual_seed(seed)\n",
        "\n",
        "    if device == \"cuda\":\n",
        "        with autocast(\"cuda\"):\n",
        "            images_list = pipe(\n",
        "                [prompt] * samples,\n",
        "                num_inference_steps=steps,\n",
        "                guidance_scale=scale,\n",
        "                generator=generator,\n",
        "            )\n",
        "    else:\n",
        "        images_list = pipe(\n",
        "            [prompt] * samples,\n",
        "            num_inference_steps=steps,\n",
        "            guidance_scale=scale,\n",
        "            generator=generator,\n",
        "        )\n",
        "    images = []\n",
        "    safe_image = Image.open(r\"/content/unsafe.png\")\n",
        "    for i, image in enumerate(images_list[\"sample\"]):\n",
        "        if(images_list[\"nsfw_content_detected\"][i]):\n",
        "            images.append(safe_image)\n",
        "        else:\n",
        "            images.append(image)\n",
        "    return prompt, images\n",
        "\n",
        "\n",
        "block = gr.Blocks(css=css)\n",
        "\n",
        "\n",
        "with block:\n",
        "    gr.HTML(header_html)\n",
        "    with gr.Group():\n",
        "        with gr.Box():\n",
        "            with gr.Row().style(mobile_collapse=False, equal_height=True):\n",
        "                audio = gr.Audio(\n",
        "                    label=\"Describe the prompt\",\n",
        "                    source=\"microphone\",\n",
        "                    type=\"filepath\"\n",
        "                )\n",
        "                btn = gr.Button(\"Generate image\").style(\n",
        "                    margin=False,\n",
        "                    rounded=(False, True, True, False),\n",
        "                )\n",
        "        \n",
        "        text_trans = gr.Textbox(label=\"Transcription,\", show_label=False, elem_id=\"text_trans\")\n",
        "\n",
        "        gallery = gr.Gallery(\n",
        "            label=\"Generated images\", show_label=False, elem_id=\"gallery\"\n",
        "        ).style(grid=[2], height=\"auto\")\n",
        "\n",
        "        advanced_button = gr.Button(\"Advanced options\", elem_id=\"advanced-btn\")\n",
        "\n",
        "        with gr.Row(elem_id=\"advanced-options\"):\n",
        "            samples = gr.Slider(label=\"Images\", minimum=1,\n",
        "                                maximum=4, value=4, step=1)\n",
        "            steps = gr.Slider(label=\"Steps\", minimum=1,\n",
        "                              maximum=50, value=45, step=1)\n",
        "            scale = gr.Slider(\n",
        "                label=\"Guidance Scale\", minimum=0, maximum=50, value=7.5, step=0.1\n",
        "            )\n",
        "            seed = gr.Slider(\n",
        "                label=\"Seed\",\n",
        "                minimum=0,\n",
        "                maximum=2147483647,\n",
        "                step=1,\n",
        "                randomize=True,\n",
        "            )\n",
        "\n",
        "        btn.click(infer, inputs=[audio, samples, steps,\n",
        "                  scale, seed], outputs=[text_trans, gallery])\n",
        "        advanced_button.click(\n",
        "            None,\n",
        "            [],\n",
        "            audio,\n",
        "            _js=\"\"\"\n",
        "            () => {\n",
        "                const options = document.querySelector(\"body > gradio-app\").querySelector(\"#advanced-options\");\n",
        "                options.style.display = [\"none\", \"\"].includes(options.style.display) ? \"flex\" : \"none\";\n",
        "            }\"\"\",\n",
        "        )\n",
        "        gr.HTML(footer_html)\n",
        "\n",
        "block.launch(debug=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 814,
          "referenced_widgets": [
            "f4e6dc169dfe42f4a9cbc1f83e6b3154",
            "4192ee41ac4d4567b90ef5237862cf86",
            "c494819b6ef541f4a19d76dbf2d08058",
            "720e7a51ef1345ab9277200568984f40",
            "3fce5084f6e24955bd3c62f39ffdf76b",
            "8a6e996e5b1341f3bbdd0b92f7b113d6",
            "f2a0135b44c342db8d7abe30d338e0e7",
            "e86560c2438a43a8a364a0cc49b1e5a6",
            "6ae4c022d9914c5792210ffb9d4aaefe",
            "84a958f9231840a5be242635c45fdc4b",
            "bb7bfb1920dc40ee8b12211ca6e85df9"
          ]
        },
        "id": "5VKhRL_av9id",
        "outputId": "18d335d2-a741-46bd-cba6-62b8a4bd9a9e"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to facebook/wav2vec2-base-960h and revision 55bb623 (https://huggingface.co/facebook/wav2vec2-base-960h).\n",
            "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
            "Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-base-960h and are newly initialized: ['wav2vec2.masked_spec_embed']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "{'trained_betas'} was not found in config. Values will be initialized to default values.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
            "Running on public URL: https://28034.gradio.app\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting, check out Spaces: https://huggingface.co/spaces\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://28034.gradio.app\" width=\"900\" height=\"500\" allow=\"autoplay; camera; microphone;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gradio/processing_utils.py:208: UserWarning: Audio data is not in 16-bit integer format.Trying to convert to 16-bit int format.\n",
            "  \"Audio data is not in 16-bit integer format.\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/47 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f4e6dc169dfe42f4a9cbc1f83e6b3154"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Keyboard interruption in main thread... closing server.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<gradio.routes.App at 0x7fbb01261910>,\n",
              " 'http://127.0.0.1:7862/',\n",
              " 'https://28034.gradio.app')"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "P_abKNmv4mKl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}