kafkaproducer1.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aomidvar/scrapper-price-comparison/blob/main/kafkaproducer1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ndl4w1uf1kqx",
        "outputId": "f4ba57e2-c540-46c5-9ee4-aad79b3c2143"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pymongo[srv] in /usr/local/lib/python3.10/dist-packages (3.13.0)\n",
            "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo[srv])\n",
            "  Downloading dnspython-2.3.0-py3-none-any.whl (283 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: dnspython\n",
            "Successfully installed dnspython-2.3.0\n"
          ]
        }
      ],
      "source": [
        "pip install pymongo[srv]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install apache-flink[avro]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "2rOzUyLEGF7m",
        "outputId": "420d7acc-5522-45a4-bc9d-77d630ad5e12"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting apache-flink[avro]\n",
            "  Downloading apache_flink-1.17.0-cp310-cp310-manylinux1_x86_64.whl (5.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m38.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[33mWARNING: apache-flink 1.17.0 does not provide the extra 'avro'\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from apache-flink[avro]) (0.10.9.7)\n",
            "Requirement already satisfied: python-dateutil<3,>=2.8.0 in /usr/local/lib/python3.10/dist-packages (from apache-flink[avro]) (2.8.2)\n",
            "Collecting apache-beam==2.43.0 (from apache-flink[avro])\n",
            "  Downloading apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (14.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.0/14.0 MB\u001b[0m \u001b[31m47.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cloudpickle==2.2.0 (from apache-flink[avro])\n",
            "  Downloading cloudpickle-2.2.0-py3-none-any.whl (25 kB)\n",
            "Collecting avro-python3!=1.9.2,<1.10.0,>=1.8.1 (from apache-flink[avro])\n",
            "  Downloading avro-python3-1.9.2.1.tar.gz (37 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: pytz>=2018.3 in /usr/local/lib/python3.10/dist-packages (from apache-flink[avro]) (2022.7.1)\n",
            "Collecting fastavro<1.4.8,>=1.1.0 (from apache-flink[avro])\n",
            "  Downloading fastavro-1.4.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from apache-flink[avro]) (2.27.1)\n",
            "Requirement already satisfied: protobuf<=3.21,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from apache-flink[avro]) (3.20.3)\n",
            "Collecting numpy<1.22.0,>=1.21.4 (from apache-flink[avro])\n",
            "  Downloading numpy-1.21.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.9/15.9 MB\u001b[0m \u001b[31m78.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pandas<1.4.0,>=1.3.0 (from apache-flink[avro])\n",
            "  Downloading pandas-1.3.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.5/11.5 MB\u001b[0m \u001b[31m99.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyarrow<9.0.0,>=5.0.0 (from apache-flink[avro])\n",
            "  Downloading pyarrow-8.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m29.4/29.4 MB\u001b[0m \u001b[31m18.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httplib2<=0.20.4,>=0.19.0 (from apache-flink[avro])\n",
            "  Downloading httplib2-0.20.4-py3-none-any.whl (96 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.6/96.6 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting apache-flink-libraries<1.17.1,>=1.17.0 (from apache-flink[avro])\n",
            "  Downloading apache-flink-libraries-1.17.0.tar.gz (240.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m240.0/240.0 MB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pemja==0.3.0 (from apache-flink[avro])\n",
            "  Downloading pemja-0.3.0-cp310-cp310-manylinux1_x86_64.whl (332 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m332.7/332.7 kB\u001b[0m \u001b[31m27.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting crcmod<2.0,>=1.7 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading crcmod-1.7.tar.gz (89 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.7/89.7 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting orjson<4.0 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading orjson-3.8.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (137 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.2/137.2 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting dill<0.3.2,>=0.3.1.1 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading dill-0.3.1.1.tar.gz (151 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.0/152.0 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting fasteners<1.0,>=0.3 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading fasteners-0.18-py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: grpcio!=1.48.0,<2,>=1.33.1 in /usr/local/lib/python3.10/dist-packages (from apache-beam==2.43.0->apache-flink[avro]) (1.54.0)\n",
            "Collecting hdfs<3.0.0,>=2.1.0 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading hdfs-2.7.0-py3-none-any.whl (34 kB)\n",
            "Collecting objsize<0.6.0,>=0.5.2 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading objsize-0.5.2-py3-none-any.whl (8.2 kB)\n",
            "Collecting pymongo<4.0.0,>=3.8.0 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading pymongo-3.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (516 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m516.2/516.2 kB\u001b[0m \u001b[31m36.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: proto-plus<2,>=1.7.1 in /usr/local/lib/python3.10/dist-packages (from apache-beam==2.43.0->apache-flink[avro]) (1.22.2)\n",
            "Requirement already satisfied: pydot<2,>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from apache-beam==2.43.0->apache-flink[avro]) (1.4.2)\n",
            "Requirement already satisfied: regex>=2020.6.8 in /usr/local/lib/python3.10/dist-packages (from apache-beam==2.43.0->apache-flink[avro]) (2022.10.31)\n",
            "Requirement already satisfied: typing-extensions>=3.7.0 in /usr/local/lib/python3.10/dist-packages (from apache-beam==2.43.0->apache-flink[avro]) (4.5.0)\n",
            "Collecting zstandard<1,>=0.18.0 (from apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading zstandard-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m78.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting find-libpython (from pemja==0.3.0->apache-flink[avro])\n",
            "  Downloading find_libpython-0.3.0-py3-none-any.whl (8.5 kB)\n",
            "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.10/dist-packages (from httplib2<=0.20.4,>=0.19.0->apache-flink[avro]) (3.0.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3,>=2.8.0->apache-flink[avro]) (1.16.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->apache-flink[avro]) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->apache-flink[avro]) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->apache-flink[avro]) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->apache-flink[avro]) (3.4)\n",
            "Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.43.0->apache-flink[avro])\n",
            "  Downloading docopt-0.6.2.tar.gz (25 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: apache-flink-libraries, avro-python3, crcmod, dill, docopt\n",
            "  Building wheel for apache-flink-libraries (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for apache-flink-libraries: filename=apache_flink_libraries-1.17.0-py2.py3-none-any.whl size=240040471 sha256=d74e26f9ed013d8fc73a20d7409521f318ae623559da8d8a82832be2ed79e0fb\n",
            "  Stored in directory: /root/.cache/pip/wheels/aa/f3/9f/f268297890997d81b90bbd0f2664d64fbd7c94dab680baab28\n",
            "  Building wheel for avro-python3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for avro-python3: filename=avro_python3-1.9.2.1-py3-none-any.whl size=43497 sha256=8b9d4a05e66a1bd403ecad9244c740e7d078a7e157efb0d1296a1e797205574a\n",
            "  Stored in directory: /root/.cache/pip/wheels/f9/dc/61/0e621c0c1c1d61c4699d6c6565de69171cbec6f244b8b380f5\n",
            "  Building wheel for crcmod (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for crcmod: filename=crcmod-1.7-cp310-cp310-linux_x86_64.whl size=37097 sha256=c8fc3d89dc9fb94f4aa1c998e2bf623d7c47ae03f06d3c94dce399bb6f63c94d\n",
            "  Stored in directory: /root/.cache/pip/wheels/85/4c/07/72215c529bd59d67e3dac29711d7aba1b692f543c808ba9e86\n",
            "  Building wheel for dill (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for dill: filename=dill-0.3.1.1-py3-none-any.whl size=78545 sha256=f8a3df9eef2816c8eaf97d69e3f18ae7bba22fe0fc487cd2b2fd42748475d349\n",
            "  Stored in directory: /root/.cache/pip/wheels/ea/e2/86/64980d90e297e7bf2ce588c2b96e818f5399c515c4bb8a7e4f\n",
            "  Building wheel for docopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13707 sha256=af135403e8bf5486584c086b7f82bc8689a475ca1ffce11b10dc3d284a35fb0c\n",
            "  Stored in directory: /root/.cache/pip/wheels/fc/ab/d4/5da2067ac95b36618c629a5f93f809425700506f72c9732fac\n",
            "Successfully built apache-flink-libraries avro-python3 crcmod dill docopt\n",
            "Installing collected packages: find-libpython, docopt, crcmod, zstandard, pymongo, pemja, orjson, objsize, numpy, httplib2, fasteners, fastavro, dill, cloudpickle, avro-python3, apache-flink-libraries, pyarrow, pandas, hdfs, apache-beam, apache-flink\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.22.4\n",
            "    Uninstalling numpy-1.22.4:\n",
            "      Successfully uninstalled numpy-1.22.4\n",
            "  Attempting uninstall: httplib2\n",
            "    Found existing installation: httplib2 0.21.0\n",
            "    Uninstalling httplib2-0.21.0:\n",
            "      Successfully uninstalled httplib2-0.21.0\n",
            "  Attempting uninstall: cloudpickle\n",
            "    Found existing installation: cloudpickle 2.2.1\n",
            "    Uninstalling cloudpickle-2.2.1:\n",
            "      Successfully uninstalled cloudpickle-2.2.1\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 9.0.0\n",
            "    Uninstalling pyarrow-9.0.0:\n",
            "      Successfully uninstalled pyarrow-9.0.0\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 1.5.3\n",
            "    Uninstalling pandas-1.5.3:\n",
            "      Successfully uninstalled pandas-1.5.3\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires pandas~=1.5.3, but you have pandas 1.3.5 which is incompatible.\n",
            "tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.21.6 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed apache-beam-2.43.0 apache-flink-1.17.0 apache-flink-libraries-1.17.0 avro-python3-1.9.2.1 cloudpickle-2.2.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.7 fasteners-0.18 find-libpython-0.3.0 hdfs-2.7.0 httplib2-0.20.4 numpy-1.21.6 objsize-0.5.2 orjson-3.8.12 pandas-1.3.5 pemja-0.3.0 pyarrow-8.0.0 pymongo-3.13.0 zstandard-0.21.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "httplib2",
                  "numpy"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "-Bq2khvD2UNW"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "337k4iNX2jJp"
      },
      "source": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install confluent-kafka"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ykHrVt-ACOYy",
        "outputId": "2a208191-a670-4375-c634-6081aeb2bc32"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting confluent-kafka\n",
            "  Downloading confluent_kafka-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.9/3.9 MB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: confluent-kafka\n",
            "Successfully installed confluent-kafka-2.1.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import List, Tuple, Any\n",
        "import os\n",
        "import requests\n",
        "from flask import Flask, request, jsonify\n",
        "from pymongo import MongoClient\n",
        "import json\n",
        "import random\n",
        "import time\n",
        "from confluent_kafka import Producer, Consumer, KafkaError\n",
        "\n"
      ],
      "metadata": {
        "id": "pYOOaR3kCp_L"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TcT_22Fk2nlK",
        "outputId": "b10fcfa7-fb0e-452e-cdfc-1824f557a405"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pinged your deployment. You successfully connected to MongoDB!\n"
          ]
        }
      ],
      "source": [
        "\n",
        "from pymongo.mongo_client import MongoClient\n",
        "from pymongo.server_api import ServerApi\n",
        "\n",
        "uri = \"mongodb+srv://scrapper:shaylin1396*M@cluster0.cco0jwv.mongodb.net/?retryWrites=true&w=majority\"\n",
        "\n",
        "# Create a new client and connect to the server\n",
        "client = MongoClient(uri, server_api=ServerApi('1'))\n",
        "\n",
        "# Send a ping to confirm a successful connection\n",
        "try:\n",
        "    client.admin.command('ping')\n",
        "    print(\"Pinged your deployment. You successfully connected to MongoDB!\")\n",
        "except Exception as e:\n",
        "    print(e)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "client.stats"
      ],
      "metadata": {
        "id": "qIwdqvg6d2hi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "76580a9e-8917-411f-8253-9a178ab3c640"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Database(MongoClient(host=['ac-ymw4xat-shard-00-02.cco0jwv.mongodb.net:27017', 'ac-ymw4xat-shard-00-01.cco0jwv.mongodb.net:27017', 'ac-ymw4xat-shard-00-00.cco0jwv.mongodb.net:27017'], document_class=dict, tz_aware=False, connect=True, retrywrites=True, w='majority', authsource='admin', replicaset='atlas-yr4frs-shard-0', ssl=True, server_api=<pymongo.server_api.ServerApi object at 0x7fc3adf32c80>), 'stats')"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "client.list_database_names()"
      ],
      "metadata": {
        "id": "Lspvw8QVwdw-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4e204b2e-d450-4bdb-9751-f3cab8610990"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['admin', 'local']"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set the configuration properties\n",
        "kafka_config = {\n",
        "    \"bootstrap.servers\": \"pkc-6ojv2.us-west4.gcp.confluent.cloud:9092\",\n",
        "    \"security.protocol\": \"SASL_SSL\",\n",
        "    \"sasl.mechanisms\": \"PLAIN\",\n",
        "    \"sasl.username\": \"RFU2BCEDGB2UJRWD\",\n",
        "    \"sasl.password\": \"rjOrdR009uQUx8F67ObMn6on0IAgEtSunMFIKR2KK17SVJqQ1gh+9CmTT9NXiAjp\"\n",
        "}"
      ],
      "metadata": {
        "id": "HKaXaLevFj4f"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a Kafka producer with the configuration properties\n",
        "producer = KafkaProducer(**kafka_config)"
      ],
      "metadata": {
        "id": "PkklixzpIqVd"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the topic to send messages to\n",
        "topic = 'KafkaTopicIN'\n",
        "# Generate and send messages in a loop\n",
        "while True:\n",
        "    # Generate a random JSON message\n",
        "    message = {\"availability\": random.choice([\"Quedan pocos\",\"Quedan pocos\"]),\n",
        "  \"brand\": random.choice([\"Onix\",\"Apple\",\"Griffin\",\"Lg\"]),\n",
        "  \"category_id\": \"112\",\n",
        "  \"date\": \"2023-02-07 12:57:10\",\n",
        "  \"price_height\": random.random()*6583,\n",
        "  \"price_low\": \"384.30\",\n",
        "  \"product_url\": random.choice([\"https://www.elpalaciodehierro.com/onix-bocina-mermaid-40513545.html\",\"https://www.elpalaciodehierro.com/apple-funda-iphone-14-pro-max-transparente-42996946.html\",\"https://www.elpalaciodehierro.com/apple-funda-case-para-iphone-14-plus-si-midnight-42997025.html\",\"https://www.elpalaciodehierro.com/apple-funda-iphone-14-pro-max-negra-42996985.html\",\"https://www.elpalaciodehierro.com/griffin-cable-usb-c-a-lightning-1.8-m---negro-42910307.html\",\"https://www.elpalaciodehierro.com/griffin-cable-usb-c-a-lightning-premium-1.5-m---plata-42910308.html\"]),\n",
        "  \"sku\": random.randint(42991111,42999999),\n",
        "  \"store_id\": \"1\",\n",
        "  \"title\": random.choice([\"Bocina Mermaid\", \"Funda iPhone 14 Pro Max Transparente\",\"Funda Case para iPhone 14 Plus Si Midnight\",\"Funda iPhone 14 Pro Max Negra\", \"Cable USB-C a Lightning 1.8 m - Negro\",\"Cable USB-C a Lightning Premium 1.5 m - Plata\",\"Funda iPhone 14 Transparente\",\"Cable USB-C a Lightning 1.8 m - Blanco\",\"Monitor Gamer 23.8 Pulgadas PlanaLED UltraGear FHD 144 Hz 24GN60R-B\"])}\n",
        "    \n",
        "    # Convert the message to a JSON string\n",
        "    message_str = json.dumps(message)\n",
        "    \n",
        "    # Send the message to Kafka\n",
        "    producer.produce(topic, message_str.encode('utf-8'))\n",
        "    \n",
        "    # Sleep for some time before sending the next message\n",
        "    time.sleep(1)"
      ],
      "metadata": {
        "id": "IaH4l0w1I6p4"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNQ+U2ssV6klUEskuFUmM9h",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}