wrangle_act.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "wrangle_act.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "zSpA3SypDCbn",
        "UWAC1VKWlEIt",
        "9Q07gBThinwI",
        "dvV1Ndy-j1kM"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zSpA3SypDCbn",
        "colab_type": "text"
      },
      "source": [
        "# Gathering Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MAqBLr9tJLwL",
        "colab_type": "code",
        "outputId": "64943cb5-bff0-471d-ad87-d97d68713ab8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ljPwQRd_JVpE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import requests\n",
        "import tweepy\n",
        "import json\n",
        "import sys\n",
        "import datetime as dt\n",
        "import matplotlib.pyplot as plt\n",
        "import re\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Si4UUNNjJZb2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fn_twitter_archive_enhanced = '/content/drive/My Drive/Colab Notebooks/Data Wrangling/Project/Data/twitter-archive-enhanced.csv'\n",
        "fn_img_pred_local_file_name = '/content/drive/My Drive/Colab Notebooks/Data Wrangling/Project/Data/image-preds.tsv'\n",
        "fn_local_json_file = '/content/drive/My Drive/Colab Notebooks/Data Wrangling/Project/Data/tweet_json.txt'\n",
        "fn_error_log = '/content/drive/My Drive/Colab Notebooks/Data Wrangling/Project/Data/json_download_log.txt'\n",
        "fn_local_json_file1 = '/content/drive/My Drive/Colab Notebooks/Data Wrangling/Project/Data/tweet-json.txt'\n",
        "fn_final_clean_file = '/content/drive/My Drive/Colab Notebooks/Data Wrangling/Project/Data/twitter_archive_master.csv'\n",
        "\n",
        "img_pred_url = 'https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv'\n",
        "\n",
        "consumer_key = 'MY CONSUMER KEY'\n",
        "consumer_secret = 'MY CONSUMER SECRET KEY'\n",
        "access_token = 'MY ACCESS TOKEN'\n",
        "access_secret = 'MY ACCESS SECRET'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJBybbC8_NLV",
        "colab_type": "code",
        "outputId": "efb667d0-2220-47ec-97eb-a5f67b11202f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 591
        }
      },
      "source": [
        "# Load the twitter enhance archive data\n",
        "df_twitter_archive = pd.read_csv(fn_twitter_archive_enhanced)\n",
        "df_twitter_archive.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>in_reply_to_status_id</th>\n",
              "      <th>in_reply_to_user_id</th>\n",
              "      <th>timestamp</th>\n",
              "      <th>source</th>\n",
              "      <th>text</th>\n",
              "      <th>retweeted_status_id</th>\n",
              "      <th>retweeted_status_user_id</th>\n",
              "      <th>retweeted_status_timestamp</th>\n",
              "      <th>expanded_urls</th>\n",
              "      <th>rating_numerator</th>\n",
              "      <th>rating_denominator</th>\n",
              "      <th>name</th>\n",
              "      <th>doggo</th>\n",
              "      <th>floofer</th>\n",
              "      <th>pupper</th>\n",
              "      <th>puppo</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>892420643555336193</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-08-01 16:23:56 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Phineas</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>892177421306343426</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-08-01 00:17:27 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Tilly. She's just checking pup on you....</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Tilly</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>891815181378084864</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-07-31 00:18:03 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
              "      <td>12</td>\n",
              "      <td>10</td>\n",
              "      <td>Archie</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>891689557279858688</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-07-30 15:58:51 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Darla. She commenced a snooze mid meal...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Darla</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>891327558926688256</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-07-29 16:00:24 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Franklin. He would like you to stop ca...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
              "      <td>12</td>\n",
              "      <td>10</td>\n",
              "      <td>Franklin</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  in_reply_to_status_id  ...  pupper puppo\n",
              "0  892420643555336193                    NaN  ...    None  None\n",
              "1  892177421306343426                    NaN  ...    None  None\n",
              "2  891815181378084864                    NaN  ...    None  None\n",
              "3  891689557279858688                    NaN  ...    None  None\n",
              "4  891327558926688256                    NaN  ...    None  None\n",
              "\n",
              "[5 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z7jzW2AL_26P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Download the image prediction data\n",
        "response = requests.get(img_pred_url)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gzF9srCo_V5I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Store the data in a local file\n",
        "with open(fn_img_pred_local_file_name, mode='wb') as file:\n",
        "  file.write(response.content)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cs8BLlBWBcXG",
        "colab_type": "code",
        "outputId": "1147b548-9134-4057-815c-482e3789418d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        }
      },
      "source": [
        "# Load the data into a dataframe\n",
        "df_image_pred = pd.read_csv(fn_img_pred_local_file_name, sep='\\t')\n",
        "df_image_pred.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>jpg_url</th>\n",
              "      <th>img_num</th>\n",
              "      <th>p1</th>\n",
              "      <th>p1_conf</th>\n",
              "      <th>p1_dog</th>\n",
              "      <th>p2</th>\n",
              "      <th>p2_conf</th>\n",
              "      <th>p2_dog</th>\n",
              "      <th>p3</th>\n",
              "      <th>p3_conf</th>\n",
              "      <th>p3_dog</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>666020888022790149</td>\n",
              "      <td>https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Welsh_springer_spaniel</td>\n",
              "      <td>0.465074</td>\n",
              "      <td>True</td>\n",
              "      <td>collie</td>\n",
              "      <td>0.156665</td>\n",
              "      <td>True</td>\n",
              "      <td>Shetland_sheepdog</td>\n",
              "      <td>0.061428</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>666029285002620928</td>\n",
              "      <td>https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>redbone</td>\n",
              "      <td>0.506826</td>\n",
              "      <td>True</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.074192</td>\n",
              "      <td>True</td>\n",
              "      <td>Rhodesian_ridgeback</td>\n",
              "      <td>0.072010</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>666033412701032449</td>\n",
              "      <td>https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>German_shepherd</td>\n",
              "      <td>0.596461</td>\n",
              "      <td>True</td>\n",
              "      <td>malinois</td>\n",
              "      <td>0.138584</td>\n",
              "      <td>True</td>\n",
              "      <td>bloodhound</td>\n",
              "      <td>0.116197</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>666044226329800704</td>\n",
              "      <td>https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Rhodesian_ridgeback</td>\n",
              "      <td>0.408143</td>\n",
              "      <td>True</td>\n",
              "      <td>redbone</td>\n",
              "      <td>0.360687</td>\n",
              "      <td>True</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.222752</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>666049248165822465</td>\n",
              "      <td>https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.560311</td>\n",
              "      <td>True</td>\n",
              "      <td>Rottweiler</td>\n",
              "      <td>0.243682</td>\n",
              "      <td>True</td>\n",
              "      <td>Doberman</td>\n",
              "      <td>0.154629</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  ... p3_dog\n",
              "0  666020888022790149  ...   True\n",
              "1  666029285002620928  ...   True\n",
              "2  666033412701032449  ...   True\n",
              "3  666044226329800704  ...   True\n",
              "4  666049248165822465  ...   True\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UWAC1VKWlEIt",
        "colab_type": "text"
      },
      "source": [
        "### Code for Downloading Twitter data. This did not work. I had Rate limit issues"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tP8h73z-TsPa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Setup to access data from twitter\n",
        "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
        "auth.set_access_token(access_token, access_secret)\n",
        "\n",
        "api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)\n",
        "\n",
        "# For each tweet, get the tweet details\n",
        "logf = open(fn_error_log, 'w')\n",
        "outfile = open(fn_local_json_file, 'w')\n",
        "for index, row in df_twitter_archive.iterrows():\n",
        "  t_id = row['tweet_id']\n",
        "  try:\n",
        "    data = api.get_status(t_id, tweet_mode='extended')._json + '\\n'\n",
        "    json.dump(data, outfile)\n",
        "    logf.write('Success|' + str(t_id) + '\\n')\n",
        "    print(str(t_id))\n",
        "  except:\n",
        "    logf.write('Error|' + str(t_id) + '\\n')\n",
        "\n",
        "logf.close()\n",
        "outfile.close()\n",
        "print('Done')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-YqBjzeIlrb_",
        "colab_type": "text"
      },
      "source": [
        "### Alternate to twitter download. Used the file provided"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bf2vavG16Eql",
        "colab_type": "code",
        "outputId": "d5eaa4c5-53d6-43ed-ba39-54af7e4fcc9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "# Gather only those tweets which are not re-tweets\n",
        "# if line contains 'retweeted_status' key, it would mean its a retweet\n",
        "j_data = []\n",
        "with open(fn_local_json_file1, 'r') as j_file:\n",
        "  for line in j_file:\n",
        "    if 'retweeted_status' not in line:\n",
        "      j_content = json.loads(line)\n",
        "      j_data.append([j_content['id'], j_content['retweet_count'], j_content['favorite_count']])\n",
        "\n",
        "df_jdata = pd.DataFrame(j_data, columns=['tweet_id', 'retweet_count', 'favorite_count'])  \n",
        "df_jdata.head()\n",
        "      "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>favorite_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>892420643555336193</td>\n",
              "      <td>8853</td>\n",
              "      <td>39467</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>892177421306343426</td>\n",
              "      <td>6514</td>\n",
              "      <td>33819</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>891815181378084864</td>\n",
              "      <td>4328</td>\n",
              "      <td>25461</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>891689557279858688</td>\n",
              "      <td>8964</td>\n",
              "      <td>42908</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>891327558926688256</td>\n",
              "      <td>9774</td>\n",
              "      <td>41048</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  retweet_count  favorite_count\n",
              "0  892420643555336193           8853           39467\n",
              "1  892177421306343426           6514           33819\n",
              "2  891815181378084864           4328           25461\n",
              "3  891689557279858688           8964           42908\n",
              "4  891327558926688256           9774           41048"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Q07gBThinwI",
        "colab_type": "text"
      },
      "source": [
        "# Assessing Data - eight (8) quality issues and two (2) tidiness issues "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QFQRgH_iiwjH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create a copy of all the extracted/downloaded data\n",
        "df_twitter_arch_clean = df_twitter_archive.copy()\n",
        "df_image_pred_clean = df_image_pred.copy()\n",
        "df_jdata_clean = df_jdata.copy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ECL02S3ci8-R",
        "colab_type": "code",
        "outputId": "bebcae4b-921c-45b7-cfe5-51dbeee0019d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 399
        }
      },
      "source": [
        "df_twitter_arch_clean.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 2356 entries, 0 to 2355\n",
            "Data columns (total 17 columns):\n",
            "tweet_id                      2356 non-null int64\n",
            "in_reply_to_status_id         78 non-null float64\n",
            "in_reply_to_user_id           78 non-null float64\n",
            "timestamp                     2356 non-null object\n",
            "source                        2356 non-null object\n",
            "text                          2356 non-null object\n",
            "retweeted_status_id           181 non-null float64\n",
            "retweeted_status_user_id      181 non-null float64\n",
            "retweeted_status_timestamp    181 non-null object\n",
            "expanded_urls                 2297 non-null object\n",
            "rating_numerator              2356 non-null int64\n",
            "rating_denominator            2356 non-null int64\n",
            "name                          2356 non-null object\n",
            "doggo                         2356 non-null object\n",
            "floofer                       2356 non-null object\n",
            "pupper                        2356 non-null object\n",
            "puppo                         2356 non-null object\n",
            "dtypes: float64(4), int64(3), object(10)\n",
            "memory usage: 313.0+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uxy272CFkxXT",
        "colab_type": "code",
        "outputId": "1ce328ec-7eb7-4575-cccc-e51b06d0d649",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 591
        }
      },
      "source": [
        "df_twitter_arch_clean.head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>in_reply_to_status_id</th>\n",
              "      <th>in_reply_to_user_id</th>\n",
              "      <th>timestamp</th>\n",
              "      <th>source</th>\n",
              "      <th>text</th>\n",
              "      <th>retweeted_status_id</th>\n",
              "      <th>retweeted_status_user_id</th>\n",
              "      <th>retweeted_status_timestamp</th>\n",
              "      <th>expanded_urls</th>\n",
              "      <th>rating_numerator</th>\n",
              "      <th>rating_denominator</th>\n",
              "      <th>name</th>\n",
              "      <th>doggo</th>\n",
              "      <th>floofer</th>\n",
              "      <th>pupper</th>\n",
              "      <th>puppo</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>892420643555336193</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-08-01 16:23:56 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Phineas</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>892177421306343426</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-08-01 00:17:27 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Tilly. She's just checking pup on you....</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Tilly</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>891815181378084864</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-07-31 00:18:03 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
              "      <td>12</td>\n",
              "      <td>10</td>\n",
              "      <td>Archie</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>891689557279858688</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-07-30 15:58:51 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Darla. She commenced a snooze mid meal...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Darla</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>891327558926688256</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-07-29 16:00:24 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Franklin. He would like you to stop ca...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
              "      <td>12</td>\n",
              "      <td>10</td>\n",
              "      <td>Franklin</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  in_reply_to_status_id  ...  pupper puppo\n",
              "0  892420643555336193                    NaN  ...    None  None\n",
              "1  892177421306343426                    NaN  ...    None  None\n",
              "2  891815181378084864                    NaN  ...    None  None\n",
              "3  891689557279858688                    NaN  ...    None  None\n",
              "4  891327558926688256                    NaN  ...    None  None\n",
              "\n",
              "[5 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SF-nsiyk_ZGy",
        "colab_type": "code",
        "outputId": "fbac5592-72bc-4bff-9cf9-a22b6da8a17c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# check if any of these tweets are re-tweeted. These need to be removed\n",
        "df_twitter_arch_clean.retweeted_status_id.notnull().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "181"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E4MEDpv0tK_O",
        "colab_type": "code",
        "outputId": "643f0e4f-499a-43c2-ae71-1e6a66e65848",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# On visual checks it was noticed that there were multiple tweets where the picture was not dogs.\n",
        "# It had the text, 'we only rate dogs' lets check how many do we have like this\n",
        "str_search = 'WE ONLY RATE DOGS'\n",
        "df_twitter_arch_clean['text_uppercase'] = df_twitter_arch_clean.text.str.upper()\n",
        "df_twitter_arch_clean[df_twitter_arch_clean['text_uppercase'].str.contains(str_search)].shape[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "63"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fh2PLAiitTr_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Drop the additional column\n",
        "df_twitter_arch_clean.drop('text_uppercase', axis=1, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "clADZHh3lq43",
        "colab_type": "code",
        "outputId": "74a8d344-edbb-4199-cca7-ab5420c5fb03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 556
        }
      },
      "source": [
        "# Rating numerator and denominator do not have null values \n",
        "# Are there any rows where the denominator is not 10\n",
        "df_temp = df_twitter_arch_clean[df_twitter_arch_clean['rating_denominator']!=10]\n",
        "df_temp.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>in_reply_to_status_id</th>\n",
              "      <th>in_reply_to_user_id</th>\n",
              "      <th>timestamp</th>\n",
              "      <th>source</th>\n",
              "      <th>text</th>\n",
              "      <th>retweeted_status_id</th>\n",
              "      <th>retweeted_status_user_id</th>\n",
              "      <th>retweeted_status_timestamp</th>\n",
              "      <th>expanded_urls</th>\n",
              "      <th>rating_numerator</th>\n",
              "      <th>rating_denominator</th>\n",
              "      <th>name</th>\n",
              "      <th>doggo</th>\n",
              "      <th>floofer</th>\n",
              "      <th>pupper</th>\n",
              "      <th>puppo</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>313</th>\n",
              "      <td>835246439529840640</td>\n",
              "      <td>8.352460e+17</td>\n",
              "      <td>26259576.0</td>\n",
              "      <td>2017-02-24 21:54:03 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>@jonnysun @Lin_Manuel ok jomny I know you're e...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>960</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>342</th>\n",
              "      <td>832088576586297345</td>\n",
              "      <td>8.320875e+17</td>\n",
              "      <td>30582082.0</td>\n",
              "      <td>2017-02-16 04:45:50 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>@docmisterio account started on 11/15/15</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>11</td>\n",
              "      <td>15</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>433</th>\n",
              "      <td>820690176645140481</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2017-01-15 17:52:40 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>The floofs have been released I repeat the flo...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://twitter.com/dog_rates/status/820690176...</td>\n",
              "      <td>84</td>\n",
              "      <td>70</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>516</th>\n",
              "      <td>810984652412424192</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2016-12-19 23:06:23 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>Meet Sam. She smiles 24/7 &amp;amp; secretly aspir...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://www.gofundme.com/sams-smile,https://tw...</td>\n",
              "      <td>24</td>\n",
              "      <td>7</td>\n",
              "      <td>Sam</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>784</th>\n",
              "      <td>775096608509886464</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2016-09-11 22:20:06 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>RT @dog_rates: After so many requests, this is...</td>\n",
              "      <td>7.403732e+17</td>\n",
              "      <td>4.196984e+09</td>\n",
              "      <td>2016-06-08 02:41:38 +0000</td>\n",
              "      <td>https://twitter.com/dog_rates/status/740373189...</td>\n",
              "      <td>9</td>\n",
              "      <td>11</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               tweet_id  in_reply_to_status_id  ...  pupper puppo\n",
              "313  835246439529840640           8.352460e+17  ...    None  None\n",
              "342  832088576586297345           8.320875e+17  ...    None  None\n",
              "433  820690176645140481                    NaN  ...    None  None\n",
              "516  810984652412424192                    NaN  ...    None  None\n",
              "784  775096608509886464                    NaN  ...    None  None\n",
              "\n",
              "[5 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Cvk64NhDXr6",
        "colab_type": "code",
        "outputId": "c6b5a1bf-523a-4e0b-d118-8bade772cfa1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "# Is this because the data is incorrect or due to the fact the data was extracted incorrectly. Check the actual tweet text\n",
        "df_temp[['tweet_id', 'text', 'rating_denominator', 'rating_numerator']].head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>text</th>\n",
              "      <th>rating_denominator</th>\n",
              "      <th>rating_numerator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>313</th>\n",
              "      <td>835246439529840640</td>\n",
              "      <td>@jonnysun @Lin_Manuel ok jomny I know you're e...</td>\n",
              "      <td>0</td>\n",
              "      <td>960</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>342</th>\n",
              "      <td>832088576586297345</td>\n",
              "      <td>@docmisterio account started on 11/15/15</td>\n",
              "      <td>15</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>433</th>\n",
              "      <td>820690176645140481</td>\n",
              "      <td>The floofs have been released I repeat the flo...</td>\n",
              "      <td>70</td>\n",
              "      <td>84</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>516</th>\n",
              "      <td>810984652412424192</td>\n",
              "      <td>Meet Sam. She smiles 24/7 &amp;amp; secretly aspir...</td>\n",
              "      <td>7</td>\n",
              "      <td>24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>784</th>\n",
              "      <td>775096608509886464</td>\n",
              "      <td>RT @dog_rates: After so many requests, this is...</td>\n",
              "      <td>11</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               tweet_id  ... rating_numerator\n",
              "313  835246439529840640  ...              960\n",
              "342  832088576586297345  ...               11\n",
              "433  820690176645140481  ...               84\n",
              "516  810984652412424192  ...               24\n",
              "784  775096608509886464  ...                9\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uQ7H2qcnEsk-",
        "colab_type": "text"
      },
      "source": [
        "Visual checks show that the data is not being picked up corectly from the text field. The field rating_denominator is incorrectly picked up. Needs to be fixed. This will impact rating_numerator as well."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dqFwnTr_jcmW",
        "colab_type": "code",
        "outputId": "8b113f48-bcf2-4a1f-a8f4-5666f0f38546",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        }
      },
      "source": [
        "# In case we are picking up the value from the text, lets check if there are any decimal values\n",
        "df_twitter_arch_clean[df_twitter_arch_clean.text.str.contains(r\"(\\d+\\.\\d*\\/\\d+)\")][['text', 'rating_numerator', 'rating_denominator']]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: UserWarning: This pattern has match groups. To actually get the groups, use str.extract.\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>rating_numerator</th>\n",
              "      <th>rating_denominator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>This is Bella. She hopes her smile made you sm...</td>\n",
              "      <td>5</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>340</th>\n",
              "      <td>RT @dog_rates: This is Logan, the Chow who liv...</td>\n",
              "      <td>75</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>695</th>\n",
              "      <td>This is Logan, the Chow who lived. He solemnly...</td>\n",
              "      <td>75</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>763</th>\n",
              "      <td>This is Sophie. She's a Jubilant Bush Pupper. ...</td>\n",
              "      <td>27</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1689</th>\n",
              "      <td>I've been told there's a slight possibility he...</td>\n",
              "      <td>5</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1712</th>\n",
              "      <td>Here we have uncovered an entire battalion of ...</td>\n",
              "      <td>26</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  ...  rating_denominator\n",
              "45    This is Bella. She hopes her smile made you sm...  ...                  10\n",
              "340   RT @dog_rates: This is Logan, the Chow who liv...  ...                  10\n",
              "695   This is Logan, the Chow who lived. He solemnly...  ...                  10\n",
              "763   This is Sophie. She's a Jubilant Bush Pupper. ...  ...                  10\n",
              "1689  I've been told there's a slight possibility he...  ...                  10\n",
              "1712  Here we have uncovered an entire battalion of ...  ...                  10\n",
              "\n",
              "[6 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jt9FRtjZoRbP",
        "colab_type": "text"
      },
      "source": [
        "On visually checking there are decimal values. The numerator column should be a float type"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8f2Z2UFFQo2",
        "colab_type": "code",
        "outputId": "1b4cc9dc-c92d-4b58-8d67-dbcec2048723",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 729
        }
      },
      "source": [
        "# Check for the values in rating_numerator as well\n",
        "df_twitter_arch_clean.rating_numerator.value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "12      558\n",
              "11      464\n",
              "10      461\n",
              "13      351\n",
              "9       158\n",
              "8       102\n",
              "7        55\n",
              "14       54\n",
              "5        37\n",
              "6        32\n",
              "3        19\n",
              "4        17\n",
              "1         9\n",
              "2         9\n",
              "420       2\n",
              "0         2\n",
              "15        2\n",
              "75        2\n",
              "80        1\n",
              "20        1\n",
              "24        1\n",
              "26        1\n",
              "44        1\n",
              "50        1\n",
              "60        1\n",
              "165       1\n",
              "84        1\n",
              "88        1\n",
              "144       1\n",
              "182       1\n",
              "143       1\n",
              "666       1\n",
              "960       1\n",
              "1776      1\n",
              "17        1\n",
              "27        1\n",
              "45        1\n",
              "99        1\n",
              "121       1\n",
              "204       1\n",
              "Name: rating_numerator, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "udcsxbGwFwBE",
        "colab_type": "text"
      },
      "source": [
        "There are values like 420, 0, 165 etc. lets check the actual text values for these tweets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y5Xe9InTF6kY",
        "colab_type": "code",
        "outputId": "5cf37d0d-7db8-4b20-9a3b-e6fb59eecad3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "# Check for tweet texts where the numerator is 0 or greater than 20\n",
        "df_temp = df_twitter_arch_clean[(df_twitter_arch_clean['rating_numerator']==0) | (df_twitter_arch_clean['rating_numerator']>20)]\n",
        "df_temp[['tweet_id', 'text', 'rating_numerator']].head(5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>text</th>\n",
              "      <th>rating_numerator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>188</th>\n",
              "      <td>855862651834028034</td>\n",
              "      <td>@dhmontgomery We also gave snoop dogg a 420/10...</td>\n",
              "      <td>420</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>189</th>\n",
              "      <td>855860136149123072</td>\n",
              "      <td>@s8n You tried very hard to portray this good ...</td>\n",
              "      <td>666</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>290</th>\n",
              "      <td>838150277551247360</td>\n",
              "      <td>@markhoppus 182/10</td>\n",
              "      <td>182</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>313</th>\n",
              "      <td>835246439529840640</td>\n",
              "      <td>@jonnysun @Lin_Manuel ok jomny I know you're e...</td>\n",
              "      <td>960</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>315</th>\n",
              "      <td>835152434251116546</td>\n",
              "      <td>When you're so blinded by your systematic plag...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               tweet_id  ... rating_numerator\n",
              "188  855862651834028034  ...              420\n",
              "189  855860136149123072  ...              666\n",
              "290  838150277551247360  ...              182\n",
              "313  835246439529840640  ...              960\n",
              "315  835152434251116546  ...                0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lOAX18m_GuAl",
        "colab_type": "text"
      },
      "source": [
        "On checking these values visually as well, seems like these are valid values. No change required."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "071mBOjkMiJJ",
        "colab_type": "code",
        "outputId": "367cc450-3e60-4845-8a09-cc2a062b121f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# check for any duplicate rows in the dataframe\n",
        "df_twitter_arch_clean.duplicated().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z0p5e2h0O3v3",
        "colab_type": "code",
        "outputId": "550b43c0-f965-4c48-c375-67bf76ea089d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "# check for the vales in the dog stage \n",
        "df_twitter_arch_clean.doggo.unique(), df_twitter_arch_clean.floofer.unique(), df_twitter_arch_clean.pupper.unique(), df_twitter_arch_clean.puppo.unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['None', 'doggo'], dtype=object),\n",
              " array(['None', 'floofer'], dtype=object),\n",
              " array(['None', 'pupper'], dtype=object),\n",
              " array(['None', 'puppo'], dtype=object))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yt2c5hDAQLj2",
        "colab_type": "code",
        "outputId": "6ed52889-a4d6-4c44-9794-8f9d06cad84b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "source": [
        "# check to see if they are mutually unique\n",
        "df_twitter_arch_clean['combined_puppy_stage'] = (df_twitter_arch_clean.doggo + df_twitter_arch_clean.floofer + df_twitter_arch_clean.pupper + df_twitter_arch_clean.puppo)\n",
        "df_twitter_arch_clean['combined_puppy_stage'] = df_twitter_arch_clean['combined_puppy_stage'].str.replace('None', '')\n",
        "df_twitter_arch_clean.combined_puppy_stage.value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                1976\n",
              "pupper           245\n",
              "doggo             83\n",
              "puppo             29\n",
              "doggopupper       12\n",
              "floofer            9\n",
              "doggopuppo         1\n",
              "doggofloofer       1\n",
              "Name: combined_puppy_stage, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PTMCS7ndcii-",
        "colab_type": "code",
        "outputId": "c3cee3bf-d14e-4d89-bb80-2ca1fe8cbf32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Check the dog name column\n",
        "df_twitter_arch_clean.name.value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "None         745\n",
              "a             55\n",
              "Charlie       12\n",
              "Lucy          11\n",
              "Oliver        11\n",
              "Cooper        11\n",
              "Penny         10\n",
              "Lola          10\n",
              "Tucker        10\n",
              "Winston        9\n",
              "Bo             9\n",
              "Sadie          8\n",
              "the            8\n",
              "Toby           7\n",
              "Daisy          7\n",
              "Bailey         7\n",
              "an             7\n",
              "Buddy          7\n",
              "Jax            6\n",
              "Dave           6\n",
              "Jack           6\n",
              "Scout          6\n",
              "Leo            6\n",
              "Stanley        6\n",
              "Rusty          6\n",
              "Koda           6\n",
              "Oscar          6\n",
              "Bella          6\n",
              "Milo           6\n",
              "Chester        5\n",
              "            ... \n",
              "Jaycob         1\n",
              "Biden          1\n",
              "Vixen          1\n",
              "Sandra         1\n",
              "Jimbo          1\n",
              "Oreo           1\n",
              "Katie          1\n",
              "Taz            1\n",
              "Mo             1\n",
              "Zuzu           1\n",
              "Berkeley       1\n",
              "space          1\n",
              "Lillie         1\n",
              "Venti          1\n",
              "Zoe            1\n",
              "Bayley         1\n",
              "Keet           1\n",
              "Willem         1\n",
              "Ivar           1\n",
              "Dietrich       1\n",
              "Yukon          1\n",
              "Patch          1\n",
              "Laela          1\n",
              "Wishes         1\n",
              "Mutt           1\n",
              "Callie         1\n",
              "Tove           1\n",
              "Kobe           1\n",
              "Grady          1\n",
              "Jebberson      1\n",
              "Name: name, Length: 957, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JN0B9O_wMRIr",
        "colab_type": "text"
      },
      "source": [
        "Are names like all, None, an valid values - lets do a visual check.\n",
        "On visually checking, \n",
        "None - there does not seem to be a name in these tweets, seems okay. Should be set to Blank\n",
        "a, all, the, actually - again here it seems to be incorrectly extracted. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tMPvQ9vdVkly",
        "colab_type": "code",
        "outputId": "1ed40507-0b73-4277-e05e-ebed2b9214a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 469
        }
      },
      "source": [
        "# It seems that the dog names starting with a lower case are incorrect. Lets see some of those values\n",
        "df_twitter_arch_clean[df_twitter_arch_clean['name'].str[0].str.islower()]['name'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "a               55\n",
              "the              8\n",
              "an               7\n",
              "very             5\n",
              "quite            4\n",
              "one              4\n",
              "just             4\n",
              "not              2\n",
              "actually         2\n",
              "mad              2\n",
              "getting          2\n",
              "his              1\n",
              "infuriating      1\n",
              "unacceptable     1\n",
              "officially       1\n",
              "space            1\n",
              "old              1\n",
              "incredibly       1\n",
              "life             1\n",
              "light            1\n",
              "my               1\n",
              "such             1\n",
              "by               1\n",
              "all              1\n",
              "this             1\n",
              "Name: name, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CiVTG-TlTh6h",
        "colab_type": "code",
        "outputId": "2089c930-e7ae-4a8c-b6c2-81ec0ced59f5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        }
      },
      "source": [
        "# Lets check the Image prediction dataframe\n",
        "df_image_pred_clean.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 2075 entries, 0 to 2074\n",
            "Data columns (total 12 columns):\n",
            "tweet_id    2075 non-null int64\n",
            "jpg_url     2075 non-null object\n",
            "img_num     2075 non-null int64\n",
            "p1          2075 non-null object\n",
            "p1_conf     2075 non-null float64\n",
            "p1_dog      2075 non-null bool\n",
            "p2          2075 non-null object\n",
            "p2_conf     2075 non-null float64\n",
            "p2_dog      2075 non-null bool\n",
            "p3          2075 non-null object\n",
            "p3_conf     2075 non-null float64\n",
            "p3_dog      2075 non-null bool\n",
            "dtypes: bool(3), float64(3), int64(2), object(4)\n",
            "memory usage: 152.1+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VVSRS44ZUnTL",
        "colab_type": "code",
        "outputId": "3f8e3a23-a22e-4996-a33d-ad7738618d6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#check for any duplicates, since there are absolutely no null entries\n",
        "df_image_pred_clean.duplicated().sum(), df_image_pred_clean.tweet_id.duplicated().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0, 0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChPJAz6vUtiZ",
        "colab_type": "code",
        "outputId": "1de881dd-fcd7-4d13-b252-db5158309561",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        }
      },
      "source": [
        "df_image_pred_clean.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>jpg_url</th>\n",
              "      <th>img_num</th>\n",
              "      <th>p1</th>\n",
              "      <th>p1_conf</th>\n",
              "      <th>p1_dog</th>\n",
              "      <th>p2</th>\n",
              "      <th>p2_conf</th>\n",
              "      <th>p2_dog</th>\n",
              "      <th>p3</th>\n",
              "      <th>p3_conf</th>\n",
              "      <th>p3_dog</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>666020888022790149</td>\n",
              "      <td>https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Welsh_springer_spaniel</td>\n",
              "      <td>0.465074</td>\n",
              "      <td>True</td>\n",
              "      <td>collie</td>\n",
              "      <td>0.156665</td>\n",
              "      <td>True</td>\n",
              "      <td>Shetland_sheepdog</td>\n",
              "      <td>0.061428</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>666029285002620928</td>\n",
              "      <td>https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>redbone</td>\n",
              "      <td>0.506826</td>\n",
              "      <td>True</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.074192</td>\n",
              "      <td>True</td>\n",
              "      <td>Rhodesian_ridgeback</td>\n",
              "      <td>0.072010</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>666033412701032449</td>\n",
              "      <td>https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>German_shepherd</td>\n",
              "      <td>0.596461</td>\n",
              "      <td>True</td>\n",
              "      <td>malinois</td>\n",
              "      <td>0.138584</td>\n",
              "      <td>True</td>\n",
              "      <td>bloodhound</td>\n",
              "      <td>0.116197</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>666044226329800704</td>\n",
              "      <td>https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Rhodesian_ridgeback</td>\n",
              "      <td>0.408143</td>\n",
              "      <td>True</td>\n",
              "      <td>redbone</td>\n",
              "      <td>0.360687</td>\n",
              "      <td>True</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.222752</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>666049248165822465</td>\n",
              "      <td>https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.560311</td>\n",
              "      <td>True</td>\n",
              "      <td>Rottweiler</td>\n",
              "      <td>0.243682</td>\n",
              "      <td>True</td>\n",
              "      <td>Doberman</td>\n",
              "      <td>0.154629</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  ... p3_dog\n",
              "0  666020888022790149  ...   True\n",
              "1  666029285002620928  ...   True\n",
              "2  666033412701032449  ...   True\n",
              "3  666044226329800704  ...   True\n",
              "4  666049248165822465  ...   True\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b7rgNFjJYs1j",
        "colab_type": "code",
        "outputId": "bd6a6190-4897-4277-adb2-ef443128815b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "df_image_pred_clean.p1.value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "golden_retriever             150\n",
              "Labrador_retriever           100\n",
              "Pembroke                      89\n",
              "Chihuahua                     83\n",
              "pug                           57\n",
              "chow                          44\n",
              "Samoyed                       43\n",
              "toy_poodle                    39\n",
              "Pomeranian                    38\n",
              "cocker_spaniel                30\n",
              "malamute                      30\n",
              "French_bulldog                26\n",
              "miniature_pinscher            23\n",
              "Chesapeake_Bay_retriever      23\n",
              "seat_belt                     22\n",
              "Siberian_husky                20\n",
              "German_shepherd               20\n",
              "Staffordshire_bullterrier     20\n",
              "Cardigan                      19\n",
              "web_site                      19\n",
              "teddy                         18\n",
              "Maltese_dog                   18\n",
              "Shetland_sheepdog             18\n",
              "Eskimo_dog                    18\n",
              "beagle                        18\n",
              "Lakeland_terrier              17\n",
              "Rottweiler                    17\n",
              "Shih-Tzu                      17\n",
              "Italian_greyhound             16\n",
              "kuvasz                        16\n",
              "                            ... \n",
              "bow                            1\n",
              "basketball                     1\n",
              "teapot                         1\n",
              "water_buffalo                  1\n",
              "hay                            1\n",
              "cuirass                        1\n",
              "fountain                       1\n",
              "ping-pong_ball                 1\n",
              "ice_lolly                      1\n",
              "grille                         1\n",
              "military_uniform               1\n",
              "agama                          1\n",
              "desktop_computer               1\n",
              "African_hunting_dog            1\n",
              "convertible                    1\n",
              "piggy_bank                     1\n",
              "king_penguin                   1\n",
              "bonnet                         1\n",
              "dining_table                   1\n",
              "nail                           1\n",
              "hare                           1\n",
              "pitcher                        1\n",
              "carousel                       1\n",
              "maillot                        1\n",
              "espresso                       1\n",
              "candle                         1\n",
              "rain_barrel                    1\n",
              "ibex                           1\n",
              "Scotch_terrier                 1\n",
              "trombone                       1\n",
              "Name: p1, Length: 378, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tCzf7jnda7Yn",
        "colab_type": "code",
        "outputId": "0968c097-6562-4810-9de1-422c3c6fe06e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        }
      },
      "source": [
        "df_image_pred_clean.tail()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>jpg_url</th>\n",
              "      <th>img_num</th>\n",
              "      <th>p1</th>\n",
              "      <th>p1_conf</th>\n",
              "      <th>p1_dog</th>\n",
              "      <th>p2</th>\n",
              "      <th>p2_conf</th>\n",
              "      <th>p2_dog</th>\n",
              "      <th>p3</th>\n",
              "      <th>p3_conf</th>\n",
              "      <th>p3_dog</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2070</th>\n",
              "      <td>891327558926688256</td>\n",
              "      <td>https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg</td>\n",
              "      <td>2</td>\n",
              "      <td>basset</td>\n",
              "      <td>0.555712</td>\n",
              "      <td>True</td>\n",
              "      <td>English_springer</td>\n",
              "      <td>0.225770</td>\n",
              "      <td>True</td>\n",
              "      <td>German_short-haired_pointer</td>\n",
              "      <td>0.175219</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2071</th>\n",
              "      <td>891689557279858688</td>\n",
              "      <td>https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>paper_towel</td>\n",
              "      <td>0.170278</td>\n",
              "      <td>False</td>\n",
              "      <td>Labrador_retriever</td>\n",
              "      <td>0.168086</td>\n",
              "      <td>True</td>\n",
              "      <td>spatula</td>\n",
              "      <td>0.040836</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2072</th>\n",
              "      <td>891815181378084864</td>\n",
              "      <td>https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Chihuahua</td>\n",
              "      <td>0.716012</td>\n",
              "      <td>True</td>\n",
              "      <td>malamute</td>\n",
              "      <td>0.078253</td>\n",
              "      <td>True</td>\n",
              "      <td>kelpie</td>\n",
              "      <td>0.031379</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2073</th>\n",
              "      <td>892177421306343426</td>\n",
              "      <td>https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Chihuahua</td>\n",
              "      <td>0.323581</td>\n",
              "      <td>True</td>\n",
              "      <td>Pekinese</td>\n",
              "      <td>0.090647</td>\n",
              "      <td>True</td>\n",
              "      <td>papillon</td>\n",
              "      <td>0.068957</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2074</th>\n",
              "      <td>892420643555336193</td>\n",
              "      <td>https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>orange</td>\n",
              "      <td>0.097049</td>\n",
              "      <td>False</td>\n",
              "      <td>bagel</td>\n",
              "      <td>0.085851</td>\n",
              "      <td>False</td>\n",
              "      <td>banana</td>\n",
              "      <td>0.076110</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                tweet_id  ... p3_dog\n",
              "2070  891327558926688256  ...   True\n",
              "2071  891689557279858688  ...  False\n",
              "2072  891815181378084864  ...   True\n",
              "2073  892177421306343426  ...   True\n",
              "2074  892420643555336193  ...  False\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RPAGx9AHbhPo",
        "colab_type": "code",
        "outputId": "5a4dfd24-8adf-43ae-e8bf-e1c23c75dd00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "# Json data file\n",
        "df_jdata_clean.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 2175 entries, 0 to 2174\n",
            "Data columns (total 3 columns):\n",
            "tweet_id          2175 non-null int64\n",
            "retweet_count     2175 non-null int64\n",
            "favorite_count    2175 non-null int64\n",
            "dtypes: int64(3)\n",
            "memory usage: 51.1 KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DZp6m96zk6Dm",
        "colab_type": "text"
      },
      "source": [
        "# Data Quality and Tidiness Issues \n",
        "\n",
        "- DataFrame for Archived Twitter data\n",
        "\n",
        "      1) Quality Issue - Base on the column retweeted_status_id, we noticed that there are around 181 retweets in this dataset. This should be removed.\n",
        "\n",
        "      2) Quality Issue - There were tweets with the Text 'We only rate dogs'. These tweets should be excluded from our analysis.\n",
        "\n",
        "      3) Quality Issue - There are rows where the rating_denominator is not 10. On checking visually for these rows, it seems that they were not parsed correctly from the text. Re-extract the Rating denominator from the text\n",
        "\n",
        "      4) Quality Issue - For the rows where rating_denominator is incorrect, it was also noticed that the rating_numerator is incorrect. This needs to be correctly extracted. \n",
        "\n",
        "      5) Tidiness Issue - Currently the rating_numerator and rating_denominator columns are type objects which would be string. We also noticed that some numerator values could be decimals. Change numerator to float and denominator to int\n",
        "\n",
        "      6) Tidiness Issue - The puppy stage columns could have been consolidated to have a categorical value. However, there are 2 rows which have both values. For these 2, concatenate using ','.\n",
        "      \n",
        "      7) Quality Issue - The puppy stage columns have text like 'None' Can be fixed as Blank.\n",
        "\n",
        "      8) Quality Issue - The timestamp column has additional details like +0000 which does not look necessary/correct. \n",
        "      \n",
        "      9) Tidiness Issue - The timestamp, retweeted_status_timestamp columns are object instead of datetime. Change timestamp to datetime type\n",
        "\n",
        "      10) Tidiness Issue - in_reply_to_status_id, in_reply_to_user_id expanded_urls contains null values. These can be excluded from analysis.\n",
        "\n",
        "      11) Quality Issue - The dog name column has 'None'. Should be Blank instead\n",
        "\n",
        "      12) Quality Issue - Are names like all, None, an valid values. On visually checking, For values where it starts with a lowercase letter: a, all, the, actually - again here it seems to be incorrectly extracted. However on visually checking these records, it seems that there are no names in these tweets. Reset these values to Blank\n",
        "      \n",
        "- DataFrame for Image Prediction data\n",
        "\n",
        "      13) Quality Issue - There are 2356 entries in the Twitter Archived dataframe where as here there are only 2075 entries. We are missing approximately around 300 records at the least not considering any mismatches. Since these are missing, they cannot be merged back at the moment.\n",
        "      \n",
        "      14) Tidiness Issue - The predicted p1, p2 and p3 have values which are not dogs. Relevant data would be to pick the prediction which is actually a dog with the highest probability. Create a scoring system to ascertain the best probaility of a dog\n",
        "\n",
        "- DataFrame for Json data\n",
        "\n",
        "      15) This is the filtered list of tweets which are not re-tweets. This will be the source list for the rest of the data when combined together\n",
        "      \n",
        "16) Tidiness Issue - Pull in the data from Image Prediction and Json data to the twitter archive dataframe for analysis\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dvV1Ndy-j1kM",
        "colab_type": "text"
      },
      "source": [
        "# Cleaning the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZpRW59pNgVti",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEFINE: Issue 1: Quality Issue - Base on the column retweeted_status_id, we noticed that there are around 181 retweets in this dataset. This should be removed\n",
        "# CLEAN: Issue 1:\n",
        "df_twitter_arch_clean = df_twitter_arch_clean[df_twitter_arch_clean.retweeted_status_id.isnull()]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yCW3SayeuCA0",
        "colab_type": "code",
        "outputId": "3f2a2999-ad0e-4808-f1cf-08779c7d8a26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# TEST CLEANED: Issue 1\n",
        "df_twitter_arch_clean.retweeted_status_id.notnull().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RqWS6N40tBHO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEFINE: Issue 2: Quality Issue - On visual checks it was noticed that there were multiple tweets where the picture was not dogs.\n",
        "#                   It had the text, 'we only rate dogs'. Remove these rows\n",
        "# CLEAN: Issue 2:\n",
        "str_search = 'WE ONLY RATE DOGS'\n",
        "df_twitter_arch_clean['text_uppercase'] = df_twitter_arch_clean.text.str.upper()\n",
        "df_twitter_arch_clean = df_twitter_arch_clean[~df_twitter_arch_clean['text_uppercase'].str.contains(str_search)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o1HIajnVt-VB",
        "colab_type": "code",
        "outputId": "57c2c45f-d53c-448c-a86e-c04b43b115f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# TEST CLEANED: Issue 2\n",
        "df_twitter_arch_clean[df_twitter_arch_clean['text_uppercase'].str.contains(str_search)].shape[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3L0czhnih2TV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEFINE: \n",
        "\n",
        "# Issue 3: Quality Issue - There are rows where the rating_denominator is not 10. On checking visually for these rows, it seems that they were not parsed correctly from the text. \n",
        "#           Re-extract the Rating denominator from the text\n",
        "\n",
        "# Issue 4: Quality Issue - For the rows where rating_denominator is incorrect, it was also noticed that the rating_numerator is incorrect. This needs to be correctly extracted.\n",
        "\n",
        "# Issue 5: Quality/Tidiness Issue - Currently the rating_numerator and rating_denominator columns are type objects which would be string. \n",
        "#           We also noticed that some numerator values could be decimals. Add clean columns for these two of type float\n",
        "\n",
        "# CLEAN: Issue 3, 4, 5:\n",
        "# For each row, get the twitter text, Split by pattern %d/%d, take the one which has 10 as the denominator\n",
        "df_twitter_arch_clean['numerator_clean'] = 0.0\n",
        "df_twitter_arch_clean['denominator_clean'] = 0.0\n",
        "df_twitter_arch_clean.numerator_clean.astype(float)\n",
        "df_twitter_arch_clean.denominator_clean.astype(float)\n",
        "\n",
        "for index, row in df_twitter_arch_clean.iterrows():\n",
        "  s_split = re.findall('\\d+\\.*\\d*/\\d+', row.text) # this pattern should give us both decimal and non-decimal values\n",
        "  for s in s_split:\n",
        "    num = float(s.split('/')[0])\n",
        "    denom = float(s.split('/')[1])\n",
        "    if(denom==10):\n",
        "      df_twitter_arch_clean.loc[index,'numerator_clean'] = num\n",
        "      df_twitter_arch_clean.loc[index,'denominator_clean'] = denom\n",
        "      break\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UhE4A_nJrebX",
        "colab_type": "code",
        "outputId": "cf716441-d4e8-4868-d0bf-0332800cd175",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "# TEST CLEANED: \n",
        "# Issue 3 and 4\n",
        "df_twitter_arch_clean[df_twitter_arch_clean['rating_denominator']!=10][['tweet_id', 'text', 'rating_numerator', 'rating_denominator', 'numerator_clean', 'denominator_clean']].head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>text</th>\n",
              "      <th>rating_numerator</th>\n",
              "      <th>rating_denominator</th>\n",
              "      <th>numerator_clean</th>\n",
              "      <th>denominator_clean</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>313</th>\n",
              "      <td>835246439529840640</td>\n",
              "      <td>@jonnysun @Lin_Manuel ok jomny I know you're e...</td>\n",
              "      <td>960</td>\n",
              "      <td>0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>342</th>\n",
              "      <td>832088576586297345</td>\n",
              "      <td>@docmisterio account started on 11/15/15</td>\n",
              "      <td>11</td>\n",
              "      <td>15</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>433</th>\n",
              "      <td>820690176645140481</td>\n",
              "      <td>The floofs have been released I repeat the flo...</td>\n",
              "      <td>84</td>\n",
              "      <td>70</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>516</th>\n",
              "      <td>810984652412424192</td>\n",
              "      <td>Meet Sam. She smiles 24/7 &amp;amp; secretly aspir...</td>\n",
              "      <td>24</td>\n",
              "      <td>7</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>902</th>\n",
              "      <td>758467244762497024</td>\n",
              "      <td>Why does this never happen at my front door......</td>\n",
              "      <td>165</td>\n",
              "      <td>150</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               tweet_id  ... denominator_clean\n",
              "313  835246439529840640  ...              10.0\n",
              "342  832088576586297345  ...               0.0\n",
              "433  820690176645140481  ...               0.0\n",
              "516  810984652412424192  ...               0.0\n",
              "902  758467244762497024  ...               0.0\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y9a1axZpuRL9",
        "colab_type": "code",
        "outputId": "64a25604-d7a9-4bc5-8d5a-b2e63f291a1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 551
        }
      },
      "source": [
        "# Check why there are 0 value denominators\n",
        "df_twitter_arch_clean[df_twitter_arch_clean.denominator_clean==0][['tweet_id', 'text']]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>342</th>\n",
              "      <td>832088576586297345</td>\n",
              "      <td>@docmisterio account started on 11/15/15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>433</th>\n",
              "      <td>820690176645140481</td>\n",
              "      <td>The floofs have been released I repeat the flo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>516</th>\n",
              "      <td>810984652412424192</td>\n",
              "      <td>Meet Sam. She smiles 24/7 &amp;amp; secretly aspir...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>902</th>\n",
              "      <td>758467244762497024</td>\n",
              "      <td>Why does this never happen at my front door......</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1120</th>\n",
              "      <td>731156023742988288</td>\n",
              "      <td>Say hello to this unbelievably well behaved sq...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1228</th>\n",
              "      <td>713900603437621249</td>\n",
              "      <td>Happy Saturday here's 9 puppers on a bench. 99...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1254</th>\n",
              "      <td>710658690886586372</td>\n",
              "      <td>Here's a brigade of puppers. All look very pre...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1274</th>\n",
              "      <td>709198395643068416</td>\n",
              "      <td>From left to right:\\nCletus, Jerome, Alejandro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1351</th>\n",
              "      <td>704054845121142784</td>\n",
              "      <td>Here is a whole flock of puppers.  60/50 I'll ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1433</th>\n",
              "      <td>697463031882764288</td>\n",
              "      <td>Happy Wednesday here's a bucket of pups. 44/40...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1598</th>\n",
              "      <td>686035780142297088</td>\n",
              "      <td>Yes I do realize a rating of 4/20 would've bee...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1634</th>\n",
              "      <td>684225744407494656</td>\n",
              "      <td>Two sneaky puppers were not initially seen, mo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1635</th>\n",
              "      <td>684222868335505415</td>\n",
              "      <td>Someone help the girl is being mugged. Several...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1663</th>\n",
              "      <td>682808988178739200</td>\n",
              "      <td>I'm aware that I could've said 20/16, but here...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1779</th>\n",
              "      <td>677716515794329600</td>\n",
              "      <td>IT'S PUPPERGEDDON. Total of 144/120 ...I think...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1843</th>\n",
              "      <td>675853064436391936</td>\n",
              "      <td>Here we have an entire platoon of puppers. Tot...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                tweet_id                                               text\n",
              "342   832088576586297345           @docmisterio account started on 11/15/15\n",
              "433   820690176645140481  The floofs have been released I repeat the flo...\n",
              "516   810984652412424192  Meet Sam. She smiles 24/7 &amp; secretly aspir...\n",
              "902   758467244762497024  Why does this never happen at my front door......\n",
              "1120  731156023742988288  Say hello to this unbelievably well behaved sq...\n",
              "1228  713900603437621249  Happy Saturday here's 9 puppers on a bench. 99...\n",
              "1254  710658690886586372  Here's a brigade of puppers. All look very pre...\n",
              "1274  709198395643068416  From left to right:\\nCletus, Jerome, Alejandro...\n",
              "1351  704054845121142784  Here is a whole flock of puppers.  60/50 I'll ...\n",
              "1433  697463031882764288  Happy Wednesday here's a bucket of pups. 44/40...\n",
              "1598  686035780142297088  Yes I do realize a rating of 4/20 would've bee...\n",
              "1634  684225744407494656  Two sneaky puppers were not initially seen, mo...\n",
              "1635  684222868335505415  Someone help the girl is being mugged. Several...\n",
              "1663  682808988178739200  I'm aware that I could've said 20/16, but here...\n",
              "1779  677716515794329600  IT'S PUPPERGEDDON. Total of 144/120 ...I think...\n",
              "1843  675853064436391936  Here we have an entire platoon of puppers. Tot..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UHPT-GIkz8Wp",
        "colab_type": "text"
      },
      "source": [
        "For the ones that are blank in our extract, on doing a visual check, we noticed some them indeed had valid values. However, some of them did not. For eg. 204/170 is a valid rating. However, text 'account started on 11/15/15' does not give us the right information. Some of them had 50/50 and 13/10 in the same text and the other had 45/50. There was no pattern to distinguish beteween this. For now, we are considering only ratings with a denominator of 10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C0B3jl1NCIwm",
        "colab_type": "code",
        "outputId": "689fdb57-6580-48cb-b984-796ac4f3c616",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 469
        }
      },
      "source": [
        "# TEST CLEANED: \n",
        "# Issue 5 - Check the column types added\n",
        "df_twitter_arch_clean.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 2121 entries, 0 to 2355\n",
            "Data columns (total 21 columns):\n",
            "tweet_id                      2121 non-null int64\n",
            "in_reply_to_status_id         77 non-null float64\n",
            "in_reply_to_user_id           77 non-null float64\n",
            "timestamp                     2121 non-null object\n",
            "source                        2121 non-null object\n",
            "text                          2121 non-null object\n",
            "retweeted_status_id           0 non-null float64\n",
            "retweeted_status_user_id      0 non-null float64\n",
            "retweeted_status_timestamp    0 non-null object\n",
            "expanded_urls                 2063 non-null object\n",
            "rating_numerator              2121 non-null int64\n",
            "rating_denominator            2121 non-null int64\n",
            "name                          2121 non-null object\n",
            "doggo                         2121 non-null object\n",
            "floofer                       2121 non-null object\n",
            "pupper                        2121 non-null object\n",
            "puppo                         2121 non-null object\n",
            "combined_puppy_stage          2121 non-null object\n",
            "text_uppercase                2121 non-null object\n",
            "numerator_clean               2121 non-null float64\n",
            "denominator_clean             2121 non-null float64\n",
            "dtypes: float64(6), int64(3), object(12)\n",
            "memory usage: 444.5+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlXSyWKiOxUg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#DEFINE \n",
        "\n",
        "# Issue 6: Tidiness Issue - The puppy stage columns could have been consolidated to have a categorical value. However, there are 2 rows which have both values. For these 2, concatenate using ','.\n",
        " \n",
        "# Issue 7: Quality Issue - The puppy stage columns have text like 'None' Can be fixed as Blank.\n",
        "\n",
        "# CLEAN Issue 7:\n",
        "#The puppy stage columns have text like 'None' Can be fixed as Blank.\n",
        "df_twitter_arch_clean.doggo.replace('None', '', inplace=True)\n",
        "df_twitter_arch_clean.floofer.replace('None', '', inplace=True)\n",
        "df_twitter_arch_clean.pupper.replace('None', '', inplace=True)\n",
        "df_twitter_arch_clean.puppo.replace('None', '', inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2CgubmbTNf2",
        "colab_type": "code",
        "outputId": "df189121-a58f-4125-85cc-53e36c01580c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# TEST Issue 7\n",
        "df_twitter_arch_clean.doggo.str.contains('None').sum(),df_twitter_arch_clean.floofer.str.contains('None').sum(), df_twitter_arch_clean.pupper.str.contains('None').sum(), df_twitter_arch_clean.puppo.str.contains('None').sum() "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0, 0, 0, 0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yJb8_zNLJatH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# CLEAN Issue 6\n",
        "# The puppy stage columns could have been consolidated to have a categorical value. However, there are 2 rows which have both values. For these 2, concatenate.\n",
        "df_twitter_arch_clean['combined_puppy_stage'] = ''\n",
        "for index, row in df_twitter_arch_clean.iterrows():\n",
        "  doggo, floofer, pupper, puppo = row.doggo, row.floofer, row.pupper, row.puppo\n",
        "  str_dogstage = ''\n",
        "  if(doggo!=''):\n",
        "    str_dogstage=doggo\n",
        "  if(floofer!=''):\n",
        "    if(str_dogstage==''):\n",
        "      str_dogstage = floofer\n",
        "    else:\n",
        "      str_dogstage = str_dogstage + ',' + floofer\n",
        "  if(pupper!=''):\n",
        "    if(str_dogstage==''):\n",
        "      str_dogstage = pupper\n",
        "    else:\n",
        "      str_dogstage = str_dogstage + ',' + pupper\n",
        "  if(puppo!=''):\n",
        "    if(str_dogstage==''):\n",
        "      str_dogstage = puppo\n",
        "    else:\n",
        "      str_dogstage = str_dogstage + ',' + puppo\n",
        "\n",
        "  df_twitter_arch_clean.loc[index,'combined_puppy_stage']=str_dogstage"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "anK70oZNVgrZ",
        "colab_type": "code",
        "outputId": "8c4555bd-f735-4f93-97eb-211d0fde8921",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "source": [
        "# TEST Issue 6\n",
        "# Check the value counts\n",
        "df_twitter_arch_clean['combined_puppy_stage'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                 1777\n",
              "pupper            224\n",
              "doggo              75\n",
              "puppo              24\n",
              "doggo,pupper       10\n",
              "floofer             9\n",
              "doggo,puppo         1\n",
              "doggo,floofer       1\n",
              "Name: combined_puppy_stage, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pO5JB4eeVsNt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEFINE \n",
        "# Issue 8 Quality Issue - The timestamp column has additional details like +0000 which does not look necessary/correct. \n",
        " \n",
        "# Issue 9 Tidiness Issue - The timestamp, retweeted_status_timestamp columns are object instead of datetime. Change timestamp to datetime type\n",
        "\n",
        "#CLEAN Issues 8 and 9\n",
        "df_twitter_arch_clean['timestamp_clean'] = pd.to_datetime(df_twitter_arch_clean['timestamp'], errors='coerce')\n",
        "df_twitter_arch_clean['retweeted_status_timestamp_clean'] = pd.to_datetime(df_twitter_arch_clean['retweeted_status_timestamp'], errors='coerce')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BubvPW-oWE2y",
        "colab_type": "code",
        "outputId": "badeefaa-c947-4348-ce0b-dbdb8bb5abd0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# TEST Issues 8 and 9\n",
        "print(str(df_twitter_arch_clean['timestamp'].dtype) + ' ' + str(df_twitter_arch_clean['timestamp_clean'].dtype))\n",
        "print(str(df_twitter_arch_clean['retweeted_status_timestamp'].dtype) + ' ' + str(df_twitter_arch_clean['retweeted_status_timestamp_clean'].dtype))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "object datetime64[ns, UTC]\n",
            "object datetime64[ns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRv5EIZKXH5a",
        "colab_type": "code",
        "outputId": "a08b0c6f-9d82-4144-ff9a-07a1abd7544e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "df_twitter_arch_clean.timestamp_clean.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0   2017-08-01 16:23:56+00:00\n",
              "1   2017-08-01 00:17:27+00:00\n",
              "2   2017-07-31 00:18:03+00:00\n",
              "3   2017-07-30 15:58:51+00:00\n",
              "4   2017-07-29 16:00:24+00:00\n",
              "Name: timestamp_clean, dtype: datetime64[ns, UTC]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QHJG6TZnYCZ3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEFINE Issue 12 Quality Issue - Are names like all, None, an valid values. On visually checking, For values where it starts with a lowercase letter: \n",
        "# a, all, the, actually - again here it seems to be incorrectly extracted. However on visually checking these records, it seems that there are no names in these tweets. \n",
        "# Reset these values to Blank\n",
        "\n",
        "# CLEAN Issue 12 \n",
        "df_twitter_arch_clean['name_clean'] = df_twitter_arch_clean['name']\n",
        "for index, row in df_twitter_arch_clean.iterrows():\n",
        "  str_name_clean = row['name_clean']\n",
        "  if(str(str_name_clean)[0].islower()):\n",
        "    df_twitter_arch_clean.loc[index,'name_clean']='None'                    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j7TkLWxQYzqa",
        "colab_type": "code",
        "outputId": "e648e35b-3c5d-46e9-a218-7d8e05043434",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# TEST Issue 12\n",
        "df_twitter_arch_clean[df_twitter_arch_clean['name_clean'].str[0].str.islower()]['name_clean'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Series([], Name: name_clean, dtype: int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jqXnQvsGbqYf",
        "colab_type": "code",
        "outputId": "813ce071-cf29-436a-955a-f7e1a6960fd2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "df_twitter_arch_clean.name_clean.value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "None         731\n",
              "Lucy          11\n",
              "Charlie       11\n",
              "Cooper        10\n",
              "Oliver        10\n",
              "Tucker         9\n",
              "Penny          9\n",
              "Sadie          8\n",
              "Winston        8\n",
              "Lola           8\n",
              "Daisy          7\n",
              "Toby           7\n",
              "Bella          6\n",
              "Bo             6\n",
              "Jax            6\n",
              "Bailey         6\n",
              "Koda           6\n",
              "Stanley        6\n",
              "Oscar          6\n",
              "Bentley        5\n",
              "Buddy          5\n",
              "Louis          5\n",
              "Rusty          5\n",
              "Milo           5\n",
              "Leo            5\n",
              "Chester        5\n",
              "Dave           5\n",
              "Scout          5\n",
              "Brody          4\n",
              "Oakley         4\n",
              "            ... \n",
              "Rooney         1\n",
              "Jed            1\n",
              "Dotsy          1\n",
              "Ester          1\n",
              "Wishes         1\n",
              "Patch          1\n",
              "Raphael        1\n",
              "Yukon          1\n",
              "Kendall        1\n",
              "Baloo          1\n",
              "Alf            1\n",
              "Nugget         1\n",
              "Robin          1\n",
              "Zuzu           1\n",
              "Berkeley       1\n",
              "Rizzy          1\n",
              "Pipsy          1\n",
              "Lillie         1\n",
              "Venti          1\n",
              "Zeek           1\n",
              "Marvin         1\n",
              "Zoe            1\n",
              "Bayley         1\n",
              "Iggy           1\n",
              "Willem         1\n",
              "Mo             1\n",
              "Ivar           1\n",
              "Leela          1\n",
              "Sprinkles      1\n",
              "Jebberson      1\n",
              "Name: name_clean, Length: 931, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bX_Q5_l1ZlLC",
        "colab": {}
      },
      "source": [
        "#DEFINE Issue 11 Quality Issue - The dog name column has 'None'. Should be Blank instead\n",
        "# CLEAN Issue 11\n",
        "df_twitter_arch_clean.name_clean.replace(['None'], '', inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "e246e5ba-bf92-42af-d525-838ef83ca45a",
        "id": "XSKl36RkZpjy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#TEST Issue 11\n",
        "df_twitter_arch_clean[df_twitter_arch_clean.name_clean=='None'].shape[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zO5VKl5Cf1-u",
        "colab_type": "code",
        "outputId": "0185650b-a3ad-463c-8d2b-3650236089ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 400
        }
      },
      "source": [
        "# DEFINE: 14 Tidiness Issue - The predicted p1, p2 and p3 have values which are not dogs. \n",
        "# Relevant data would be to pick the prediction which is actually a dog with the highest probability. \n",
        "# Create a scoring system to ascertain the best probaility of a dog\n",
        "\n",
        "# CLEAN: 14\n",
        "\n",
        "truefalsemap = {\n",
        "    True: 1,\n",
        "    False: 0\n",
        "}\n",
        "\n",
        "df_image_pred_clean['p1_score'] = df_image_pred_clean['p1_conf'] * df_image_pred_clean['p1_dog'].map(truefalsemap)\n",
        "df_image_pred_clean['p2_score'] = df_image_pred_clean['p2_conf'] * df_image_pred_clean['p2_dog'].map(truefalsemap)\n",
        "df_image_pred_clean['p3_score'] = df_image_pred_clean['p3_conf'] * df_image_pred_clean['p3_dog'].map(truefalsemap)\n",
        "df_image_pred_clean.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>jpg_url</th>\n",
              "      <th>img_num</th>\n",
              "      <th>p1</th>\n",
              "      <th>p1_conf</th>\n",
              "      <th>p1_dog</th>\n",
              "      <th>p2</th>\n",
              "      <th>p2_conf</th>\n",
              "      <th>p2_dog</th>\n",
              "      <th>p3</th>\n",
              "      <th>p3_conf</th>\n",
              "      <th>p3_dog</th>\n",
              "      <th>p1_score</th>\n",
              "      <th>p2_score</th>\n",
              "      <th>p3_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>666020888022790149</td>\n",
              "      <td>https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Welsh_springer_spaniel</td>\n",
              "      <td>0.465074</td>\n",
              "      <td>True</td>\n",
              "      <td>collie</td>\n",
              "      <td>0.156665</td>\n",
              "      <td>True</td>\n",
              "      <td>Shetland_sheepdog</td>\n",
              "      <td>0.061428</td>\n",
              "      <td>True</td>\n",
              "      <td>0.465074</td>\n",
              "      <td>0.156665</td>\n",
              "      <td>0.061428</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>666029285002620928</td>\n",
              "      <td>https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>redbone</td>\n",
              "      <td>0.506826</td>\n",
              "      <td>True</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.074192</td>\n",
              "      <td>True</td>\n",
              "      <td>Rhodesian_ridgeback</td>\n",
              "      <td>0.072010</td>\n",
              "      <td>True</td>\n",
              "      <td>0.506826</td>\n",
              "      <td>0.074192</td>\n",
              "      <td>0.072010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>666033412701032449</td>\n",
              "      <td>https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>German_shepherd</td>\n",
              "      <td>0.596461</td>\n",
              "      <td>True</td>\n",
              "      <td>malinois</td>\n",
              "      <td>0.138584</td>\n",
              "      <td>True</td>\n",
              "      <td>bloodhound</td>\n",
              "      <td>0.116197</td>\n",
              "      <td>True</td>\n",
              "      <td>0.596461</td>\n",
              "      <td>0.138584</td>\n",
              "      <td>0.116197</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>666044226329800704</td>\n",
              "      <td>https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Rhodesian_ridgeback</td>\n",
              "      <td>0.408143</td>\n",
              "      <td>True</td>\n",
              "      <td>redbone</td>\n",
              "      <td>0.360687</td>\n",
              "      <td>True</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.222752</td>\n",
              "      <td>True</td>\n",
              "      <td>0.408143</td>\n",
              "      <td>0.360687</td>\n",
              "      <td>0.222752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>666049248165822465</td>\n",
              "      <td>https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>miniature_pinscher</td>\n",
              "      <td>0.560311</td>\n",
              "      <td>True</td>\n",
              "      <td>Rottweiler</td>\n",
              "      <td>0.243682</td>\n",
              "      <td>True</td>\n",
              "      <td>Doberman</td>\n",
              "      <td>0.154629</td>\n",
              "      <td>True</td>\n",
              "      <td>0.560311</td>\n",
              "      <td>0.243682</td>\n",
              "      <td>0.154629</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  ...  p3_score\n",
              "0  666020888022790149  ...  0.061428\n",
              "1  666029285002620928  ...  0.072010\n",
              "2  666033412701032449  ...  0.116197\n",
              "3  666044226329800704  ...  0.222752\n",
              "4  666049248165822465  ...  0.154629\n",
              "\n",
              "[5 rows x 15 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pln7-5UkedWc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# CLEAN: 14 Contd.\n",
        "\n",
        "# Store the data in a separate column called predicted_dog\n",
        "# in case none of the prediction is a dog, keep it blank\n",
        "df_image_pred_clean['predicted_dog'] = ''\n",
        "for index, row in df_image_pred_clean.iterrows():\n",
        "  if((row['p1_score']>row['p2_score']) & (row['p1_score']>row['p3_score'])):\n",
        "    df_image_pred_clean.loc[index,'predicted_dog']=row['p1']\n",
        "  elif((row['p2_score']>row['p1_score']) & (row['p2_score']>row['p3_score'])):\n",
        "    df_image_pred_clean.loc[index,'predicted_dog']=row['p2']\n",
        "  elif((row['p3_score']>row['p1_score']) & (row['p3_score']>row['p3_score'])):\n",
        "    df_image_pred_clean.loc[index,'predicted_dog']=row['p3']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tlkyCcr94sy3",
        "colab_type": "code",
        "outputId": "36261700-bf86-436a-ff2b-5ad461a6d908",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# TEST 14\n",
        "df_image_pred_clean['predicted_dog'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                               388\n",
              "golden_retriever               166\n",
              "Labrador_retriever             110\n",
              "Pembroke                        94\n",
              "Chihuahua                       94\n",
              "pug                             61\n",
              "toy_poodle                      48\n",
              "chow                            47\n",
              "Samoyed                         45\n",
              "Pomeranian                      41\n",
              "malamute                        32\n",
              "French_bulldog                  32\n",
              "cocker_spaniel                  31\n",
              "Chesapeake_Bay_retriever        30\n",
              "miniature_pinscher              26\n",
              "Cardigan                        22\n",
              "Staffordshire_bullterrier       22\n",
              "German_shepherd                 21\n",
              "Eskimo_dog                      20\n",
              "Siberian_husky                  20\n",
              "Shih-Tzu                        20\n",
              "Rottweiler                      19\n",
              "Lakeland_terrier                18\n",
              "beagle                          18\n",
              "Shetland_sheepdog               18\n",
              "kuvasz                          18\n",
              "Maltese_dog                     18\n",
              "Italian_greyhound               17\n",
              "basset                          17\n",
              "West_Highland_white_terrier     16\n",
              "                              ... \n",
              "giant_schnauzer                  4\n",
              "Saluki                           4\n",
              "Gordon_setter                    4\n",
              "Ibizan_hound                     4\n",
              "Weimaraner                       4\n",
              "Afghan_hound                     4\n",
              "Rhodesian_ridgeback              4\n",
              "Leonberg                         3\n",
              "cairn                            3\n",
              "komondor                         3\n",
              "Brabancon_griffon                3\n",
              "toy_terrier                      3\n",
              "briard                           3\n",
              "Irish_water_spaniel              3\n",
              "curly-coated_retriever           3\n",
              "Scottish_deerhound               3\n",
              "Greater_Swiss_Mountain_dog       3\n",
              "Sussex_spaniel                   2\n",
              "Appenzeller                      2\n",
              "groenendael                      2\n",
              "Australian_terrier               2\n",
              "black-and-tan_coonhound          2\n",
              "wire-haired_fox_terrier          2\n",
              "Japanese_spaniel                 1\n",
              "standard_schnauzer               1\n",
              "silky_terrier                    1\n",
              "Irish_wolfhound                  1\n",
              "EntleBucher                      1\n",
              "clumber                          1\n",
              "Scotch_terrier                   1\n",
              "Name: predicted_dog, Length: 113, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ACZdu-ColDGt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEFINE: 16 Tidiness Issue - Pull in the data from Image Prediction and Json data to the twitter archive dataframe for analysis\n",
        "\n",
        "# CLEAN 16\n",
        "\n",
        "# Lets put all the data together in the twitter archive dataframe\n",
        "# Merge Archived tweet data with the downloaded Json data\n",
        "# Note, here we will want only the tweets which are not re-tweets which is what we have in the json dataframe\n",
        "# Therefore use and innerjoin where entries from both twitter archive and json data match\n",
        "df_master_data = pd.merge(df_twitter_arch_clean, df_jdata_clean, on='tweet_id', how='inner')\n",
        "\n",
        "# For the image prediction merge, we will do a left join to get the details for the main dataset\n",
        "df_master_data = pd.merge(df_master_data, df_image_pred_clean, on='tweet_id', how='left')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CAkQg32f2Ca2",
        "colab_type": "code",
        "outputId": "b13c1dcc-58ed-4a38-d94e-daa8df04d80f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 816
        }
      },
      "source": [
        "# TEST 16\n",
        "df_master_data.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 2121 entries, 0 to 2120\n",
            "Data columns (total 41 columns):\n",
            "tweet_id                            2121 non-null int64\n",
            "in_reply_to_status_id               77 non-null float64\n",
            "in_reply_to_user_id                 77 non-null float64\n",
            "timestamp                           2121 non-null object\n",
            "source                              2121 non-null object\n",
            "text                                2121 non-null object\n",
            "retweeted_status_id                 0 non-null float64\n",
            "retweeted_status_user_id            0 non-null float64\n",
            "retweeted_status_timestamp          0 non-null object\n",
            "expanded_urls                       2063 non-null object\n",
            "rating_numerator                    2121 non-null int64\n",
            "rating_denominator                  2121 non-null int64\n",
            "name                                2121 non-null object\n",
            "doggo                               2121 non-null object\n",
            "floofer                             2121 non-null object\n",
            "pupper                              2121 non-null object\n",
            "puppo                               2121 non-null object\n",
            "combined_puppy_stage                2121 non-null object\n",
            "text_uppercase                      2121 non-null object\n",
            "numerator_clean                     2121 non-null float64\n",
            "denominator_clean                   2121 non-null float64\n",
            "timestamp_clean                     2121 non-null datetime64[ns, UTC]\n",
            "retweeted_status_timestamp_clean    0 non-null datetime64[ns]\n",
            "name_clean                          2121 non-null object\n",
            "retweet_count                       2121 non-null int64\n",
            "favorite_count                      2121 non-null int64\n",
            "jpg_url                             1940 non-null object\n",
            "img_num                             1940 non-null float64\n",
            "p1                                  1940 non-null object\n",
            "p1_conf                             1940 non-null float64\n",
            "p1_dog                              1940 non-null object\n",
            "p2                                  1940 non-null object\n",
            "p2_conf                             1940 non-null float64\n",
            "p2_dog                              1940 non-null object\n",
            "p3                                  1940 non-null object\n",
            "p3_conf                             1940 non-null float64\n",
            "p3_dog                              1940 non-null object\n",
            "p1_score                            1940 non-null float64\n",
            "p2_score                            1940 non-null float64\n",
            "p3_score                            1940 non-null float64\n",
            "predicted_dog                       1940 non-null object\n",
            "dtypes: datetime64[ns, UTC](1), datetime64[ns](1), float64(13), int64(5), object(21)\n",
            "memory usage: 696.0+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJ6yPA93o7mB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEFINE: 10. Tidiness Issue Lets drop some of the additional columns to make it look tidier\n",
        "\n",
        "# CLEAN\n",
        "cols_to_drop = ['in_reply_to_status_id', 'in_reply_to_user_id', 'doggo', 'floofer', 'pupper', 'puppo', 'retweeted_status_timestamp_clean', 'retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp', 'expanded_urls', 'p1', 'p1_dog', 'p1_conf', 'p2', 'p2_dog', 'p2_conf', 'p3', 'p3_dog', 'p3_conf' ]\n",
        "df_mastr_clean = df_master_data.copy()\n",
        "df_mastr_clean.drop(cols_to_drop, axis=1, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_HaJcRo2WIL",
        "colab_type": "code",
        "outputId": "36f9b384-7048-4d39-81ea-eee78ee5118e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 469
        }
      },
      "source": [
        "# TEST\n",
        "df_mastr_clean.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 2121 entries, 0 to 2120\n",
            "Data columns (total 21 columns):\n",
            "tweet_id                2121 non-null int64\n",
            "timestamp               2121 non-null object\n",
            "source                  2121 non-null object\n",
            "text                    2121 non-null object\n",
            "rating_numerator        2121 non-null int64\n",
            "rating_denominator      2121 non-null int64\n",
            "name                    2121 non-null object\n",
            "combined_puppy_stage    2121 non-null object\n",
            "text_uppercase          2121 non-null object\n",
            "numerator_clean         2121 non-null float64\n",
            "denominator_clean       2121 non-null float64\n",
            "timestamp_clean         2121 non-null datetime64[ns, UTC]\n",
            "name_clean              2121 non-null object\n",
            "retweet_count           2121 non-null int64\n",
            "favorite_count          2121 non-null int64\n",
            "jpg_url                 1940 non-null object\n",
            "img_num                 1940 non-null float64\n",
            "p1_score                1940 non-null float64\n",
            "p2_score                1940 non-null float64\n",
            "p3_score                1940 non-null float64\n",
            "predicted_dog           1940 non-null object\n",
            "dtypes: datetime64[ns, UTC](1), float64(6), int64(5), object(9)\n",
            "memory usage: 364.5+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "reW96E4EpvaN",
        "colab_type": "code",
        "outputId": "807d5b64-d7ea-4bf5-a2a5-8f37245eae41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 591
        }
      },
      "source": [
        "df_mastr_clean.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>timestamp</th>\n",
              "      <th>source</th>\n",
              "      <th>text</th>\n",
              "      <th>rating_numerator</th>\n",
              "      <th>rating_denominator</th>\n",
              "      <th>name</th>\n",
              "      <th>combined_puppy_stage</th>\n",
              "      <th>text_uppercase</th>\n",
              "      <th>numerator_clean</th>\n",
              "      <th>denominator_clean</th>\n",
              "      <th>timestamp_clean</th>\n",
              "      <th>name_clean</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>favorite_count</th>\n",
              "      <th>jpg_url</th>\n",
              "      <th>img_num</th>\n",
              "      <th>p1_score</th>\n",
              "      <th>p2_score</th>\n",
              "      <th>p3_score</th>\n",
              "      <th>predicted_dog</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>892420643555336193</td>\n",
              "      <td>2017-08-01 16:23:56 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Phineas</td>\n",
              "      <td></td>\n",
              "      <td>THIS IS PHINEAS. HE'S A MYSTICAL BOY. ONLY EVE...</td>\n",
              "      <td>13.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>2017-08-01 16:23:56+00:00</td>\n",
              "      <td>Phineas</td>\n",
              "      <td>8853</td>\n",
              "      <td>39467</td>\n",
              "      <td>https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>892177421306343426</td>\n",
              "      <td>2017-08-01 00:17:27 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Tilly. She's just checking pup on you....</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Tilly</td>\n",
              "      <td></td>\n",
              "      <td>THIS IS TILLY. SHE'S JUST CHECKING PUP ON YOU....</td>\n",
              "      <td>13.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>2017-08-01 00:17:27+00:00</td>\n",
              "      <td>Tilly</td>\n",
              "      <td>6514</td>\n",
              "      <td>33819</td>\n",
              "      <td>https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.323581</td>\n",
              "      <td>0.090647</td>\n",
              "      <td>0.068957</td>\n",
              "      <td>Chihuahua</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>891815181378084864</td>\n",
              "      <td>2017-07-31 00:18:03 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
              "      <td>12</td>\n",
              "      <td>10</td>\n",
              "      <td>Archie</td>\n",
              "      <td></td>\n",
              "      <td>THIS IS ARCHIE. HE IS A RARE NORWEGIAN POUNCIN...</td>\n",
              "      <td>12.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>2017-07-31 00:18:03+00:00</td>\n",
              "      <td>Archie</td>\n",
              "      <td>4328</td>\n",
              "      <td>25461</td>\n",
              "      <td>https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.716012</td>\n",
              "      <td>0.078253</td>\n",
              "      <td>0.031379</td>\n",
              "      <td>Chihuahua</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>891689557279858688</td>\n",
              "      <td>2017-07-30 15:58:51 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Darla. She commenced a snooze mid meal...</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>Darla</td>\n",
              "      <td></td>\n",
              "      <td>THIS IS DARLA. SHE COMMENCED A SNOOZE MID MEAL...</td>\n",
              "      <td>13.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>2017-07-30 15:58:51+00:00</td>\n",
              "      <td>Darla</td>\n",
              "      <td>8964</td>\n",
              "      <td>42908</td>\n",
              "      <td>https://pbs.twimg.com/media/DF_q7IAWsAEuuN8.jpg</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.168086</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>Labrador_retriever</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>891327558926688256</td>\n",
              "      <td>2017-07-29 16:00:24 +0000</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
              "      <td>This is Franklin. He would like you to stop ca...</td>\n",
              "      <td>12</td>\n",
              "      <td>10</td>\n",
              "      <td>Franklin</td>\n",
              "      <td></td>\n",
              "      <td>THIS IS FRANKLIN. HE WOULD LIKE YOU TO STOP CA...</td>\n",
              "      <td>12.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>2017-07-29 16:00:24+00:00</td>\n",
              "      <td>Franklin</td>\n",
              "      <td>9774</td>\n",
              "      <td>41048</td>\n",
              "      <td>https://pbs.twimg.com/media/DF6hr6BUMAAzZgT.jpg</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.555712</td>\n",
              "      <td>0.225770</td>\n",
              "      <td>0.175219</td>\n",
              "      <td>basset</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id                  timestamp  ...  p3_score       predicted_dog\n",
              "0  892420643555336193  2017-08-01 16:23:56 +0000  ...  0.000000                    \n",
              "1  892177421306343426  2017-08-01 00:17:27 +0000  ...  0.068957           Chihuahua\n",
              "2  891815181378084864  2017-07-31 00:18:03 +0000  ...  0.031379           Chihuahua\n",
              "3  891689557279858688  2017-07-30 15:58:51 +0000  ...  0.000000  Labrador_retriever\n",
              "4  891327558926688256  2017-07-29 16:00:24 +0000  ...  0.175219              basset\n",
              "\n",
              "[5 rows x 21 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 124
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b11CHtiwj6dj",
        "colab_type": "text"
      },
      "source": [
        "# Analyze and visualize - three (3) insights and one (1) visualization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nnX2SFOUIiXR",
        "colab_type": "code",
        "outputId": "0c8f5c8d-6e2c-4075-aa3b-14fab7c6fc6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# What is the average dog rating\n",
        "df_mastr_clean.numerator_clean.mean()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "12.161848184818481"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PEilFp0gujpl",
        "colab_type": "code",
        "outputId": "ae6afd95-1795-4425-d24d-5a8fb90c8595",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# What is the average rating by dog type as based on the image predictions\n",
        "df_mastr_clean[df_mastr_clean.predicted_dog!=''].groupby('predicted_dog')['numerator_clean'].mean().sort_values(ascending=False).head(1)\n",
        "# Insight 1: This shows that the dog type Clumber most probably has the highest average rating as per this twitter account"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predicted_dog\n",
              "Saluki    12.5\n",
              "Name: numerator_clean, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 132
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sh8cFBpIvDuF",
        "colab_type": "code",
        "outputId": "ad5e71d2-a032-4ead-a987-61147fb9c375",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "df_mastr_clean[df_mastr_clean.predicted_dog!=''].groupby('predicted_dog')['numerator_clean'].mean().sort_values().head(1)\n",
        "# Insight 1 contd. the dog type Japanese Spaniel most probably has the lowest average rating as per this twitter account"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predicted_dog\n",
              "Japanese_spaniel    5.0\n",
              "Name: numerator_clean, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hn-kLo0hzCKL",
        "colab_type": "code",
        "outputId": "a1126f87-1d56-441b-e062-46989f8c21c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Insight 2: Which type of dogs were retweeted the most\n",
        "\n",
        "df_mastr_clean[df_mastr_clean.predicted_dog!=''].groupby('predicted_dog')['retweet_count'].mean().sort_values(ascending=False).head(1)\n",
        "\n",
        "# Bedlington Terrier dogs were the most re-tweeted"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predicted_dog\n",
              "Bedlington_terrier    8740.2\n",
              "Name: retweet_count, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 134
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FoGBB6U-0T9_",
        "colab_type": "code",
        "outputId": "3e9844ac-4d26-424c-fde8-ececb31e5258",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Insight 2: Contd.: Which type of dogs were retweeted the least\n",
        "df_mastr_clean[df_mastr_clean.predicted_dog!=''].groupby('predicted_dog')['retweet_count'].mean().sort_values(ascending=True).head(1)\n",
        "\n",
        "# groenendael dogs were the least re-tweeted"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predicted_dog\n",
              "groenendael    276.5\n",
              "Name: retweet_count, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 135
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fvkxuXpG30FZ",
        "colab_type": "code",
        "outputId": "2b1b4f63-ae41-4c47-e62d-bc4ca66fa9ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Insight 3: Which type of dogs were favorited the most\n",
        "df_mastr_clean.groupby('predicted_dog')['favorite_count'].mean().sort_values(ascending=False).head(1)\n",
        "\n",
        "# Saluki dogs were the most favorited\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predicted_dog\n",
              "Bedlington_terrier    24438.4\n",
              "Name: favorite_count, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YfYeKGgq3hFt",
        "colab_type": "code",
        "outputId": "7367418e-b87e-4e99-89e4-4682dd31e99f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Insight 3: Contd: Which type of dogs were favorited the least\n",
        "df_mastr_clean.groupby('predicted_dog')['favorite_count'].mean().sort_values(ascending=True).head(1)\n",
        "\n",
        "# Brabancon_griffon dogs were the least favorited\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predicted_dog\n",
              "Brabancon_griffon    885.0\n",
              "Name: favorite_count, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rpjLa5oQ4s65",
        "colab_type": "code",
        "outputId": "7a0c1da3-e3b9-4654-b03b-87bebf0b04d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "#Insight 4: Which is the most common dog name\n",
        "df_mastr_clean['name_clean'].value_counts().sort_values(ascending=False).head(5)\n",
        "\n",
        "# Charlie and Lucy seems to the most common dog names followed by Cooper and Oliver"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           731\n",
              "Charlie     11\n",
              "Lucy        11\n",
              "Oliver      10\n",
              "Cooper      10\n",
              "Name: name_clean, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dADe75JI6frD",
        "colab_type": "code",
        "outputId": "a8e8766d-c4db-49dd-ceea-ad7ccd577643",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "#Insight 5: which is the most favorited dog stage\n",
        "df_mastr_clean.groupby('combined_puppy_stage')['favorite_count'].mean().sort_values(ascending=False).head(1)\n",
        "\n",
        "#puppo seems to be most popular"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "combined_puppy_stage\n",
              "doggo,puppo    47844.0\n",
              "Name: favorite_count, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oaa9sWfP78Vs",
        "colab_type": "code",
        "outputId": "e5c3db44-46b6-469f-aad2-07cfc5179e36",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# When was the first tweet and the last tweet in this dataset\n",
        "df_mastr_clean.timestamp_clean.min()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Timestamp('2015-11-15 22:32:08+0000', tz='UTC')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZhKY6ZaM3sb",
        "colab_type": "code",
        "outputId": "50f66c25-d9b4-4233-e9c3-491da4a996f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_mastr_clean.timestamp_clean.max()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Timestamp('2017-08-01 16:23:56+0000', tz='UTC')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qc92N73GM53s",
        "colab_type": "code",
        "outputId": "3a9ca85d-739c-485a-c22d-c05b21f74121",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 351
        }
      },
      "source": [
        "#Visualization 1\n",
        "# What was the frequency of tweets based on this dataset\n",
        "ax = df_mastr_clean.groupby(df_mastr_clean['timestamp_clean'].dt.strftime('%Y %b'))['tweet_id'].count().plot.bar()\n",
        "ax.set_title('Tweeting Frequency')\n",
        "ax.set_xlabel('Timeline')\n",
        "ax.set_ylabel('No. of Tweets')"
      ],
      "execution_count": 145,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'No. of Tweets')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 145
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAE9CAYAAAARRPXgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X28ZXPd//HX2xByN2FiMiMSTSoG\nQ4qulAj1MyrCJUk0KYWfSlS/i64uXeOq6J6m3HblriKKopBQ0Yz7cdfkbmYajLtxFzV8fn98v4ft\nWGfvtfc666xzzryfj8d+nL3XXt/1/a59Pnt91vp+11pbEYGZmVl/SzXdADMzG56cIMzMrJAThJmZ\nFXKCMDOzQk4QZmZWyAnCzMwKOUHYEknSBpIebbodZsOZE4TVTtITLY/nJP2j5fVeQ9SG+yRt3fc6\nIu6IiLE11XWmpGf6rfcuddRlVqelm26AjX4RsWLfc0l3A/tHxO+aa9GQ+EpE/Fe7GSQtHRGLh6pB\nZt3yEYQ1StKKkp6WtHJ+/ZW89718fv01SdPz8+UlfVPS3HxE8B1Jy7Ys632SbpT0qKQrJG2Yp/8U\neCVwcd6bP0jSJEmLW8r+WdKR+e9jki6U9IqW9/fP9S6UdFj/I5Iu1vc+SZ+VNBt4LE+bKOk8SQ9K\nulPSAS3zryDpJ3mdbpJ0hKQ5+b3lJIWkCS3znynpS50+k5a2/F9JN0talOt5Wcv7u+Wyj0v6q6Rt\nJe0t6ap+6/QFSWd1+1nY8OcEYY2KiCeAG4G35UlvB+YBW7a8vjw/PxaYALwJeB2wAXA4gKQtge8D\n+wKrAT8GfpH30ncDHgC2j4gVI+LbAzTn34G9gPHAWODgvOxNct275fonAKtXWO3dge2A1SSNAS4E\n/gi8CtgB+IKkt+d5jwbWBNYBdgY+UraSdp9Jy2y7AtsCrwXeTPoMkPRvwAzSZ7BKnmcucA7wJknr\ntixjb+C0su2ykcMJwoaDy4G356OB9YHj8+uVgI2Aq/JGbT/g4Ih4NCIWAdOBPfIyPg58NyJmRcSz\nETEDWBbYrIt2/DAi/hYRTwI/Aybn6bsBP4+IP0fEM8CX6Pzd+WLea39U0rx+7x0XEX+PiH8AWwPL\nRcQxEfHPiLgDOLllvT5I6q56NCLuAr7XxfqU+UyOi4j7I2IhKVH1rfN+wAkRcVlEPBcR9+ZxmyeB\nnwMfApA0hZR8LuqiXTZCOEHYcHA5sA1pD3YmcCnpyGEr4KaIeIy0d70MMLtvwwv8gtR1BPBq0p73\noy3vjwPW6qId97U8fwroGzt5FWnvGYDcnkUdlnV0RIzNjwn93pvb8vzVwDr92n0osKakpUhHD63z\n31N+dUp9JgOt80TgbwMs91TSkRakRHGGx1JGJw9S23BwJbAx8B5SsrgemARszwvdSwuAxcB6EfFQ\nwTLmAhdExDcGqKPKbYsXkLqVAMjjJatUWF5rW+YCt0XEm4pmlHQ/L95Yr93y9j+BfwEvb5m2JnBz\ny7LbfSbtzAXWG+C9y4HlchfWnqT/m41CPoKwxkXEo8Bs4BPA5RHxHOlIYn9ygoiIfwEnAd+StLqS\niZK2y4uZAXxa0pT83oqSdpbUt/G8H3hNj008G/iApM3zIO5/As/1uKz+rgSQdEgedF5a0kaSNm2p\n+4uSVpH0auCTfQXz53QTsJekMZJ2Bt7SsuxOn0k7PwI+LunfJC2VP+sNcr1BGs+YATwUETMrfgY2\nTDlB2HBxOSDg2pbXK5A3oNkhwN9JyWMR8BvS4CoRcRVwEPAD4FHgDtKAa9/e+tHA0bmr5VPdNCwi\nrgM+B5wLzCcdUSwCnulqDYuX/S9gJ+CtpO6jhaQxmL6uni8BDwL3Ahfw0sHgT5EGvR8BdgF+1bLs\nTp9Ju3ZdARxAGuReBFxCy1FUbsebSInCRin5B4PMupNPf30YeFVELBjiuncgDTy/dijrLWjHSqSj\nskkRcW+TbbH6+AjCrITcNbO8pBVJp7xePdTJYZj5NPB7J4fRrfYEkftGr5P0q/x6XUlXS5oj6ay+\nC3MkLZtfz8nvr1N328y6sBvpjJ95pLOAhuQWIcORpPtI40OHNd0Wq1ftXUySDgWmACtHxHslnQ2c\nExFnSjoBuCEijpf0SWCjiDhA0h7A+yJi91obZ2ZmA6r1CCLfAuA9pDMikCTgnaSLkCCdT913E7Op\n+TX5/W3z/GZm1oC6r4P4JukwdKX8ejXg0ZaLavoO18l/5wJExGJJi/L8D7YuUNI0YBrACiussNmk\nSZNqXQEzs9Fm1qxZD0bEuE7z1ZYgJL0XeCAiZknaZrCWm28XMANgypQpMXOmT8E2M+uGpFJX5Nd5\nBLEVsLOknYDlgJWBbwFj9cJtjieQzisn/50IzMv33VkFKLpi1szMhkBtYxARcURETIiIdUg3Hrs0\nIvYCLiPdQRJgH+C8/Pz8/Jr8/qXhizTMzBrTxHUQnwcOVbqn/WrAiXn6iaTbH88h3azs8AbaZmZm\n2ZDcrC8ifg/8Pj+/E9iiYJ6nSeeam5nZMOArqc3MrJAThJmZFXKCMDOzQk4QZmZWyAnCzMwKjaqf\nHF3n8Avavn/3dP8yoplZWT6CMDOzQk4QZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAT\nhJmZFXKCMDOzQk4QZmZWyAnCzMwKOUGYmVmh2hKEpOUkXSPpBkmzJX05Tz9F0l2Srs+PyXm6JH1b\n0hxJN0ratK62mZlZZ3XezfUZ4J0R8YSkZYArJf06v/e5iPhZv/l3BNbPjzcDx+e/ZmbWgNqOICJ5\nIr9cJj+iTZGpwGm53J+BsZLG19U+MzNrr9YxCEljJF0PPAD8NiKuzm8dnbuRjpO0bJ62FjC3pfi8\nPK3/MqdJmilp5sKFC+tsvpnZEq3WBBERz0bEZGACsIWkNwJHAJOAzYFVgc93ucwZETElIqaMGzdu\n0NtsZmbJkJzFFBGPApcBO0TEgtyN9AxwMrBFnm0+MLGl2IQ8zczMGlDnWUzjJI3Nz5cHtgNu6xtX\nkCRgF+DmXOR84MP5bKYtgUURsaCu9pmZWXt1nsU0HjhV0hhSIjo7In4l6VJJ4wAB1wMH5PkvBHYC\n5gBPAfvW2DYzM+ugtgQRETcCmxRMf+cA8wdwYF3tMTOz7vhKajMzK+QEYWZmhZwgzMyskBOEmZkV\ncoIwM7NCThBmZlbICcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgzMyskBOEmZkVcoIwM7NCThBmZlbI\nCcLMzAo5QZiZWSEnCDMzK+QEYWZmhZwgzMysUG0JQtJykq6RdIOk2ZK+nKevK+lqSXMknSXpZXn6\nsvn1nPz+OnW1zczMOqvzCOIZ4J0RsTEwGdhB0pbAMcBxEfFa4BFgvzz/fsAjefpxeT4zM2tIbQki\nkifyy2XyI4B3Aj/L008FdsnPp+bX5Pe3laS62mdmZu0tXefCJY0BZgGvBb4H/A14NCIW51nmAWvl\n52sBcwEiYrGkRcBqwIP9ljkNmAaw9tprD1pb1zn8grbv3z39PYNWl5nZSFDrIHVEPBsRk4EJwBbA\npEFY5oyImBIRU8aNG1e5jWZmVmxIzmKKiEeBy4C3AGMl9R25TADm5+fzgYkA+f1VgIeGon1mZvZS\ndZ7FNE7S2Px8eWA74FZSotg1z7YPcF5+fn5+TX7/0oiIutpnZmbt1TkGMR44NY9DLAWcHRG/knQL\ncKak/wKuA07M858I/FjSHOBhYI8a22ZmZh3UliAi4kZgk4Lpd5LGI/pPfxrYra72mJlZd3wltZmZ\nFXKCMDOzQk4QZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoWcIMzMrJAThJmZFXKCMDOzQk4QZmZW\nyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoWcIMzMrFDHBCFpS0kvz8/3lPQ/kibW3zQzM2tSmSOIGcA/\nJG0EfB6YD/y41laZmVnjyiSIxRERwFTguxHxLWDlToUkTZR0maRbJM2WdHCefpSk+ZKuz4+dWsoc\nIWmOpNslvbvXlTIzs+rK/Cb1k5I+B3wI2EbSUsAyJcotBj4TEddKWgmYJem3+b3jIuLrrTNL2hDY\nA3gD8Crgd5I2iIhny66MmZkNnjJHELsDAg6IiAXABODYToUiYkFEXJufPw7cCqzVpshU4MyIeCYi\n7gLmAFuUaJ+ZmdWgTIL4VET8T0RcBhAR9wLrd1OJpHWATYCr+5Yp6UZJJ0l6RZ62FjC3pdg8ChKK\npGmSZkqauXDhwm6aYWZmXSiTIHYomPaeshVIWhH4OXBIRDwGHA+sB0wGFgDfKLssgIiYERFTImLK\nuHHjuilqZmZdGHAMQtLHgQOADSRd2/LWSsC1xaVesoxlSMnhJxFxDkBE3N/y/g+BX+WX84HW02cn\n5GlmZtaAdoPUZwOXAP8NHN4y/fGIeKDTgiUJOBG4NSKObZk+Po9lALwPuDk/Px84XdKxpEHq9YFr\nyq6ImZkNrgETREQ8AjwC7CZpS2CDiDhN0qqS1s5jEe1sBewN3CTp+jztC8CekiYDAdwNfDzXN1vS\n2cAtpDOgDvQZTGZmzel4mqukL5E29usBpwHLA6cDW7crFxFXks5+6u/CNmWOBo7u1CYzM6tfmUHq\nXYGdgCcBImI+JS6UMzOzka1MgngmX0kdAH33ZTIzs9GtTII4R9L3gFUk7QtcDJxUb7PMzKxpHccg\nIuIYSTsC/wQ2Bo6OiF/X3jIzM2tUmXsxAdwEPB0Rl0laTtIKEfFknQ0zM7Nmlfk9iI+SrlH4UZ60\nNnBenY0yM7PmlRmDOAjYEngMICLuAF5ZZ6PMzKx5ZRLE0xHxz74XksZQfH2DmZmNImUSxFWSDgOW\nk/QO4CxeuH+SmZmNUmUSxGHA48BtwMGk+zN9sc5GmZlZ88qcxbQ1cFJEHF93Y8zMbPgocwQxDZgt\n6UpJ/y1pR0m+1YaZ2ShX5kK5vQAkvRrYBfgBMJ5yv0ttZmYjVJm7ue4BvI30C3CLgBOAK2pul5mZ\nNazMGMTxwB3Ad4HLImJevU0yM7PhoMwYxKqkcYixwDck/VnSyfU2y8zMmlYmQbycdOX0GsCawGrA\ny+pslJmZNa9MF9M1wFXAlcCPIuLuWltkZmbDwoBHEJK+ChARb4iIaRFxWjfJQdJESZdJukXSbEkH\n5+mrSvqtpL/mv6/I0yXp25LmSLpR0qYV183MzCpo18W0Q8VlLwY+ExEbkm72d6CkDYHDgUsiYn3S\nVdmH5/l3BNbPj2mkwXEzM2tIuy6mMXnvvvDGfBHxcLsFR8QCYEF+/rikW4G1gKnANnm2U4HfA5/P\n00/LP2/6Z0ljJY3PyzEzsyHWLkFMAmZRnCACeE3ZSiStA2wCXA2s0bLRv480+A0pecxtKTYvT3tR\ngpA0jXSEwdprr122CWZm1qV2CeKWiNikagWSVgR+DhwSEY9JL+SbiAhJ0c3yImIGMANgypQpXZU1\nM7Pyypzm2jNJy5CSw08i4pw8+X5J4/P744EH8vT5wMSW4hPyNDMza0C7BPGtKgtWOlQ4Ebg1Io5t\neet8YJ/8fB9e+PnS84EP57OZtgQWefzBzKw5A3YxRcQpFZe9FbA3cJOk6/O0LwDTgbMl7QfcA3ww\nv3chsBMwB3gK2Ldi/WZmVkGZC+V6EhFXMvBPk25bMH8AB9bVHhtc6xx+Qdv3757+niFqiZnVpd2F\ncsfkv7sNXXPMzGy4aDcGsVMeRzhiqBpjZmbDR7supt8AjwArSnqM1F0UfX8jwr8qZ2Y2ig14BBER\nn4uIscAFEbFyRKzU+ncI22hmZg0o85OjUyWtAWyeJ10dEQvrbZaZmTWt44VyeZD6GmA30imp10ja\nte6GmZlZs8qc5volYPOIeABA0jjgd8DP6myYmZk1q8ytNpbqSw7ZQyXLmZnZCFbmCOI3ki4Czsiv\ndydd9WxmZqNYmUHqz0l6P7B1njQjIs6tt1lmZta0UrfayHdiPafjjGZmNmp4LMHMzAo5QZiZWSEn\nCDMzK9RTgpB01CC3w8zMhplejyBmDWorzMxs2OkpQUTELwe7IWZmNryUuRfTBEnnSloo6QFJP5c0\nYSgaZ2ZmzSlzBHEycD4wHngV8Ms8rS1JJ+WEcnPLtKMkzZd0fX7s1PLeEZLmSLpd0ru7XxUzMxtM\nZRLEuIg4OSIW58cpwLgS5U4BdiiYflxETM6PCwEkbQjsAbwhl/m+pDGl1sDMzGpRJkE8JOlDksbk\nx4dIN+xrKyL+ADxcsh1TgTMj4pmIuAuYA2xRsqyZmdWgTIL4KOl3IO4DFgC7AvtWqPNTkm7MXVCv\nyNPWAua2zDMvT3sJSdMkzZQ0c+FC/26RmVldOiaIiLgnInaOiHER8cqI2CUi7u2xvuOB9YDJpGTz\njW4XEBEzImJKREwZN65MT5eZmfViwJv1SfqPNuUiIr7SbWURcX/L8n8I/Cq/nA9MbJl1Qp5mZmYN\naXcE8WTBA2A/4PO9VCZpfMvL9wF9ZzidD+whaVlJ6wLrk37m1MzMGjLgEUREPN/9I2kl4GDS2MOZ\nlOgaknQGsA2wuqR5wJHANpImAwHcDXw81zVb0tnALcBi4MCIeLa3VTIzs8HQ9vcgJK0KHArsBZwK\nbBoRj5RZcETsWTD5xDbzHw0cXWbZZmZWv3ZjEF8D3g/MAN4UEU8MWavMzKxx7cYgPkO6cvpLwN8l\nPZYfj0t6bGiaZ2ZmTWk3BuHfijAzW4KV+k1qs9FincMvaPv+3dPfM0QtMRv+fJRgZmaFnCDMzKyQ\nE4SZmRVygjAzs0JOEGZmVsgJwszMCvk01yWYT/k0s3Z8BGFmZoWcIMzMrJAThJmZFXKCMDOzQk4Q\nZmZWyAnCzMwKOUGYmVmh2hKEpJMkPSDp5pZpq0r6raS/5r+vyNMl6duS5ki6UdKmdbXLzMzKqfMI\n4hRgh37TDgcuiYj1gUvya4AdgfXzYxpwfI3tMjOzEmpLEBHxB+DhfpOnAqfm56cCu7RMPy2SPwNj\nJY2vq21mZtbZUI9BrBERC/Lz+4A18vO1gLkt883L015C0jRJMyXNXLhwYX0tNTNbwjU2SB0RAUQP\n5WZExJSImDJu3LgaWmZmZjD0N+u7X9L4iFiQu5AeyNPnAxNb5puQp9ko1e5Ggb5JoNnwMNRHEOcD\n++Tn+wDntUz/cD6baUtgUUtXlJmZNaC2IwhJZwDbAKtLmgccCUwHzpa0H3AP8ME8+4XATsAc4Clg\n37raZWZm5dSWICJizwHe2rZg3gAOrKstZmbWPf9gkI04Hr8wGxq+1YaZmRVygjAzs0JOEGZmVsgJ\nwszMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKyQL5QzGwF8caA1wQnCzIaddgkRnBSHiruYzMyskI8g\nzGzUcZfc4HCCMBvlvLG0XrmLyczMCjlBmJlZIXcxDQPuAjCz4cgJwmwI+LRNG4kaSRCS7gYeB54F\nFkfEFEmrAmcB6wB3Ax+MiEeaaF+3mvzy++hj6Hgjb3UajvHV5BjEOyJickRMya8PBy6JiPWBS/Jr\nMzNryHDqYpoKbJOfnwr8Hvh8U40xsyXTcNyTb0pTRxABXCxplqRpedoaEbEgP78PWKOooKRpkmZK\nmrlw4cKhaKuZ2RKpqSOIrSNivqRXAr+VdFvrmxERkqKoYETMAGYATJkypXAeMzOrrpEEERHz898H\nJJ0LbAHcL2l8RCyQNB54oIm2mZn1arR1Tw15gpC0ArBURDyen28P/CdwPrAPMD3/PW+o22ZmLzba\nNnjWnSaOINYAzpXUV//pEfEbSX8Bzpa0H3AP8MEG2mZmZtmQJ4iIuBPYuGD6Q8C2Q90eMzMrNpxO\nczUzsx7VcdGsb9ZnZmaFnCDMzKyQE4SZmRVygjAzs0JOEGZmVsgJwszMCjlBmJlZIScIMzMr5ARh\nZmaFnCDMzKyQE4SZmRVygjAzs0JOEGZmVsgJwszMCjlBmJlZIScIMzMr5ARhZmaFnCDMzKzQsEsQ\nknaQdLukOZIOb7o9ZmZLqmGVICSNAb4H7AhsCOwpacNmW2VmtmRauukG9LMFMCci7gSQdCYwFbil\n0VaZWdfWOfyCtu/fPf09Q9QS65Uiouk2PE/SrsAOEbF/fr038OaI+FTLPNOAafnl64Db2yxydeDB\nHpszEss2WbfXeWSUbbJur/PwKfvqiBjXcSkRMWwewK7Aj1pe7w18t8LyZi5JZUdqu73O/ry8zsOn\nbOtjWI1BAPOBiS2vJ+RpZmY2xIZbgvgLsL6kdSW9DNgDOL/hNpmZLZGG1SB1RCyW9CngImAMcFJE\nzK6wyBlLWNkm6/Y6j4yyTdbtdR4ZZZ83rAapzcxs+BhuXUxmZjZMOEGYmVkhJ4iKJI2R9JOm21GF\npJUlrdR0O+omaSlJH2y6Hd2StFuZaTXU23hsOzabNarGICStCyyIiKfz6+WBNSLi7hJlDwXOioiu\nT6uVdCXwzoj4Zw9lNy2YvAi4JyIWlyhfZZ03B04CVgIEPAp8NCJmlV+D8iT9Ehgw4CJi5y6WtRbw\nalpOtIiIP5QoNzMippStp1/Zc4ATgV9HxHNdlq0SX9dGxKadpnVYxvuBrUmf/5URcW7Jco3E9lDH\nZq5zUOJzpMVm2+WOsgQxE3hrXzDnU2WviojNS5Q9Evgg8DBwFvDTiLi/ZL2nAa8nnZL7ZN/0iDi2\nRNk/A5sCN5K+CG8EZgOrAJ+IiIs7lK+yzjcCB0bEFfn11sD3I2KjEmXfDxwDvDK3W0BExMptyry9\n3TIj4vJO9eblHAPsTroFy7MvFO/8BZY0nXSF6Vm8+H/1cImy7wL2BbYEfgqcHBHtruRvLdt1fEna\nEdgplzur5a2VgQ0jYouSdX8feC1wRp60O/C3iDiwRNlGYrtKbOb5G4nPkRibbZc7yhLE9RExud+0\nGyJi4y6WsRHpH/wBYF5EvKtEmSOLpkfEl0uUPQf4f32n8+abE/4ncBhwTv/1KSjf8zpLui4iNuk3\nrdSeqaQ5wP+JiFs7zTvYJN0ObBQRz/RQ9q6CyRERr+liGasAewJfBOYCPwT+NyL+VaJs6fiStDEw\nmRQP/9Hy1uPAZRHxSMn23ga8PvKXXdJSwOyIeH2Jso3EdpXYzPM2Ep8jOTaLDKvrIAbBQkk7R8T5\nAJKm0v29TB4A7gMeIu19dNT3ZZG0cnoZj3dR3wat13pExC2SJkXEnZLKlO96nVsO/S+X9APSnmWQ\nNly/L9nu+3v98uUvwkv2TLr4ItwJLAN0/SWMiHW7LdNK0mrAh0i3gbkO+Amp62YfYJsSiygdXxFx\nA3CDpHOBJyPi2dyGMcCyXTR7DrA2cE9+PTFP62ioY3uQYhOai8+RHJsvMdoSxAHATyR9j/QPngd8\nuExBSZ8kHcqPIx2ifSwiSt1FVtIU4GRSfymSFlG+v3S2pOOBM/Pr3YFbJC0LlMn6vazzN/q9bt1L\nLHtIOVPSWcAvaPkyRMQ5Jcq29rMuB+wGrFqyXoCngOslXdKv7oPKFJb0RtLt5JdrKXtaiXLnkm4Q\n+WPS3umC/NZZuauvXdme4wu4GHgX8ER+vXye9taS5VcCbpV0Den/uwXp/3c+tO9bbyC2ByM2obn4\nHHGx2Xa5o6mLqY+kFQEi4olO87aU+W/SIOL1PdRXpS9/eeCTpEwPcBXwfeBp4OVl16GXda5C0skF\nkyMiPtrj8mZFxGYl592naHpEnFqi7JGkvakNgQtJvz1yZUTsWqLsOyLisjJtLChbJb6KuhFfMq1N\n+Z771odDbPeiqfgcibHZdrmjKUFIWgP4KvCqiNgx93m+JSJOLFl+Y+Bt+eUV+RC/TLlK/aVVVFnn\nqp9XhTa3fi5LkfbYPtHNWFGFum8CNgaui4iN82fwvxGxXcnyPe3h5bK9xtdVwKcj4tr8ejPSXY7f\nUqZ8LvNqYP2I+F3ecC9dpruoqdhuKjZz3Y3EZ5OxOZDR1sV0Culw+Iv59R2kMwLKbCwPIv3ORN8h\n6P9KmhER3ylRb2F/aV+g9X2xB6i3an/8KfS4zlXKSloO2A94Ay8OyDJ7aK3dCIuBu0ndL6VU/Mz+\nERHPSVqc+9Uf4MV3EG5Xb+EeHlCmC6BKfB0C/FTS30ln46xJirFSJH0s170qsB7pLsknANuWKN5U\nbJ9C73HdWHyOxNhsZ7QliNUj4mxJR8DzN/97tlOhbH/SjxM9Cc+frvYnoMwXuG/Pov8ZH5uQguWd\nbcpW7Y+vss5Vyv4YuA14N+nMlL2AUoOCEfGOknUMpMpnNlPSWNLZHbNI/fp/Kll2V17Yw9u3bw+v\nZNme4ysi/iJpEqmPGeD2Ls9KOZA07nB1Xt5fJZU6AYPmYrtKbEJz8TkSY3NAoy1BPJlH8vtO59uS\ndGFOGeKF85bJz0udRlQUUJLWiBLXUUTEQ/0mfVPSLF58WmM7Vda5StnXRsRukqZGxKmSTgeuaFdA\n6WKxgTwD/A24ODpc6FPlM4uIT+anJ0j6DbByRNzYqVzW8x4ePcZX3pAfSNoThnQdwfdy3WU9ExH/\nVD5zSNLSlBzwbTC2q8QmNBSfIzQ2BzTaEsShpAt61sv9tuNImbWMk4Gr89kAALtQ8nC2T87+HwD+\nnXRx0atKlCnq7+zm/1JlnauU7duDfTT3fd5H59OC290y4RWkLo+P0uFwvpfPTMVX9T7/XruukhZV\n9vC6ji9JWwGnk7pb+roKNgOukbRXRFxVsu7LJX0BWF7SdqSB41+WLNvXlqGO7SqxCQ3F5wiNzYHb\nNpoGqeH5vaPXkfbOujoUz4N/W+WXV0TEdSXKLA9MJX1xNiEF2S7AHzrtCefyrWce9PV3fj26uAqy\n4jr3VFbS/sDPgY1IG78Vgf+IiBPK1j3Acm/sdIZML5+ZpOeAm3nhGpHWvfeIiHZdJUXLW4fu9vC6\nji+lK5E/0X8+SZOBH0TEm0vWuxSpP3570npfRPpp37Zf/qZju2JcNxKfIzU2B1zWaEkQ+XD034FJ\nedKtwOlR4jL1CnWeTjor5WLSud6XAnOi+gUvpQ7he11nSV+NiC/k59tFxG+rtHckkHQIaQ90Eel/\ndW7Z0yzb7eFB+4HaKiTdEhEbdvtem+W9jNRVNT8i2nZRNRXbjs3hFZujIkFIej0pgC8iXUEo0h7P\ndqQbjd3WpuzjvNAfq5bnSwMqBwLhAAANtklEQVQvi4gBDw8lXU86jDwNODMi5km6s4szkFqX9aJD\n+IhoewhfcZ2fP01R3d/0bVDGEXoxGHVLeg3pp2ynkq4s/mp0uDahyh5exfi6lXSfrUf6TV8V+GNE\nTCou+fx8JwDfiYjZSrdg+BNp7GNV4LMRcUabso3EdpXYzGUaic+RGJtljJYxiK8AB0fE2a0TJX0A\nOJoUnIUi4kX9jkoXnB0IfBxoe8fLiJisdHbJnsDvJD0IrNTFEcCAh/CdylJhnSsalHGEpuqOdJuH\n80hXI+8NbAB0unjtUNIe3j/ocg+vSnwBxwEXS/os0LcnuBnpJnTHlaj+bRFxQH6+L3BHROwiaU3g\n17xw876idjcZ21U0FZ8jLjZLiYgR/yD1T3b9Xr/5xgJHke6l8l/Aaj20YzPg68C9pD28dvOeTrqZ\n1omkvf4xwF1Dsc6k23EcCnym5fnzj0H4f9w4wPQ3k/pGIX0JvkwaLD0GWGWQYmGgul8DfIF0qufP\nSF+q5btcdusyzgYmd1G2p/gC3kvaqD5E2kv8A+lWCmXKXtfy/ALgI0XvlVzWkMR23bHZIUYOAiYO\nRh0jKTbbPUZLF9OAh6KdDlMlrU4Kxt1J95//TkR0czpd0TJF2nsbcG+p6iF8xXUuvENnnyhxp85e\nSJoNbBzpnPYZpPvW/Iy0d7VxRLy/jnpz3c+Rbjt9HvAY/U7zjBK3r87LeQOpG2Bv4LDodwRXMP+g\nx1dZecD0G8B84DJgUkTclwd/b44OXVQDLLPW2G4qNnPdi0i32f4b6ejqpxGxsK76WuptJDZLLXOU\nJIh5QNGHKOCQiBjwfGBJTwILSWc6vOTWA2X/Ob1oOYTfnbR3+DrgjVHuEL7ndW6KpFsj32K6fxJT\nF/cW6rHuo2j/YzADbnj69Q3PJR3KXxAR/yhRb5PxtQHwbdKV19+MiFPy9HcD20fEZ2qsu+fYboqk\n60hHSu8itXtn0imjZ5BuT97NnWy7qfcoGojNUm0bJQmi572OKv+cwZRPgdyT1Ec5LyLa3qmzyT2t\nXkn6KXBhRJysdDO170XEzLwh+0mU+JGjJlTZwxsu8dWkbmO7KQU7LcuQblmxJ/CuiBjXWOMGMFhH\nHwMufzQkiNGkzCF8kyS9Gbg1Ih7LA5GHk3417BbSWRcDdp/kM2m+RTp98sFcbm5+HBQdbl5Xpe4q\nvJEfHEMR20r3vDo3Iub2UPYlNyZsee/lEfFUHfVWUXdsOkFU0NQGq0mDMY6gdCuAdUln0c0r2+3Q\n5BhGE5qMr6Y2eFVVGUeQtEFE3DHU9Q5nThAVjNQNVsWjgFrGESStGB1Oz2tyDKMJDQ/qNzVgWykx\n1TWO0Ck+mxq/qNtSTTdghFsqIhbn51Mi4pCIuDIf1nV9QdEQOom0sYHU5bMK6VTTp0iDqe3cLGnf\n/PwGpV8c6xsQ7el3b7Myv67Wc92S3pyPXJC0vKQvS/qlpGNy19dwVCm+Kq7znaTbgn+FtOG7RdJv\nJO0jqd05/1V9hXTPqiskfVJSt/3+ERHPRcTFEbEf6Z5R3wd2IK1TrzrFZ8/1SjpI0rA7qQRG6RGE\n0q9ebUE6le/iuspWGXSt2n3Q1FFAlXEEDXy1qYAvRkTb2yJXrLuRvfGK/6dKg/pV1rnKgG3VcQAq\n7IlXHEfoOT4r1jtsu6dGRYKQdE1EbJGff4x0peq5pJuT/TIiptdUtrENVsUvf+WziXoZR5D0NPA1\n0k3M+vu/ETG20zIq1F0lKVbZyFf5P1Ud1K+yzo1s8Kokpjx/lXGEnuOzYr09J8Xax4qipqsGh/LB\ni68Y/QswLj9fAbiprrIt5VYm/VjHZsAaJcvc2vL82n7vXV9neVKX0imkL/DVpO6ZO4HLSRutXv8P\nK3Z4/4/AZgO8N7diDHSq+6fAvvn5yaQuG0i3M/hLh7KzST/RCTAD+Cbpd5aPJH2Ba/s/9xpfg7DO\nG1T4X1xH6r7ennQ19ULgN8A+wEqdyrZ57+U1x0gt8Vmi3v5xsQwpSZwBLOxQdhHwd9JvXXyyb/s1\nWI9BW1CTD+AG0v1OVgNm9nuv7S0FqpStGBQ9f3kHo3yet6cNT5vl3dvh/dcNFMBV6y9Rd89JscpG\nfjD+T73EV9V1rlJ3xQ1ez4lpEGKklvgsUW/PSZEKybjMY7TcrG8V0iGZgJA0PiIWKN0YrdOvdlUp\n284twNpt3t8f+JakL5G6D/4kqa/7YP8Sy69anoh4jJQgn1fibI12/bQrdqhvwHviR7luoip1LwI+\n0kv3FHlwPCJOJg+Oxwtdcp0G5iv/nwbQKb6qrnOVul/0vYn0Ow7nA+dLenm7BUebbppOsZnnaSQ+\nq9RLm98XjzZdeS/MEs+Rbst+cb8uua+TfmipZ6NiDGIgORjXiIi76ihbddA1L6PSl3ewv/yS7o2I\nAb/8FftpVwGOIN3V85WkC3weIF0FOj0iHu3QtkEZwyhYbqekWGksIC+jl3GTyvHVZtlVdgQ6Ddj2\n3B/fTqfYzPM0Ep8NxmbPY0Wl6h/NCQLK7XX0WrapoKhavuKX/4/ApyNiVsF7c6P9fa8uIv2GxakR\ncV+etibpcHjbiNh+oLJV6+6w3I4bnjzfYCfjRuIrL7u2HYEO9dYWm7l8I/HZVGzWlYyfX/4SkCBK\nffl7Kdv0BqvX8hX3sl4HPBwFZ6Wo86+F3R4Rr+v2vUGqu5a98Yo7ILXGV1M7Ah3aVGtiaio+R1ts\n9hkVYxBV+v8q9h3uCwz0855Taqy3avlrgV8M8OVv2y9ecRzhHkmHkfbQ7s/1rQF8hNRd01bFur/K\nwBueKheMtu2Pbyq+sirr3FRs9xyb0Fx8jsTYLGNUJAiqfcA9l204KJr68lcZR9iddA3B5ZJemafd\nTxrA7PgLXxXr7nnDU3GD11R8QXM7Ao3EJjQXnyM0NjsaFV1MFfsdq5StMqhVtfugli6ATqqOIzRV\nd8UugCpdco3EVy5fZZ0bi+0qmorPkRibpURN5xwP5YMK5y9XLHsR8HlgzZZpa+ZpF9dV7yC0exVg\nOnAbaW/tIeDWPG1sh7KVft4VmES6iniFftN3KFG28k/L9hhfPV9A1VR8DcI6NxLbVWJzMGKk1/gc\nibFZavl1BtlofzQVFIPQ7ipf/ouBw1q/6MAauezvOpQ9CLgd+AVwNzC15b1rS7S7St1VkmJtF/jV\nGV8V17mpDV6lpNhUfI7W2KzlnzzUj4ofcJWyjQTFILS75y8/6arzY1rqfTjXewywaoeyN5GvwgXW\nAWYCB+fXHa9ar1h3I3vjTcVX1XVuKrarxGaT8TkSY7NU/DZZ+aCtRLUvQpWyjQVFU1/+iv+n2f1e\nr0i6LcCxlLwvUYW6qyTFKhu8RuJrENa5kdhuKjabjM+mYrNU2+r8wIfqUfEDbupQuuqeUiNf/ly+\n137aS4HJ/aYtDZwGPFvyc+u17ip7xFU2eI11Qza1sW0yNpuMz5EWm6U+yzoDdKgeFT/gqofxQx4U\ng1G+wmddpZ92Qmsg93tvq5rrrrJHXGWD10h8VV3nJmN7JMbnSIzNUp9nnf+soXpU/ICrlG0kKAap\nfK9f/krjCBX/z43UXWWD11R8DcI6NxnbVZJiUzEy4mKz1PLrDLLR/mhyY1mx3VW+/E2OI1Squ9cN\nT9UNXpPxVWGdm9rgVT3TralxhFEZm7V8WE08ev2AK/5zGgmKQWh3lbM1Ko8jVPgf91x31Q1PE7E5\nCPHV2I5AE7HZZHyO1NjsuF5NVj6I/5wqX4QqZRsLiqa+/FQcR6j4f67SR1x1w9PrBq+R+Kq6zk3F\ndpXYbDI+R2Jsllqvuj6woXxU/CJUKdtkUIzIo4AGY6RKUqyywWskvgZhnRuJbcfm0MVmqbY1/eEM\ngw94pPZZjsijgAZjpMoecZUNXpNjNk11tzg2h+j/VCU2yzyq3Ep2OLlf0uS+F5Hugf5eYHXgTTWW\nraJqvT2Xj4h5kW8oVvDeVSXqHok+DLxonSNicUR8GPi3DmWXyp8vEXE3sA2wo6RjoePP0jYVX1Bt\nnatwbHanqdjsaLTczXUCsLgosCRt1S6wqpStomq9TbV7SSTpUuDQiLi+ZdrSwEnAXhExpk3ZJe7/\ntCSuc1OqxGap5Y+GBGFWJ2/wbLiqOzadIMzMrNBoGYMwM7NB5gRhZmaFRstvUpv1TNJqwCX55ZrA\ns0Dfzz8+FRFvHYQ6PgJMiYhPSTogL/e0qss1q5MThC3xIuIhYDKApKOAJyLi6zXWd0JdyzYbTO5i\nMmtD0hP57zaSLpd0nqQ7JU2XtJekayTdJGm9PN84ST+X9Jf82KpgmUdJ+mx+/ntJx+Tl3CHpbXn6\nGElfy8u4UdLHh3K9zcAJwqwbGwMHAK8H9gY2iIgtgB8Bn87zfAs4LiI2Bz6Q3+tk6bycQ4Aj87T9\ngEV5OZsDH5O07qCtiVkJ7mIyK+8vEbEAQNLfSPfih3S7g3fk5+8CNpSev4h1ZUkrdljuOfnvLNLt\nEgC2BzaStGt+vQqwPnBXlRUw64YThFl5z7Q8f67l9XO88F1aCtgyIp5uLdiSMNot99mW5Qj4dERc\nVKXBZlW4i8lscF3MC91NtN6TqEsXAZ+QtExezgaSVhiE9pmV5gRhNrgOAqbkgeVbSGMWvfgRcAtw\nraSbgR/gI34bYr7VhpmZFfIRhJmZFXKCMDOzQk4QZmZWyAnCzMwKOUGYmVkhJwgzMyvkBGFmZoX+\nP6E2prctPUXhAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pO2BlfAsNuAf",
        "colab_type": "code",
        "outputId": "2bbabdd8-b877-4895-b318-7abcc854bd98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        }
      },
      "source": [
        "# Visualization 2\n",
        "# Which type of dog was the most tweeted about. Since there are many dog types\n",
        "# Lets pare it down to the top 10 most frequently tweeted dog type\n",
        "df_tweet_countby_dogtype = df_mastr_clean[df_mastr_clean['predicted_dog']!=''].groupby('predicted_dog')['tweet_id'].count().sort_values(ascending=False).head(10)\n",
        "df_tweet_countby_dogtype"
      ],
      "execution_count": 146,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predicted_dog\n",
              "golden_retriever      150\n",
              "Labrador_retriever     99\n",
              "Pembroke               90\n",
              "Chihuahua              89\n",
              "pug                    55\n",
              "toy_poodle             47\n",
              "Pomeranian             41\n",
              "chow                   41\n",
              "Samoyed                37\n",
              "malamute               30\n",
              "Name: tweet_id, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 146
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-vs2HvSPBNm",
        "colab_type": "code",
        "outputId": "8a9505e6-b7c3-4f6c-87f3-093be4017fe5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        }
      },
      "source": [
        "# lets do a pie chart for this information\n",
        "ax = df_tweet_countby_dogtype.plot.pie(autopct='%1.0f%%')\n",
        "ax.set_title('Percentage of tweets by Dog Breed')"
      ],
      "execution_count": 144,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Percentage of tweets by Dog Breed')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 144
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAD7CAYAAAD+dIjEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnWd4XMXVgN+zu+qSZctVsmzkXuVe\nKG7YQIgNBEIPkFATnJAvJJQ4jYjQDIFUEkJCAIfeQ3HouPeKm2TAtlxlW7ZlW33b+X7Mlb0WKrvS\nalflvs+zj+7OnTvn7NXuuTNnZs4RVcXGxsamNeKItgI2NjY2TYVt4GxsbFottoGzsbFptdgGzsbG\nptViGzgbG5tWi23gbGxsWi22gWsjiOEZESkSkZXR1icciIiKSN9o69HSEJEs6965oq1LU9OiDJyI\n5ItIuYiUiMgBEXlWRJKjrVcglo7nRFuPGpgAnAtkquq46idF5HoRWRxppZqDkbI+u8/6XpWIyA7r\nYdA/QvK2i8jMppDV1mlRBs7iQlVNBkYBY4Bfh9pAW3hy1cBpQL6qlkZbkWbKMut7lQqcA5QDa0Rk\naFPKs2ReCjwiIiNrqthGv69hoSUaOABUdS/wPjAUQERSReTfIlIgIntF5H4RcVrnrheRJSLyRxE5\nDORY5beISK6IFIvIFhEZZZVniMgbIlJoPc3/r0quiOSIyKsi8h/rus0iMsY69xzQE3jXejLfbZW/\nJiL7ReSYiCwUkSEB7XUUkXdF5LiIrLL0XhxwfqCIfCwiR0Rkq4hcUds9sfR+x6r7lYjcYpXfBDwF\nnGHpdW+16wYB/wg4f1REell/HVadf4nIwYBrnhOR2+u799b5G637XCQiH4rIaVb5QqvK55bcK0Wk\nk4i8Z8k+IiKLqnSohelWD+iQiPxeRBwiEmtdmx2gQxcRKRORznW0har6VHWbqv4QWID1XbHauMj6\nfx8VkfnWfas6N0pE1lnfiddE5BURub8uWQEy1wG5wCCrraoh5E0isgv4zCo/XUSWWvI/F5EpAfLr\n+v47ReRR6x5tB2YEo1erQFVbzAvIB86xjnsAm4H7rPdvAU8CSUAXYCXwA+vc9YAX+DHgAhKAy4G9\nwFhAgL6YXo4DWAPcA8QCvYHtwDestnKACmA64AQeApbXpGNA2Y1AChAH/AlYH3DuZeuVCAwGdgOL\nrXNJ1vsbLL1HAoeAwbXcn4XA34F4YARQCEwNuAeL67i3XzsP7AJGW8dbrfswKODcyCDu/beArzA/\nXhemx700QIYCfQPeP4QxtjHWayIgteiswDwgDfNg+QK42Tr3d+DhgLo/Ad4N9rMH/N8OWMf9gVLM\nMD8GuNv6XLHWa6clIwb4NuAG7g9GHuY7eBTob73Psj7bf6x7mgB0Bw5jvncOS4/DQOcg/ge3AnmY\n30yadc8UcEX7N93kNiPaCoSkrDEeJdaXYaf1JU4AugKVQEJA3auBeQFfqF3V2voQ+EkNMsbXUPcX\nwDPWcQ7wScC5wUB5NR3PqeMztLe+XKkYA+kBBgScv5+TBu5KYFG1658EfltDuz0AH5ASUPYQ8GzA\nPQjVwD0H/AzohjFwj1g/ll7W/8ARxL1/H7gp4JwDKANOs95XN3C/A94OLKtDZwXOD3j/Q+DTwP8j\nlnEEVgNXBPvZrfLzAY91/Bvg1WqfYy8wBZhkHUvA+cXUbeC81j0stj7HXwN0zbLKegdc83PguRq+\nw98L4n/wGXBrwLnzaCMGriWO7S9W1U8CC6yhSAxQICJVxQ5M76eKwGMwBmFbDe2fBmSIyNGAMiew\nKOD9/oDjMiBeRFyq6q3emDVMeADTY+wM+K1TnTDG2VWHnqcB46vp4sIYnupkAEdUtTigbCfGT9lQ\nFgAXAXswvcP5wHWYHuwiVfVbw8267v1pwJ9F5LGAdgXTI9lZg8zfYx4iH1nt/VNVZ9ehY+D92om5\nD6jqChEpA6aISAGmh/5O/R/5FLoDR6zjjEB9rc++26rjA/aqZT1q0KsmlqvqBAAR6Qq8BDyIeZjW\n1MZpwOUicmFAWQymN1bf/yCDr9+nNkFLNHA1sRvzBOtUk5GxqB42ZTfQp5a2dqhqvwbqUl3OdzDD\ntHMwvbtUoAjzIy/EPMkzMcMrMIY3UJcFqnpuEHL3AWkikhJg5HpiehYN0RuMgfs9xsAtwPRK/oEx\ncAsCdKzr3u8GHlDVF4JSwuh+B3CHGAf/ZyKySlU/reWSKlcFmM+7L+DcHOBazAPpdVWtCEaHAC7h\n5INtHxDo0xNL9l7MvesuIhJg5Gp7gH4NVT0gIm8AMznVwFU3mM+p6i3VrxeRdOr+HxRw6veqZzB6\ntQZa7CRDIKpaAHwEPCYi7SxHcx8RmVzHZU8Bd4rIaDH0tXojK4FiEfm5iCRYDtqhIjI2SHUOYPx2\nVaRgvnyHMX62BwP09gFvAjkikigiA4HvBlz7HtBfRK4TkRjrNTbQuR3Q1m5gKfCQiMSLyDDgJuD5\nEPTOFJHYgDa/xMwmXosxtMetepdiGbgg7v0/gF+INbFiOcMvr+1+icgF1v9CgGOY3pGf2rlLRDqI\nSA+MD+yVgHPPY4zUtRh/Vr1Y/+9eIvJXzPCzakLmVWCGiEwTkRiMEa7E3PNllp63iYhLRL4FfG0p\nTh0yO1p6bq6j2vPAhSLyDUvHeBGZIiKZQfwPXgX+T0QyRaQDMCtY3Vo6rcLAWXwX4+zdgukhvQ6k\n11ZZVV/DDB1fxPhB/gukWUbnAoyTfgfGqf8UpucVDA8Bv7Zmuu7E/LB2Yp70W4Dl1erfZrW9HzP0\nfAnzw6nqzZwHXIXpQewHHsZMVtTE1Rj/zT6M0/m31YfzdfAZ5ge2X0QOBZQvAA5bBrTqvQBrA+rU\neu9V9S1L55dF5DiwCfhmwLU5wBzrfl0B9AM+wfhalwF/V9V5dej9NmZSaD0wF/h31QlL57WYntCi\nGq8+yRkiUgIcxwzF2wFjVXWj1dZWjKH8K+Y7cSFmyZJbVd2YiYWbMH61azEPp8r65FkyczG9+R/X\nVtn6LN8CfmnV3Q3cxcnfcF3f/39h/HWfW/fjzXruRatBVGsamdhECxF5GOimqt+Lti6tARF5Gtin\nqiGvl2yk3BXAP1T1mUjKtTmV1uKDa7FYw9JYYCNmucBNwM1RVaqVICJZmJ5VjQtowyxrMmam+RBw\nDTAM+KCp5drUjW3gok8KZliagfFHPYYZdtk0AhG5D/gp8JCq7oiAyAEYX1cSZr3gZZZvzCaK2ENU\nGxubVktrmmSwsbGxOQXbwNnY2LRabANnY2PTarENnI2NTavFNnA2NjatFtvA2djYtFpsA2djY9Nq\nsQ2cjY1Nq8U2cDY2Nq0W28DZ2Ni0WmwDZ2Nj02qxDZyNjU2rxTZwNjY2rRbbwNnY2LRaomrgrHDN\nwdbNsUKAh0v29SLyeLjaC0Jeloh8p47zGSLyeqT0sbFpC7SqHpyINFkAz2DarqdOFibDVo3Xqeo+\nVb2sgerVS1PeGxub5kqzM3AicqGIrBCRdSLyiZUzsorhIrJMRL4UkVus+lNEZJGIvINJuIGI/FdE\n1ojIZhH5fkDbN4jIFyKyEjgroDxLRD4TkQ0i8qmI9LTKnxWRf1jx9R+pRd8cEXlORJYAz1kZj34v\nIqus9n5gVZ0NTBSR9SLyU6sH+Y6IfAZ8aumwyWqzxjZE5GURmREg+1kRuayO+l+7NzY2bYpoZp0G\nSmoo68DJSMM3A49ZxzmYrEAJmKTJuzFhvqcApUCvgDbSrL8JmCxOHTEZhnZhki/HAkuAx6167wLf\ns45vBP5rHT+LyY7krOMz5GCyOiVY778P/No6jsNkVO9l6flewHXXY/KNVumaBWyqp41LgDlWeax1\nDxLqkXnKvbFf9qstvZrjsCUTeEVMMttYTOq+Kt5W1XKgXETmYXJPHgVW6qlx9/9PRC6xjntgUtF1\nA+araiGAiLwC9LfqnIFJTgImdV9gb+01NakE6+IdSy8waf6GiUjVcDPVku+u4bqPVfVIDeW1tfE+\nJkt8HHA+sFBVy0WkLpnV742NTZuhORq4vwJ/UNV3RGQKpodURfUEElXvS6sKrGvOAc5Q1TIRmQ/E\nN0Kf0vqrnFJHgB+r6oeBFSy9gm27xjasduYD3wCuBF4OQmYw+tvYtEqanQ8O0/vYax1Xzw36LSuj\nd0fM8GtVLdcXWcZtIHC6Vb4CmCwiHa3M5IHZ1ZdikiuDSflWX5LguvgQmGnJQET6i0gSJrl0SiPb\nAJO5/QZgIifT0tVV38amzRLtHlyiiOwJeP8HTI/tNREpwmRb7xVwfgMwD+ODu09V94lIf07lA+BW\nEcnF5KlcDqCqBSKSg8mWfhSTCb2KHwPPiMhdmKzhNzTiMz2F8aetFRGx2rvY0t0nIp9jfHtFDWgD\n4CPMMPptNRnV66tvY9NmsdMG2tjYtFqi3YOzaWnkpAqQXO2VVMN7N6aXesT6a45zjh2PgtY2bRS7\nBxckInID8JNqxUtU9UfR0KfJyUlNBwYBg62/Va+umEmNhuLFuAiqjN92zBq9LcBm4CtyjtU3a21j\nExS2gWvr5KS6MMttTgeGcNKQtY+GOtv86YumuR9LxKwtXA2sBDbkz55hf1FtQsY2cG2RnNRsYBpm\nOc0kgp/dbXJe9U5ecLf3B5OrFR/ETK58CHyUP3vGwchrZtMSsQ1cWyAnNRY4G7gIuADoGV2FaufH\n7ttWv+s/c0wdVRQzA/4RZsZ8Yf7sGf6IKGfT4rANXGsmJ3UacAswnWbUS6uL8RWPHzxAWpcQLtkL\nPA/MyZ89I7eJ1LJpodgGrrWRk9oes8/1VmBAdJUJDZ9KYZ/KFzo3oomVwBzgpfzZM+paZ2jTRrAN\nXGshJ3UM8EPMjoyEKGvTIA5q6ppxlU+MDkNTlcA7wB/zZ89YFob2bFootoFryeSkJgBXAzOBuvxW\nLYIFvmHzv+eZNSXczQIP5c+e8bV9vTatH3uhb0skJzUOuA34JZAWZW3CxlL/kNgmaHYyMDlr1tw1\nwEPAW/akRNvB7sG1JMwugiuBBzl1j26rYEblA19t1l59m1hMHsbQPW8butZPc4wmYlMTOakTMYED\nXqIVGjdVKvO0Z1YERA3ETESsyJo19/T6Ktu0bOweXHMnJ3UA8DDwrWir0pSUalzekMpnBkZYrGKW\nmNydP3vG/gjLtokAtoFrruSkdgbuxaxja/W+0k3+rEUXuB+cGCXxxcD9wJ/yZ8+oKfKyTQvFHqI2\nR3JSv4XZfD6TNmDcAFb5B0TTH5aC6SVvzJo1d1IU9bAJM3YPrjmRk5oI/BGTRKZNcZP7js8/9Y8e\nHm09AD8mJ8c9+bNneKKtjE3jsHtwzYWc1JGYCBptzrgBrPIPyIq2DhYOYBawLGvW3Ba1E8Tm69gG\nLtrkpAo5qXdjZkgj7WRvFnjVsec4yanR1qMao4G1WbPm3hptRWwajj1EjSY5qd2B/wBTo61KNNmj\nnVZOqPzLuGjrUQfvAjfmz55xKNqK2ISG3YOLFjmpVYlo2rRxA9jg710WbR3q4UJgZdasuYOjrYhN\naNgGLhqYIembtKJtVo1hqX9IY/LWRopewNKsWXPPi7YiNsFjG7hIkpPqJCf1CcyShMbkNWhVLPUP\nSY+2DkGSCvwva9bcH0ZbEZvgsA1cpMhJTcb4cmyndQCqFG/X9GYbYbgGnMDfsmbN/XPWrLnOSAoW\nkWdF5LIayqeIyHuR1CVA9vUiklHH+d+JyDmR1CkQ28BFgOw52R2HZ/V4f3FCfPdo69LcOE7iDpCW\n2Jv9P+CdrFlzE6OtSFMjInUZ8uuBGg2ciDhV9R5V/aRJFDMy6lwIbxu4JiZ7TnZ3YKFfZMIPu3bu\nmRsbsy3aOjUnvtTMo9HWoRFMB95rjJETkd+IyFYRWSwiL4nInSIyQkSWi8gGEXlLRDrUcN35IpIn\nImuBbweUJ4nI0yKyUkTWici3rPLrReRNEflARL4UkUfq0atERB4Tkc+BM0RktIgsEJE1IvKhiKRb\nvckxwAsisl5EEkQkX0QetvS6PLDXWUsbA0VkZYDcLBHZWFt9q3y+iPxJRFbz9VSep2AbuCYke052\nH2AxJrcoKtL+6oxuCftczoLoatZ8WOEf1BJ7b4GcDbybNWtuyFGURWQscCkwHPgmJ4OW/gf4uaoO\nAzYCv612XTzwL8zs7migW8DpXwGfqeo4S7ffi0iSdW4EJtxWNnCliPSoQ70kYIWqDgdWAH8FLlPV\n0cDTwAOq+jomteM1qjpCVcutaw+r6ihVfTlA55ha2sgDYkWkKkLOlcArtdUP0C9WVceo6mN1fAbb\nwDUV2XOye2KiyWYFlvtEMi7qnl56zOFoyT2XsLHIP7Q1zCRPpWFG7izgbVWtUNVijI82CWivqgus\nOnMwqR0DGQjsUNUv1SxkfT7g3HnALBFZD8wH4jmZRe1TVT2mqhWYvc6n1aGbD3jDOh4ADAU+ttr9\nNZBZx7Wv1FBWVxuvYgwb1t9XgpBZk4yv0SY2ckea7DnZaZiUdjX63Codjr7TM9M3fLZ7b3yc0hKW\nSDQJqvg/9/dtLbHtpgFvZ82ae1H+7BkVUdRDgEtVdesphSLjMbkqqvBR9++/QlV9AW1uVtUzgtSh\ntBa9amvjFeA1EXkTUFX9UkSy65FZk4yvYffgwkz2nOwE4D1MdvhaOe50Druwe8bnPvNFa5O4ce0s\nJ641OenPxRi5YB9aS4ALRSReRJIxOWtLgSIRqQoddR1mJBBIHpAlIn2s91cHnPsQ+LGImbgRkZEN\n+BzV2Qp0FpEzrDZjRGSIda6Y4FJS1tqGqm7D/A5+w8meWV0yg8Y2cGEke062E/MPCupJVxDjGn9t\nRtelTatV82Wvdm6NvsjzgBeyZs2t17eoqqsw2b82AO9j/G3HgO9hfGcbMH6z31W7rgITlGGu5cw/\nGHD6PiAG2CAim633jUJV3cBlwMPWpMN64Ezr9LPAP6omGRrYBpjfzbWY4Wow9YPC3osaRrLnZD8F\n3BTqdeeWls3/w8FDU8KvUfPmDd/E+Xd4Zk6Jth5NxO/zZ8+4u75KIpKsqiUikggsBL6vqmubXr22\ngd2DCxPZc7LvowHGDeDjpMQpf+yQujDMKjV7FvuGJkdbhybkrqxZc4MJffVPy4m+FnjDNm7hxe7B\nhYHsOdm3Ak80qhFV3z2Hj6y+vLh0fHi0av6cUfHX/QV07FZ/zRaLFzgnf/aM6j60ZoOIrADiqhVf\np6obo6FPuLENXCPJnpM9DrPWLabRjalW/P1A4RcTyyuGNbqtOrjx7XLe+8JLlyRh0w9NJypnfgX/\nWuuhc6JxHT04LY7p/WJYssvLzLkVxDrhpUsT6NfRydEK5YrXyvjg2kQcDdyE4Fc53LvyhY5h+1DN\nl0JgTP7sGbuirUhbxB6iNoLsOdntMGn8Gm/cAETif9S1c88tsTFfhaW9Wrh+RAwfXPv1ycufnh7L\n+luTWX9rMtP7mY/02DI3/7smkT+dH88/VpsI3vcvrOSXE+MabNwAjpDSVn7wnYH/NmQhsE3jsQ1c\n43gC6B3OBq3dDkl7Xc594Ww3kEmnuUhLCM44xTihzKOUeczxtiN+dh/3MyWrcUsoc/2nHW9UAy2L\nkUCdK+5tmgbbwDWQ7DnZ1wPfaYq2/SLpF3XPKD/qcBQ1Rfu18fhKN8OeKOHGt8spKjeui19MiOO7\nb1Xw0OJKbhsXy68+q+D+s6u7bEJnmX9wW1tkPjNr1tzzo61EW8M2cA0ge052f+DxppThdkif6ZkZ\neypEyuuv3Xhmjoll2/8ls/7WJNKThTs+MovxR3RzsvzmJOZ9L4ntRX7Skx0ocOXrZVz7ZjkHShqW\n7W+Jf2iXMKrfUng6a9bc1rA1rcVgG7gQyZ6THQu8jNkz2KQUOx3ZF2amb4zEboeuyQ6cDsEhwi2j\nY1m591SRqsr9Cyv5zeQ47l1QySPnxHPLqBj+siL0PMmquDfraVlhUr0lkQ78I9pKtCVsAxc6D2N8\nKhFhv8s17jsR2O1QUHyyJ/ZWroehXU79avzncw/T+xnfXZkHHGJeZQ3IHFpO3A4vrvBMzLQ8Ls+a\nNffaaCvRVrCXiYRA9pzsMzFLQiIe4mdaadn8P4Vpt8PVb5QxP9/HoTKla5Jw75Q45u/0sX6/DwGy\n2jt48oJ40lOMkSvzKDNeLOOjaxOJcQqLdnr54f/M0pEXv53AgE6hBbbd4u+5ZLp79lnh+CwtlGNA\ndv7sGbujrUhrxzZwQZI9J9sBrMTE34oK1x89vvCOoqPVQ+e0OP7jPXfBPd4bJkdbjyjzXv7sGRdG\nW4nWjj1EDZ4biKJxA3g2NWXCyynJy6OpQzhY7B/aLto6NAMusDN0NT12Dy4IrAW9XwBdo60LqpWP\nHyjMm1xeMTzaqjSUERVPFh0l5WthuNsgW4Bh+bNntNmQWU1Ng3pwIuKzwqNsEpHXrEgIjcKKGd+o\npRciUtJYPWrhNzQH4wYgEndb185Zm2Jjv4y2Kg3Bq44C27idYDB2lrUmpaFD1HIrBvtQwE2E/kli\niOiwOntOdj9MBqXmg0jqNRldk/e4nHujrUqoHKS97Vg/lXuzZs21DX4TEQ5jsQjoCyAi11rZfNaL\nyJNV6casDD2/F5HNIvKJiIyzMuNsF5GLAtrqYZV/KSK/ta7NEpN16D/AJqvO1SKy0epBPlxdIRHp\nJCLLRGSG9f4uEVklJkvRvSF+vj8CsaHflqbFL5L+rcyMyiKH40i0dQmFTf5eZdHWoZnRkWpJZWzC\nR6MMnJichN8ENorIIEzCiLNUdQRmceo1VtUkTKafIZgQx/djwjtfwqnRSsdhsgwNw6Qcq8oy1A/4\nu3W9B7MWbSom2ulYEbk4QKeuwFzgHlWdKyLnWdePs+qPFpGgZiKz52R/A5gRwi2JKG6R3tN7ZOwr\nF2kxRmOJf2ibzUFRBz/KmjW3f7SVaI001MAlWEH6VgO7gH9jkm6MBlZZ56ZxciO6G5OEBUxY5gWq\n6rGOswLa/VhVD1vpx94EJljlO1W1avZwLDBfVQtV1Qu8wMmsQzHAp8DdqvqxVXae9VqHCSo4EGPw\nguH+IOtFjRKHY+iFmembvCb2WLNnqX9Ia47/1lBcwKxoK9EaaawPboSq/tiKny7AnIDyAaqaY9X3\n6MnpWj9Wdh9V9XNqZp/qU7pV74PKoIP5ka8BvhFQJsBDAXr1VdV/19dQ9pzscziZp7JZc8DlGved\njG7Loq1HfahS+pVm9Ky/ZpvkmqxZc2vMwmbTcMLpsP8UuExEugCISJqI1JV3sSbOta5LAC7GZB2q\nzkpgsuVnc2IyClVFTFXgRmCgiPzcKvsQuNHKWoSIdK/SsR5a1BM1Ny524o+7dGq2kWMBiknYoTia\nbJLIe7yQ/S/9gn1PzWTfUz/k+Oq3ASia9zR7/3Ur+56+jYNv3o+/wky2V+zZwr6nb6Ngzu14jpj5\nGn9FCQde+Q3m2RtRYoHbIy20tRO2L5uqbsEkZ/3Iygb0MWZzcSisxCSb3YCJT7+6BjkFGOMzD/gc\nWKOqbwec92GM3lQR+aGqfgS8CCwTkY3A69ST5ix7TvZYzBC7RTE/KXHyw2ntm62R+0q7N+2EiMNJ\nh7NvIuPmJ+h23aMUr52L+9Au4rNGkHHT38i48XFi0rpzbPlrABxf9RZdLsuhw7RbKF7/PgDHlr5C\n6hmXE+HJ+iq+nzVrbmo0BLdWGhSTS1VrTBaiqq9QQ8bpwPoBw9ZTzqnqs5gUZNWvzcdkuA4sewkT\nSbdGOapaScAwVVX/DPy5ts9TAz8LoW6z4vl2KRMzvd5l1xwvCTZJb8RY6R/YpO27ktNwJZtoRI64\nRGI69sBXfJiEXqNO1InLGEDpVjMwEIcL9VainkrE4cJTVIC3+BDxPZs0YnxdtANmArOjpUBrw96q\nVY3sOdnpmJnclomIY3Zah1GfJSasj7Yq1Vnsz45YLDTvsQO4D2wnLmPAKeUlGz4mobdxraaefjmH\n3vsDx5a/RsqoCzi68D+0nxj1QB8/yZo1t/ERRW0A28DVxEzClWMhWojE/aRLp14bY2O/iLYqVaji\nX+vv1ysSsvzucgrfepC0abfgiDu5yebY0lfA4SRp8BQAYrv2Jv27j9Ht6ofwHtuP0+r9Fb79MIfe\nfRRfaUQDKlfRjSaKFN0WsQ1cAFYwy2ByWTZ/RFKvzeiautvl2hNtVQA8uHaVEd/kQULV56XwrQdJ\nGjyFxAEnE6GXbPyEsm0r6XThnUi1ZDmqanxvZ17F0SUv0mHKDSQP/wbH17zb1OrWxvXREtzasA3c\nqVxEc9lzGgb8Il2/lZnuOeJwHI62Lvu04/6mlqGqHH7/z8R07EG7cZecKC/fvobjK96gy6X34Ij5\n+jrj0k2fkdB7DM6EFNRTCSIgYo6jw8SsWXNDXYFgUwNtLfFHfVwebQXCjUek1/QeGZvn7dqbkKDa\n6KAIDWWd9q1oahmVe7dQunkeMZ2z2PfMjwHoMOm7HPnkn6jPw4FXfg2YiYaO37gNAL+ngpJNn9D1\nivsAaDf2Yg6+loM4XXS68K6mVrk2BLML6MFoKdBasMMlWWTPyU4ADgI1zhC3dLp4vas+3L1vpCtK\nD7U7PT9Y+bpv8rhoyG6h5ObPnjE42kq0dOwh6knOp5UaN4CDLtfYqzK6RS1Y5nL/4B7Rkt1CGZQ1\na25UA6y2BmwDd5LLoq1AU7M1LnbCj7p2nh9puX6laI92DnXRtw1cF20FWjq2gQOy52THARdEW49I\nsDAxYcpDaR0iutuhiJT8SMprRVyVNWtuxBMctSbsSQbDeZhV5E3Knn/voXh9Ma52Lvo9YAKaHHjj\nAMfXHUdEcLZzknlzJjEdYji26hgH3zqIM9lJz//riSvZReXBSg68foCeP2zcfvUX2yVP7OH1Lrv2\neHFEdjts9fc4Hgk5rZCumBSVa6OtSEvF7sEZIjI87TChA1l3ZJ1S1ml6J/rd34++9/Wl3Yh2HHz7\nIACHPzlMn9/2IW1KGseWHQPg4BsH6frtMKxiEXE8nNZ+9CeJCesa31j9LPUPCS2voE0g50RbgZZM\nmzdw2XOyXZj1b01O0oAknEmn/tadCSff+yv9JxahikNQr+J3+xGnULq1FFeqi7huYdrFIxL70y6d\n+nweF7s1PA3WzhL/0M5NLaMa7y7xAAAgAElEQVQVc260FWjJtHkDh4ny2z6aChx4/QB5P8vj6LKj\ndLnERHLqPKMzOx7ZQfG6YlJPT+XgOwfpfFGY7YRIu++md+2w0+VqsjwJqng2aa+IbNFqpUzImjXX\njoLcQGwDB6dHW4Gul3Vl4B8G0v6M9hz+1Gw6SB6aTN97+3LaT0/j+LrjpAxLwb3fza7Hd7H36b34\nK8MTr8wv0uWSzHTfYYfjUFgarEYFsfkeXM0up0ULIh44K9pKtFRsAwfjo61AFalnpHJ89an+eH+l\nn6OLj9JxWkcO/vcgmbdkktg/kaPLjoZNrkcka0aPjANlIsFGTg6aXdrlYLjbbIPYw9QGYhu4KPfg\nKvef3O9YvLaYuPRTfWyH3j9Ex3M6Ii7B77Z6bcLJ4zBR6nAMuSAzPddjkvqEjTX+/i0iV0Qzx55o\naCAR2aolInUGkFTVPzS5EjWQPSe7I9AkQ7Oa2P3EbkrzSvGWeHG1c9Hl4i6UbCgxRk4gtmMsGddn\nENPBRGvyFHnY+8xesn6WBcCxlcc4+N+DOBOtpSPtwr/Kp6/bveTNvfvPFLMfstHMdP9k3fv+8SPD\n0VYbxgMk5c+eEdaHT1sgUuvgqkKED8BkxXrHen8hJkx5tIjo3sgeM7++Wyltcu0xIGM6xJwwbgCp\n41JJHde0Ea2/io0964ddO89/4kDhlHC0t8I/0E4y03hiMNngNkZbkZZGRIaoqnqvqt4LZAKjVPUO\nVb0Dk2Ywmj+AqE8wNEcWJyZMeaBj43c7+FT2HyG1Yzh0siE72gq0RCLtg+uKyZFahZvoxl9rNhMM\nzY2XU5In/addytLGtFFI+2YRbLOVYBu4BhDprVr/AVaKyFvW+4upIdFMBBkbRdnNGxH5fVr7Md28\n3rXnlZWPqv+Cr7PJn1USbrXaMLaBawAR7cGp6gPADUCR9bpBVR+KpA5VZM/Jbg9ELAlKi0Qk9o4u\nnfqtjYvNbcjlS/1D7fVv4cM2cA0gIgZORNpZf9OAfOA567XTKosGmVGS27IQSbk+vWunfJdrV6iX\nLvUPsUMkhY+eds7U0IlUD+5F6+8aYHXAq+p9NLANXJCoSOdLMtP1kNNRGPQ1StlWzbTzCoSX/tFW\noKURqVnUC6y/vVS1d8Crl6r2rqonIkMioY+FbeBCwCty2ozMjMJSkaD8aiUk7FAc9kLy8NJqEiJF\niub2BXwugrJsAxciZQ7H4AsyM7YGs9thu6YfiYRObYwu0VagpdHcDFwko5faBq4BHHI5R1/ePX2l\nQp1bYFb4B9rZjMKPbeBCpLkZuEj+KGwD10C2xcac9YOunRfWVWexPzuqIahaKfYQNUSam4GLJLaB\nawTLEhMm/66W3Q6q6Bp/fzsGXPixe3Ah0twMnLv+KmEjI4KyWiWvpSRPejo1ZUn1ci/OXaUkpNR0\njU2jsHtwIRJRAycin9ZVpqqR3BsatSzvrQYR+WOH9uM+SEpcE1hcoB0LoqVSK8fuwYVIRLZqiUg8\nxqB0EpEOnJxMaAd0j4QONRATJbmtC5GYuzp3HNDZ68sdXVk5CGC99qms7zKbBpEQbQVaGpHqwf0A\ns6h3ICYF2hrr9TbweIR0OEH2nGwnzW943nIRSb4hvUun7TGunQCL/UPtH2LTYKf5DJFILfT9s6r2\nAu60FvdWvYarasQNHGDvkQwzKtL50u7pUuh0FC73D7b9m02DnX4xRCL9RHhaRH4N9FTV74tIP2CA\nqr4XSSV+/5RXuhXxJWZZiqoZMJvYxoIfc0IBVKw6oEgN5adei4LfKuPENUJV3GRVCSg/+R6sOion\n2kRP1hGrjarrzLmT1wDgD5QlUnV9Vd2T8sxL/HLiswTWE381Hfwn5Qe2Jda5E2UAr8T1Oxw3ypfk\nT0z6yicuu5ccTlSPRVuFlkbEDRxmaHqm9X4v8BoQUQN3WiF+oF8kZUaeyK+zrYxJKVwy/KL1Nx4+\n5Hb2+nfShoThMfOYVrmDPn384oyWr7U1YcfXC5FIG7g+qnqliFwNoKplUpXpOLL4oiCzVeNxJhxb\ndvq9R9xl78YOP35+yhsrPWn9eq/aPyXzF2NFSMjXXts+5bw9qxmfcpx2QxGx3QShY39vQyTSBs4t\nIglUjcpE+gDRmHGzvyhhxOeILVt2+u92+p1xw/zeAj3mPnjgfPcI79zt/rMOHuyVP3zEB74s547+\nN/Fkn5t4knLiS5brWevnM61iO317+8XZLBZdq8/HkZnX4OjUhQ4P/oVjD/wS746viD19Iik3/xiA\nkuf+hatXX+InnB0NFe0MZSESaR/Jb4EPgB4i8gLwKXB3hHVgUF6uHwhv3r02il+c7mXjc7Z4YxKH\nqb/iGHj67y/fkZCuHQZn+DssLi1N67Ns6ZWnHS3qtkDVPNgSqEg+m0/H3csvJz3HFZkP6h3bpumH\nC9rp0bWoRnKx9ymUvfkirp5mA4Zn2xcQF0fHp17Fu3Uz/pJifIcL8eRuipZxA7B9cCES6Yi+HwPf\nBq4HXgLGqOr8SOoQQMTSBbZWFPGvGPvrNe641DEAfs+2LwDHgfL87gDneoaPFZXdqs64jRvPnbw1\nb8IaVb6WCPo08vvcyD8nP8FNo/7NNZ7v6+Or+mneQqd6d0fqs/gKD+BevpiE6ZcAIC4XVFaifj/q\n9YLTSekzT5B8/a2RUqkmiqIpvCUS0SGq5W/7JtBbVX8nIj1FZJyqRiN1YAH2yvBGsXr03UvKE7tM\nrHrvc+eVAZR6j3VX1SMx4kw72zMk97PYTT0ACgt7jTl6NP3wiJFzV8bHl9WYsjGeyqTJzBs7mXkA\n7NaeOz7hvF2rOD35GO2HIhJX03WNpfhvvyf5Bz9By8oAcJ3WG0dqB4784Griz52Bb+9uVP3E9B/U\nFOKDxQ5BFSKR9sH9HTM0nAr8DigG3iA6yV8KgOFRkNsqWDfstgXFKT0nB5b5vfs6Vx1X+svy451J\nab39XUev9+cvOeIoOQvA44nvuGrlpR179VqzsHvmlrEida/O78GuXjfwVK8beIpK4spW6Bmr5nFO\n+Tb6ZfnEFZaUk5XLFuJon0ZM/8G4158MMJ1y210njot++RPa/exXlDz/FN5tXxA7+nQSL/h2OMSH\ngm3gQiTSBm68qo4SkXUAqlok0ZtN2xcluS2ezYO+N78obdCUwDLVymLwDKh6f6Ryf3FGYh8AprtH\nDn4+blEhwgkDuGPH6EkHD/baNnzEB16n0zeAIIijMnES88dOYj4Au7XHjk85b9cqzkg6anp38Q35\nPO5N66lcuoDCFYvB7cZfVsqxB39F6i8fAKBiyTxi+g9Cy8vx7dtD+98+QtHdPyThnG8i8RHdtGEb\nuBCJ9CSDR0ScnJxF7Uz0nP32hvAG8GWfby880HXclOrlfvf2rQSstD9Qnn9ir288sR1O9/b7qvo1\n1gREVlHABEQo9GB3r+v59+S/cfOYp7nGf6v+ZdUA3bLAqd6dobSTcsv/0fnVD+n80v9I/c1sYkeO\nPWHc1Ouh7I0XSbrqe6i74uQuar/P+OYiy+GmFiAiU0SkSdelisgIEZnelDKqiHQP7i/AW0AXEXkA\nuAz4dYR1qMI2cCGys8e5S3ZnTp1Y0zmfO7c08P3+8vxTtmsN9fU8Y6Nr18pSqTzF96bqjNu08dzJ\nnTrnrxk4cHGmiDYoJFAclYkTWTB2IiZE3R7NzP+U83au5Iyko3RocO+u7L+vknDehUh8Aq7e/dGK\nCg7fdDmx4yfgSI54RKiQM5s1U0YAY4D/NbUgUY3sincRGQhMwzwLP1XVBuXcbCy5Awd9G+P/swmC\nfd3OWJk34JpRiNT4UKwoenwzuAOTBukVWXeXiMgJK1BCRcHLcUuSENrV1EZMTMXhESPnbqttAqKh\nVBJbvorTN8/jnNIv6Z/lk5iWmu1r3P6zR6yqr5KIZGGWYy3H7BpaBTwD3IuZWLvGqvpnIB4ox+Qo\n3ioiUzB7xi8QkXG11Lkek7Q9CbMj6FHM/u7rMOtap6vqERGZb7W1WkQ6YTLo9Qe+wkRG2Qs8hNnJ\n9FdgKCbKT46qvt2wW3QqkZ5FvQ9YCDyrqqX11W9ibB9ckBR2GrYub8A1w2ozbqruEnBX96OJ21+x\nI86ZMKyqIJn49Gxfz0UbXbtq7AUGTEAs6p65ZUx9ExDBEoc7YQILx0zARFnfq913fca5O1ZwZmIR\naUMxi89bAjtCqNsXuBy4EWPgvgNMAC4Cfgl8F5ioql4ROQd4ELi0Wht5ddQZCozEGL+vgJ+r6kgR\n+aPV9p9qUkpV3SJyD2aJ2G0AIvIg8Jmq3igi7YGVIvJJOGxEpIeo24Grgb+ISDGwCFgYLmsdIva+\nviAoSu23ZeOQ7/era4jn92zfCoyuXn7MfbCoS8KpnaVx3r4TvnDu+7xSvLXOYO/YMXpiqBMQodCd\nvT2v49me1/EsbmIqVuv41Z9xbumXDOjplZjmGmr9yP6zR4SydnOHqm4EEJHNmNGSishGIAtIBeZY\nAS+UmuMj1lVnnqoWA8Uicgx41yrfCAwjNM4DLhKRO6338UBPoNGju4gaOFV9BnhGRLoBVwB3At8H\nIu7MGJSXuyd34KDDQMdIy24pFCf3+GrdiJ90QyS5rno+d15xTeUHyne6qhs4QWSGe3S7N2NXVCLU\nuqbNmoBwDxn62YL27fdPEmmajGuxeOLPZPGYM1kMQIFm7PqUc/NXcGbCEToOQaS5RH7+IsT6gVsg\n/QHv/Zjf/X0YI3WJNaSdX0MbddWpr30wW8uqJjLr8oEKcKmqbq2jToOIdMjyp0RkKfAE5iZcBnSI\npA7VWFN/lbZJWULn3atG352MSFp9df3evZ1qKi8o31HjQuo0Te7Vx991WX3tqjpjN208d3Je3sR1\nqnKgfq0bTzr7el7LnEl/5Qdjn+Fqx236hzWDdeMCl3pCGR42BXlhbi8V4wMDs7OooXXqIp+TPfvL\nAsqLObVT8yHw46rAGyIysgGyaiTSy0Q6YpYSHMWs6TmkqtHcQFyvw7YtUhHb/sCKsb9WxNGtvrqq\nnjK0ssZh5FH3gd6qWl7TucmewWc51RFUr+RQYdaoFcsvc1VUJEV0x0ssnvgzWDL6V+RMnsNVvR7V\n23ZP17cXpumhlaiWRVIXwv8wfgR4yFqTWttILpg6dfEoMNO6PvAhOA8YLCLrReRKTE8xBthgDafv\na4CsGon4LCqAiAwCvgH8FHCqalSiSeQOHHQxZtlKRPhVQQELSktIczp5p1dvAP5yqJDPiksQgY5O\nJw+mp9PFFcNHxcf566FDpDqdPN49k/ZOJ7vcbv50qJA/ZDRdaDWPK6loyRn3H/I7Y4OKl+dzb13r\nKZ07qrbz3z7t9s0xjrghNZ0rkKItc2PXDkCCj1Sb1WvtwszMzfXugGhqPLgq1zBu02ecW7KVgT28\nEtu7iUWO33/2iGhsaWzRRNTAicgFwERgEtAeM429SFWfjpgSAeQOHJQJRGxD9+qyMhIdDmYV7Dth\n4Ep8PpKd5vf9XNERtlW6yenWje/t2sk/MnvwcXExx/0+ru2Qxp379nJbp85kxTbN5g+vM65kyRkP\n7PS5Emo0SDXhLnl7vt+zbUpt56elX7uwU3z3SbWd/1/M2gX7nEWTaztfE0lJRduHj3jf0xQTEA1l\nP932zOOc7cuYEH+YTkMQSQpj824gZf/ZI6IWaaWlEulZ1G9jxtt/VtV9ACLycIR1OIE10bAfqHco\nFg7GJCay13Pqd7TKuAGU+/WEJ90hgluVClVcIqwuK6OTy9Vkxs0vrspl4+/90udKCMn/4ffuqdNH\nd7Bil3SKr73Hea5n+Nj/OBbsVtEewcosLe3Q20xAzFvQvn1BgyYgrvnOLhISBadDcDrh709k8q9/\nHmblynL69I1l1izjPvzk42KOHfdz6aWpdbbXjf2ZV/N85tU8jxeXe62OWfsp5xVvZVCmR2L7hKpf\nNT63jVvDiLSBG6GqN1Yr+ybw8wjrEchq4IIoyudPhYW8c/wYyQ4Hz/Yw+8dvSevITbt30cXl4uH0\nDH66by+PNtHQ1C8O7/Jx96z3xKaMD+U6VU85Wjmwrjr7y7Z3Gtz+jFrPx+BMDIw4ErxsZ+ymjedM\n7tQpf83AQQ3bAfHYYxmkppoHTEmJny+/dPOvpzJ57NFCtm930727iw8/LOah2ekhtevCGzuO5aPG\nsRyAA9q1qncXd4jOQ+qbla4Be2jaQCIyySAiM631NwNEZEPAawewIRI61MHq+qs0Lbd37sxnffpy\nQbtUXjhqQn6dmZTE61m9+HtmDz4rKWFSUjI73W5u37uXe/YXUO4PzxZeBV01+hcrKhI6hmTcAPye\n/DzqyVB2uLKgt6p66qrT2991dJo/eUmo8gEOHcoabU1ArGjI9VU4HOD1KapKRaUflwtee/UYF1+c\nisvVuBUqXTmQeRUvTPozM8fP4arY2/WRddm6fkGMur+2P7cWljdKgTZMpGZRXwQuBN6x/la9Rqvq\ntRHSoTYa9MNqCi5o146Pi09dUlbu9/Pf48e4ukMHHj90iAfT0xmVkMh7x4+HRea6EbcvLE3OOKsh\n1/rcufUq4ccX51Pv9vrqTXePHIxS2BA9zA6Ib4/fvWvIQlWCmt0UgZ/fXcDMW/fw3nvHSUx0MH5c\nIrf+YC8d01wkJTnIza3krAnhdKWZ3t1YVoycxX2Tn+Xqvn/UmXsv0jcWddKDKzALZ2vik7Aq0YaI\nyBBVTbqzY5hdDM2NBRjd6nayNBH5bvcJv9pnJcX0jj117evTR45wTfsOxIhQoX4E81Sq0Mb34DYO\nuXnB0fb9QnLwB+L37mkfTL1iz5HCDnFd65wQsCKOLFse82XnuurVRX7+qEmFhb2CmoD4058y6NTZ\nRVGRj5/fXUDPnjFceVV7rrzKfKTHHi3k+us78L+5x1m9ppzevWO59trwL9nswsHuV/Ji9yt5ES9O\nz3odve5TzjuWy+DuHonrB2zcf/aI/WEX3EZo85myB+XlenIHDpqL2avXpNy5by8ry8o46vNx9rav\nuK1jJxaWlrDD7caBkBHj4rddT853HPR62FhRzo86mSVE13TowBU782nncPLX7o3zx23td+WCws4j\nG2zcVL0VaEWd/rcqCit2+zvE1e8iMxFHdq8slYoGb7YPdgKiU2fz1e/QwclZExLJy6tk2DCz8uTL\nLytRhcweMTz17yM8/HA6v3/kIHv2eMjMrGlHU3hw4YsZw8qRYyyXW6F23reCM140wTdsGkKbN3AW\n/yUCBq6mSYJL29feCeriiuEfmSd97+entOP8lBoDcYTE9qzpi/ZmTKx16UYwWP63oH55BeXbO/Q3\naRvq5cLK0T1ejltyvLaII8FwcgJi59qBgxZliOgps+Tl5X5UITHRQXm5nzWry7n2upO9s2efKeKn\nP+uEz6dUuTpFhMrKyIYu7ExhxgW8swz+GFG5rQnbwBneByqoe79cq2BPxqTl+adNP5NG5qP1ufOO\nBlu3sGJPb1X1i0i9Pt/6Io6EwqFDp41avqzrkZGj/rciPr70xCRKUZGPnN+anV8+nzJ1WjLjxpkt\np0sWl9J/QCydOpmfRt8+sdx88256946jT58mSQdRF0doRj7ilkhUdjI0R3IHDnoPmBFtPZqSA51H\nr9k8+IbscCRdrjj6xHq0POix02VZd+xwiiuoSB2K6vNxCzfUFXEkVLKy1i7M7LF5jAjNZfN8MLww\nbeq2aE/CtWgivRe1OfPfaCvQlBzuMGjj5sE3DAqHcVP1VqLlQfnfqij1HA06gnJVxBE0fEnB8/NH\nTVq39oL9Pp8r3JvWm5JXoq1AS8c2cCd5h1aaDPpYStbWz4f9qGe4Qv/4Pbu2EuJwvrBiT51r4apj\nIo50C+v6LzMBcUXvoiPpDcoBEWEOYFwnNo3ANnAWg/JyD9IK/R2lid3y14y6Iw2RsC2D8blzQ87u\nVFC+I+RJg8meQWcGG3EkWFSdsZs2nTM5L3fSOlVpzssvnps2dVs0I+20CmwDdypzoq1AOCmPSytY\nOeaXLsTR4LVlNeH37grZWBZW7Ao5Uq4DR8z57hFeFF+o19aHmYC4LLaxOyCakKgEoGht2AbuVF7C\nxKpr8bhjUg4tH39PuTqcYQ1Fpepzh+p/A3D7K9r71RdymPh07TA4w99hcajXBYPXG59mdkAMXRTs\nDogIsWLa1G1RScbU2rANXACD8nLLaAW9OK8z/viy0+8tVEdM2GOU+b278qBhsdjKvMcblAfjXM/w\nsQ6VJkuZl58/cmIzm4B4JtoKtBaCMnAi0k1EXhaRbSKyRkT+JyLfry1BrBWafLB1XBIORUUkS0Q2\nhaOtevhHBGQ0GT5HTPnS03+3w+eMG9Qk7bvzihp67aHKfQ2aFY3BmTjFM6TehCsVFRW8+uqrPP74\n4/ztb39j9+7dfPzxxzzxxBO89dbJuKYbNmxg+fJT5y+qJiCOHMmYH+UJiHLg5SjKb1XUa+CsOOlv\nAfNVtY+qjgZ+AdS690ZVb1bVLeFTM3IMysvNAz6Kth4NwS8Oz/LxOZu8MUlhWz/2NRmenaGG+jnB\n/rLtDd653tvfdVSaP7nOoeoHH3xA3759ue2227j11ltJSUlh//79zJw5E6fTyYEDB/B4PKxfv56x\nY8d+7XpVZ+zmTdOm5EZ3AuK5aVO3HYuS7FZHMD24swGPqp7o2ajq55iUf8ki8rqI5InICwFJI+aL\nyIm9OSLygIh8LiLLRaSrVfasiFwWUKfE+pssIp+KyFoR2Sgi3wrQxSki/xKRzSLykVj5LAPliUgn\nEcm3jrNEZJHV1loROTPI+/JYkPWaDYr4V4791arKuPZf/+WGS4b6PGhZyP63Kg5U7GxUwuXp7pFD\naos4UlFRwc6dOxk50sTrdDqdJCQk4PP5UFU8Hg9Op5OlS5cybtw4nM7ao6Qfjt4EhB/4fYRltmqC\nMXBDqT3hxUjgdmAw0BuoKexOErBcVYdjkj7fUo+8CuASVR2FMa6PVRlOTBbtv6nqEMxkQPVEtdU5\nCJxrtXUl8Jd66gMwKC/3I6Ifpy4kVo+6c0lZYrdgDXiD8Hv35GH+nw2iwlfaWdXf4OxYVsSRGmOo\nHT16lMTERN5++22efPJJ3nnnHUSEfv368eSTT5KcnExcXBx79+5l4MD6bXTVBMSuyE5AvDFt6rZ6\nY8SJyK+sh/wGK3FLyLH8IoGI5ATkOo0Kjd2LulJV9wCIyHpMQtnqwwg3UOWrWwOcW0+bAjwoIpMw\nT7TunBwO71DV9QFtZdXTVgzwuIiMAHxA/3rqB/IYLWTC4fPsmfOL22VNaWo5DVn/Vp1yX8muRFe7\nkKPvVlFbxBG/309BQQHf/OY3yczM5P3332fx4sVMnTqVs84yz9133nmHs88+m7Vr17Jt2za6du3K\npEl1xxzYmT9yYuHBrB0jRn5Q4XR6m8SvGcDs+iqIyBmYCNSjVLVSRDpRT9DRtkwwPbjN1JC13CLQ\naeyjZoPp0ZMbXgPrnEgKa23CrvonXQN0xgTDHIFZ0V21ar42ebUlmP2pdf1wYAyhfRFeAr4MoX5U\n2DLgugWHOw6dEglZfs/ORkd/PFJZUNrYNi6sHN0D5ZRgm+3ataNdu3ZkZppVMYMHD2b//pNutIKC\nAlSVjh07smXLFi6//HKKioo4fPhwvfLKyjr0Wrb0ij5NPAHx7rSp29YGUS8dk26zEkBVD6nqPhG5\nR0RWicgmEflnNXfRH0VktYjkishYEXlTRL4UkfurGhWRn1nXbhKR262y31UdW+8fEJGfWMd3WfI2\niMi9AXV+JSJfiMhiIOpJgYIxcJ8BcSLy/aoCERmGyY7VGPI5aTgvwvS2wASePKiqHhE5GwjGbxPY\nVmCC2VSgQFX9wHUQfHq6QXm5HuDuYOtHg696X7xwf/rpDY7pFgqqfi9a2mD/WxX7y/Mbne4vmfj0\nYb7T1p9SlpxMamoqhw6ZydYdO3bQqdPJVJzz5s1j6tSp+P1+/AHh3j2e4HaQVZuACHpfbQjkBFnv\nI6CHZUT+LiJV///HVXWsqg7FLOMJzDPiVtUxmBUCbwM/wrierheRjiIyGrgBGA+cDtxiJV9+Gvgu\nnOiEXAU8LyLnYdxF4zAhs0aLyCSrnaussulAk/mDg6VeA2f1vi4BzrGWiWwGHgIaO8v0L2CyiHwO\nnAFUPdlfAMZYORy+S3AZvWtLMPt34HuWjIEBMoJiUF7ufzF+w2bHrsypS3f1OGdCpOT5vXu2Ag2e\nQa1if3l+WBYej/X2mRinrs8Dy775zW/y5ptv8sQTT7B//34mTjTP4Ly8PDIyMkhJSSE+Pp5u3brx\nxBNP4PP56NYttIRq1gREfEV5cjj3yb4TZO8NVS3BPMy/DxQCr4jI9cDZIrLC+t1MBQJTP75j/d0I\nbFbVAqsHuB3oAUwA3lLVUqv9N4GJqpoPHLaM3XnAOlU9bB2fB6wD1mJ+W/0wnZ63VLVMVY8HyI0a\ndrikesgdOGg0sApCT03XVBR0Hb8qd+B1IxCpNbzsZxteZ2ne/xCEjLReXDvlbl5Y8Cj7jmxnaM/T\nuWj8zQB8sPZ50jtkMbxX3bbSU/rhfJ9785Rw6H9F1t1HRKTOdIPBcERKdrwZuyIDIeKB2gBOy1q3\nqEePTaMbGYLJDQydNnVbg9wh1kqEHwDDgDGqultEcgBUNUdE5gN3qupqEZliHV9gXTsfuBMzOdhR\nVe+xyu8DClX1L2Iyz5+JSa05R1X/JyKPAV+o6pPVdLkdSAto5w/APlV9tCGfLRzYOxnqYVBe7hpM\nr7JZcKjj0M9zB143tC7jdrS0kAWb3uLubz/Br674N371syT3PWJcsfzy8qfYWbiV8soSjpUeJv9A\nbr3GDcDnyW90762KSn9ZfjjaSdPkXn393ZaFo62GsDN/5MS1ay444PO5GrOt6g+hGDcRGSAi/QKK\nRgBbreNDYlISXvb1K+tkEXCxiCSKSVh9iVUGZg3s+Zjh5odW2YfAjZYsRKS7iHTBjHYuFpEEEUnB\nJJaKKnZE3+D4BWZJSiv/uJYAABWQSURBVKP9R43haGqf3A1Db+2Ftf6vLnx+Hx5vJU6HC7e3AkXx\neN341Y/P78PhcDJ39bPMGPO9euWq+n1oadgcxkWVB4rTE0/dRXasopi733+ErYd2IMCj02fx4ZeL\nmLd9BUO69ONPF/wKgDc3f8SRsqPcPPYKACZ5Bk3Y4Tj4hU/8ocyQhw1rAsI9eMj8BR067As1CfUe\n4P56a51KMvBXEWmPmVz7CjNcPQpswriOVoXSoKquFZFnOZl/9SlVXWedc4vIPOCoqvqsso9EZBCw\nzJrLKAGutdp5Bfgcs0QrJD2aAnuIGiS5AwfdD/wqWvKLk7pvWzVmVnvE0TGY+vM2vsG7K58m1hXH\nwMwxXD/tl7y+5G98WfA54/qdw4Duo1iw6S2umXJXvW35Pbu3uEteG9zoD2HRv92YZSM7TjslG/RP\n5z7AuMzhXD38Atw+DwdLDnP3+4/w4lV/4K73H+bGMZeR1T6TG974Oc9d/igxzpPP5gIp2jI3du0A\nJPhJpKagY8dd6wYNXthNRIPNFH31tKnbmvW2LGtyYS1wuao2+1UF1bGHqMEzm8ZPrDSI8vhOe1aN\n+XlisMatrLKYjflLufc7L/DAta/i9paz8ouPueysH/GLy/7JtOFX8N6qZ5gx9gY+WPsC//74dyzJ\nnVtrez53XoPyldbG/vL8UwzA8coSVuz+nKuGmYjxsc4Y2se3w+P3oqqUeyqJcbh4cuXLXD/q0lOM\nGzRtxJFQOHy458jlyy6LLw9uAmJBCzBugzE9xE9bonED28AFzaC83BJgZqTlVsamHlw+7jc+xBls\nr4C8PWvpmNKNlIT2OJ0uhveayI4DJ7cGb8hfQo/O/an0lHPo+D5uOvce1m1fiNtTUWN7Ps+OsA7N\nj3sOnaYBSY53Hy0gLbE9P/vfQ5z/zE3c9f7DOESY2ud0zn/2Jrokp5ESl8T6gi2c37/m1UlNHXEk\nWLze+A6rV11y+q6d2YtUa5219wE/jqReDUFVt6hqb1W9I9q6NBTbBxcCg/Jy/5s7cNC/gZsiIc/j\nSjy6bHzOUXW4QvIvpSV3YcfBXNyeCmJccWzdu5aenY0LzefzMm/jG8w8/0EOHt97wmGk6sfr935t\nJbSq+tGSU/xvB4+X8PzydSfeHy4p4xtD+3O8vIKt+wvJaN+Oq8ebfDRrdu6htNLDpP6nxLsUt79i\ne5wzYTiA1+9j0/4vue+c2xmZMZjffvJn/rb8Be6adDMzx5tsjne9/zB3TLiRlz5/j4U7VjGwS29+\ncuZJ/2EMzsSzPUMPfRq7sWco96qp2LlzxMTCwqwdI0a+X9MOiEemTd22MSqKtTHsHlzo/ATTbW9S\nfI7Y0mWn37vb74wN2Xme1XUQI3tN4uE3b+XB125GVTlrkBn+Ldz8NuP7n0dsTDzd03rj9lbywGs3\n06NTPxLjvj5Rqr59WzELpk/QpV0yPztvIj87byK3nzOBWJeTod27svfoce74xiScDgcFR4/j8fpY\ntWMPZ/X9+lrtY+7CExEz0lM6k57SmZEZxs03fcAUNh04Gal804EvUFX6pPXkvbx5PHHxvews+v/2\nzj06qure459fEiCEVxCMAoGC+ApSRB5WlKAGrFpA9FqlLq2i9l6jsV5rq8vVdulpXW3TpVZXe9Pm\nXp/1UR/X1kfprS5tFu+nPA0mPKMgAgES3kkmM/O7f+wzZIBk8pozZxj2Z62zMjmzZ++dZPKd/fjt\n7+9rqmq2HVPnsHBOq44jieTIkexhixfNPLumZuBc1aP5PlYDj/vZr1MJO4JrJ3mVFYcrzs+7DXPm\n1pPfX1jSA4sv+UVlMCOrpSNyrTJ1/Cymjp91wv0rRzX5E4gId075ecx6Qg0V1UCLZzA3Vu+hX48s\nsrp2JRQOG+eOUIj0tDTmrN/CxHOGkp524uforrov0nO6m8FWTs9+DOidw+a9WxnebwgLv1zBOf2H\nHi371PwXKL76YRrDQcJqdCJNhLrgifZy3wmMGflat3m7EeJq095RVNO6rCuffHm/fltXnZ83r29a\nmt42uWBzuxLwWDqOHcF1gLzKiqXAr7yoW5HQkosfWxno2rvD4hZPQsHY62+rt37N6CEDyeySQd6A\nHJ75eAG9MruR2SWDrTX7GDmo+ZMCO+qqcqK/f2LKf/LD2U9w1YuzWFe9ifsnfB+ADzfMZ9SZ53Fm\nr/70yezFiJyzmfLCHTSEAozIOfuEejPpkn1J8FzPR9jtZe/eIRctWnjLU5MLNq/zuy+nEjZMpINU\nnJ+XgQmGvCRedSro8rGPLjzUa3DCjmDFQlXDDfueOQBkN/d8MBTml3//hIevuZxemcceJnh7+Vou\nPfsbbK/dz/qdexiY3YspI5riUwUJ3jT04aCItCv9YFt5o9vCExxHfOZ9x3Gu97sTpxp2BNdB8ior\ngpgD/HGxZAdYdeED85JF3AA0tGMjLYgbQOXOanL79jlB3LbX7geU03v1YM22Hdx+6Rj2HDrC7oNN\nm4qKZgQ1sMWrvjfnOOIj24C7/O7EqYgVuE6QV1mxCWPg2elhcPmIu+bs63teQpxB2kooUBkz7i8y\nPT2eD8s3cPXI8wiHlcgMIU2gMXRs9r8DgTZ4FXUQ13FkTeslPacBmOk4Tqe99CztxwpcJ8mrrHgT\niL1S3wobzr5pbnXO2Cvi06P4EW7c0uL0sSEYZMOuPXzzuDW28u07ye3bhz7dM+netQsDs3vz1Efz\naAyFGZh9bO7nXfVfetNxl/HB4ROPdxxJMArMchzHt/Oypzp2DS5OVJyf9z+0bsd+Al8MuXrBlmHT\nL6PJlj0pUFVt2PdMLdBp14+WOD0z9/OCAbfG7QhYc/jsOPJTx3F+40O7Fhc7gosf99HkttAmtg+4\nbOmWYdMnJJu4AWho5yY8FDeAvfU7hqtq0Ms2fHQcec6Km/9YgYsT7qbDTRgnhVap7j965fpzbxmN\niK8HxFsiFKj82us2woS6hTTo2UZDhEmNeRMzNG196yXjxoeYDzyLz1iBiyN5lRUHgakYG5wWqck+\nd135BT84FxFfjBrbQrhxS0L6drCxptrrNtJIy7g6MDqEEmq9dKdZA9zsOI6nI1NL27ACF2fyKiu2\nY0TuYHPPH+g5eOPqCx8YhGsWmIyoqmp4/zmtl+w8u+u3JUJ0GKB9RwwKn+b1Ma5twFTHcZr921sS\njxU4D8irrFgLXAvHxmEd7n7Gl5+OfaQ3xqwwadFQ9WagTdZMnWVH3RZP1/mimdI4ykvHkS+Byx3H\n2e5R/ZYOYAXOI/IqKxYCU4BagPpu2TuXjf9ZOpLW4ZygiSIR628R9tR/NczNeuY5EccRD6quwohb\nlQd1WzqBFTgPyausWA4UBLr03LjkYuewpqXHJaOU14QbN7eY7yHeBLWxZ5iQtwFxUQwL54zpF1/H\nkY0YcUvYz2BpO1bgPCavsmL1kosfnx5O75Iw0egsGt534il2DzncuM+LPKMtcm1gzEiUeLgUrwIm\nOo6zrdWSFl+wApcA7nlh2npM7te1fvelNcLB6s2QWKuh3fVfJdQ+KJMu2RM67zgyF7jCcZx27QKL\nyMtuqj9LArAClyCKSgu+BiYBc3zuSkwSuf4WYWddVe/WS8WXC0KDJ/TUzKUdfPmbwDWO4yTLYX5L\nC1iBSyBFpQX7MTkm3/K7Ly0Rbtyc8MDj6vqtw1ovFX+mNYwd0k7HkSDwI8dxbnEcp/kEFschIreL\nyFoRWSMir7q3J4nIIhHZEhnNieFJESkXkc/chMuISImIXOc+fldEXnQf3yUinngSphJW4BJMUWlB\nQ1FpwfeA+4E2/ZMkkkSvvwEEwvXZYQ3FDI72AtdxZHUbi+8CJjuO82xb6xeRCzBGDAWqeiHG7h5g\nADARmIbJ1gbwb5gkzhdidt+fFJEBGM/BSKadQUDk7G4+JtGyJQZW4HyiqLSgBJMtvNzvvkQIh/ZU\ngea0XjL+HAkeTLjAAYwPDs9vg+PIYmCs4zjtFZQC4H9VdQ+AqkYsk95T1bCqfg5EwoYmAm+oakhV\nd2HW+MbjCpybwu9zYJcrfBOARe3szymHFTgfKSotKMe8iUv87gtAKFDpi8gA7G3YfmKChQQgiEwN\njO2Ntjia/hNmMyGeAbzRP2tMowVV3Y4xHb0GM2KbD9wMHIpOvWhpHitwPlNUWlBfVFpwPzAD8MwA\nsi2EA5t8O/i/o66qh19tu44jx2847AdudxznPsdxAh2sugy4SUT6AYhIrFMb84GZIpIuIqdjNqSW\nuc8tAR6kSeB+4n61tIIVuCShqLTgA2AU8C+/+qDh2rP8antX3Rcn5hZMIJMa8y6Lchz5JzDScZxX\nY72mNVR1HSY50VwRWQP8LkbxdzFhRGswwviIqkYclecDGaq6CViJsbGyAtcGrOFlklFSWJYGPAw8\nASQsODgc2vtl4MCffRWZm4c+vEt8PMr2dVrN0v/ruuq/Hcd5ya8+WOKLFbgkpaSwbAzwX5jFZM9p\nrFs4P1S/NL/1kt4xffB9y7Myeo33qfm/AA/lFufv8ql9iwfYxM9JSlFpwUrg0pLCslswoQRDvGwv\nHNjk+3JFTcOOI1kZvRLd7Cbg3tzi/E8S3bDFe3x/U1tiU1Ra8AZwPvAYcLiV4h1GwzW+BNtGs7Ou\nypMcqS2wHbgXGGHFLXWxU9STiJLCsjOBnwL/AfFLohIO1WwNHHjZ0xFiW+iRkb192uB7BnncTDVm\nRPyn3OL8pAu0tsQXK3AnISWFZYMxEfJ3EoeNiMa6RQtC9UuSIuH0zUMfqRWRvh5UXQs8Cfw+tzjf\ns5GwJbmwAncSU1JYdhbwKHAL0GEL9IYDryzQ0J6kELgZQ+5flZne46I4VlmDCaR+Orc4f38c67Wc\nBFiBSwFKCst6AjOBHwCXtPf19bXPfAWaFGack864ae6ArLMuj0NVKzG70G/mFufXxaE+y0mIFbgU\no6SwbARwN3A70L+18uFQ7VeBAy8lhbgBnNt73OKL+k3uaGjMIeBt4Lnc4vwlceyW5STFCpwHiMhQ\nTG7MFcAYYB1GcD4HxqnqHhEZBzylqle4R3P+AgzEHOy+ChgbOaTdEUoKy7oC12FGdVfRwo55sG7J\ngmD9oqSYngL07tK/6trcu9uzoxsCFgB/Bt6262uWaGwcnHecB9ytqgtdD69YiYAfB8pU9Tcicg1m\nBNYpikoLAsA7wDslhWVDgDsw6QzHAUfPnIYaN3a2qbhyoHHPUFU9KCKxAuJqMB8g/wA+zC3Or4lR\n1nIKYwXOO7ap6kL38WvAAzHKTgRuAFDVD0WkNp4dKSot2Io5+vVESWFZNjAZM6q7SkN7fT2e1QwS\nCNdXdUvvPuq4++UYQZsNLM4tzk9IPlXLyY0VOO84fu6vGEfYyFQxkUGtRykqLdgH/NW9eHrm7wYD\nlwGXul8vJGqE5wc1DTt2Dsg6qxbjprEMWJJbnN8pKycRCQGfYd7zFcAdqnqk0531CBH5JTBPVW0Q\nciewa3Ae4K7BVQGXqupiEXke8091LfC0qv5TRJ4BLnLX4EqArar6WxH5NvARcHpn1uA6ytMzp2UB\nw4Gzmvk6FOgap6YagN3utREjPpGr6sdvzY7rG1NEDqlqT/fx68AKVY3l7hGvdjNUNeh1O5bmsQLn\nAVGbDJ8CYzGbC993H7+AyXg/B7PhcIWI5ABvYNxdF2OsrIeqqi8mkC3x9MxpaUAuRuzOAHocd3XB\njFAjVxjjcVdNk5hVA7t//NbshCZsOU7gCoFRqnqfiDwE3OUWe15Vn436+y3BjGyXAy8BvwBygFtV\ndZmI9AD+AIzE/OyOqr4vIrMwFuQ9MaPhqcD7QF+33M/dckMx1kwL3Ha2AzNUtU5EXgZmq+o7IvIY\nMB3ojnHxvUdVVUTmAEuBKzGmmHerqrVRikZV7RXnCzPSKW9H+W4Yvy8w7iGr/f4ZUu3COOCCmaK+\njzmHOhYzYuyBEaN1wEXu3y8IfBMj1CuAFzHuuzMwluMAvwZucx9nAxvcumYBXwGnRbXZ233cH3PA\nX6LaGe0+93ZUfS8D33Ufnxb1c7wKTHcfz8HMCAC+A3zi9+852S67BpccDAHeFpE0IAD8u8/9SUW6\ni0gkwcx8zEj6XuBdVT0MICJ/wyRz+QCoUtXP3PvrgH+pqorIZxhhAvg2cJ2I/MT9PpMm15ePtSkH\ngwC/FpFJmFHtIJpyMVSpaqRfK6LqjuZKEXkEyMKYXa4D/u4+97dWXntKYwXOA1T1C8y0pa3lN2JG\nDhbvqFPV0dE3RGKmQ4heHghHfR+m6f9GgBtVdX30C0XkWxzr/HIrJpn2WFVtFJEvaNpkim4nhJmG\nRteVCfwRs5yxTUQcjt2gaoh6rf1/Pg5rl2Q5lZkPXC8iWe562g20zwr8I+CH4iqliLT0IdUHqHbF\n7UqgPaE5ETHbIyI9ge+247WnPFbgEoyIZItIrKDfpEBEnMjUS0RejiQoTiVUdSVmrWsZZrH+eVVd\n1Y4qIrbya91p7BMtlHsdGOdOb28HKtvRx33Ac5g4wI8wGx6WNmJ3UROMu3M2W1XbPIX1A3cqdEhV\nn4re0fO3VxZL+7AjuMRTDAwXkdUi8qR7lYvIZyIyE0BEXhGR6yMvEJHXRWRGc5WJyCwReV9E5ojI\nRhF5POq5h9y6y0XkwTbc/5mIbBCRBZijZs21N1ZE5orIChH5yE1CbLEkJXZRMvE8CoxU1dEiciNQ\niDk90B9YLiLzMDt8PwLeE5E+mBipO2LUeTFmU+OIW8c/MCcn7gS+hVkMXyoiczEfai3d/x4wGvO+\nWInZmTuKiHTBxH3NUNXdriD/iqY4MoslqbAC5y8TgTdUNQTscoVmvKp+ICJ/dF1GbgT+qrGj4T9W\n1b1wNNRhIkbgmguBkBbup7n3j7j3P2imnfMwQvqxu66eDuzo1G/AYvEQK3DJyyvAbZhR1Z2tlG3u\n3KsXCLBOVROSytBi6Sx2DS7xHAQiVkDzgZkiku6O1iZhdvTA7O49CKCqn7dS51UicpqIdAeuBxbS\ncghES/fnufe7u1ZF05tpZz1wuohMADNlFZEL2v8rsFgSgx3BJRhV3SsiC0WkHHMOcS2wBjPqekRV\nd7rldolIBfBeG6pdhnEHyQVeU9VPwYR30CSYR0MgYtx/y+1LNc2EI6hqwA0X+b27NpgBPIuJrLdY\nkg4bJpKkiEgW5pzkGFVtMVmKe7B7nKren6i+WSwnC3aKmoSIyBSMvdIfYombxWKJjR3BnSSIyNXA\nb4+7XaWqN/jRH4vlZMAKnMViSVnsFNVisaQsVuAsFkvKYgXOYrGkLFbgLBZLymIFzmKxpCxW4CwW\nS8piBc5isaQsVuAsFkvKYgXOYrGkLFbgLBZLymIFzmKxpCxW4CwWS8piBc5isaQsVuAsFkvKYgXO\nYrGkLFbgLBZLyvL/8Sf+sGpioD4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RFTtxcWAP2FI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_mastr_clean.to_csv(fn_final_clean_file, index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yMRW0WaoTcsH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}