analysis_type2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialising Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#initialising libraries\n",
    "import json\n",
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline \n",
    "import glob\n",
    "import os\n",
    "import re\n",
    "\n",
    "pd.set_option(\"display.max_colwidth\", 70000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setting folder for analysis and concat all datewise tweet files.\n",
    "input dialog box asks for path, just go to the parent folder where `json` files are located and select that folder location. \n",
    "the rest of the things will be taken care of automatically.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Enter 1 for single file or 2 for multiple files:  2\n",
      "Enter the folder path:  /media/ambijat/Datrat/ipytoneeee/CNUS\n"
     ]
    }
   ],
   "source": [
    "nof = int(input(\"Enter 1 for single file or 2 for multiple files: \"))\n",
    "\n",
    "if nof == 1:\n",
    "    file_path = input('Enter the file path: ')\n",
    "    tweets = pd.read_csv(file_path, lineterminator='\\n')\n",
    "elif nof == 2:\n",
    "    folder_path2 = input(\"Enter the folder path: \")\n",
    "    folder_path = folder_path2 + \"/\" + \"datewise\"\n",
    "    all_files = glob.iglob(os.path.join(folder_path, \"*.csv\"))\n",
    "    tweets = pd.concat((pd.read_csv(f, lineterminator='\\n', sep = ';') for f in all_files))\n",
    "else:\n",
    "    print(\"Path not found.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inspecting the datatype."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user                    object\n",
       "reuser                  object\n",
       "full_text               object\n",
       "range                    int64\n",
       "created_at              object\n",
       "id                       int64\n",
       "in_reply_to_user_id    float64\n",
       "user.location           object\n",
       "date                    object\n",
       "date2                    int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### total number of tweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "116017\n"
     ]
    }
   ],
   "source": [
    "print(len(tweets))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### choosing `full_text` selection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrench= tweets['full_text'].copy()\n",
    "t1 = \"full_text\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Processing tweets for various quantitative analysis.\n",
    "##### sending to a text file for word count analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrench.to_csv(r'wrench.txt', header=False, index=None, sep=' ', mode='a')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using `bash` script for extracting the frequency of words in overall collection of tweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm: cannot remove '*.tmp': No such file or directory\n",
      "sort: write failed: 'standard output': Broken pipe\n",
      "sort: write error\n"
     ]
    }
   ],
   "source": [
    "!rm *.tmp\n",
    "\n",
    "!cat wrench.txt | sed 's/[[:space:]]/\\n/g' | sed 's/^[ \\t]*//' | sed 's/[ \\t]*$//' | \\\n",
    "sed '/^$/d'| tr -cd '[:alnum:] [:space:]' >> rvd2.tmp\n",
    "\n",
    "!awk 'length > 2' < rvd2.tmp | awk '{print tolower($0)}' > rvd22.tmp\n",
    "\n",
    "!grep -vwF -f exclude.exd < rvd22.tmp > rvd11.tmp\n",
    "\n",
    "\n",
    "!cat rvd11.tmp | sort | uniq -c | sort -nr | head -n 250 > lsfdout.txt\n",
    "\n",
    "!rm *.tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### this gives the frequency of words used in the tweets. The treemap can be developed.\n",
    "##### maximum number of times words occuring and listing them as rating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>92210 hong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80664 kong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57871 china</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15115 law</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13399 security</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12065 chinas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10767 johnson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8715 people</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8421 national</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8156 que</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>7724 support</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>7660 boris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>7513 kongs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>7509 protests</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>6635 trump</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>6392 chinese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>6256 time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>6202 british</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>6130 tiananmen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5955 beijing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5485 back</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>5369 citizens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>5365 step</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>5363 democracy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>5177 vigil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>4650 america</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4637 said</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>4544 government</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>4511 las</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>4464 reuters</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 words\n",
       "0           92210 hong\n",
       "1           80664 kong\n",
       "2          57871 china\n",
       "3            15115 law\n",
       "4       13399 security\n",
       "5         12065 chinas\n",
       "6        10767 johnson\n",
       "7          8715 people\n",
       "8        8421 national\n",
       "9             8156 que\n",
       "10        7724 support\n",
       "11          7660 boris\n",
       "12          7513 kongs\n",
       "13       7509 protests\n",
       "14          6635 trump\n",
       "15        6392 chinese\n",
       "16           6256 time\n",
       "17        6202 british\n",
       "18      6130 tiananmen\n",
       "19        5955 beijing\n",
       "20           5485 back\n",
       "21       5369 citizens\n",
       "22           5365 step\n",
       "23      5363 democracy\n",
       "24          5177 vigil\n",
       "25        4650 america\n",
       "26           4637 said\n",
       "27     4544 government\n",
       "28            4511 las\n",
       "29        4464 reuters"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating = pd.read_csv('lsfdout.txt', header=None, names=[\"words\"])\n",
    "rating.head(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Using `str.lstrp` instead of `str.replace` for removing intial whitespace and then splitting the columns.\n",
    "##### take note of `str.split` where `n=1` represents splitting at the first instance of space and not on later ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>wordlist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>92210</td>\n",
       "      <td>hong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80664</td>\n",
       "      <td>kong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57871</td>\n",
       "      <td>china</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15115</td>\n",
       "      <td>law</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13399</td>\n",
       "      <td>security</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12065</td>\n",
       "      <td>chinas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10767</td>\n",
       "      <td>johnson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8715</td>\n",
       "      <td>people</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8421</td>\n",
       "      <td>national</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8156</td>\n",
       "      <td>que</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>7724</td>\n",
       "      <td>support</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>7660</td>\n",
       "      <td>boris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>7513</td>\n",
       "      <td>kongs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>7509</td>\n",
       "      <td>protests</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>6635</td>\n",
       "      <td>trump</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>6392</td>\n",
       "      <td>chinese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>6256</td>\n",
       "      <td>time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>6202</td>\n",
       "      <td>british</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>6130</td>\n",
       "      <td>tiananmen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5955</td>\n",
       "      <td>beijing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    count   wordlist\n",
       "0   92210       hong\n",
       "1   80664       kong\n",
       "2   57871      china\n",
       "3   15115        law\n",
       "4   13399   security\n",
       "5   12065     chinas\n",
       "6   10767    johnson\n",
       "7    8715     people\n",
       "8    8421   national\n",
       "9    8156        que\n",
       "10   7724    support\n",
       "11   7660      boris\n",
       "12   7513      kongs\n",
       "13   7509   protests\n",
       "14   6635      trump\n",
       "15   6392    chinese\n",
       "16   6256       time\n",
       "17   6202    british\n",
       "18   6130  tiananmen\n",
       "19   5955    beijing"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r2 = rating.copy()\n",
    "r2.words = r2.words.str.lstrip()\n",
    "r2.dropna(inplace = True) \n",
    "new = r2['words'].str.split(\" \", n = 1, expand = True) \n",
    "r2['count']= new[0] \n",
    "r2['wordlist']= new[1] \n",
    "r2.drop(columns =['words'], inplace = True) \n",
    "r2.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### exporting the most words to csv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "r2b = r2.head(50).copy()\n",
    "mostword = os.path.join(folder_path2 + \"/\" + \"processed2\", 'mostword.csv')\n",
    "r2b.to_csv(mostword, index = False, header = True, encoding = 'utf-8', sep = ';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### maximum number of location used for tweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hong Kong                  11905\n",
      "香港                          1464\n",
      "United States                998\n",
      "London, England              503\n",
      "London                       460\n",
      "United Kingdom               395\n",
      "India                        375\n",
      "USA                          359\n",
      "Internet                     319\n",
      "Canada                       302\n",
      "England, United Kingdom      277\n",
      "California, USA              273\n",
      "Washington, DC               246\n",
      "New York, NY                 221\n",
      "New York, USA                221\n",
      "Argentina                    188\n",
      "Los Angeles, CA              186\n",
      "Venezuela                    186\n",
      "Toronto, Ontario             183\n",
      "HK                           183\n",
      "Name: user.location, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(tweets['user.location'].value_counts().nlargest(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#exporting the most userloc to csv file.\n",
    "userloc = pd.DataFrame(tweets['user.location'].value_counts().nlargest(30))\n",
    "userloc.reset_index(level=0, inplace=True)\n",
    "mostloc = os.path.join(folder_path2 + \"/\" + \"processed2\", 'mostloc.csv')\n",
    "userloc.to_csv(mostloc, index = False, header = True, encoding = 'utf-8', sep = ';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Making use of frequency of words to grasp relevant tweets by boolean exercise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "For Custom words Press 1 or 0 to Continue:  1\n",
      "Enter first word:  hong\n",
      "Enter second word:  china\n",
      "Enter third word:  law\n"
     ]
    }
   ],
   "source": [
    "sof = 0\n",
    "sof = int(input(\"For Custom words Press 1 or 0 to Continue: \"))\n",
    "\n",
    "if sof == 1:\n",
    "    k1 = input(\"Enter first word: \")\n",
    "    k2 = input(\"Enter second word: \")\n",
    "    k3 = input(\"Enter third word: \")\n",
    "elif sof == 0:\n",
    "    k1 = r2.wordlist[0]\n",
    "    k2 = r2.wordlist[1]\n",
    "    k3 = r2.wordlist[2]\n",
    "else:\n",
    "    print(\"No choice given.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### `k1`, `k2`, & `k3` are the 3 maximun frequency words in the tweet text.\n",
    "##### refer to `cell no. 5` for `t1` variable. `case=False` stands for ignorecase."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets[k1] = tweets[t1].str.contains(k1, case=False)\n",
    "tweets[k2] = tweets[t1].str.contains(k2, case=False)\n",
    "tweets[k3] = tweets[t1].str.contains(k3, case=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The above method can also be done by defining a function. \n",
    "##### Same example is repeated. `t1` refers to `full_text`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wordcheck(series, key):\n",
    "    return series.str.contains(key, case=False)\n",
    "\n",
    "for key in (k1,k2,k3):\n",
    "    tweets[key +'2'] = wordcheck(tweets[t1], key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Inspecting the output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>reuser</th>\n",
       "      <th>full_text</th>\n",
       "      <th>range</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>user.location</th>\n",
       "      <th>date</th>\n",
       "      <th>date2</th>\n",
       "      <th>hong</th>\n",
       "      <th>china</th>\n",
       "      <th>law</th>\n",
       "      <th>hong2</th>\n",
       "      <th>china2</th>\n",
       "      <th>law2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>namjoonayo</td>\n",
       "      <td>cannacae</td>\n",
       "      <td>RT @cannacae: i know BLM is important rn but i URGE you to look at Hong Kong for even a second. America is taking away its preferential tre…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:57 +0000 2020</td>\n",
       "      <td>1267606846824542200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:57+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ngdasokau</td>\n",
       "      <td>BarrySheerman</td>\n",
       "      <td>RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:57 +0000 2020</td>\n",
       "      <td>1267606845763367000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:57+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>geoffreypcdog</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: Seven former foreign secretaries urge Boris Johnson to lead global response to 'deteriorating' Hong Kong situation http…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:57 +0000 2020</td>\n",
       "      <td>1267606843322331100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:57+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Marisa15061974</td>\n",
       "      <td>WDVL4</td>\n",
       "      <td>RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:56 +0000 2020</td>\n",
       "      <td>1267606839547625500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:56+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tony13945454</td>\n",
       "      <td>nytimesworld</td>\n",
       "      <td>RT @nytimesworld: The gathering to remember Tiananmen, held annually since 1990, had become a major rallying point for Hong Kongers worried…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:55 +0000 2020</td>\n",
       "      <td>1267606835428593700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:55+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>SenatorLankford</td>\n",
       "      <td>RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:50 +0000 2020</td>\n",
       "      <td>1267606815530823700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:50+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Alankawing</td>\n",
       "      <td>bopinion</td>\n",
       "      <td>RT @bopinion: Taiwan, the U.S., Canada and Australia — democracies that have spoken up in support of the rights of Hong Kongers — need to b…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:48 +0000 2020</td>\n",
       "      <td>1267606806215356400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:48+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Tony13945454</td>\n",
       "      <td>MailOnline</td>\n",
       "      <td>RT @MailOnline: Hong Kong's pro-Beijing government is criticised for 'stifling its citizens' freedom of expression' https://t.co/Gro13tb3A1</td>\n",
       "      <td>139</td>\n",
       "      <td>Mon Jun 01 23:59:48 +0000 2020</td>\n",
       "      <td>1267606805720400000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:48+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chowcch1</td>\n",
       "      <td>bopinion</td>\n",
       "      <td>RT @bopinion: Taiwan, the U.S., Canada and Australia — democracies that have spoken up in support of the rights of Hong Kongers — need to b…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:48 +0000 2020</td>\n",
       "      <td>1267606805284180000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:48+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…</td>\n",
       "      <td>139</td>\n",
       "      <td>Mon Jun 01 23:59:45 +0000 2020</td>\n",
       "      <td>1267606795637461000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Queens, NY</td>\n",
       "      <td>2020-06-01 23:59:45+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>next_china</td>\n",
       "      <td>RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:45 +0000 2020</td>\n",
       "      <td>1267606793041047600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:45+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>KelvinWo4</td>\n",
       "      <td>BarrySheerman</td>\n",
       "      <td>RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:40 +0000 2020</td>\n",
       "      <td>1267606775550754800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:40+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>GayFairyCat</td>\n",
       "      <td>cannacae</td>\n",
       "      <td>RT @cannacae: i know BLM is important rn but i URGE you to look at Hong Kong for even a second. America is taking away its preferential tre…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:39 +0000 2020</td>\n",
       "      <td>1267606769066373000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>New horizons in Floren</td>\n",
       "      <td>2020-06-01 23:59:39+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on dissent in Hong Kong. Answer me this @SenatorCollins: has Trump learned his lesson yet? The blood of innocent ppl is on ur hands now. U should've voted to convict</td>\n",
       "      <td>280</td>\n",
       "      <td>Mon Jun 01 23:59:35 +0000 2020</td>\n",
       "      <td>1267606754113859600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Queens, NY</td>\n",
       "      <td>2020-06-01 23:59:35+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>WSJPolitics</td>\n",
       "      <td>RT @WSJPolitics: President Trump said Friday that China’s decision to impose a national security law was “absolutely smothering Hong Kong f…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:35 +0000 2020</td>\n",
       "      <td>1267606752339480600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:35+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Tony13945454</td>\n",
       "      <td>bopinion</td>\n",
       "      <td>RT @bopinion: Taiwan, the U.S., Canada and Australia — democracies that have spoken up in support of the rights of Hong Kongers — need to b…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:29 +0000 2020</td>\n",
       "      <td>1267606725554663400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:29+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>kathyktk</td>\n",
       "      <td>USAenEspanol</td>\n",
       "      <td>RT @USAenEspanol: .@SecPompeo: El presidente Trump nos ha pedido que revisemos cada tratamiento preferencial que tenía Hong Kong y trabajem…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:24 +0000 2020</td>\n",
       "      <td>1267606707292835800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:24+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>TornadoMario347</td>\n",
       "      <td>cannacae</td>\n",
       "      <td>RT @cannacae: i know BLM is important rn but i URGE you to look at Hong Kong for even a second. America is taking away its preferential tre…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:24 +0000 2020</td>\n",
       "      <td>1267606706323849200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:24+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Tony13945454</td>\n",
       "      <td>Andrew_Adonis</td>\n",
       "      <td>RT @Andrew_Adonis: We need a cross-party alliance in Britain on robust action to defend the people of Hong Kong and contain Xi Jinping’s in…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:24 +0000 2020</td>\n",
       "      <td>1267606705426186200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:24+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Jeffrey44012270</td>\n",
       "      <td>BarrySheerman</td>\n",
       "      <td>RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:23 +0000 2020</td>\n",
       "      <td>1267606704218226700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:23+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               user           reuser  \\\n",
       "0        namjoonayo         cannacae   \n",
       "1         ngdasokau    BarrySheerman   \n",
       "2     geoffreypcdog   benedictrogers   \n",
       "3    Marisa15061974            WDVL4   \n",
       "4      Tony13945454     nytimesworld   \n",
       "5    kangaroolooloo  SenatorLankford   \n",
       "6        Alankawing         bopinion   \n",
       "7      Tony13945454       MailOnline   \n",
       "8          Chowcch1         bopinion   \n",
       "9   FidelisToJusti1  FidelisToJusti1   \n",
       "10   kangaroolooloo       next_china   \n",
       "11        KelvinWo4    BarrySheerman   \n",
       "12      GayFairyCat         cannacae   \n",
       "13  FidelisToJusti1              NaN   \n",
       "14   kangaroolooloo      WSJPolitics   \n",
       "15     Tony13945454         bopinion   \n",
       "16         kathyktk     USAenEspanol   \n",
       "17  TornadoMario347         cannacae   \n",
       "18     Tony13945454    Andrew_Adonis   \n",
       "19  Jeffrey44012270    BarrySheerman   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                   full_text  \\\n",
       "0                                                                                                                                               RT @cannacae: i know BLM is important rn but i URGE you to look at Hong Kong for even a second. America is taking away its preferential tre…   \n",
       "1                                                                                                                                               RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…   \n",
       "2                                                                                                                                               RT @benedictrogers: Seven former foreign secretaries urge Boris Johnson to lead global response to 'deteriorating' Hong Kong situation http…   \n",
       "3                                                                                                                                               RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…   \n",
       "4                                                                                                                                               RT @nytimesworld: The gathering to remember Tiananmen, held annually since 1990, had become a major rallying point for Hong Kongers worried…   \n",
       "5                                                                                                                                               RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…   \n",
       "6                                                                                                                                               RT @bopinion: Taiwan, the U.S., Canada and Australia — democracies that have spoken up in support of the rights of Hong Kongers — need to b…   \n",
       "7                                                                                                                                                RT @MailOnline: Hong Kong's pro-Beijing government is criticised for 'stifling its citizens' freedom of expression' https://t.co/Gro13tb3A1   \n",
       "8                                                                                                                                               RT @bopinion: Taiwan, the U.S., Canada and Australia — democracies that have spoken up in support of the rights of Hong Kongers — need to b…   \n",
       "9                                                                                                                                                RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…   \n",
       "10                                                                                                                                              RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…   \n",
       "11                                                                                                                                              RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…   \n",
       "12                                                                                                                                              RT @cannacae: i know BLM is important rn but i URGE you to look at Hong Kong for even a second. America is taking away its preferential tre…   \n",
       "13  What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on dissent in Hong Kong. Answer me this @SenatorCollins: has Trump learned his lesson yet? The blood of innocent ppl is on ur hands now. U should've voted to convict   \n",
       "14                                                                                                                                              RT @WSJPolitics: President Trump said Friday that China’s decision to impose a national security law was “absolutely smothering Hong Kong f…   \n",
       "15                                                                                                                                              RT @bopinion: Taiwan, the U.S., Canada and Australia — democracies that have spoken up in support of the rights of Hong Kongers — need to b…   \n",
       "16                                                                                                                                              RT @USAenEspanol: .@SecPompeo: El presidente Trump nos ha pedido que revisemos cada tratamiento preferencial que tenía Hong Kong y trabajem…   \n",
       "17                                                                                                                                              RT @cannacae: i know BLM is important rn but i URGE you to look at Hong Kong for even a second. America is taking away its preferential tre…   \n",
       "18                                                                                                                                              RT @Andrew_Adonis: We need a cross-party alliance in Britain on robust action to defend the people of Hong Kong and contain Xi Jinping’s in…   \n",
       "19                                                                                                                                              RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…   \n",
       "\n",
       "    range                      created_at                   id  \\\n",
       "0     140  Mon Jun 01 23:59:57 +0000 2020  1267606846824542200   \n",
       "1     140  Mon Jun 01 23:59:57 +0000 2020  1267606845763367000   \n",
       "2     140  Mon Jun 01 23:59:57 +0000 2020  1267606843322331100   \n",
       "3     140  Mon Jun 01 23:59:56 +0000 2020  1267606839547625500   \n",
       "4     140  Mon Jun 01 23:59:55 +0000 2020  1267606835428593700   \n",
       "5     140  Mon Jun 01 23:59:50 +0000 2020  1267606815530823700   \n",
       "6     140  Mon Jun 01 23:59:48 +0000 2020  1267606806215356400   \n",
       "7     139  Mon Jun 01 23:59:48 +0000 2020  1267606805720400000   \n",
       "8     140  Mon Jun 01 23:59:48 +0000 2020  1267606805284180000   \n",
       "9     139  Mon Jun 01 23:59:45 +0000 2020  1267606795637461000   \n",
       "10    140  Mon Jun 01 23:59:45 +0000 2020  1267606793041047600   \n",
       "11    140  Mon Jun 01 23:59:40 +0000 2020  1267606775550754800   \n",
       "12    140  Mon Jun 01 23:59:39 +0000 2020  1267606769066373000   \n",
       "13    280  Mon Jun 01 23:59:35 +0000 2020  1267606754113859600   \n",
       "14    140  Mon Jun 01 23:59:35 +0000 2020  1267606752339480600   \n",
       "15    140  Mon Jun 01 23:59:29 +0000 2020  1267606725554663400   \n",
       "16    140  Mon Jun 01 23:59:24 +0000 2020  1267606707292835800   \n",
       "17    140  Mon Jun 01 23:59:24 +0000 2020  1267606706323849200   \n",
       "18    140  Mon Jun 01 23:59:24 +0000 2020  1267606705426186200   \n",
       "19    140  Mon Jun 01 23:59:23 +0000 2020  1267606704218226700   \n",
       "\n",
       "    in_reply_to_user_id           user.location                       date  \\\n",
       "0                   NaN                     NaN  2020-06-01 23:59:57+00:00   \n",
       "1                   NaN               Hong Kong  2020-06-01 23:59:57+00:00   \n",
       "2                   NaN                     NaN  2020-06-01 23:59:57+00:00   \n",
       "3                   NaN                     NaN  2020-06-01 23:59:56+00:00   \n",
       "4                   NaN               Hong Kong  2020-06-01 23:59:55+00:00   \n",
       "5                   NaN               Hong Kong  2020-06-01 23:59:50+00:00   \n",
       "6                   NaN                     NaN  2020-06-01 23:59:48+00:00   \n",
       "7                   NaN               Hong Kong  2020-06-01 23:59:48+00:00   \n",
       "8                   NaN                     NaN  2020-06-01 23:59:48+00:00   \n",
       "9                   NaN              Queens, NY  2020-06-01 23:59:45+00:00   \n",
       "10                  NaN               Hong Kong  2020-06-01 23:59:45+00:00   \n",
       "11                  NaN                     NaN  2020-06-01 23:59:40+00:00   \n",
       "12                  NaN  New horizons in Floren  2020-06-01 23:59:39+00:00   \n",
       "13                  NaN              Queens, NY  2020-06-01 23:59:35+00:00   \n",
       "14                  NaN               Hong Kong  2020-06-01 23:59:35+00:00   \n",
       "15                  NaN               Hong Kong  2020-06-01 23:59:29+00:00   \n",
       "16                  NaN                     NaN  2020-06-01 23:59:24+00:00   \n",
       "17                  NaN                     NaN  2020-06-01 23:59:24+00:00   \n",
       "18                  NaN               Hong Kong  2020-06-01 23:59:24+00:00   \n",
       "19                  NaN                     NaN  2020-06-01 23:59:23+00:00   \n",
       "\n",
       "      date2   hong  china    law  hong2  china2   law2  \n",
       "0   1062020   True  False  False   True   False  False  \n",
       "1   1062020  False   True  False  False    True  False  \n",
       "2   1062020   True  False  False   True   False  False  \n",
       "3   1062020  False   True  False  False    True  False  \n",
       "4   1062020   True  False  False   True   False  False  \n",
       "5   1062020   True   True  False   True    True  False  \n",
       "6   1062020   True  False  False   True   False  False  \n",
       "7   1062020   True  False  False   True   False  False  \n",
       "8   1062020   True  False  False   True   False  False  \n",
       "9   1062020  False   True  False  False    True  False  \n",
       "10  1062020   True   True   True   True    True   True  \n",
       "11  1062020  False   True  False  False    True  False  \n",
       "12  1062020   True  False  False   True   False  False  \n",
       "13  1062020   True   True  False   True    True  False  \n",
       "14  1062020   True   True   True   True    True   True  \n",
       "15  1062020   True  False  False   True   False  False  \n",
       "16  1062020   True  False  False   True   False  False  \n",
       "17  1062020   True  False  False   True   False  False  \n",
       "18  1062020   True  False  False   True   False  False  \n",
       "19  1062020  False   True  False  False    True  False  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Finding the number of tweets for which the `k1` term occurs, `k2` occurs and `k3` occurs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91180\n",
      "68439\n",
      "17002\n",
      "91180\n",
      "68439\n",
      "17002\n"
     ]
    }
   ],
   "source": [
    "print(tweets[k1].value_counts()[True])\n",
    "print(tweets[k2].value_counts()[True])\n",
    "print(tweets[k3].value_counts()[True])\n",
    "#\n",
    "print(tweets[k1 +'2'].value_counts()[True])\n",
    "print(tweets[k2 +'2'].value_counts()[True])\n",
    "print(tweets[k3 +'2'].value_counts()[True])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now finding the tweets which are combination of one or more than one keywords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets[k2+k1] = tweets[t1].str.contains(k2 or k1, case=False)\n",
    "tweets[k3+k1] = tweets[t1].str.contains(k3 or k1, case=False)\n",
    "\n",
    "tweets[k2+k1+'2'] = tweets[t1].str.contains(k2 or k1, case=False)\n",
    "tweets[k3+k1+'2'] = tweets[t1].str.contains(k3 or k1, case=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### If you notice these scores then `k2+k1` and `k3+k1` are the ones that have similar scores for their combination, just as their individual `k2` and `k3` scores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68439\n",
      "17002\n",
      "68439\n",
      "17002\n"
     ]
    }
   ],
   "source": [
    "print(tweets[k2+k1].value_counts()[True])\n",
    "print(tweets[k3+k1].value_counts()[True])\n",
    "#\n",
    "print(tweets[k2+k1+'2'].value_counts()[True])\n",
    "print(tweets[k3+k1+'2'].value_counts()[True])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The following score is basically `and` boolean operator in between `k2+k1` and `k3+k1` and `k1` values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57681\n",
      "14506\n",
      "57681\n",
      "14506\n"
     ]
    }
   ],
   "source": [
    "print(tweets[tweets[k2+k1]][k1].value_counts()[True])\n",
    "print(tweets[tweets[k3+k1]][k1].value_counts()[True])\n",
    "#\n",
    "print(tweets[tweets[k2+k1+'2']][k1].value_counts()[True])\n",
    "print(tweets[tweets[k3+k1+'2']][k1].value_counts()[True])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exporting the tweets that contain most of the three keywords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "separ= tweets[(tweets[k2+k1+'2'] == True) | (tweets[k3+k1+'2'] == True)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>reuser</th>\n",
       "      <th>full_text</th>\n",
       "      <th>range</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>user.location</th>\n",
       "      <th>date</th>\n",
       "      <th>date2</th>\n",
       "      <th>hong</th>\n",
       "      <th>china</th>\n",
       "      <th>law</th>\n",
       "      <th>hong2</th>\n",
       "      <th>china2</th>\n",
       "      <th>law2</th>\n",
       "      <th>chinahong</th>\n",
       "      <th>lawhong</th>\n",
       "      <th>chinahong2</th>\n",
       "      <th>lawhong2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ngdasokau</td>\n",
       "      <td>BarrySheerman</td>\n",
       "      <td>RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:57 +0000 2020</td>\n",
       "      <td>1267606845763367000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:57+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Marisa15061974</td>\n",
       "      <td>WDVL4</td>\n",
       "      <td>RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:56 +0000 2020</td>\n",
       "      <td>1267606839547625500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:56+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>SenatorLankford</td>\n",
       "      <td>RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:50 +0000 2020</td>\n",
       "      <td>1267606815530823700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:50+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…</td>\n",
       "      <td>139</td>\n",
       "      <td>Mon Jun 01 23:59:45 +0000 2020</td>\n",
       "      <td>1267606795637461000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Queens, NY</td>\n",
       "      <td>2020-06-01 23:59:45+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>next_china</td>\n",
       "      <td>RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:45 +0000 2020</td>\n",
       "      <td>1267606793041047600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:45+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               user           reuser  \\\n",
       "1         ngdasokau    BarrySheerman   \n",
       "3    Marisa15061974            WDVL4   \n",
       "5    kangaroolooloo  SenatorLankford   \n",
       "9   FidelisToJusti1  FidelisToJusti1   \n",
       "10   kangaroolooloo       next_china   \n",
       "\n",
       "                                                                                                                                       full_text  \\\n",
       "1   RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…   \n",
       "3   RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…   \n",
       "5   RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…   \n",
       "9    RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…   \n",
       "10  RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…   \n",
       "\n",
       "    range                      created_at                   id  \\\n",
       "1     140  Mon Jun 01 23:59:57 +0000 2020  1267606845763367000   \n",
       "3     140  Mon Jun 01 23:59:56 +0000 2020  1267606839547625500   \n",
       "5     140  Mon Jun 01 23:59:50 +0000 2020  1267606815530823700   \n",
       "9     139  Mon Jun 01 23:59:45 +0000 2020  1267606795637461000   \n",
       "10    140  Mon Jun 01 23:59:45 +0000 2020  1267606793041047600   \n",
       "\n",
       "    in_reply_to_user_id user.location                       date    date2  \\\n",
       "1                   NaN     Hong Kong  2020-06-01 23:59:57+00:00  1062020   \n",
       "3                   NaN           NaN  2020-06-01 23:59:56+00:00  1062020   \n",
       "5                   NaN     Hong Kong  2020-06-01 23:59:50+00:00  1062020   \n",
       "9                   NaN    Queens, NY  2020-06-01 23:59:45+00:00  1062020   \n",
       "10                  NaN     Hong Kong  2020-06-01 23:59:45+00:00  1062020   \n",
       "\n",
       "     hong  china    law  hong2  china2   law2  chinahong  lawhong  chinahong2  \\\n",
       "1   False   True  False  False    True  False       True    False        True   \n",
       "3   False   True  False  False    True  False       True    False        True   \n",
       "5    True   True  False   True    True  False       True    False        True   \n",
       "9   False   True  False  False    True  False       True    False        True   \n",
       "10   True   True   True   True    True   True       True     True        True   \n",
       "\n",
       "    lawhong2  \n",
       "1      False  \n",
       "3      False  \n",
       "5      False  \n",
       "9      False  \n",
       "10      True  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separ.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73387\n"
     ]
    }
   ],
   "source": [
    "print(len(separ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hong Kong                         7445\n",
      "香港                                 930\n",
      "United States                      617\n",
      "London, England                    355\n",
      "London                             329\n",
      "United Kingdom                     266\n",
      "India                              255\n",
      "Internet                           224\n",
      "USA                                222\n",
      "Canada                             196\n",
      "England, United Kingdom            186\n",
      "California, USA                    168\n",
      "Washington, DC                     161\n",
      "Argentina                          142\n",
      "New York, NY                       140\n",
      "New York, USA                      137\n",
      "Buenos Aires, Argentina            132\n",
      "Venezuela                          122\n",
      "Australia                          118\n",
      "HK                                 116\n",
      "UK                                 115\n",
      "Toronto, Ontario                   113\n",
      "Los Angeles, CA                    105\n",
      "New Delhi, India                   101\n",
      "Worldwide                           98\n",
      "Singapore                           90\n",
      "Hong Kong                           84\n",
      "Earth                               83\n",
      "Florida, USA                        83\n",
      "Texas, USA                          82\n",
      "hong kong                           79\n",
      "Melbourne, Victoria                 70\n",
      "Chicago, IL                         69\n",
      "England                             66\n",
      "Malaysia                            64\n",
      "Asia                                63\n",
      "Global                              63\n",
      "HONG KONG                           60\n",
      "Beijing                             59\n",
      "London, UK                          58\n",
      "Ciudad Autónoma de Buenos Aire      57\n",
      "Ecuador                             57\n",
      "Córdoba, Argentina                  56\n",
      "México                              55\n",
      "San Francisco, CA                   55\n",
      "New Jersey, USA                     53\n",
      "she/her                             53\n",
      "Ontario, Canada                     53\n",
      "New York                            52\n",
      "Chile                               52\n",
      "Name: user.location, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(separ['user.location'].value_counts().nlargest(50))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Separating and counting the `Retweets`.\n",
    "##### this regex expression `^(?=.*\\bRT\\b\\s@\\b).*$` is good only for direct expression and not for reference variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "separ['RT'] = separ[t1].str.contains(r\"^(?:RT\\s@\\w+:)\", case=False) == True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13523\n"
     ]
    }
   ],
   "source": [
    "print(separ['RT'].value_counts()[False])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59864\n"
     ]
    }
   ],
   "source": [
    "print(separ['RT'].value_counts()[True])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extracting the `urls` from the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "separ['URLS'] = separ[t1].str.extract(r'(https?://[^\\s<>\"]+|www\\.[^\\s<>\"]+)', expand = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>reuser</th>\n",
       "      <th>full_text</th>\n",
       "      <th>range</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>user.location</th>\n",
       "      <th>date</th>\n",
       "      <th>date2</th>\n",
       "      <th>...</th>\n",
       "      <th>law</th>\n",
       "      <th>hong2</th>\n",
       "      <th>china2</th>\n",
       "      <th>law2</th>\n",
       "      <th>chinahong</th>\n",
       "      <th>lawhong</th>\n",
       "      <th>chinahong2</th>\n",
       "      <th>lawhong2</th>\n",
       "      <th>RT</th>\n",
       "      <th>URLS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ngdasokau</td>\n",
       "      <td>BarrySheerman</td>\n",
       "      <td>RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:57 +0000 2020</td>\n",
       "      <td>1267606845763367000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:57+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Marisa15061974</td>\n",
       "      <td>WDVL4</td>\n",
       "      <td>RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:56 +0000 2020</td>\n",
       "      <td>1267606839547625500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-01 23:59:56+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>SenatorLankford</td>\n",
       "      <td>RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:50 +0000 2020</td>\n",
       "      <td>1267606815530823700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:50+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…</td>\n",
       "      <td>139</td>\n",
       "      <td>Mon Jun 01 23:59:45 +0000 2020</td>\n",
       "      <td>1267606795637461000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Queens, NY</td>\n",
       "      <td>2020-06-01 23:59:45+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>next_china</td>\n",
       "      <td>RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…</td>\n",
       "      <td>140</td>\n",
       "      <td>Mon Jun 01 23:59:45 +0000 2020</td>\n",
       "      <td>1267606793041047600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>2020-06-01 23:59:45+00:00</td>\n",
       "      <td>1062020</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               user           reuser  \\\n",
       "1         ngdasokau    BarrySheerman   \n",
       "3    Marisa15061974            WDVL4   \n",
       "5    kangaroolooloo  SenatorLankford   \n",
       "9   FidelisToJusti1  FidelisToJusti1   \n",
       "10   kangaroolooloo       next_china   \n",
       "\n",
       "                                                                                                                                       full_text  \\\n",
       "1   RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…   \n",
       "3   RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…   \n",
       "5   RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…   \n",
       "9    RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…   \n",
       "10  RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…   \n",
       "\n",
       "    range                      created_at                   id  \\\n",
       "1     140  Mon Jun 01 23:59:57 +0000 2020  1267606845763367000   \n",
       "3     140  Mon Jun 01 23:59:56 +0000 2020  1267606839547625500   \n",
       "5     140  Mon Jun 01 23:59:50 +0000 2020  1267606815530823700   \n",
       "9     139  Mon Jun 01 23:59:45 +0000 2020  1267606795637461000   \n",
       "10    140  Mon Jun 01 23:59:45 +0000 2020  1267606793041047600   \n",
       "\n",
       "    in_reply_to_user_id user.location                       date    date2  \\\n",
       "1                   NaN     Hong Kong  2020-06-01 23:59:57+00:00  1062020   \n",
       "3                   NaN           NaN  2020-06-01 23:59:56+00:00  1062020   \n",
       "5                   NaN     Hong Kong  2020-06-01 23:59:50+00:00  1062020   \n",
       "9                   NaN    Queens, NY  2020-06-01 23:59:45+00:00  1062020   \n",
       "10                  NaN     Hong Kong  2020-06-01 23:59:45+00:00  1062020   \n",
       "\n",
       "    ...    law  hong2  china2   law2  chinahong  lawhong  chinahong2  \\\n",
       "1   ...  False  False    True  False       True    False        True   \n",
       "3   ...  False  False    True  False       True    False        True   \n",
       "5   ...  False   True    True  False       True    False        True   \n",
       "9   ...  False  False    True  False       True    False        True   \n",
       "10  ...   True   True    True   True       True     True        True   \n",
       "\n",
       "    lawhong2    RT  URLS  \n",
       "1      False  True   NaN  \n",
       "3      False  True   NaN  \n",
       "5      False  True   NaN  \n",
       "9      False  True   NaN  \n",
       "10      True  True   NaN  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separ.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user                    object\n",
       "reuser                  object\n",
       "full_text               object\n",
       "range                    int64\n",
       "created_at              object\n",
       "id                       int64\n",
       "in_reply_to_user_id    float64\n",
       "user.location           object\n",
       "date                    object\n",
       "date2                    int64\n",
       "hong                      bool\n",
       "china                     bool\n",
       "law                       bool\n",
       "hong2                     bool\n",
       "china2                    bool\n",
       "law2                      bool\n",
       "chinahong                 bool\n",
       "lawhong                   bool\n",
       "chinahong2                bool\n",
       "lawhong2                  bool\n",
       "RT                        bool\n",
       "URLS                    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(separ)\n",
    "separ.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding the range of the dates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1062020 2062020 3062020]\n"
     ]
    }
   ],
   "source": [
    "#separ['date2'] = separ.date.dt.strftime('%d%m%Y')\n",
    "collect = separ['date2'].unique()\n",
    "print(collect)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selecting columns for further processing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "separ3a = separ.iloc[:, [0, 1, 2, 3, 7, 9, 13, 14, 15, -1, -2, -3, -4]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>reuser</th>\n",
       "      <th>full_text</th>\n",
       "      <th>range</th>\n",
       "      <th>user.location</th>\n",
       "      <th>date2</th>\n",
       "      <th>hong2</th>\n",
       "      <th>china2</th>\n",
       "      <th>law2</th>\n",
       "      <th>URLS</th>\n",
       "      <th>RT</th>\n",
       "      <th>lawhong2</th>\n",
       "      <th>chinahong2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ngdasokau</td>\n",
       "      <td>BarrySheerman</td>\n",
       "      <td>RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…</td>\n",
       "      <td>140</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Marisa15061974</td>\n",
       "      <td>WDVL4</td>\n",
       "      <td>RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…</td>\n",
       "      <td>140</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>SenatorLankford</td>\n",
       "      <td>RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…</td>\n",
       "      <td>140</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>FidelisToJusti1</td>\n",
       "      <td>RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…</td>\n",
       "      <td>139</td>\n",
       "      <td>Queens, NY</td>\n",
       "      <td>1062020</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>kangaroolooloo</td>\n",
       "      <td>next_china</td>\n",
       "      <td>RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…</td>\n",
       "      <td>140</td>\n",
       "      <td>Hong Kong</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               user           reuser  \\\n",
       "1         ngdasokau    BarrySheerman   \n",
       "3    Marisa15061974            WDVL4   \n",
       "5    kangaroolooloo  SenatorLankford   \n",
       "9   FidelisToJusti1  FidelisToJusti1   \n",
       "10   kangaroolooloo       next_china   \n",
       "\n",
       "                                                                                                                                       full_text  \\\n",
       "1   RT @BarrySheerman: Will the Johnson Government now wake up to the need for a coordinated global response to the malevolence of China’s acti…   \n",
       "3   RT @WDVL4: China pidió a empresas estatales que suspendan compras de soja y carne de cerdo de EEUU luego de que Washington anunciase que el…   \n",
       "5   RT @SenatorLankford: while Americans protest injustice-China bans in Hong Kong any remembrance of the 1989 massacre in Tiananmen Square- fr…   \n",
       "9    RT @FidelisToJusti1: What @realDonaldTrump is threatening if governors don't do what he wants is to emulate Communist China's crackdown on…   \n",
       "10  RT @next_china: Hong Kong’s finance workers are seeking to organize a strike to protest against China’s plans to implement new security law…   \n",
       "\n",
       "    range user.location    date2  hong2  china2   law2 URLS    RT  lawhong2  \\\n",
       "1     140     Hong Kong  1062020  False    True  False  NaN  True     False   \n",
       "3     140           NaN  1062020  False    True  False  NaN  True     False   \n",
       "5     140     Hong Kong  1062020   True    True  False  NaN  True     False   \n",
       "9     139    Queens, NY  1062020  False    True  False  NaN  True     False   \n",
       "10    140     Hong Kong  1062020   True    True   True  NaN  True      True   \n",
       "\n",
       "    chinahong2  \n",
       "1         True  \n",
       "3         True  \n",
       "5         True  \n",
       "9         True  \n",
       "10        True  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separ3a.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exporting tweets with most `urls` frequency.\n",
    "#### filter the dataframe for tweets with urls of highest frequency."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "mostur = pd.DataFrame(separ3a['URLS'].value_counts().nlargest(60))\n",
    "mostur.reset_index(level=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(mostur)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>URLS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>1056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://t.co/TysDBVW8H9</td>\n",
       "      <td>673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/Ek680whc2t</td>\n",
       "      <td>547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://t.co/6BNz7v8w5m</td>\n",
       "      <td>506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://t.co/ZxeHlOhqvh</td>\n",
       "      <td>477</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     index  URLS\n",
       "0  https://t.co/A11HGtqK0v  1056\n",
       "1  https://t.co/TysDBVW8H9   673\n",
       "2  https://t.co/Ek680whc2t   547\n",
       "3  https://t.co/6BNz7v8w5m   506\n",
       "4  https://t.co/ZxeHlOhqvh   477"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mostur.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `URL` with highest frequency."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://t.co/A11HGtqK0v\n"
     ]
    }
   ],
   "source": [
    "mostur_0 = mostur['index'].iloc[0]\n",
    "print(mostur_0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### exporting the most urls to csv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/ambijat/Datrat/ipytoneeee/CNUS/processed2/mostur.csv\n"
     ]
    }
   ],
   "source": [
    "mosturls = os.path.join(folder_path2 + \"/\" + \"processed2\", 'mostur.csv')\n",
    "print(mosturls)\n",
    "mostur.to_csv(mosturls, index = False, header = True, encoding = 'utf-8', sep = ';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using the most `URL` to identify tweets and exporting it as a dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "separ3b = separ3a.loc[separ['full_text'].str.contains(mostur_0, flags=re.IGNORECASE, regex=True)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>reuser</th>\n",
       "      <th>full_text</th>\n",
       "      <th>range</th>\n",
       "      <th>user.location</th>\n",
       "      <th>date2</th>\n",
       "      <th>hong2</th>\n",
       "      <th>china2</th>\n",
       "      <th>law2</th>\n",
       "      <th>URLS</th>\n",
       "      <th>RT</th>\n",
       "      <th>lawhong2</th>\n",
       "      <th>chinahong2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>legloa1s8</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>hongkong</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>LazyLionLovesHK</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>Joelie92228745</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>Lambert11679267</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>Hot_L_</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1062020</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                user          reuser  \\\n",
       "75         legloa1s8  benedictrogers   \n",
       "202  LazyLionLovesHK  benedictrogers   \n",
       "270   Joelie92228745  benedictrogers   \n",
       "283  Lambert11679267  benedictrogers   \n",
       "299           Hot_L_  benedictrogers   \n",
       "\n",
       "                                                                                                                          full_text  \\\n",
       "75   RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "202  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "270  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "283  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "299  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "\n",
       "     range user.location    date2  hong2  china2  law2  \\\n",
       "75     126      hongkong  1062020   True    True  True   \n",
       "202    126           NaN  1062020   True    True  True   \n",
       "270    126           NaN  1062020   True    True  True   \n",
       "283    126           NaN  1062020   True    True  True   \n",
       "299    126           NaN  1062020   True    True  True   \n",
       "\n",
       "                        URLS    RT  lawhong2  chinahong2  \n",
       "75   https://t.co/A11HGtqK0v  True      True        True  \n",
       "202  https://t.co/A11HGtqK0v  True      True        True  \n",
       "270  https://t.co/A11HGtqK0v  True      True        True  \n",
       "283  https://t.co/A11HGtqK0v  True      True        True  \n",
       "299  https://t.co/A11HGtqK0v  True      True        True  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separ3b.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reducing the number of columns for export."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "separ3c = separ3b.iloc[:, [0, 1, 2, 3, 9, -1, -2, -3, -4]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>reuser</th>\n",
       "      <th>full_text</th>\n",
       "      <th>range</th>\n",
       "      <th>URLS</th>\n",
       "      <th>chinahong2</th>\n",
       "      <th>lawhong2</th>\n",
       "      <th>RT</th>\n",
       "      <th>URLS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>legloa1s8</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>LazyLionLovesHK</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>Joelie92228745</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>Lambert11679267</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>Hot_L_</td>\n",
       "      <td>benedictrogers</td>\n",
       "      <td>RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v</td>\n",
       "      <td>126</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>https://t.co/A11HGtqK0v</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                user          reuser  \\\n",
       "75         legloa1s8  benedictrogers   \n",
       "202  LazyLionLovesHK  benedictrogers   \n",
       "270   Joelie92228745  benedictrogers   \n",
       "283  Lambert11679267  benedictrogers   \n",
       "299           Hot_L_  benedictrogers   \n",
       "\n",
       "                                                                                                                          full_text  \\\n",
       "75   RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "202  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "270  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "283  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "299  RT @benedictrogers: BBC News - Hong Kong: Boris Johnson urged to form alliance over China security law https://t.co/A11HGtqK0v   \n",
       "\n",
       "     range                     URLS  chinahong2  lawhong2    RT  \\\n",
       "75     126  https://t.co/A11HGtqK0v        True      True  True   \n",
       "202    126  https://t.co/A11HGtqK0v        True      True  True   \n",
       "270    126  https://t.co/A11HGtqK0v        True      True  True   \n",
       "283    126  https://t.co/A11HGtqK0v        True      True  True   \n",
       "299    126  https://t.co/A11HGtqK0v        True      True  True   \n",
       "\n",
       "                        URLS  \n",
       "75   https://t.co/A11HGtqK0v  \n",
       "202  https://t.co/A11HGtqK0v  \n",
       "270  https://t.co/A11HGtqK0v  \n",
       "283  https://t.co/A11HGtqK0v  \n",
       "299  https://t.co/A11HGtqK0v  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separ3c.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "mosturltweet = os.path.join(folder_path2 + \"/\" + \"processed2\", 'mosturl_tweet.csv')\n",
    "separ3c.to_csv(mosturltweet, index = False, header = True, encoding = 'utf-8', sep = ';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `Separ3a` is split into two dataframe `sp2` and `sp3` of `_original` and `_retweets` and exported to `csv` format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp2 = separ3a[separ3a['RT'] == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13523\n"
     ]
    }
   ],
   "source": [
    "print(len(sp2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp3= separ3a[separ3a['RT'] == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59864\n"
     ]
    }
   ],
   "source": [
    "print(len(sp3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### exporting the original tweets to csv file. `str(collect[0])` and `str(collect[-1])` are conversion of `int64` to `string`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/ambijat/Datrat/ipytoneeee/CNUS/processed2/hongchina10620203062020_original.csv\n"
     ]
    }
   ],
   "source": [
    "k1k2_original = os.path.join(folder_path2 + \"/\" + \"processed2\", k1+k2+ str(collect[0]) + str(collect[-1]) +'_original.csv')\n",
    "print(k1k2_original)\n",
    "sp2.to_csv(k1k2_original, index = False, header = True, encoding = 'utf-8', sep = ';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### exporting the retweets to csv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/ambijat/Datrat/ipytoneeee/CNUS/processed2/hongchina10620203062020_retweets.csv\n"
     ]
    }
   ],
   "source": [
    "k1k2_retweets = os.path.join(folder_path2 + \"/\" + \"processed2\", k1+k2+ str(collect[0]) + str(collect[-1]) +'_retweets.csv')\n",
    "print(k1k2_retweets)\n",
    "sp3.to_csv(k1k2_retweets, index = False, header = True, encoding = 'utf-8', sep = ';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm: cannot remove '*.tmp': No such file or directory\n"
     ]
    }
   ],
   "source": [
    "!rm *.tmp\n",
    "!rm *.txt"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}