Elasticsearch Kibana Twitter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import sys\n",
    "import tweepy\n",
    "from tweepy import Stream\n",
    "from tweepy import OAuthHandler# authentication\n",
    "from tweepy.streaming import StreamListener# twitter tweets listener\n",
    "from textwrap import TextWrapper\n",
    "from datetime import datetime\n",
    "from elasticsearch import Elasticsearch\n",
    "import pandas as pd\n",
    "from pandas import DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#consumer key, consumer secret, access token, access secret.\n",
    "ckey='xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'\n",
    "csecret='xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'\n",
    "atoken='xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'\n",
    "asecret='xxxxxxxxxxxxxxxxxxx not telling this either'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth = OAuthHandler(ckey, csecret)\n",
    "auth.set_access_token(atoken, asecret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#without using elastic search(using only tweepy) \n",
    "class listener(StreamListener):\n",
    "\n",
    "    def on_data(self, data):\n",
    "        print(data)\n",
    "        return(True)\n",
    "\n",
    "    def on_error(self, status):\n",
    "        print(status)\n",
    "\n",
    "\n",
    "\n",
    "twitterStream = Stream(auth, listener())\n",
    "twitterStream.filter(track=[\"USA\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "host = {\"host\" : \"localhost\", \"port\" : 9200}\n",
    "es = Elasticsearch(hosts = [host])# Connect to the elastic cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True, 'shards_acknowledged': True, 'index': 'tweeting'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.create(index='tweeting',ignore =400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True, 'shards_acknowledged': True, 'index': 'twitter_index'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.create(index='twitter_index',ignore =400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StreamApi(StreamListener):\n",
    "\n",
    "    def on_data(self, data):\n",
    "        json_data=json.loads(data)\n",
    "        print(json_data)\n",
    "        \n",
    "        es.index(index=\"tweeting\",\n",
    "                doc_type=\"teet\",\n",
    "                body={\"author\": json_data[\"user\"],\n",
    "                       \"date\": json_data[\"created_at\"],\n",
    "                       \"message\": json_data[\"text\"]\n",
    "                       })\n",
    "                #ignore=400\n",
    "        return True\n",
    "      # on failure\n",
    "    def on_error(self, status):\n",
    "        print(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    listener = StreamApi()\n",
    "    \n",
    "    auth = OAuthHandler(ckey, csecret)\n",
    "    auth.set_access_token(atoken, asecret)\n",
    "    \n",
    "    streamer  = Stream(auth, listener)\n",
    "    streamer.filter(track=['one','two','three'])        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.exists(index=\"tweeting\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".kibana_1\n",
      "inspections\n",
      "tweeting\n",
      "twitter_index\n",
      "twt_ind\n"
     ]
    }
   ],
   "source": [
    "for index in es.indices.get('*'):\n",
    "    print (index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 0,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': 0, 'max_score': None, 'hits': []}}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first = es.search(index=\"tweeting\", body={\"from\":0,\"size\":1,\"query\":{\"match_phrase\":{\"message\":\"snow day\"}}})\n",
    "first# got 0 hits so no tweets on 'snow day'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 5,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': 5,\n",
       "  'max_score': 2.1918683,\n",
       "  'hits': [{'_index': 'tweeting',\n",
       "    '_type': 'teet',\n",
       "    '_id': 'HhqzvWcB5XnUAWVlkXSP',\n",
       "    '_score': 2.1918683,\n",
       "    '_source': {'author': {'id': 2401508816,\n",
       "      'id_str': '2401508816',\n",
       "      'name': 'CLM Burundi',\n",
       "      'screen_name': 'clmburundi',\n",
       "      'location': 'BUJUMBURA, BURUNDI',\n",
       "      'url': None,\n",
       "      'description': 'CLM. CHRIST-CENTERED. CELL BASED. TRANSFORMATION.',\n",
       "      'translator_type': 'none',\n",
       "      'protected': False,\n",
       "      'verified': False,\n",
       "      'followers_count': 392,\n",
       "      'friends_count': 38,\n",
       "      'listed_count': 3,\n",
       "      'favourites_count': 124,\n",
       "      'statuses_count': 666,\n",
       "      'created_at': 'Fri Mar 21 12:41:09 +0000 2014',\n",
       "      'utc_offset': None,\n",
       "      'time_zone': None,\n",
       "      'geo_enabled': True,\n",
       "      'lang': 'en',\n",
       "      'contributors_enabled': False,\n",
       "      'is_translator': False,\n",
       "      'profile_background_color': 'C0DEED',\n",
       "      'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "      'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "      'profile_background_tile': False,\n",
       "      'profile_link_color': '1DA1F2',\n",
       "      'profile_sidebar_border_color': 'C0DEED',\n",
       "      'profile_sidebar_fill_color': 'DDEEF6',\n",
       "      'profile_text_color': '333333',\n",
       "      'profile_use_background_image': True,\n",
       "      'profile_image_url': 'http://pbs.twimg.com/profile_images/531927109774893057/c3IsLpZo_normal.jpeg',\n",
       "      'profile_image_url_https': 'https://pbs.twimg.com/profile_images/531927109774893057/c3IsLpZo_normal.jpeg',\n",
       "      'profile_banner_url': 'https://pbs.twimg.com/profile_banners/2401508816/1503932184',\n",
       "      'default_profile': True,\n",
       "      'default_profile_image': False,\n",
       "      'following': None,\n",
       "      'follow_request_sent': None,\n",
       "      'notifications': None},\n",
       "     'date': 'Mon Dec 17 19:45:14 +0000 2018',\n",
       "     'message': 'Exactly one week from today, we will be reminded on how people of every social status praised, bowed before in hono… https://t.co/GZUaDThZU2'}}]}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second = es.search(index=\"tweeting\", body={\"from\":0,\"size\":1,\"query\":{\"match_phrase\":{\"date\":\"Mon Dec 17 19:45:14 +0000 2018\"}}})\n",
    "second # 5 tweets on dec 17th and at that precise time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.delete(index='twitter_index')#deleting an index created for practising"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".kibana_1\n",
      "inspections\n",
      "tweeting\n",
      "twt_ind\n"
     ]
    }
   ],
   "source": [
    "for index in es.indices.get('*'):\n",
    "    print (index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "OLHS_Library=es.get(index=\"tweeting\", doc_type='teet',id='LRqzvWcB5XnUAWVllXQO') # from kibana i got the id for a single document \n",
    "#and used it to retrieve a document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'author': {'id': 3020368912, 'id_str': '3020368912', 'name': 'OLHS Library', 'screen_name': 'OLHSLRC', 'location': 'Motherwell', 'url': 'http://www.ourladyshighschool.co.uk', 'description': 'What goes on in the Library, plus news, competitions, stories, resources, #infolit and all sorts of interesting stuff. Tweets mainly by Mrs Macfadyen.', 'translator_type': 'none', 'protected': False, 'verified': False, 'followers_count': 867, 'friends_count': 1832, 'listed_count': 42, 'favourites_count': 716, 'statuses_count': 4926, 'created_at': 'Thu Feb 05 19:16:56 +0000 2015', 'utc_offset': None, 'time_zone': None, 'geo_enabled': False, 'lang': 'en', 'contributors_enabled': False, 'is_translator': False, 'profile_background_color': '000000', 'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme15/bg.png', 'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme15/bg.png', 'profile_background_tile': False, 'profile_link_color': '9266CC', 'profile_sidebar_border_color': '000000', 'profile_sidebar_fill_color': '000000', 'profile_text_color': '000000', 'profile_use_background_image': False, 'profile_image_url': 'http://pbs.twimg.com/profile_images/858975703508844545/aEZlxfUQ_normal.jpg', 'profile_image_url_https': 'https://pbs.twimg.com/profile_images/858975703508844545/aEZlxfUQ_normal.jpg', 'profile_banner_url': 'https://pbs.twimg.com/profile_banners/3020368912/1505419985', 'default_profile': False, 'default_profile_image': False, 'following': None, 'follow_request_sent': None, 'notifications': None}, 'date': 'Mon Dec 17 19:45:15 +0000 2018', 'message': 'An application to @ScotSpaceSchool might lead to you appearing on an official @NASA poster one day. Find out more h… https://t.co/O3sPW0Td87'}\n"
     ]
    }
   ],
   "source": [
    "print (OLHS_Library['_source'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'took': 2,\n",
       " 'timed_out': False,\n",
       " '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0},\n",
       " 'hits': {'total': 1,\n",
       "  'max_score': 0.0,\n",
       "  'hits': [{'_index': 'twt_ind',\n",
       "    '_type': 'twt',\n",
       "    '_id': 'fRgXumcBVgY_VFLmkCXV',\n",
       "    '_score': 0.0,\n",
       "    '_source': {'author': {'id': 2461904382,\n",
       "      'id_str': '2461904382',\n",
       "      'name': 'Michael',\n",
       "      'screen_name': 'miguel_lilith',\n",
       "      'location': 'México',\n",
       "      'url': None,\n",
       "      'description': 'Deportes Fútbol',\n",
       "      'translator_type': 'none',\n",
       "      'protected': False,\n",
       "      'verified': False,\n",
       "      'followers_count': 0,\n",
       "      'friends_count': 31,\n",
       "      'listed_count': 0,\n",
       "      'favourites_count': 9,\n",
       "      'statuses_count': 16,\n",
       "      'created_at': 'Thu Apr 24 16:38:04 +0000 2014',\n",
       "      'utc_offset': None,\n",
       "      'time_zone': None,\n",
       "      'geo_enabled': False,\n",
       "      'lang': 'es',\n",
       "      'contributors_enabled': False,\n",
       "      'is_translator': False,\n",
       "      'profile_background_color': 'C0DEED',\n",
       "      'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "      'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "      'profile_background_tile': False,\n",
       "      'profile_link_color': '1DA1F2',\n",
       "      'profile_sidebar_border_color': 'C0DEED',\n",
       "      'profile_sidebar_fill_color': 'DDEEF6',\n",
       "      'profile_text_color': '333333',\n",
       "      'profile_use_background_image': True,\n",
       "      'profile_image_url': 'http://pbs.twimg.com/profile_images/823520432791949312/ivQQ5ZM2_normal.jpg',\n",
       "      'profile_image_url_https': 'https://pbs.twimg.com/profile_images/823520432791949312/ivQQ5ZM2_normal.jpg',\n",
       "      'profile_banner_url': 'https://pbs.twimg.com/profile_banners/2461904382/1485177546',\n",
       "      'default_profile': True,\n",
       "      'default_profile_image': False,\n",
       "      'following': None,\n",
       "      'follow_request_sent': None,\n",
       "      'notifications': None},\n",
       "     'date': 'Mon Dec 17 02:56:00 +0000 2018',\n",
       "     'message': '@ClubAmerica Vamos america'}}]}}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.search(index='twt_ind',body={\n",
    "        'query':{\n",
    "            'bool':{#using booleanto combination of queries search with authors not using english language and have zero followers.\n",
    "                'must_not':{\n",
    "                    'match':{\n",
    "                        'author.lang':'en'\n",
    "                    }\n",
    "                },\n",
    "                \"filter\":{\n",
    "                    \"range\":{\n",
    "                        \"author.followers_count\":{\n",
    "                            \"lte\":0\n",
    "                        }\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    })\n",
    "#filtered using REST API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trying to include top 5 tweets owners name and follower count information into panda dataframe from elastic search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = es.search(index=\"tweeting\", doc_type='teet', body={\"query\": {\"match_all\": {}}}, size=5, _source=('author.screen_name','author.followers_count'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'_index': 'tweeting',\n",
       "  '_type': 'teet',\n",
       "  '_id': 'FxqzvWcB5XnUAWVlj3Q3',\n",
       "  '_score': 1.0,\n",
       "  '_source': {'author': {'screen_name': 'kaylacampion',\n",
       "    'followers_count': 146}}},\n",
       " {'_index': 'tweeting',\n",
       "  '_type': 'teet',\n",
       "  '_id': 'GhqzvWcB5XnUAWVlkHRW',\n",
       "  '_score': 1.0,\n",
       "  '_source': {'author': {'screen_name': 'GecoTree', 'followers_count': 723}}},\n",
       " {'_index': 'tweeting',\n",
       "  '_type': 'teet',\n",
       "  '_id': 'IhqzvWcB5XnUAWVlknRx',\n",
       "  '_score': 1.0,\n",
       "  '_source': {'author': {'screen_name': 'sbthemoor',\n",
       "    'followers_count': 1887}}},\n",
       " {'_index': 'tweeting',\n",
       "  '_type': 'teet',\n",
       "  '_id': 'JxqzvWcB5XnUAWVlk3SU',\n",
       "  '_score': 1.0,\n",
       "  '_source': {'author': {'screen_name': 'ZalUIbaorimi',\n",
       "    'followers_count': 17973}}},\n",
       " {'_index': 'tweeting',\n",
       "  '_type': 'teet',\n",
       "  '_id': 'KhqzvWcB5XnUAWVllHRT',\n",
       "  '_score': 1.0,\n",
       "  '_source': {'author': {'screen_name': 'NerdsSocialClub',\n",
       "    'followers_count': 2188}}}]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=data['hits']['hits']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1=pd.DataFrame.from_dict(df)\n",
    "source=df1._source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>followers_count</th>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>screen_name</th>\n",
       "      <td>kaylacampion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>followers_count</th>\n",
       "      <td>723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>screen_name</th>\n",
       "      <td>GecoTree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>followers_count</th>\n",
       "      <td>1887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>screen_name</th>\n",
       "      <td>sbthemoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>followers_count</th>\n",
       "      <td>17973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>screen_name</th>\n",
       "      <td>ZalUIbaorimi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>followers_count</th>\n",
       "      <td>2188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>screen_name</th>\n",
       "      <td>NerdsSocialClub</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          author\n",
       "followers_count              146\n",
       "screen_name         kaylacampion\n",
       "followers_count              723\n",
       "screen_name             GecoTree\n",
       "followers_count             1887\n",
       "screen_name            sbthemoor\n",
       "followers_count            17973\n",
       "screen_name         ZalUIbaorimi\n",
       "followers_count             2188\n",
       "screen_name      NerdsSocialClub"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat(map(pd.DataFrame.from_dict, source))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "references\n",
    "https://www.linkedin.com/pulse/stream-twitter-data-elasticsearch-dashboard-kibana-python-tripathi/\n",
    "\n",
    "https://realpython.com/twitter-sentiment-python-docker-elasticsearch-kibana/\n",
    "\n",
    "https://www.rittmanmead.com/blog/2015/08/three-easy-ways-to-stream-twitter-data-into-elasticsearch/\n",
    "\n",
    "http://nitin-panwar.github.io/Elasticsearch-tutorial-for-beginners-using-Python/\n",
    "\n",
    "https://tryolabs.com/blog/2015/02/17/python-elasticsearch-first-steps/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}