scraper_tweets.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Tweet Scraper:\n",
        "\n",
        "Using Twitter API to scrape tweets relating #Bitcoin or #BTC\n",
        "\n",
        "\n",
        "\n",
        "*   Loads config.py file that has all our variables (api keys, AWS keys, etc)\n",
        "*   Loads requirements.txt that stores all dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [],
      "source": [
        "%pip install -r requirements.txt\n",
        "from config import *\n",
        "import tweepy\n",
        "from tweepy import OAuthHandler\n",
        "import json\n",
        "import pandas as pd\n",
        "import csv\n",
        "import string\n",
        "import os\n",
        "import time\n",
        "import datetime\n",
        "import boto3\n",
        "\n",
        "\n",
        "def Scrapper():\n",
        "  btc_tweets = tweepy.Cursor(api.search_tweets ,q=\"#bitcoin -filter:retweets since:\" + str(yesterday)+ \" until:\" + str(today) , lang=\"en\", tweet_mode = 'extended').items()\n",
        "  data = []\n",
        "  \n",
        "  for tweet in btc_tweets:\n",
        "    text = tweet._json[\"full_text\"]\n",
        "    favorite_count = tweet.favorite_count\n",
        "    retween_count = tweet.retweet_count\n",
        "    created_at = tweet.created_at\n",
        "    screen_name = tweet.user.screen_name\n",
        "    line = {'screen_name': screen_name,'text': text, 'favorite_count': favorite_count, 'retweet_count': retween_count, 'created_at': created_at}\n",
        "    data.append(line)\n",
        "\n",
        "  btc_df = pd.DataFrame(data)\n",
        "  btc_df.to_csv(\"btc_tweets.csv\", mode='a', header=False)\n",
        "\n",
        "  client = boto3.client('s3', aws_access_key_id = aws_access_id, aws_secret_access_key = aws_secret_acess)\n",
        "\n",
        "  client.upload_file('btc_tweets.csv', upload_file_bucket, upload_file_key)\n",
        "\n",
        "Scrapper()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "Dataeng.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "824480eebd21a519a881ff5077d4bf7e625c132ad9908a62de3ad1328dc859e8"
    },
    "kernelspec": {
      "display_name": "Python 3.9.7 ('base')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}