diff --git a/.gitignore b/.gitignore index 72364f9..3582338 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# Byte-compiled / optimized / DLL files +# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class @@ -81,9 +81,20 @@ celerybeat-schedule # virtualenv venv/ ENV/ +Include/ +tcl/ +Scripts/ # Spyder project settings .spyderproject # Rope project settings .ropeproject + +# Build / Install / Tests +pip* +*.whl +tweets.txt +files.txt +/images +/mp4 \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..27ce464 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include *.md +include *.txt \ No newline at end of file diff --git a/README.md b/README.md index 88ada0c..0b56835 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,97 @@ # DMArchiver -A tool to archive the direct messages from your private conversations on Twitter +A tool to archive **all** the direct messages from your private conversations on Twitter. + +## Introduction +Have you ever need to retrieve old information from a chat with your friends on Tweeter? Or maybe you would just like to backup all these cheerful moments and keep them safe. + +I have made this tool to retrieve all the tweets from my private conversations and transform them in an _IRC-like_ log for archiving. Emoji are currently kept with their description to prevent encoding issues. + +**Output sample:** +``` +[2016-09-07 10:35:55] [Media-image] https://ton.twitter.com/1.1/ton/data/dm/773125478562429059/773401254876366208/mfeDmXXj.jpg I am so a Dexter fan... +[2016-09-07 10:37:12] He is so sexy. [Flushed face] I love him. [Heavy red heart] +[2016-09-07 10:38:10] You guys are ridiculous! [Face with tears of joy] +``` + +This tool is also able to **download all the uploaded images** in their original resolution and, as a bonus, also retrieve the **GIFs** you used in your conversation as MP4 files (the format used by Tweeter to optimized them and save space). + +This tool does not leverage the Twitter API because of its very restrictive limitations in regard of the handling of the Direct Messages. Actually, it is currently possible to retrieve only the latest 200 messages of a private conversation. + +Because it is still possible to retrieve the older messages from a Direct Conversation by scrolling up, this script only simulates this behavior to retrieve automatically the messages. + +## Prerequisites +You need two things to retrieve your tweets: +- Your `auth-token` value, which is present in your `auth-token` cookie when you are connected on Twitter. This is a 40 characters string like `9e6ecbd088g0baaf2fa9cf2c690aca8ff8027b9f`. Do not share your `auth-token` with anyone, it is like a password to authenticate on the Tweeter site or API. +- The `conversation-id` which is the identifier of the conversation you want to backup. This one is just a little bit harder to find: + - Open the _Network_ panel in the _Development Tools_ of your favorite browser. + - Open a the desired conversation on Tweeter and have a look in the requests. + - Identify a request with the following arguments: + `https://twitter.com/messages/with/conversation?id=645754097571131337&max_entry_id=78473919348771337` + - Use the `id` value as your `conversation-id`. This identifier can contain special characters such as '-'. + +I will try to ease this setup in a future version. :wink: + +## Installation +### Using pip +`$ pip install dmarchiver` + +## Usage + +### Command line tool +``` +$ dmarchiver [-h] [-di] [-dg] conversation-id auth-token +$ dmarchiver-script.py [-h] [-di] [-dg] conversation-id auth-token + +$ dmarchiver --help + positional arguments: + conversation-id Conversation ID + auth-token Authentication token + + optional arguments: + -h, --help show this help message and exit + -di, --download-images + Download images + -dg, --download-gifs Download GIFs (as MP4) +``` + +The script output is a `tweets.txt` file with the conversation formatted in an _IRC-like_ style. + +If the switches have been set for the download of images and gifs, the files can be respectively found in the `/images` and `/mp4` folders. + +### Module import +```python +>>> from dmarchiver import Crawler +>>> crawler = Crawler('CONVERSATION_ID', 'AUTH_TOKEN') +>>> crawler.crawl() +``` + +## Development setup +```shell +$ git clone https://github.com/Mincka/DMArchiver.git +$ cd dmarchiver +$ virtualenv venv +$ source venv/bin/activate # "venv/Scripts/Activate.bat" on Windows +$ pip install -r requirements.txt +$ python setup.py install +``` + +## Troubleshooting +You may encounter building issues with the `lxml` library on Windows. The most simple and straightforward fix is to download and install a precompiled binary from [this site](http://www.lfd.uci.edu/~gohlke/pythonlibs/#lxml) and install the package locally: +`$ pip install lxml-3.6.4-cp35-cp35m-win_amd64.whl` + +## License + +Copyright (C) 2016 Julien EHRHART + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . \ No newline at end of file diff --git a/dmarchiver/__init__.py b/dmarchiver/__init__.py new file mode 100644 index 0000000..c29dc42 --- /dev/null +++ b/dmarchiver/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +""" + This module allows to archive direct messages from Twitter + without having to deal with the API limitations. +""" + +__version__ = "0.0.2" diff --git a/dmarchiver/cmdline.py b/dmarchiver/cmdline.py new file mode 100644 index 0000000..614a4a1 --- /dev/null +++ b/dmarchiver/cmdline.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +""" + Direct Messages Archiver - Command Line + + Usage: + # dmarchiver [-h] [-di] [-dg] conversation-id auth-token + # dmarchiver-script.py [-h] [-di] [-dg] conversation-id auth-token + + positional arguments: + conversation-id Conversation ID + auth-token Authentication token + + optional arguments: + -h, --help show this help message and exit + -di, --download-images + Download images + -dg, --download-gifs Download GIFs (as MP4) +""" + +import argparse +from dmarchiver.core import Crawler + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("conversation_id", metavar='conversation-id', nargs=1, help="Conversation ID") + parser.add_argument("auth_token", metavar='auth-token', nargs=1, help="Authentication token") + parser.add_argument("-di", "--download-images", help="Download images", action="store_true") + parser.add_argument("-dg", "--download-gifs", help="Download GIFs (as MP4)", action="store_true") + + args = parser.parse_args() + + crawler = Crawler(args.conversation_id[0], args.auth_token[0]) + crawler.crawl(args.download_images, args.download_gifs) \ No newline at end of file diff --git a/dmarchiver/core.py b/dmarchiver/core.py new file mode 100644 index 0000000..320279c --- /dev/null +++ b/dmarchiver/core.py @@ -0,0 +1,418 @@ +# -*- coding: utf-8 -*- + +""" + Direct Messages Archiver + + Usage: + + >>> from dmarchiver import Crawler + >>> crawler = Crawler('CONVERSATION_ID', 'AUTH_TOKEN') + >>> crawler.crawl() +""" + +import argparse +import collections +import datetime +from enum import Enum +import lxml.html +from lxml.cssselect import CSSSelector +import os +import re +import requests +from requests.packages.urllib3.exceptions import InsecureRequestWarning +import shutil + +__all__ = ['Crawler'] + +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + +class Conversation(object): + conversation_id = None + tweets = collections.OrderedDict() + + def __init__(self, conversation_id): + self.tweets = collections.OrderedDict() + self.conversation_id = conversation_id + + def print_conversation(self): + items = list(self.tweets.items()) + items.reverse() + + for tweet in items: + if type(tweet[1]).__name__ == 'DirectMessage': + irc_formatted_date = datetime.datetime.utcfromtimestamp( + int(tweet[1].timestamp)).strftime('%Y-%m-%d %H:%M:%S') + print('[{0}] <{1}> '.format(irc_formatted_date, tweet[1].author), end='') + for element in tweet[1].elements: + print('{0} '.format(element), end='') + print('\r') + elif type(tweet[1]).__name__ == 'DMConversationEntry': + print('[DMConversationEntry] {0}\r'.format(tweet[1])) + + def write_conversation(self, filename): + file_buffer = '' + + items = list(self.tweets.items()) + items.reverse() + + for tweet in items: + if type(tweet[1]).__name__ == 'DirectMessage': + irc_formatted_date = datetime.datetime.utcfromtimestamp( + int(tweet[1].timestamp)).strftime('%Y-%m-%d %H:%M:%S') + file_buffer += '[{0}] <{1}> '.format(irc_formatted_date, tweet[1].author) + for element in tweet[1].elements: + file_buffer += '{0} '.format(element) + file_buffer += '{0}'.format(os.linesep) + elif type(tweet[1]).__name__ == 'DMConversationEntry': + file_buffer += '[DMConversationEntry] {0}\r'.format(tweet[1]) + + # Convert all '\n' of the buffer to os.linesep + file_buffer = file_buffer.replace('\n', os.linesep) + + with open(filename, "wb") as myfile: + myfile.write(file_buffer.encode('UTF-8')) + +class DMConversationEntry(object): + tweet_id = '' + _text = '' + + def __init__(self, tweet_id, text): + self.tweet_id = tweet_id + self._text = text.strip() + + def __str__(self): + return self._text + +class DirectMessage(object): + + tweet_id = '' + timestamp = '' + author = '' + elements = [] + + def __init__(self, tweet_id, timestamp, author): + self.tweet_id = tweet_id + self.timestamp = timestamp + self.author = author + +class DirectMessageText(object): + _text = '' + + def __init__(self, text): + self._text = text + + def __str__(self): + return self._text + +class DirectMessageTweet(object): + _tweet_url = '' + + def __init__(self, tweet_url): + self._tweet_url = tweet_url + + def __str__(self): + return self._tweet_url + +class DirectMessageCard(object): + _card_url = '' + + def __init__(self, card_url): + self._card_url = card_url + + def __str__(self): + return self._card_url + +class MediaType(Enum): + image = 1 + gif = 2 + video = 3 + unknown = 4 + +class DirectMessageMedia(object): + _media_preview_url = '' + _media_url = '' + _media_type = '' + + def __init__(self, media_url, media_preview_url, MediaType): + self._media_url = media_url + self._media_preview_url = media_preview_url + self._media_type = MediaType + + def __repr__(self): + # Todo + return "{0}('{1}','{2}')".format( + self.__class__.__name__, + self._media_url, + self._media_preview_url) + + def __str__(self): + if self._media_preview_url != '': + return '[Media-{0}] {1} [Media-preview] {2}'.format(self._media_type.name, self._media_url, self._media_preview_url) + else: + return '[Media-{0}] {1}'.format(self._media_type.name, self._media_url) + + +class Crawler(object): + _twitter_base_url = 'https://twitter.com' + _url = 'https://twitter.com/messages/with/conversation' + _headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0', + 'Accept': 'application/json, text/javascript, */*; q=0.01', + 'X-Requested-With': 'XMLHttpRequest'} + + def __init__(self, conversation_id, auth_token): + self.conversation_id = conversation_id + self._auth_token = auth_token + self._payload = {'id': self.conversation_id} + self._cookies = { + 'auth_token': self._auth_token} + + def _extract_dm_text_url(self, element, expanding_mode='only_expanded'): + raw_url = '' + if expanding_mode == 'only_expanded': + raw_url = element.get('data-expanded-url') + elif expanding_mode == 'only_short': + raw_url = element.get('href') + elif expanding_mode == 'short_and_expanded': + raw_url = '{0} [{1}]'.format(element.get( + 'href'), element.get('data-expanded-url')) + return raw_url + + def _extract_dm_text_hashtag(self, element): + raw_hashtag = element.text_content() + if element.tail is not None: + raw_hashtag += element.tail + return raw_hashtag + + def _extract_dm_text_atreply(self, element): + raw_atreply = element.text_content() + if element.tail is not None: + raw_atreply += element.tail + return raw_atreply + + # Todo: Implement parsing options + def _extract_dm_text_emoji(self, element): + raw_emoji = '[{0}]'.format(element.get('title')) + if element.tail is not None: + raw_emoji += element.tail + return raw_emoji + + def _parse_dm_text(self, element): + dm_text = '' + text_tweet = element.cssselect("p.tweet-text")[0] + for text in text_tweet.iter('p', 'a','img'): + if text.tag == 'a': + # External link + if 'twitter-timeline-link' in text.classes: + dm_text += self._extract_dm_text_url(text) + # #hashtag + elif 'twitter-hashtag' in text.classes: + dm_text += self._extract_dm_text_hashtag(text) + # @identifier + elif 'twitter-atreply' in text.classes: + dm_text += self._extract_dm_text_atreply(text) + else: + # Unable to identify the link type, raw HTML output + dm_text += lxml.html.tostring(text).decode('UTF-8') + # Emoji + elif text.tag == 'img' and 'Emoji' in text.classes: + dm_text += self._extract_dm_text_emoji(text) + else: + if text.text is not None: + #print('++ Text: ' + text.text) + dm_text += text.text + return DirectMessageText(dm_text) + + def _parse_dm_media(self, element, tweet_id, session, auth_token, download_images, download_gif): + media_url = '' + media_preview_url = '' + media_type = MediaType.unknown + + img_url = element.find('.//img') + gif_url = element.cssselect('div.PlayableMedia--gif') + video_url = element.cssselect('div.PlayableMedia--video') + if img_url is not None: + media_type = MediaType.image + media_url = img_url.get('data-full-img') + media_filename_re = re.findall(tweet_id + '/(.+)/(.+)$', media_url) + media_filename = tweet_id + '-' + \ + media_filename_re[0][0] + '-' + media_filename_re[0][1] + #print('++ Media: ' + media_url) + + if download_images: + cookies = {'auth_token': auth_token} + r = session.get(media_url, cookies=cookies, verify=False, stream=True) + if r.status_code == 200: + os.makedirs('images', exist_ok=True) + with open('images/' + media_filename, 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + elif len(gif_url) > 0: + media_type = MediaType.gif + media_style = gif_url[0].find('div').get('style') + media_preview_url = re.findall('url\(\'(.*?)\'\)', media_style)[0] + media_url = media_preview_url.replace( + 'dm_gif_preview', 'dm_gif').replace('.jpg', '.mp4') + media_filename_re = re.findall('dm_gif/(.+)/(.+)$', media_url) + media_filename = media_filename_re[0][ + 0] + '-' + media_filename_re[0][1] + + if download_gif: + r = session.get(media_url, verify=False, stream=True) + if r.status_code == 200: + os.makedirs('mp4', exist_ok=True) + with open('mp4/' + media_filename, 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + elif len(video_url) > 0: + media_type = MediaType.video + media_style = video_url[0].find('div').get('style') + media_preview_url = re.findall('url\(\'(.*?)\'\)', media_style)[0] + media_url = 'https://twitter.com/i/videos/dm/' + tweet_id + else: + print('Unknown media') + + return DirectMessageMedia(media_url, media_preview_url, media_type) + + def _parse_dm_tweet(self, element): + tweet_url = '' + tweet_url = element.cssselect('a.QuoteTweet-link')[0] + tweet_url = '{0}{1}'.format( + self._twitter_base_url, tweet_url.get('href')) + #print('++ Tweet link: ' + self.twitter_base_url + tweet_url.get('href')) + return DirectMessageTweet(tweet_url) + + def _parse_dm_card(self, element): + card_url = '' + card = element.cssselect( + 'div[class^=" card-type-"], div[class*=" card-type-"]')[0] + card_url = '{0} - Type: {1}'.format( + card.get('data-card-url'), card.get('data-card-name')) + #print('++ Card URL: ' + card.get('data-card-url') + ' ' + card.get('data-card-name')) + return DirectMessageCard(card_url) + + def _process_tweets(self, session, auth_token, tweets, download_images, download_gif): + conversation_set = collections.OrderedDict() + + orderedTweets = sorted(tweets, reverse=True) + + # DirectMessage-message + # -- DirectMessage-text + # -- DirectMessage-media + # -- DirectMessage-tweet + # -- DirectMessage-card + + for tweet_id in orderedTweets: + dm_author = '' + irc_formatted_date = '' + dm_element_text = '' + value = tweets[tweet_id] + document = lxml.html.fragment_fromstring(value) + + #print('------------------') + #print('#' + tweet_id) + + dm_container = document.cssselect('div.DirectMessage-container') + + # Generic messages such as "X has join the group" or "The group has + # been renamed" + dm_conversation_entry = document.cssselect( + 'div.DMConversationEntry') + + if len(dm_container) > 0: + dm_avatar = dm_container[0].cssselect('img.DMAvatar-image')[0] + dm_author = dm_avatar.get('alt') + + #print(dm_author) + + dm_footer = document.cssselect('div.DirectMessage-footer') + time_stamp = dm_footer[0].cssselect('span._timestamp')[ + 0].get('data-time') + + # DirectMessage-text, div.DirectMessage-media, + # div.DirectMessage-tweet_id, div.DirectMessage-card... + dm_elements = document.cssselect( + 'div.DirectMessage-message > div[class^="DirectMessage-"], div.DirectMessage-message > div[class*=" DirectMessage-"]') + + message = DirectMessage(tweet_id, time_stamp, dm_author) + + # Required array cleanup + message.elements = [] + + for dm_element in dm_elements: + dm_element_type = dm_element.get('class') + if 'DirectMessage-text' in dm_element_type: + element_object = self._parse_dm_text(dm_element) + message.elements.append(element_object) + #print(element_object) + elif dm_element_type == 'DirectMessage-media': + element_object = self._parse_dm_media(dm_element, tweet_id, session, auth_token, download_images, download_gif) + message.elements.append(element_object) + #print(element_object) + elif dm_element_type == 'DirectMessage-tweet': + element_object = self._parse_dm_tweet(dm_element) + message.elements.append(element_object) + #print(element_object) + elif dm_element_type == 'DirectMessage-card': + element_object = self._parse_dm_card(dm_element) + message.elements.append(element_object) + #print(element_object) + else: + print('Unknown element type') + + elif len(dm_conversation_entry) > 0: + #print(dm_conversation_entry[0].text) + dm_element_text = dm_conversation_entry[0].text.strip() + message = DMConversationEntry(tweet_id, dm_element_text) + + if message is not None: + conversation_set[tweet_id] = message + return conversation_set + + def crawl(self, download_images=False, download_gif=False): + session = requests.Session() + conversation = Conversation(self.conversation_id) + processed_tweet_counter = 0 + + while True: + r = session.get( + self._url, + headers = self._headers, + cookies = self._cookies, + params = self._payload, + verify = False) + + #print(r.text) + json = r.json() + + if 'max_entry_id' not in json: + # End of thread + print('Begin of thread reached') + break + + #print('max_entry_id: ' + json['max_entry_id']) + + # To call back to make the loop + #print('min_entry_id: ' + json['min_entry_id']) + + self._payload = {'id': self.conversation_id, + 'max_entry_id': json['min_entry_id']} + + tweets = json['items'] + + # Get tweets for the current request + conversation_set = self._process_tweets(session, self._auth_token, tweets, download_images, download_gif) + + # Append to the whole conversation + for tweet_id in conversation_set: + processed_tweet_counter += 1 + conversation.tweets[tweet_id] = conversation_set[tweet_id] + print('Processed tweets: {0}\r'.format(processed_tweet_counter), end='') + + print('Total processed tweets: {0}'.format(processed_tweet_counter)) + + #print('Printing conversation') + #conversation.print_conversation() + + print('Writing conversation') + conversation.write_conversation('tweets.txt') \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..65d0413 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +cssselect==0.9.2 +lxml==3.6.4 +requests==2.11.1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..50e9b60 --- /dev/null +++ b/setup.py @@ -0,0 +1,43 @@ +from setuptools import setup, find_packages + +import dmarchiver + +setup( + + name='dmarchiver', + version=dmarchiver.__version__, + + packages=find_packages(), + + install_requires=['requests>=2.11.1', 'lxml>=3.6.4', 'cssselect>=0.9.2'], + + author="Julien EHRHART", + author_email="julien.ehrhart@live.com", + + description="A tool to archive the direct messages from your private conversations on Twitter.", + + long_description=open('README.md').read(), + + include_package_data=True, + + url='https://github.com/Mincka/DMArchiver', + + classifiers=[ + "Programming Language :: Python", + "Development Status :: 3 - Alpha", + "License :: OSI Approved", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.5", + "Topic :: System :: Archiving", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + ], + + entry_points = { + 'console_scripts': [ + 'dmarchiver = dmarchiver.cmdline:main', + ], + }, + + license="GNU General Public License v3 (GPLv3)", +)