Skip to content
This repository has been archived by the owner on Feb 28, 2023. It is now read-only.

Commit

Permalink
Closes #42
Browse files Browse the repository at this point in the history
  • Loading branch information
Mincka committed Nov 11, 2017
1 parent 7843435 commit 1577327
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 4 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ $ dmarchiver --help
-u, --username Username (e-mail or handle)
-p, --password Password
-d, --delay Delay between requests (seconds)
-s, --save-session Save the session locally
-di, --download-images
Download images
-dg, --download-gifs Download GIFs (as MP4)
Expand Down
2 changes: 1 addition & 1 deletion dmarchiver/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
without having to deal with the API limitations.
"""

__version__ = "0.2.1"
__version__ = "0.2.2"
11 changes: 10 additions & 1 deletion dmarchiver/cmdline.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
-u, --username Username (e-mail or handle)
-p, --password Password
-d, --delay Delay between requests (seconds)
-s, --save-session Save the session locally
-di, --download-images
Download images
-dg, --download-gifs Download GIFs (as MP4)
Expand Down Expand Up @@ -42,6 +43,11 @@ def main():
parser.add_argument("-u", "--username", help="Username (e-mail or handle)")
parser.add_argument("-p", "--password", help="Password")
parser.add_argument("-d", "--delay", type=float, default=0, help="Delay between requests (seconds)")
parser.add_argument(
"-s",
"--save_session",
help="Save the session locally.",
action="store_true")
parser.add_argument(
"-di",
"--download-images",
Expand All @@ -65,6 +71,9 @@ def main():

args = parser.parse_args()

if args.save_session:
print('Warning: Session saving is enabled. Your authentication cookie (Twitter credentials) will be kept in the dmarchiver_session.dat file.')

if args.username is None:
username = input('Enter your username or email: ')
else:
Expand All @@ -78,7 +87,7 @@ def main():

crawler = Crawler()
try:
crawler.authenticate(username, password, args.raw_output)
crawler.authenticate(username, password, args.save_session, args.raw_output)
except PermissionError as err:
print('Error: {0}'.format(err.args[0]))
print('Exiting.')
Expand Down
28 changes: 26 additions & 2 deletions dmarchiver/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import datetime
from enum import Enum
import os
import pickle
import re
import shutil
from sys import platform
Expand Down Expand Up @@ -265,12 +266,30 @@ class Crawler(object):
'X-Requested-With': 'XMLHttpRequest'}

_max_id_found = False
_session = None

def authenticate(self, username, password, raw_output):
def authenticate(self, username, password, save_session, raw_output):
login_url = self._twitter_base_url + '/login'
sessions_url = self._twitter_base_url + '/sessions'
messages_url = self._twitter_base_url + '/messages'

self._session = requests.Session()
if save_session:
try:
with open('dmarchiver_session.dat', 'rb') as file:
self._session = pickle.load(file)
print('dmarchiver_session.dat found. Reusing a previous session, ignoring the provided credentials.')
# Test if the session is still valid
response = self._session.get(messages_url, headers=self._http_headers, allow_redirects=False)
if response.status_code == 200:
return
else:
self._session = None
print('Previous session is invalid. Creating a new session with provided credentials.')
except FileNotFoundError:
print('dmarchiver_session.dat not found. Creating a new session with provided credentials.')

if save_session is False or self._session is None:
self._session = requests.Session()

if raw_output:
raw_output_file = open(
Expand Down Expand Up @@ -299,6 +318,11 @@ def authenticate(self, username, password, raw_output):
cookies = requests.utils.dict_from_cookiejar(self._session.cookies)
if 'auth_token' in cookies:
print('Authentication succeedeed.{0}'.format(os.linesep))

if save_session:
# Saving the session locally
with open('dmarchiver_session.dat', "wb") as file:
pickle.dump(self._session, file)
else:
raise PermissionError(
'Your username or password was invalid. Note: DMArchiver does not support multi-factor authentication or application passwords.')
Expand Down

0 comments on commit 1577327

Please sign in to comment.