Unable to locate google drive file map

.gitignore

@@ -0,0 +1,3 @@
+pkgxtra/pkcs1/__pycache__/
+pkgxtra/__pycache__/
+pkgxtra/gpsoauth/__pycache__/

README.md

@@ -4,15 +4,19 @@ Allows WhatsApp users on Android to extract their backed up WhatsApp data from G
 ###### BRANCH UPDATES:
 v1.0 - Initial release.  
 v1.1 - Added Python 3 support.  
+v2.0 - Fixed gDriveFileMap after Whatsapp q requirements update
+       Fixed downloadurl (the script is working again!)    
+v2.5 - Added multithreading support
 
 
 ###### PREREQUISITES:
  1. O/S: Windows Vista, Windows 7, Windows 8, Windows 10, Mac OS X or Linux  
- 2. Python 2.x or 3.x - If not installed: https://www.python.org/downloads/  
+ 2. Python 3.x - If not installed: https://www.python.org/downloads/  
  3. Android device with WhatsApp installed and the Google Drive backup feature enabled  
  4. Google services device id (if you want to reduce the risk of being logged out of Google)  
      Search Google Play for "device id" for plenty of apps that can reveal this information  
  5. Google account login credentials (username and password)  
+ 6. Whatsapp cellphone number as shown in backup tab on google drive website.
 
 
 ###### INSTRUCTIONS:
@@ -30,3 +34,6 @@ v1.1 - Added Python 3 support.
 
 ###### CREDITS:
  AUTHOR: TripCode  
+
+###### CREDITS:
+ CONTRIBUTORS: DrDeath1122 from XDA for the multithreading backbone part

WhatsAppGDExtract.py

@@ -1,21 +1,37 @@
 #!/usr/bin/env python
-
-from configparser import ConfigParser
+ 
+import configparser
 import json
 import os
 import re
 import requests
 import sys
-
+import queue
+import threading
+import time
+from pkgxtra.gpsoauth import google
+exitFlag = False
+
+
 def getGoogleAccountTokenFromAuth():
-    payload = {'Email':gmail, 'Passwd':passw, 'app':client_pkg, 'client_sig':client_sig, 'parentAndroidId':devid}
+
+    b64_key_7_3_29 = (b"AAAAgMom/1a/v0lblO2Ubrt60J2gcuXSljGFQXgcyZWveWLEwo6prwgi3"
+                      b"iJIZdodyhKZQrNWp5nKJ3srRXcUW+F1BD3baEVGcmEgqaLZUNBjm057pK"
+                      b"RI16kB0YppeGx5qIQ5QjKzsR8ETQbKLNWgRY0QRNVz34kMJR3P/LgHax/"
+                      b"6rmf5AAAAAwEAAQ==")
+
+    android_key_7_3_29 = google.key_from_b64(b64_key_7_3_29)
+    encpass = google.signature(gmail, passw, android_key_7_3_29)
+    payload = {'Email':gmail, 'EncryptedPasswd':encpass, 'app':client_pkg, 'client_sig':client_sig, 'parentAndroidId':devid}
     request = requests.post('https://android.clients.google.com/auth', data=payload)
     token = re.search('Token=(.*?)\n', request.text)
+
     if token:
        return token.group(1)
     else:
        quit(request.text)
-
+
+
 def getGoogleDriveToken(token):
     payload = {'Token':token, 'app':pkg, 'client_sig':sig, 'device':devid, 'google_play_services_version':client_ver, 'service':'oauth2:https://www.googleapis.com/auth/drive.appdata https://www.googleapis.com/auth/drive.file', 'has_permission':'1'}
     request = requests.post('https://android.clients.google.com/auth', data=payload)
@@ -24,12 +40,18 @@ def getGoogleDriveToken(token):
        return token.group(1)
     else:
        quit(request.text)
-
+ 
 def rawGoogleDriveRequest(bearer, url):
     headers = {'Authorization': 'Bearer '+bearer}
     request = requests.get(url, headers=headers)
     return request.text
-
+
+def gDriveFileMapRequest(bearer):
+    header = {'Authorization': 'Bearer '+bearer}
+    url = "https://www.googleapis.com/drive/v2/files?mode=restore&spaces=appDataFolder&maxResults=1000&fields=items(description%2Cid%2CfileSize%2Ctitle%2Cmd5Checksum%2CmimeType%2CmodifiedDate%2Cparents(id)%2Cproperties(key%2Cvalue))%2CnextPageToken&q=title%20%3D%20'"+celnumbr+"-invisible'%20or%20title%20%3D%20'gdrive_file_map'%20or%20title%20%3D%20'Databases%2Fmsgstore.db.crypt12'%20or%20title%20%3D%20'Databases%2Fmsgstore.db.crypt11'%20or%20title%20%3D%20'Databases%2Fmsgstore.db.crypt10'%20or%20title%20%3D%20'Databases%2Fmsgstore.db.crypt9'%20or%20title%20%3D%20'Databases%2Fmsgstore.db.crypt8'"
+    request = requests.get(url, headers=header)
+    return request.text
+
 def downloadFileGoogleDrive(bearer, url, local):
     if not os.path.exists(os.path.dirname(local)):
         os.makedirs(os.path.dirname(local))
@@ -43,48 +65,49 @@ def downloadFileGoogleDrive(bearer, url, local):
             for chunk in request.iter_content(1024):
                 asset.write(chunk)
     print('Downloaded: "'+local+'".')
-
+ 
 def gDriveFileMap():
     global bearer
-    data = rawGoogleDriveRequest(bearer, 'https://www.googleapis.com/drive/v2/files')
+    data = gDriveFileMapRequest(bearer)
     jres = json.loads(data)
     backups = []
     for result in jres['items']:
         try:
             if result['title'] == 'gdrive_file_map':
-                backups.append((result['description'], rawGoogleDriveRequest(bearer, result['downloadUrl'])))
+                backups.append((result['description'], rawGoogleDriveRequest(bearer, 'https://www.googleapis.com/drive/v2/files/'+result['id']+'?alt=media')))
         except:
             pass
     if len(backups) == 0:
         quit('Unable to locate google drive file map for: '+pkg)
     return backups
-
+ 
 def getConfigs():
-    global gmail, passw, devid, pkg, sig, client_pkg, client_sig, client_ver
-    config = ConfigParser()
+    global gmail, passw, devid, pkg, sig, client_pkg, client_sig, client_ver, celnumbr
+    config = configparser.RawConfigParser()
     try:
         config.read('settings.cfg')
         gmail = config.get('auth', 'gmail')
         passw = config.get('auth', 'passw')
         devid = config.get('auth', 'devid')
+        celnumbr = config.get('auth', 'celnumbr')
         pkg = config.get('app', 'pkg')
         sig = config.get('app', 'sig')
         client_pkg = config.get('client', 'pkg')
         client_sig = config.get('client', 'sig')
         client_ver = config.get('client', 'ver')
-    except(ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+    except(configparser.NoSectionError, configparser.NoOptionError):
         quit('The "settings.cfg" file is missing or corrupt!')
-
+ 
 def jsonPrint(data):
     print(json.dumps(json.loads(data), indent=4, sort_keys=True))
-
+ 
 def localFileLog(md5):
     logfile = 'logs'+os.path.sep+'files.log'
     if not os.path.exists(os.path.dirname(logfile)):
         os.makedirs(os.path.dirname(logfile))
     with open(logfile, 'a') as log:
         log.write(md5+'\n')
-
+ 
 def localFileList():
     logfile = 'logs'+os.path.sep+'files.log'
     if os.path.isfile(logfile):
@@ -93,31 +116,99 @@ def localFileList():
     else:
         open(logfile, 'w')
         return localFileList()
-
+ 
 def createSettingsFile():
     with open('settings.cfg', 'w') as cfg:
-        cfg.write('[auth]\ngmail = [email protected]\npassw = yourpassword\ndevid = 0000000000000000\n\n[app]\npkg = com.whatsapp\nsig = 38a0f7d505fe18fec64fbf343ecaaaf310dbd799\n\n[client]\npkg = com.google.android.gms\nsig = 38918a453d07199354f8b19af05ec6562ced5788\nver = 9877000')
-
+        cfg.write('[auth]\ngmail = [email protected]\npassw = yourpassword\ndevid = 0000000000000000\ncelnumbr = BACKUPPHONENUMBER\n\n[app]\npkg = com.whatsapp\nsig = 38a0f7d505fe18fec64fbf343ecaaaf310dbd799\n\n[client]\npkg = com.google.android.gms\nsig = 38918a453d07199354f8b19af05ec6562ced5788\nver = 9877000')
+ 
 def getSingleFile(data, asset):
     data = json.loads(data)
     for entries in data:
         if entries['f'] == asset:
             return entries['f'], entries['m'], entries['r'], entries['s']
 
+class myThread (threading.Thread):
+    def __init__(self, threadID, name, q):
+        threading.Thread.__init__(self)
+        self.threadID = threadID
+        self.name = name
+        self.q = q
+    def run(self):
+        print ('Initiated: ' + self.name)
+        process_data(self.name, self.q)
+        print ('Terminated: ' + self.name)
+
+def process_data(threadName, q):
+    while not exitFlag:
+        queueLock.acquire()
+        if not workQueue.empty():
+            data = q.get()
+            queueLock.release()
+            getMultipleFilesThread(data['bearer'], data['entries_r'], data['local'], data['entries_m'], threadName)
+        else:
+            queueLock.release()
+        time.sleep(1)
+
+def getMultipleFilesThread(bearer, entries_r, local, entries_m, threadName):
+        url = 'https://www.googleapis.com/drive/v2/files/'+entries_r+'?alt=media'
+        if not os.path.exists(os.path.dirname(local)):
+            try:
+                os.makedirs(os.path.dirname(local))
+            #Other thead was trying to create the same 'local'
+            except (FileExistsError):
+                pass
+
+        if os.path.isfile(local):
+            os.remove(local)
+        headers = {'Authorization': 'Bearer '+bearer}
+        request = requests.get(url, headers=headers, stream=True)
+        request.raw.decode_content = True
+        if request.status_code == 200:
+            with open(local, 'wb') as asset:
+                for chunk in request.iter_content(1024):
+                    asset.write(chunk)
+        print(threadName + '=> Downloaded: "'+local+'".')
+        logfile = 'logs'+os.path.sep+'files.log'
+        if not os.path.exists(os.path.dirname(logfile)):
+            os.makedirs(os.path.dirname(logfile))
+        with open(logfile, 'a') as log:
+            log.write(entries_m+'\n')
+
+queueLock = threading.Lock()
+workQueue = queue.Queue(9999999)
+
 def getMultipleFiles(data, folder):
+    threadList = ["Thread-1", "Thread-2", "Thread-3", "Thread-4", "Thread-5", "Thread-6", "Thread-7", "Thread-8", "Thread-9", "Thread-10", "Thread-11", "Thread-12", "Thread-13", "Thread-14", "Thread-15", "Thread-16", "Thread-17", "Thread-18", "Thread-19", "Thread-20"]
+    threads = []
+    threadID = 1
+    global exitFlag
+    for tName in threadList:
+        thread = myThread(threadID, tName, workQueue)
+        thread.start()
+        threads.append(thread)
+        threadID += 1
     files = localFileList()
     data = json.loads(data)
+    queueLock.acquire()
     for entries in data:
         if any(entries['m'] in lists for lists in files) == False or 'database' in entries['f'].lower():
             local = folder+os.path.sep+entries['f'].replace("/", os.path.sep)
             if os.path.isfile(local) and 'database' not in local.lower():
                 quit('Skipped: "'+local+'".')
             else:
-                downloadFileGoogleDrive(bearer, 'https://www.googleapis.com/drive/v2/files/'+entries['r']+'?alt=media', local)
-                localFileLog(entries['m'])
-
+                workQueue.put({'bearer':bearer, 'entries_r':entries['r'], 'local':local, 'entries_m':entries['m']})
+    queueLock.release()
+    while not workQueue.empty():
+        pass
+    exitFlag = True
+    for t in threads:
+        t.join()
+    print ("File List Downloaded")
+
 def runMain(mode, asset, bID):
     global bearer
+    global exitFlag
+
     if os.path.isfile('settings.cfg') == False:
         createSettingsFile()
     getConfigs()
@@ -154,12 +245,13 @@ def runMain(mode, asset, bID):
             localFileLog(m)
     elif mode == 'sync':
         for i, drive in enumerate(drives):
+            exitFlag = False
             folder = 'WhatsApp'
             if len(drives) > 1:
                 print('Backup: '+str(i))
                 folder = 'WhatsApp-' + str(i)
             getMultipleFiles(drive[1], folder)
-
+ 
 def main():
     args = len(sys.argv)
     if  args < 2 or str(sys.argv[1]) == '-help' or str(sys.argv[1]) == 'help':
@@ -188,6 +280,7 @@ def main():
         runMain('pull', str(sys.argv[2]), bID)
     else:
         quit('\nUsage: python '+str(sys.argv[0])+' -help|-vers|-info|-list|-sync|-pull file [backupID]\n')
-
+ 
 if __name__ == "__main__":
     main()
+

pkgxtra/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.0.01'

pkgxtra/gpsoauth/__init__.py

@@ -0,0 +1,103 @@
+import requests
+
+from . import google
+
+
+# The key is distirbuted with Google Play Services.
+# This one is from version 7.3.29.
+b64_key_7_3_29 = (b"AAAAgMom/1a/v0lblO2Ubrt60J2gcuXSljGFQXgcyZWveWLEwo6prwgi3"
+                  b"iJIZdodyhKZQrNWp5nKJ3srRXcUW+F1BD3baEVGcmEgqaLZUNBjm057pK"
+                  b"RI16kB0YppeGx5qIQ5QjKzsR8ETQbKLNWgRY0QRNVz34kMJR3P/LgHax/"
+                  b"6rmf5AAAAAwEAAQ==")
+
+android_key_7_3_29 = google.key_from_b64(b64_key_7_3_29)
+
+auth_url = 'https://android.clients.google.com/auth'
+useragent = 'gpsoauth-portify/1.0'
+
+
+def _perform_auth_request(data):
+    res = requests.post(auth_url, data,
+                        headers={'User-Agent': useragent})
+
+    return google.parse_auth_response(res.text)
+
+
+def perform_master_login(email, password, android_id,
+                         service='ac2dm', device_country='us', operatorCountry='us',
+                         lang='en', sdk_version=17):
+    """
+    Perform a master login, which is what Android does when you first add a Google account.
+
+    Return a dict, eg::
+
+        {
+            'Auth': '...',
+            'Email': '[email protected]',
+            'GooglePlusUpgrade': '1',
+            'LSID': '...',
+            'PicasaUser': 'My Name',
+            'RopRevision': '1',
+            'RopText': ' ',
+            'SID': '...',
+            'Token': 'oauth2rt_1/...',
+            'firstName': 'My',
+            'lastName': 'Name',
+            'services': 'hist,mail,googleme,...'
+        }
+    """
+
+    data = {
+        'accountType': 'HOSTED_OR_GOOGLE',
+        'Email':   email,
+        'has_permission':  1,
+        'add_account': 1,
+        'EncryptedPasswd': google.signature(email, password, android_key_7_3_29),
+        'service': service,
+        'source':  'android',
+        'androidId':   android_id,
+        'device_country':  device_country,
+        'operatorCountry': device_country,
+        'lang':    lang,
+        'sdk_version': sdk_version
+    }
+
+    return _perform_auth_request(data)
+
+
+def perform_oauth(email, master_token, android_id, service, app, client_sig,
+                  device_country='us', operatorCountry='us', lang='en', sdk_version=17):
+    """
+    Use a master token from master_login to perform OAuth to a specific Google service.
+
+    Return a dict, eg::
+
+        {
+            'Auth': '...',
+            'LSID': '...',
+            'SID': '..',
+            'issueAdvice': 'auto',
+            'services': 'hist,mail,googleme,...'
+        }
+
+    To authenticate requests to this service, include a header
+    ``Authorization: GoogleLogin auth=res['Auth']``.
+    """
+
+    data = {
+        'accountType': 'HOSTED_OR_GOOGLE',
+        'Email':   email,
+        'has_permission':  1,
+        'EncryptedPasswd': master_token,
+        'service': service,
+        'source':  'android',
+        'androidId':   android_id,
+        'app': app,
+        'client_sig': client_sig,
+        'device_country':  device_country,
+        'operatorCountry': device_country,
+        'lang':    lang,
+        'sdk_version': sdk_version
+    }
+
+    return _perform_auth_request(data)