Improve jdownloader

- Improve queue
- handle invalid directory in jdownloader incase file name is too long
- Otherr minor fixes

Signed-off-by: anasty17 <[email protected]>

bot/__main__.py

@@ -264,7 +264,7 @@ async def main():
     if config_dict["DATABASE_URL"]:
         await database.db_load()
     await gather(
-        jdownloader.initiate(),
+        jdownloader.boot(),
         sync_to_async(clean_all),
         bot_settings.initiate_search_tools(),
         restart_notification(),

bot/helper/common.py

@@ -1053,6 +1053,9 @@ async def substitute(self, dl_path):
                 else:
                     res = ""
                 name = sub(rf"{pattern}", res, name, flags=I if sen else 0)
+                if len(name.encode()) > 255:
+                    LOGGER.error(f"Substitute: {name} is too long")
+                    return dl_path
             new_path = ospath.join(up_dir, name)
             await move(dl_path, new_path)
             return new_path
@@ -1074,5 +1077,8 @@ async def substitute(self, dl_path):
                         else:
                             res = ""
                         file_ = sub(rf"{pattern}", res, file_, flags=I if sen else 0)
+                        if len(file_.encode()) > 255:
+                            LOGGER.error(f"Substitute: {file_} is too long")
+                            continue
                     await move(f_path, ospath.join(dirpath, file_))
             return dl_path

bot/helper/ext_utils/bot_utils.py

@@ -171,14 +171,6 @@ def update_user_ldata(id_, key, value):
     user_data[id_][key] = value
 
 
-async def retry_function(func, *args, **kwargs):
-    try:
-        return await func(*args, **kwargs)
-    except:
-        await sleep(0.2)
-        return await retry_function(func, *args, **kwargs)
-
-
 async def cmd_exec(cmd, shell=False):
     if shell:
         proc = await create_subprocess_shell(cmd, stdout=PIPE, stderr=PIPE)

bot/helper/ext_utils/jdownloader_booter.py

@@ -2,19 +2,11 @@
 from aioshutil import rmtree
 from json import dump
 from random import randint
-from asyncio import sleep, wait_for
 from re import match
 
-from bot import config_dict, LOGGER, jd_lock, bot_name
-from .bot_utils import cmd_exec, new_task, retry_function
+from bot import config_dict, LOGGER, bot_name
+from .bot_utils import cmd_exec, new_task
 from myjd import MyJdApi
-from myjd.exception import (
-    MYJDException,
-    MYJDAuthFailedException,
-    MYJDEmailForbiddenException,
-    MYJDEmailInvalidException,
-    MYJDErrorEmailNotConfirmedException,
-)
 
 
 class JDownloader(MyJdApi):
@@ -23,23 +15,16 @@ def __init__(self):
         self._username = ""
         self._password = ""
         self._device_name = ""
+        self.is_connected = False
         self.error = "JDownloader Credentials not provided!"
-        self.device = None
-        self.set_app_key("mltb")
-
-    @new_task
-    async def initiate(self):
-        self.device = None
-        async with jd_lock:
-            is_connected = await self.jdconnect()
-            if is_connected:
-                await self.boot()
-                await self.connectToDevice()
 
     @new_task
     async def boot(self):
         await cmd_exec(["pkill", "-9", "-f", "java"])
-        self.device = None
+        if not config_dict["JD_EMAIL"] or not config_dict["JD_PASS"]:
+            self.is_connected = False
+            self.error = "JDownloader Credentials not provided!"
+            return
         self.error = "Connecting... Try agin after couple of seconds"
         self._device_name = f"{randint(0, 1000)}@{bot_name}"
         if await path.exists("/JDownloader/logs"):
@@ -56,13 +41,33 @@ async def boot(self):
             "devicename": f"{self._device_name}",
             "email": config_dict["JD_EMAIL"],
         }
+        remote_data = {
+            "localapiserverheaderaccesscontrollalloworigin": "",
+            "deprecatedapiport": 3128,
+            "localapiserverheaderxcontenttypeoptions": "nosniff",
+            "localapiserverheaderxframeoptions": "DENY",
+            "externinterfaceenabled": True,
+            "deprecatedapilocalhostonly": True,
+            "localapiserverheaderreferrerpolicy": "no-referrer",
+            "deprecatedapienabled": True,
+            "localapiserverheadercontentsecuritypolicy": "default-src 'self'",
+            "jdanywhereapienabled": True,
+            "externinterfacelocalhostonly": False,
+            "localapiserverheaderxxssprotection": "1; mode=block",
+        }
         await makedirs("/JDownloader/cfg", exist_ok=True)
         with open(
             "/JDownloader/cfg/org.jdownloader.api.myjdownloader.MyJDownloaderSettings.json",
             "w",
         ) as sf:
             sf.truncate(0)
             dump(jdata, sf)
+        with open(
+            "/JDownloader/cfg/org.jdownloader.api.RemoteAPIConfig.json",
+            "w",
+        ) as rf:
+            rf.truncate(0)
+            dump(remote_data, rf)
         if not await path.exists("/JDownloader/JDownloader.jar"):
             pattern = r"JDownloader\.jar\.backup.\d$"
             for filename in await listdir("/JDownloader"):
@@ -74,72 +79,11 @@ async def boot(self):
             await rmtree("/JDownloader/update")
             await rmtree("/JDownloader/tmp")
         cmd = "java -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 -Djava.awt.headless=true -jar /JDownloader/JDownloader.jar"
+        self.is_connected = True
         _, __, code = await cmd_exec(cmd, shell=True)
+        self.is_connected = False
         if code != -9:
             await self.boot()
 
-    async def jdconnect(self):
-        if not config_dict["JD_EMAIL"] or not config_dict["JD_PASS"]:
-            return False
-        try:
-            await self.connect(config_dict["JD_EMAIL"], config_dict["JD_PASS"])
-            LOGGER.info("MYJDownloader is connected!")
-            return True
-        except (
-            MYJDAuthFailedException,
-            MYJDEmailForbiddenException,
-            MYJDEmailInvalidException,
-            MYJDErrorEmailNotConfirmedException,
-        ) as err:
-            self.error = f"{err}".strip()
-            LOGGER.error(f"Failed to connect with jdownloader! ERROR: {self.error}")
-            self.device = None
-            return False
-        except MYJDException as e:
-            self.error = f"{e}".strip()
-            LOGGER.error(
-                f"Failed to connect with jdownloader! Retrying... ERROR: {self.error}"
-            )
-            return await self.jdconnect()
-
-    async def connectToDevice(self):
-        self.error = "Connecting to device..."
-        await sleep(0.5)
-        while True:
-            self.device = None
-            if not config_dict["JD_EMAIL"] or not config_dict["JD_PASS"]:
-                self.error = "JDownloader Credentials not provided!"
-                await cmd_exec(["pkill", "-9", "-f", "java"])
-                return False
-            try:
-                await self.update_devices()
-                if not (devices := self.list_devices()):
-                    continue
-                for device in devices:
-                    if self._device_name == device["name"]:
-                        self.device = self.get_device(f"{self._device_name}")
-                        break
-                else:
-                    continue
-            except:
-                continue
-            break
-        await self.device.enable_direct_connection()
-        self.error = ""
-        LOGGER.info("JDownloader Device have been Connected!")
-        return True
-
-    async def check_jdownloader_state(self):
-        try:
-            await wait_for(retry_function(self.device.jd.version), timeout=10)
-        except:
-            is_connected = await self.jdconnect()
-            if not is_connected:
-                raise MYJDException(self.error)
-            await self.boot()
-            isDeviceConnected = await self.connectToDevice()
-            if not isDeviceConnected:
-                raise MYJDException(self.error)
-
 
 jdownloader = JDownloader()

bot/helper/ext_utils/task_manager.py

@@ -1,4 +1,4 @@
-from asyncio import Event, sleep
+from asyncio import Event
 
 from bot import (
     config_dict,
@@ -95,13 +95,13 @@ async def check_running_tasks(listener, state="dl"):
 async def start_dl_from_queued(mid: int):
     queued_dl[mid].set()
     del queued_dl[mid]
-    await sleep(0.7)
+    non_queued_dl.add(mid)
 
 
 async def start_up_from_queued(mid: int):
     queued_up[mid].set()
     del queued_up[mid]
-    await sleep(0.7)
+    non_queued_up.add(mid)
 
 
 async def start_from_queued():

bot/helper/listeners/jdownloader_listener.py

@@ -1,7 +1,7 @@
 from asyncio import sleep
 
 from bot import intervals, jd_lock, jd_downloads
-from ..ext_utils.bot_utils import new_task, retry_function
+from ..ext_utils.bot_utils import new_task
 from ..ext_utils.jdownloader_booter import jdownloader
 from ..ext_utils.status_utils import get_task_by_gid
 
@@ -10,9 +10,8 @@
 async def remove_download(gid):
     if intervals["stopAll"]:
         return
-    await retry_function(
-        jdownloader.device.downloads.remove_links,
-        package_ids=jd_downloads[gid]["ids"],
+    await jdownloader.device.downloads.remove_links(
+        package_ids=jd_downloads[gid]["ids"]
     )
     if task := await get_task_by_gid(gid):
         await task.listener.on_download_error("Download removed manually!")
@@ -25,8 +24,7 @@ async def _on_download_complete(gid):
     if task := await get_task_by_gid(gid):
         if task.listener.select:
             async with jd_lock:
-                await retry_function(
-                    jdownloader.device.downloads.cleanup,
+                await jdownloader.device.downloads.cleanup(
                     "DELETE_DISABLED",
                     "REMOVE_LINKS_AND_DELETE_FILES",
                     "SELECTED",
@@ -37,8 +35,7 @@ async def _on_download_complete(gid):
             return
         async with jd_lock:
             if gid in jd_downloads:
-                await retry_function(
-                    jdownloader.device.downloads.remove_links,
+                await jdownloader.device.downloads.remove_links(
                     package_ids=jd_downloads[gid]["ids"],
                 )
                 del jd_downloads[gid]
@@ -52,10 +49,6 @@ async def _jd_listener():
             if len(jd_downloads) == 0:
                 intervals["jd"] = ""
                 break
-            try:
-                await jdownloader.check_jdownloader_state()
-            except:
-                continue
             try:
                 packages = await jdownloader.device.downloads.query_packages(
                     [{"finished": True, "saveTo": True}]
@@ -64,8 +57,6 @@ async def _jd_listener():
                 continue
 
             all_packages = {pack["uuid"]: pack for pack in packages}
-            if not all_packages:
-                continue
             for d_gid, d_dict in list(jd_downloads.items()):
                 if d_dict["status"] == "down":
                     for index, pid in enumerate(d_dict["ids"]):

bot/helper/listeners/task_listener.py

@@ -228,8 +228,6 @@ async def on_download_complete(self):
             await event.wait()
             if self.is_cancelled:
                 return
-            async with queue_dict_lock:
-                non_queued_up.add(self.mid)
             LOGGER.info(f"Start from Queued/Upload: {self.name}")
 
         self.size = await get_path_size(up_dir)

bot/helper/mirror_leech_utils/download_utils/aria2_download.py

@@ -8,8 +8,6 @@
     config_dict,
     aria2_options,
     aria2c_global,
-    non_queued_dl,
-    queue_dict_lock,
 )
 from ...ext_utils.bot_utils import bt_selection_buttons, sync_to_async
 from ...ext_utils.task_manager import check_running_tasks
@@ -83,8 +81,6 @@ async def add_aria2c_download(listener, dpath, header, ratio, seed_time):
         await event.wait()
         if listener.is_cancelled:
             return
-        async with queue_dict_lock:
-            non_queued_dl.add(listener.mid)
         async with task_dict_lock:
             task = task_dict[listener.mid]
             task.queued = False

bot/helper/mirror_leech_utils/download_utils/direct_downloader.py

@@ -6,8 +6,6 @@
     aria2c_global,
     task_dict,
     task_dict_lock,
-    non_queued_dl,
-    queue_dict_lock,
 )
 from ...ext_utils.bot_utils import sync_to_async
 from ...ext_utils.task_manager import check_running_tasks, stop_duplicate_check
@@ -45,8 +43,6 @@ async def add_direct_download(listener, path):
         await event.wait()
         if listener.is_cancelled:
             return
-        async with queue_dict_lock:
-            non_queued_dl.add(listener.mid)
 
     a2c_opt = {**aria2_options}
     [a2c_opt.pop(k) for k in aria2c_global if k in aria2_options]

bot/helper/mirror_leech_utils/download_utils/gd_download.py

@@ -1,6 +1,6 @@
 from secrets import token_urlsafe
 
-from bot import task_dict, task_dict_lock, LOGGER, non_queued_dl, queue_dict_lock
+from bot import task_dict, task_dict_lock, LOGGER
 from ...ext_utils.bot_utils import sync_to_async
 from ...ext_utils.task_manager import check_running_tasks, stop_duplicate_check
 from ...mirror_leech_utils.gdrive_utils.count import GoogleDriveCount
@@ -38,8 +38,6 @@ async def add_gd_download(listener, path):
         await event.wait()
         if listener.is_cancelled:
             return
-        async with queue_dict_lock:
-            non_queued_dl.add(listener.mid)
 
     drive = GoogleDriveDownload(listener, path)
     async with task_dict_lock: