Dev

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 *$py.class
 tests/alist/
 local_test*
+tmp/
 
 # C extensions
 *.so

alist_sync/base_sync.py

@@ -3,10 +3,8 @@
 import os
 
 from alist_sync.alist_client import AlistClient
-from alist_sync.common import sha1_6, timeout_input
-from alist_sync.config import cache_dir
-from alist_sync.models import SyncJob, AlistServer
-from alist_sync.scan_dir import scan_dir
+from alist_sync.common import timeout_input
+from alist_sync.models import AlistServer
 
 logger = logging.getLogger("alist-sync.base")
 
@@ -19,37 +17,7 @@ def __init__(self, alist_info: AlistServer, sync_dirs: list[str | os.PathLike]):
 
         self.sync_dirs = sync_dirs
         self.sync_dirs.sort()
-
-        self.sync_task_cache_file = cache_dir.joinpath(
-            f"sync_task_{sha1_6(self.sync_dirs)}.json"
-        )
-        self.sync_job = SyncJob(
-            alist_info=alist_info,
-        )
-        self.load_from_cache()
-
-    def __del__(self):
-        self.client.close()
-        self.save_to_cache()
-        self.clear_cache()
-
-    def load_from_cache(self):
-        """从缓存中加载"""
-        if not self.sync_task_cache_file.exists():
-            return
-        self.sync_job = SyncJob.model_validate_json(
-            self.sync_task_cache_file.read_text()
-        )
-
-    def save_to_cache(self):
-        """保存到缓存中"""
-        self.sync_task_cache_file.write_text(self.sync_job.model_dump_json(indent=2))
-
-    def clear_cache(self, force=False):
-        """清除缓存"""
-        if force:
-            self.sync_task_cache_file.unlink(missing_ok=True)
-            self.sync_task_cache_file.unlink(missing_ok=True)
+        self.alist_info = alist_info
 
     async def create_storages(self, storages):
         """创建后端存储"""
@@ -70,33 +38,8 @@ async def create_storages(self, storages):
                     f"创建存储失败：%s, message: %s", st["mount_path"], res.message
                 )
 
-    async def scans(self):
-        """扫描目录"""
-
-        async def scan(path):
-            self.sync_job.sync_dirs.setdefault(path, await scan_dir(self.client, path))
-
-        for sync_dir in self.sync_dirs:
-            asyncio.create_task(scan(sync_dir), name=f"{id(self)}_scan_{sync_dir}")
-
-        while True:
-            await asyncio.sleep(1)
-            if not [
-                i.get_name()
-                for i in asyncio.all_tasks()
-                if f"{id(self)}_scan" in i.get_name()
-            ]:
-                break
-
-        logger.info("扫描完成。")
-
     def run(self):
         asyncio.run(self.async_run())
 
     async def async_run(self):
-        await self.create_storages(self.sync_job.alist_info.storages())
-        if not self.sync_job.sync_dirs.values():
-            await self.scans()
-            self.save_to_cache()
-        else:
-            logger.info(f"一件从缓存中找到 %d 个 SyncDir", len(self.sync_job.sync_dirs))
+        await self.create_storages(self.alist_info.storages())

alist_sync/checker.py

@@ -3,7 +3,8 @@
 from alist_sdk import Item
 from pydantic import BaseModel
 
-from alist_sync.models import SyncDir
+from alist_sync.alist_client import AlistClient
+from alist_sync.scanner import Scanner, scan_dirs
 
 
 class Checker(BaseModel):
@@ -13,52 +14,47 @@ class Checker(BaseModel):
     cols: list[PurePosixPath]
 
     @classmethod
-    def checker(cls, *scanned_dirs: SyncDir):
+    def checker(cls, scanner: Scanner):
         _result = {}
-        for scanned_dir in scanned_dirs:
-            for item in scanned_dir.items:
-                r_path = item.full_name.relative_to(
-                    scanned_dir.base_path
-                )
+        for base_path, items in scanner.items.items():
+            items: list[Item]
+            base_path: str | PurePosixPath
+            for item in items:
+                r_path = item.full_name.relative_to(base_path)
                 try:
                     _result[PurePosixPath(r_path)].setdefault(
-                        PurePosixPath(scanned_dir.base_path),
-                        item
+                        PurePosixPath(base_path), item
                     )
                 except KeyError:
-                    _result[PurePosixPath(r_path)] = {
-                        PurePosixPath(scanned_dir.base_path): item
-                    }
+                    _result[PurePosixPath(r_path)] = {PurePosixPath(base_path): item}
 
         return cls(
-            matrix=_result,
-            cols=[PurePosixPath(t.base_path) for t in scanned_dirs]
+            matrix=_result, cols=[PurePosixPath(t) for t in scanner.items.keys()]
         )
 
     def model_dump_table(self):
-        """"""
+        """Table 打印"""
         from rich.console import Console
         from rich.table import Table
 
         console = Console()
         table = Table(show_header=True, header_style="bold magenta")
-        table.add_column('r_path', style="dim red", )
+        table.add_column(
+            "r_path",
+            style="dim red",
+        )
         for col in self.cols:
-            table.add_column(str(col), justify="center", vertical='middle')
+            table.add_column(str(col), justify="center", vertical="middle")
 
         for r_path, raw in self.matrix.items():
             table.add_row(
-                str(r_path),
-                *["True" if raw.get(tt) else "False" for tt in self.cols]
+                str(r_path), *["True" if raw.get(tt) else "False" for tt in self.cols]
             )
         console.print(table)
 
 
-if __name__ == '__main__':
-    import json
-    from pathlib import Path
-
-    checker = Checker.checker(*[SyncDir(**s) for s in json.load(
-        Path(__file__).parent.parent.joinpath('tests/resource/SyncDirs-m.json').open())
-                                ])
-    checker.model_dump_table()
+async def check_dir(*dirs, client: AlistClient) -> Checker:
+    """"""
+    return Checker.checker(
+        await Scanner.scans(*dirs, client=client)
+    )

alist_sync/common.py

@@ -77,6 +77,7 @@ def timeout_input(msg, default, timeout=3):
     events = sel.select(timeout)
     if events:
         key, _ = events[0]
+        # noinspection PyUnresolvedReferences
         return key.fileobj.readline().rstrip()
     else:
         sys.stdout.write("\n")

alist_sync/job_copy.py

@@ -2,9 +2,10 @@
 from pathlib import PurePosixPath
 from typing import Literal, Optional, Iterator
 
-from pydantic import BaseModel, computed_field
+from pydantic import computed_field
 
 from alist_sync.alist_client import AlistClient, get_status
+from alist_sync.models import BaseModel
 from alist_sync.checker import Checker
 from alist_sync.common import get_alist_client
 from alist_sync.jobs import JobBase

alist_sync/jobs.py

@@ -8,35 +8,18 @@
 """
 import asyncio
 import logging
-from pathlib import Path, PurePosixPath
+from pathlib import PurePosixPath
 from typing import Iterator
 
-from pydantic import BaseModel
-
+from alist_sync.models import BaseModel
 from alist_sync.alist_client import AlistClient
 from alist_sync.checker import Checker
 from alist_sync.common import get_alist_client
-from alist_sync.config import cache_dir
 
 logger = logging.getLogger("alist-sync.jobs")
 
 
 class JobBase(BaseModel):
-    @classmethod
-    def from_json_file(cls, file: Path):
-        return cls.model_validate_json(Path(file).read_text(encoding="utf-8"))
-
-    @classmethod
-    def from_cache(cls):
-        class_name = cls.__name__
-        file = cache_dir.joinpath(f"{class_name}.json")
-        return cls.from_json_file(file)
-
-    def save_to_cache(self):
-        class_name = self.__class__.__name__
-        file = cache_dir.joinpath(f"{class_name}.json")
-        file.write_text(self.model_dump_json(indent=2), encoding="utf-8")
-
     @staticmethod
     def create_task(_s, _t, checker: Checker) -> Iterator[BaseModel]:
         raise NotImplementedError

alist_sync/models.py

@@ -3,15 +3,37 @@
 from pathlib import PurePosixPath, Path
 from typing import Optional
 
-from alist_sdk import Item
-from pydantic import BaseModel, Field
+from pydantic import BaseModel as _BaseModel
 
 __all__ = [
+    "BaseModel",
     "AlistServer",
-    "SyncDir",
-    "SyncJob",
 ]
 
+from alist_sync.config import cache_dir
+
+
+class BaseModel(_BaseModel):
+    """基础模型"""
+
+    @classmethod
+    def from_json_file(cls, file: Path):
+        """从文件中读取json"""
+        if not file.exists():
+            raise FileNotFoundError(f"找不到文件：{file}")
+        return cls.model_validate_json(Path(file).read_text(encoding="utf-8"))
+
+    @classmethod
+    def from_cache(cls):
+        class_name = cls.__name__
+        file = cache_dir.joinpath(f"{class_name}.json")
+        return cls.from_json_file(file)
+
+    def save_to_cache(self):
+        class_name = self.__class__.__name__
+        file = cache_dir.joinpath(f"{class_name}.json")
+        file.write_text(self.model_dump_json(indent=2), encoding="utf-8")
+
 
 class AlistServer(BaseModel):
     """"""
@@ -66,30 +88,6 @@ def is_storage(_st):
             raise KeyError("给定的")
 
 
-class SyncDir(BaseModel):
-    """同步目录模型，定义一个同步目录"""
-
-    base_path: str  # 同步基础目录
-    items: list[Item]  # Item列表
-
-    items_relative: Optional[list] = Field([], exclude=True)  # Item列表相对路径
-
-    def in_items(self, path) -> bool:
-        """判断path是否在items中"""
-        if not self.items_relative:
-            self.items_relative = [
-                i.full_name.relative_to(self.base_path) for i in self.items
-            ]
-        return path in self.items_relative
-
-
-class SyncJob(BaseModel):
-    """同步任务"""
-
-    alist_info: AlistServer  # Alist Info
-    sync_dirs: dict[str, SyncDir] = {}  # 同步目录
-
-
 class Config(BaseModel):
     """配置"""
 

alist_sync/run_copy.py

@@ -1,10 +1,11 @@
 import logging
 
 from alist_sync.base_sync import SyncBase
-from alist_sync.checker import Checker
+from alist_sync.checker import check_dir
 from alist_sync.common import async_all_task_names
 from alist_sync.job_copy import CopyJob
 from alist_sync.models import AlistServer
+from alist_sync.scanner import scan_dirs
 
 logger = logging.getLogger("alist-sync.copy-to-target")
 
@@ -22,7 +23,7 @@ def __init__(
     async def async_run(self):
         """异步运行"""
         await super().async_run()
-        checker = Checker.checker(*self.sync_job.sync_dirs.values())
+        checker = await check_dir(*self.sync_dirs, client=self.client)
         copy_job = CopyJob.from_checker(self.source_path, self.targets_path, checker)
         await copy_job.start(self.client)
         logger.info("当前全部的Task %s", async_all_task_names())

alist_sync/run_mirror.py

@@ -12,7 +12,7 @@
 import logging
 
 from alist_sync.base_sync import SyncBase
-from alist_sync.checker import Checker
+from alist_sync.checker import check_dir
 from alist_sync.common import async_all_task_names
 from alist_sync.job_copy import CopyJob
 from alist_sync.job_remove import RemoveJob
@@ -33,7 +33,7 @@ async def async_run(self):
         # 创建复制列表
         await super().async_run()
 
-        checker = Checker.checker(*self.sync_job.sync_dirs.values())
+        checker = await check_dir(*self.sync_dirs, client=self.client)
         copy_job = CopyJob.from_checker(self.source_path, self.targets_path, checker)
         delete_job = RemoveJob.from_checker(
             self.source_path, self.targets_path, checker

alist_sync/run_sync.py

@@ -11,4 +11,5 @@
 
 class Sync:
     def __init__(self, alist_info: AlistServer, dirs: list[str] = None):
+        # TODO 重构Base 以完成更高的效率
         pass