diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f56385d..49fd30ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/lang/zh-CN/ ## [Unreleased] +### Fixed + +- 修复跳过插件测试时无法获取插件版本与时间的问题 + ## [3.0.4] - 2023-09-06 ### Changed diff --git a/src/utils/store_test/models.py b/src/utils/store_test/models.py index e5000aef..5d183a4f 100644 --- a/src/utils/store_test/models.py +++ b/src/utils/store_test/models.py @@ -26,7 +26,7 @@ class Plugin(TypedDict): supported_adapters: list[str] | None valid: bool time: str - version: str | None + version: str skip_test: bool diff --git a/src/utils/store_test/store.py b/src/utils/store_test/store.py index e671ad5b..1853a13c 100644 --- a/src/utils/store_test/store.py +++ b/src/utils/store_test/store.py @@ -53,6 +53,10 @@ def __init__( def should_skip(self, key: str) -> bool: """是否跳过测试""" + if key.startswith("git+http"): + print(f"插件 {key} 为 Git 插件，无法测试，已跳过") + return True + # 如果强制测试，则不跳过 if self._force: return False @@ -76,27 +80,34 @@ def skip_plugin_test(self, key: str) -> bool: return self._previous_plugins[key].get("skip_test", False) return False - async def run( - self, key: str | None = None, config: str | None = None, data: str | None = None + async def test_plugins( + self, + key: str | None = None, + config: str | None = None, + data: str | None = None, ): - """测试商店内插件情况""" + """测试并更新插件商店中的插件信息""" new_results: dict[str, TestResult] = {} new_plugins: dict[str, Plugin] = {} if key: - if self.should_skip(key): - return - - print(f"正在测试插件 {key} ...") - new_results[key], new_plugin = await validate_plugin( - plugin=self._store_plugins[key], - config=config or "", - skip_test=self.skip_plugin_test(key), - data=data, - previous_plugin=self._previous_plugins.get(key), - ) - if new_plugin: - new_plugins[key] = new_plugin + try: + if self.should_skip(key): + # 直接返回上次测试的结果 + return self._previous_results, self._previous_plugins + + print(f"正在测试插件 {key} ...") + new_results[key], new_plugin = await validate_plugin( + plugin=self._store_plugins[key], + config=config or "", + skip_test=self.skip_plugin_test(key), + data=data, + previous_plugin=self._previous_plugins.get(key), + ) + if new_plugin: + new_plugins[key] = new_plugin + except Exception as e: + print(f"测试插件 {key} 失败：{e}") else: test_plugins = list(self._store_plugins.items())[self._offset :] plugin_configs = { @@ -111,23 +122,26 @@ async def run( if i > self._limit: print(f"已达到测试上限 {self._limit}，测试停止") break - if key.startswith("git+http"): - continue - if self.should_skip(key): + try: + if self.should_skip(key): + continue + + print(f"{i}/{self._limit} 正在测试插件 {key} ...") + + new_results[key], new_plugin = await validate_plugin( + plugin=plugin, + config=plugin_configs.get(key, ""), + skip_test=self.skip_plugin_test(key), + previous_plugin=self._previous_plugins.get(key), + ) + if new_plugin: + new_plugins[key] = new_plugin + except Exception as e: + # 如果测试中遇到意外错误，则跳过该插件 + print(f"测试插件 {key} 失败：{e}") continue - print(f"{i}/{self._limit} 正在测试插件 {key} ...") - - new_results[key], new_plugin = await validate_plugin( - plugin=plugin, - config=plugin_configs.get(key, ""), - skip_test=self.skip_plugin_test(key), - previous_plugin=self._previous_plugins.get(key), - ) - if new_plugin: - new_plugins[key] = new_plugin - i += 1 results: dict[str, TestResult] = {} @@ -148,9 +162,18 @@ async def run( elif key in self._previous_plugins: plugins[key] = self._previous_plugins[key] + return results, plugins + + async def run( + self, key: str | None = None, config: str | None = None, data: str | None = None + ): + """测试商店内插件情况""" + + results, plugins = await self.test_plugins(key, config, data) + # 保存测试结果与生成的列表 - dump_json(RESULTS_PATH, results) dump_json(ADAPTERS_PATH, self._store_adapters) dump_json(BOTS_PATH, self._store_bots) dump_json(DRIVERS_PATH, self._store_drivers) dump_json(PLUGINS_PATH, list(plugins.values())) + dump_json(RESULTS_PATH, results) diff --git a/src/utils/store_test/utils.py b/src/utils/store_test/utils.py index 54f2363b..a4f106ef 100644 --- a/src/utils/store_test/utils.py +++ b/src/utils/store_test/utils.py @@ -25,7 +25,7 @@ def dump_json(path: Path, data: dict | list): @cache -def get_pypi_data(project_link: str) -> dict[str, Any] | None: +def get_pypi_data(project_link: str) -> dict[str, Any]: """获取 PyPI 数据""" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" @@ -34,16 +34,16 @@ def get_pypi_data(project_link: str) -> dict[str, Any] | None: r = httpx.get(url, headers=headers) if r.status_code == 200: return r.json() + raise ValueError(f"获取 PyPI 数据失败：{r.text}") -def get_latest_version(project_link: str) -> str | None: +def get_latest_version(project_link: str) -> str: """获取插件的最新版本号""" - if data := get_pypi_data(project_link): - return data["info"]["version"] + data = get_pypi_data(project_link) + return data["info"]["version"] -def get_upload_time(project_link: str) -> str | None: +def get_upload_time(project_link: str) -> str: """获取插件的上传时间""" - if data := get_pypi_data(project_link): - if len(data["urls"]) != 0: - return data["urls"][0]["upload_time_iso_8601"] + data = get_pypi_data(project_link) + return data["urls"][0]["upload_time_iso_8601"] diff --git a/src/utils/store_test/validation.py b/src/utils/store_test/validation.py index 85b7c68c..5333f564 100644 --- a/src/utils/store_test/validation.py +++ b/src/utils/store_test/validation.py @@ -12,7 +12,7 @@ from src.utils.validation import PublishType, validate_info from .models import Metadata, Plugin, StorePlugin, TestResult -from .utils import get_upload_time +from .utils import get_latest_version, get_upload_time def extract_metadata(path: Path) -> Metadata | None: @@ -54,14 +54,15 @@ async def validate_plugin( project_link = plugin["project_link"] module_name = plugin["module_name"] is_official = plugin["is_official"] - # 获取插件上传时间 - upload_time_str = get_upload_time(project_link) + # 从 PyPI 获取信息 + pypi_version = get_latest_version(project_link) + pypi_time = get_upload_time(project_link) # 如果传递了 data 参数 # 则直接使用 data 作为插件数据 # 并且将 skip_test 设置为 True if data: - # 无法获取到插件版本 - version = None + # 跳过测试时无法获取到测试的版本 + test_version = None # 因为跳过测试，测试结果无意义 plugin_test_result = True plugin_test_output = "已跳过测试" @@ -71,8 +72,8 @@ async def validate_plugin( # 为插件数据添加上所需的信息 new_plugin = json.loads(data) new_plugin["valid"] = True - new_plugin["version"] = version - new_plugin["time"] = upload_time_str or now_time_str + new_plugin["version"] = pypi_version + new_plugin["time"] = pypi_time new_plugin["skip_test"] = True new_plugin = cast(Plugin, new_plugin) @@ -96,7 +97,7 @@ async def validate_plugin( plugin_test_result, plugin_test_output = await test.run() metadata = extract_metadata(test.path) - version = extract_version(test.path) + test_version = extract_version(test.path) # 测试并提取完数据后删除测试文件夹 shutil.rmtree(test.path) @@ -136,8 +137,9 @@ async def validate_plugin( # 插件验证过程中无法获取是否是官方插件，因此需要从原始数据中获取 new_plugin["is_official"] = is_official new_plugin["valid"] = validation_info_result["valid"] - new_plugin["version"] = version - new_plugin["time"] = upload_time_str or now_time_str + # 优先使用测试中获取的版本号 + new_plugin["version"] = test_version or pypi_version + new_plugin["time"] = pypi_time new_plugin["skip_test"] = should_skip new_plugin = cast(Plugin, new_plugin) else: @@ -155,7 +157,7 @@ async def validate_plugin( result: TestResult = { "time": now_time_str, - "version": version, + "version": test_version, "results": { "validation": validation_result, "load": plugin_test_result, diff --git a/tests/utils/store_test/test_store_test.py b/tests/utils/store_test/test_store_test.py index 500ec1b4..0b1d0b3a 100644 --- a/tests/utils/store_test/test_store_test.py +++ b/tests/utils/store_test/test_store_test.py @@ -263,3 +263,136 @@ async def test_store_test_with_key_not_in_previous( assert not mocked_api["project_link_wordcloud"].called assert not mocked_api["project_link_treehelp"].called assert not mocked_api["project_link_datastore"].called + + +async def test_store_test_raise( + mocked_store_data: dict[str, Path], mocked_api: MockRouter, mocker: MockerFixture +): + """测试插件，但是测试过程中报错 + + 第一个插件因为版本号无变化跳过 + 第二插件测试中报错跳过 + 第三个插件测试中也报错 + + 最后数据没有变化 + """ + from src.utils.store_test.store import StoreTest + + mocked_validate_plugin = mocker.patch("src.utils.store_test.store.validate_plugin") + mocked_validate_plugin.side_effect = Exception + + test = StoreTest(0, 1, False) + await test.run() + + mocked_validate_plugin.assert_has_calls( + [ + mocker.call( + plugin={ + "module_name": "nonebot_plugin_treehelp", + "project_link": "nonebot-plugin-treehelp", + "author": "he0119", + "tags": [], + "is_official": False, + }, + config="", + skip_test=False, + previous_plugin={ + "module_name": "nonebot_plugin_treehelp", + "project_link": "nonebot-plugin-treehelp", + "name": "帮助", + "desc": "获取插件帮助信息", + "author": "he0119", + "homepage": "https://github.com/he0119/nonebot-plugin-treehelp", + "tags": [], + "is_official": False, + "type": "application", + "supported_adapters": None, + "valid": True, + "time": "2023-06-22 12:10:18", + }, + ), + mocker.call( + plugin={ + "module_name": "nonebot_plugin_wordcloud", + "project_link": "nonebot-plugin-wordcloud", + "author": "he0119", + "tags": [], + "is_official": False, + }, + config="", + skip_test=False, + previous_plugin=None, + ), # type: ignore + ], + ) + + # 数据没有更新，只是被压缩 + assert ( + mocked_store_data["results"].read_text(encoding="utf8") + == '{"nonebot-plugin-datastore:nonebot_plugin_datastore":{"time":"2023-06-26T22:08:18.945584+08:00","version":"1.0.0","results":{"validation":true,"load":true,"metadata":true},"inputs":{"config":""},"outputs":{"validation":"通过","load":"datastore","metadata":{"name":"数据存储","description":"NoneBot 数据存储插件","usage":"请参考文档","type":"library","homepage":"https://github.com/he0119/nonebot-plugin-datastore","supported_adapters":null}}},"nonebot-plugin-treehelp:nonebot_plugin_treehelp":{"time":"2023-06-26T22:20:41.833311+08:00","version":"0.3.0","results":{"validation":true,"load":true,"metadata":true},"inputs":{"config":""},"outputs":{"validation":"通过","load":"treehelp","metadata":{"name":"帮助","description":"获取插件帮助信息","usage":"获取插件列表\\n/help\\n获取插件树\\n/help -t\\n/help --tree\\n获取某个插件的帮助\\n/help 插件名\\n获取某个插件的树\\n/help --tree 插件名\\n","type":"application","homepage":"https://github.com/he0119/nonebot-plugin-treehelp","supported_adapters":null}}}}' + ) + assert ( + mocked_store_data["adapters"].read_text(encoding="utf8") + == '[{"module_name":"nonebot.adapters.onebot.v11","project_link":"nonebot-adapter-onebot","name":"OneBot V11","desc":"OneBot V11 协议","author":"yanyongyu","homepage":"https://onebot.adapters.nonebot.dev/","tags":[],"is_official":true}]' + ) + assert ( + mocked_store_data["bots"].read_text(encoding="utf8") + == '[{"name":"CoolQBot","desc":"基于 NoneBot2 的聊天机器人","author":"he0119","homepage":"https://github.com/he0119/CoolQBot","tags":[],"is_official":false}]' + ) + assert ( + mocked_store_data["drivers"].read_text(encoding="utf8") + == '[{"module_name":"~none","project_link":"","name":"None","desc":"None 驱动器","author":"yanyongyu","homepage":"/docs/advanced/driver","tags":[],"is_official":true},{"module_name":"~fastapi","project_link":"nonebot2[fastapi]","name":"FastAPI","desc":"FastAPI 驱动器","author":"yanyongyu","homepage":"/docs/advanced/driver","tags":[],"is_official":true}]' + ) + assert ( + mocked_store_data["plugins"].read_text(encoding="utf8") + == '[{"module_name":"nonebot_plugin_datastore","project_link":"nonebot-plugin-datastore","name":"数据存储","desc":"NoneBot 数据存储插件","author":"he0119","homepage":"https://github.com/he0119/nonebot-plugin-datastore","tags":[],"is_official":false,"type":"library","supported_adapters":null,"valid":true,"time":"2023-06-22 11:58:18"},{"module_name":"nonebot_plugin_treehelp","project_link":"nonebot-plugin-treehelp","name":"帮助","desc":"获取插件帮助信息","author":"he0119","homepage":"https://github.com/he0119/nonebot-plugin-treehelp","tags":[],"is_official":false,"type":"application","supported_adapters":null,"valid":true,"time":"2023-06-22 12:10:18"}]' + ) + + +async def test_store_test_with_key_raise( + mocked_store_data: dict[str, Path], mocked_api: MockRouter, mocker: MockerFixture +): + """测试指定插件，但是测试过程中报错""" + from src.utils.store_test.store import StoreTest + + mocked_validate_plugin = mocker.patch("src.utils.store_test.store.validate_plugin") + mocked_validate_plugin.side_effect = Exception + + test = StoreTest(0, 1, False) + await test.run(key="nonebot-plugin-wordcloud:nonebot_plugin_wordcloud") + + mocked_validate_plugin.assert_called_once_with( + plugin={ + "module_name": "nonebot_plugin_wordcloud", + "project_link": "nonebot-plugin-wordcloud", + "author": "he0119", + "tags": [], + "is_official": False, + }, + config="", + skip_test=False, + data=None, + previous_plugin=None, + ) + + # 数据没有更新，只是被压缩 + assert ( + mocked_store_data["results"].read_text(encoding="utf8") + == '{"nonebot-plugin-datastore:nonebot_plugin_datastore":{"time":"2023-06-26T22:08:18.945584+08:00","version":"1.0.0","results":{"validation":true,"load":true,"metadata":true},"inputs":{"config":""},"outputs":{"validation":"通过","load":"datastore","metadata":{"name":"数据存储","description":"NoneBot 数据存储插件","usage":"请参考文档","type":"library","homepage":"https://github.com/he0119/nonebot-plugin-datastore","supported_adapters":null}}},"nonebot-plugin-treehelp:nonebot_plugin_treehelp":{"time":"2023-06-26T22:20:41.833311+08:00","version":"0.3.0","results":{"validation":true,"load":true,"metadata":true},"inputs":{"config":""},"outputs":{"validation":"通过","load":"treehelp","metadata":{"name":"帮助","description":"获取插件帮助信息","usage":"获取插件列表\\n/help\\n获取插件树\\n/help -t\\n/help --tree\\n获取某个插件的帮助\\n/help 插件名\\n获取某个插件的树\\n/help --tree 插件名\\n","type":"application","homepage":"https://github.com/he0119/nonebot-plugin-treehelp","supported_adapters":null}}}}' + ) + assert ( + mocked_store_data["adapters"].read_text(encoding="utf8") + == '[{"module_name":"nonebot.adapters.onebot.v11","project_link":"nonebot-adapter-onebot","name":"OneBot V11","desc":"OneBot V11 协议","author":"yanyongyu","homepage":"https://onebot.adapters.nonebot.dev/","tags":[],"is_official":true}]' + ) + assert ( + mocked_store_data["bots"].read_text(encoding="utf8") + == '[{"name":"CoolQBot","desc":"基于 NoneBot2 的聊天机器人","author":"he0119","homepage":"https://github.com/he0119/CoolQBot","tags":[],"is_official":false}]' + ) + assert ( + mocked_store_data["drivers"].read_text(encoding="utf8") + == '[{"module_name":"~none","project_link":"","name":"None","desc":"None 驱动器","author":"yanyongyu","homepage":"/docs/advanced/driver","tags":[],"is_official":true},{"module_name":"~fastapi","project_link":"nonebot2[fastapi]","name":"FastAPI","desc":"FastAPI 驱动器","author":"yanyongyu","homepage":"/docs/advanced/driver","tags":[],"is_official":true}]' + ) + assert ( + mocked_store_data["plugins"].read_text(encoding="utf8") + == '[{"module_name":"nonebot_plugin_datastore","project_link":"nonebot-plugin-datastore","name":"数据存储","desc":"NoneBot 数据存储插件","author":"he0119","homepage":"https://github.com/he0119/nonebot-plugin-datastore","tags":[],"is_official":false,"type":"library","supported_adapters":null,"valid":true,"time":"2023-06-22 11:58:18"},{"module_name":"nonebot_plugin_treehelp","project_link":"nonebot-plugin-treehelp","name":"帮助","desc":"获取插件帮助信息","author":"he0119","homepage":"https://github.com/he0119/nonebot-plugin-treehelp","tags":[],"is_official":false,"type":"application","supported_adapters":null,"valid":true,"time":"2023-06-22 12:10:18"}]' + ) diff --git a/tests/utils/store_test/test_validate_plugin.py b/tests/utils/store_test/test_validate_plugin.py index cdc8ad33..740454bd 100644 --- a/tests/utils/store_test/test_validate_plugin.py +++ b/tests/utils/store_test/test_validate_plugin.py @@ -165,7 +165,7 @@ async def test_validate_plugin_with_data( "type": "application", "time": "2023-09-01T00:00:00+00:00Z", "valid": True, - "version": None, + "version": "0.0.1", "skip_test": True, }