diff --git a/config/config.ini b/config/config.ini index b26a025c77dc..250421dda892 100644 --- a/config/config.ini +++ b/config/config.ini @@ -9,12 +9,13 @@ online_search_page_num = 1 urls_limit = 10 open_keep_all = False open_sort = True -sort_timeout = 5 +sort_timeout = 10 open_ffmpeg = True open_filter_resolution = True min_resolution = 1920x1080 -response_time_weight = 0.5 -resolution_weight = 0.5 +delay_weight = 0.25 +speed_weight = 0.5 +resolution_weight = 0.25 recent_days = 30 ipv_type = 全部 ipv_type_prefer = 自动 diff --git a/docs/config.md b/docs/config.md index 5d0f883ccebd..b85640161391 100644 --- a/docs/config.md +++ b/docs/config.md @@ -17,8 +17,9 @@ | open_m3u_result | True | 开启转换生成 m3u 文件类型结果链接，支持显示频道图标 | | open_filter_resolution | True | 开启分辨率过滤，低于最小分辨率（min_resolution）的接口将会被过滤 | | min_resolution | 1920x1080 | 接口最小分辨率，需要开启 open_filter_resolution 才能生效 | -| response_time_weight | 0.5 | 响应时间权重值（所有权重值总和应为 1） | -| resolution_weight | 0.5 | 分辨率权重值 （所有权重值总和应为 1） | +| speed_weight | 0.5 | 速率权重值（所有权重值总和应为 1） | +| delay_weight | 0.25 | 响应时间权重值（所有权重值总和应为 1） | +| resolution_weight | 0.25 | 分辨率权重值 （所有权重值总和应为 1） | | recent_days | 30 | 获取最近时间范围内更新的接口（单位天），适当减小可避免出现匹配问题 | | ipv_type | 全部 | 生成结果中接口的协议类型，可选值：ipv4、ipv6、全部、all | | ipv_type_prefer | 自动 | 接口协议类型偏好，优先将该类型的接口排在结果前面，可选值：IPv4、IPv6、自动、auto | diff --git a/docs/config_en.md b/docs/config_en.md index 3dfc444f5931..5b6fa8d429ce 100644 --- a/docs/config_en.md +++ b/docs/config_en.md @@ -17,8 +17,9 @@ | open_m3u_result | True | Enable the conversion to generate m3u file type result links, supporting the display of channel icons | | open_filter_resolution | True | Enable resolution filtering, interfaces with resolution lower than the minimum resolution (min_resolution) will be filtered | | min_resolution | 1920x1080 | Minimum interface resolution, requires enabling open_filter_resolution to take effect | -| response_time_weight | 0.5 | Response time weight value (the sum of all weight values should be 1) | -| resolution_weight | 0.5 | Resolution weight value (the sum of all weight values should be 1) | +| speed_weight | 0.5 | Speed weight value (the sum of all weight values should be 1) | +| delay_weight | 0.25 | Response time weight value (the sum of all weight values should be 1) | +| resolution_weight | 0.25 | Resolution weight value (the sum of all weight values should be 1) | | recent_days | 30 | Retrieve interfaces updated within a recent time range (in days), reducing appropriately can avoid matching issues | | ipv_type | all | The protocol type of interface in the generated result, optional values: ipv4, ipv6, all | | ipv_type_prefer | auto | Interface protocol type preference, prioritize interfaces of this type in the results, optional values: IPv4, IPv6, auto | diff --git a/tkinter_ui/default.py b/tkinter_ui/default.py index 89ea30be77fc..768463b2229b 100644 --- a/tkinter_ui/default.py +++ b/tkinter_ui/default.py @@ -1,9 +1,10 @@ +import os import tkinter as tk -from utils.config import config -from tkinter import ttk -from tkinter import scrolledtext from tkinter import filedialog -import os +from tkinter import scrolledtext +from tkinter import ttk + +from utils.config import config class DefaultUI: @@ -292,20 +293,20 @@ def init_ui(self, root): frame_default_sort_params_column2 = tk.Frame(frame_default_sort_params) frame_default_sort_params_column2.pack(side=tk.RIGHT, fill=tk.Y) - self.response_time_weight_label = tk.Label( + self.delay_weight_label = tk.Label( frame_default_sort_params_column1, text="响应时间权重:", width=12 ) - self.response_time_weight_label.pack(side=tk.LEFT, padx=4, pady=8) - self.response_time_weight_scale = tk.Scale( + self.delay_weight_label.pack(side=tk.LEFT, padx=4, pady=8) + self.delay_weight_scale = tk.Scale( frame_default_sort_params_column1, from_=0, to=1, orient=tk.HORIZONTAL, resolution=0.1, - command=self.update_response_time_weight, + command=self.update_delay_weight, ) - self.response_time_weight_scale.pack(side=tk.LEFT, padx=4, pady=8) - self.response_time_weight_scale.set(config.response_time_weight) + self.delay_weight_scale.pack(side=tk.LEFT, padx=4, pady=8) + self.delay_weight_scale.set(config.delay_weight) self.resolution_weight_label = tk.Label( frame_default_sort_params_column2, text="分辨率权重:", width=12 @@ -458,19 +459,19 @@ def update_min_resolution(self, event): def update_urls_limit(self, event): config.set("Settings", "urls_limit", self.urls_limit_entry.get()) - def update_response_time_weight(self, event): - weight1 = self.response_time_weight_scale.get() + def update_delay_weight(self, event): + weight1 = self.delay_weight_scale.get() weight2 = 1 - weight1 self.resolution_weight_scale.set(weight2) - config.set("Settings", "response_time_weight", str(weight1)) + config.set("Settings", "delay_weight", str(weight1)) config.set("Settings", "resolution_weight", str(weight2)) def update_resolution_weight(self, event): weight1 = self.resolution_weight_scale.get() weight2 = 1 - weight1 - self.response_time_weight_scale.set(weight2) + self.delay_weight_scale.set(weight2) config.set("Settings", "resolution_weight", str(weight1)) - config.set("Settings", "response_time_weight", str(weight2)) + config.set("Settings", "delay_weight", str(weight2)) def update_open_update_time(self): config.set("Settings", "open_update_time", str(self.open_update_time_var.get())) @@ -518,7 +519,7 @@ def change_entry_state(self, state): "open_filter_resolution_checkbutton", "min_resolution_entry", "urls_limit_entry", - "response_time_weight_scale", + "delay_weight_scale", "resolution_weight_scale", "open_update_time_checkbutton", "open_url_info_checkbutton", diff --git a/tkinter_ui/tkinter_ui.py b/tkinter_ui/tkinter_ui.py index 628279b4d055..013ed82f7868 100644 --- a/tkinter_ui/tkinter_ui.py +++ b/tkinter_ui/tkinter_ui.py @@ -1,5 +1,5 @@ -import sys import os +import sys sys.path.append(os.path.dirname(sys.path[0])) import tkinter as tk @@ -55,7 +55,7 @@ def save_config(self): "open_sort": self.default_ui.open_sort_var.get(), "open_filter_resolution": self.default_ui.open_filter_resolution_var.get(), "min_resolution": self.default_ui.min_resolution_entry.get(), - "response_time_weight": self.default_ui.response_time_weight_scale.get(), + "delay_weight": self.default_ui.delay_weight_scale.get(), "resolution_weight": self.default_ui.resolution_weight_scale.get(), "ipv_type": self.default_ui.ipv_type_combo.get(), "url_keywords_blacklist": self.default_ui.url_keywords_blacklist_text.get( diff --git a/updates/proxy/request.py b/updates/proxy/request.py index d335655cd91a..52899847f196 100644 --- a/updates/proxy/request.py +++ b/updates/proxy/request.py @@ -1,12 +1,14 @@ from asyncio import Semaphore +from concurrent.futures import ThreadPoolExecutor + from tqdm import tqdm from tqdm.asyncio import tqdm_asyncio -from utils.config import config -from utils.speed import get_speed_requests -from concurrent.futures import ThreadPoolExecutor + from driver.utils import get_soup_driver from requests_custom.utils import get_soup_requests, close_session +from utils.config import config from utils.retry import retry_func +from utils.speed import get_delay_requests def get_proxy_list(page_count=1): @@ -71,7 +73,7 @@ async def get_proxy_list_with_test(base_url, proxy_list): async def get_speed_task(url, timeout, proxy): async with semaphore: - return await get_speed_requests(url, timeout=timeout, proxy=proxy) + return await get_delay_requests(url, timeout=timeout, proxy=proxy) response_times = await tqdm_asyncio.gather( *(get_speed_task(base_url, timeout=30, proxy=url) for url in proxy_list), diff --git a/utils/config.py b/utils/config.py index a4ccf7869c3c..da70ff7fe23a 100644 --- a/utils/config.py +++ b/utils/config.py @@ -1,8 +1,8 @@ -import os import configparser +import os +import re import shutil import sys -import re def resource_path(relative_path, persistent=False): @@ -63,7 +63,7 @@ def ipv_type(self): @property def open_ipv6(self): return ( - "ipv6" in self.ipv_type or "all" in self.ipv_type or "全部" in self.ipv_type + "ipv6" in self.ipv_type or "all" in self.ipv_type or "全部" in self.ipv_type ) @property @@ -297,8 +297,12 @@ def subscribe_urls(self): ] @property - def response_time_weight(self): - return self.config.getfloat("Settings", "response_time_weight", fallback=0.5) + def delay_weight(self): + return self.config.getfloat("Settings", "delay_weight", fallback=0.5) + + @property + def speed_weight(self): + return self.config.getfloat("Settings", "speed_weight", fallback=0.5) @property def resolution_weight(self): @@ -370,7 +374,7 @@ def copy(self): for src_file in files_to_copy: dest_path = os.path.join(dest_folder, os.path.basename(src_file)) if os.path.abspath(src_file) == os.path.abspath( - dest_path + dest_path ) or os.path.exists(dest_path): continue shutil.copy(src_file, dest_folder) diff --git a/utils/speed.py b/utils/speed.py index 73a59cd20796..742a1c3831b6 100644 --- a/utils/speed.py +++ b/utils/speed.py @@ -15,18 +15,22 @@ logger = get_logger(constants.log_path) -async def get_speed_with_download(url, timeout=config.sort_timeout): +async def get_speed_with_download(url: str, timeout: int = config.sort_timeout) -> dict[str, float | None]: """ Get the speed of the url with a total timeout """ start_time = time() total_size = 0 total_time = 0 + info = {'speed': None, 'delay': None} try: async with ClientSession( connector=TCPConnector(ssl=False), trust_env=True ) as session: async with session.get(url, timeout=timeout) as response: + if response.status == 404: + return info + info['delay'] = int(round((time() - start_time) * 1000)) async for chunk in response.content.iter_any(): if chunk: total_size += len(chunk) @@ -35,25 +39,33 @@ async def get_speed_with_download(url, timeout=config.sort_timeout): finally: end_time = time() total_time += end_time - start_time - average_speed = (total_size / total_time if total_time > 0 else 0) / 1024 - return average_speed + info['speed'] = (total_size / total_time if total_time > 0 else 0) / 1024 / 1024 + return info -async def get_speed_m3u8(url, timeout=config.sort_timeout): +async def get_speed_m3u8(url: str, timeout: int = config.sort_timeout) -> dict[str, float | None]: """ Get the speed of the m3u8 url with a total timeout """ - url = quote(url, safe=':/?$&=@') - m3u8_obj = m3u8.load(url) - speed_list = [] - start_time = time() - for segment in m3u8_obj.segments: - if time() - start_time > timeout: - break - ts_url = segment.absolute_uri - speed = await get_speed_with_download(ts_url, timeout) - speed_list.append(speed) - return sum(speed_list) / len(speed_list) if speed_list else 0 + info = {'speed': None, 'delay': None} + try: + url = quote(url, safe=':/?$&=@') + m3u8_obj = m3u8.load(url, timeout=2) + speed_list = [] + start_time = time() + for segment in m3u8_obj.segments: + if time() - start_time > timeout: + break + ts_url = segment.absolute_uri + download_info = await get_speed_with_download(ts_url, timeout) + speed_list.append(download_info['speed']) + if info['delay'] is None and download_info['delay'] is not None: + info['delay'] = download_info['delay'] + info['speed'] = sum(speed_list) / len(speed_list) if speed_list else 0 + except: + pass + finally: + return info def get_info_yt_dlp(url, timeout=config.sort_timeout): @@ -72,9 +84,9 @@ def get_info_yt_dlp(url, timeout=config.sort_timeout): return ydl.sanitize_info(ydl.extract_info(url, download=False)) -async def get_speed_yt_dlp(url, timeout=config.sort_timeout): +async def get_delay_yt_dlp(url, timeout=config.sort_timeout): """ - Get the speed of the url by yt_dlp + Get the delay of the url by yt_dlp """ try: start_time = time() @@ -92,9 +104,9 @@ async def get_speed_yt_dlp(url, timeout=config.sort_timeout): return float("inf"), None -async def get_speed_requests(url, timeout=config.sort_timeout, proxy=None): +async def get_delay_requests(url, timeout=config.sort_timeout, proxy=None): """ - Get the speed of the url by requests + Get the delay of the url by requests """ async with ClientSession( connector=TCPConnector(ssl=False), trust_env=True @@ -176,9 +188,9 @@ def get_video_info(video_info): return frame_size, resolution -async def check_stream_speed(url_info): +async def check_stream_delay(url_info): """ - Check the stream speed + Check the stream delay """ try: url = url_info[0] @@ -195,13 +207,14 @@ async def check_stream_speed(url_info): return float("inf") -speed_cache = {} +cache = {} async def get_speed(url, ipv6_proxy=None, callback=None): """ Get the speed (response time and resolution) of the url """ + data = {'speed': None, 'delay': None, 'resolution': None} try: cache_key = None url_is_ipv6 = is_ipv6(url) @@ -210,19 +223,20 @@ async def get_speed(url, ipv6_proxy=None, callback=None): matcher = re.search(r"cache:(.*)", cache_info) if matcher: cache_key = matcher.group(1) - if cache_key in speed_cache: - return speed_cache[cache_key][0] + if cache_key in cache: + return cache[cache_key][0] if ipv6_proxy and url_is_ipv6: - speed = (0, None) - # elif '.m3u8' in url: - # speed = await get_speed_m3u8(url) + data['speed'] = float("inf") + data['delay'] = float("-inf") + elif '.m3u8' in url: + data.update(await get_speed_m3u8(url)) else: - speed = await get_speed_yt_dlp(url) - if cache_key and cache_key not in speed_cache: - speed_cache[cache_key] = speed - return speed + data.update(await get_speed_with_download(url)) + if cache_key and cache_key not in cache: + cache[cache_key] = data + return data except: - return float("inf"), None + return data finally: if callback: callback() @@ -239,32 +253,44 @@ def sort_urls_by_speed_and_resolution(name, data, logger=None): continue cache_key_match = re.search(r"cache:(.*)", url.partition("$")[2]) cache_key = cache_key_match.group(1) if cache_key_match else None - if cache_key and cache_key in speed_cache: - cache = speed_cache[cache_key] - if cache: - response_time, cache_resolution = cache + if cache_key and cache_key in cache: + cache_item = cache[cache_key] + if cache_item: + speed, delay, cache_resolution = cache_item['speed'], cache_item['delay'], cache_item['resolution'] resolution = cache_resolution or resolution - if response_time != float("inf"): + if speed is not None: url = remove_cache_info(url) try: if logger: logger.info( - f"Name: {name}, URL: {url}, Date: {date}, Resolution: {resolution}, Response Time: {response_time} ms" + f"Name: {name}, URL: {url}, Date: {date}, Delay: {delay} ms, Speed: {speed:.2f} M/s, Resolution: {resolution}" ) except Exception as e: print(e) - filter_data.append((url, date, resolution, origin)) + filter_data.append( + { + "url": url, + "date": date, + "delay": delay, + "speed": speed, + "resolution": resolution, + "origin": origin + } + ) def combined_key(item): - _, _, resolution, origin = item + speed, delay, resolution, origin = item["speed"], item["delay"], item["resolution"], item["origin"] if origin == "important": - return -float("inf") + return float("inf") else: - resolution_value = get_resolution_value(resolution) if resolution else 0 return ( - config.response_time_weight * response_time - - config.resolution_weight * resolution_value + config.speed_weight * (speed * 1024 if speed is not None else float("-inf")) + - config.delay_weight * (delay if delay is not None else float("inf")) + + config.resolution_weight * (get_resolution_value(resolution) if resolution else 0) ) - filter_data.sort(key=combined_key) - return filter_data + filter_data.sort(key=combined_key, reverse=True) + return [ + (item["url"], item["date"], item["resolution"], item["origin"]) + for item in filter_data + ]