From fdb54edf5fcc0e72ed4456f33132427524d850d0 Mon Sep 17 00:00:00 2001 From: "guorong.zheng" <360996299@qq.com> Date: Thu, 5 Sep 2024 17:14:15 +0800 Subject: [PATCH 1/2] chore:url cache --- updates/fofa/request.py | 14 ++++++++++---- updates/hotel/request.py | 3 ++- utils/channel.py | 7 ++++++- utils/speed.py | 4 ++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/updates/fofa/request.py b/updates/fofa/request.py index 04712cc769..4d3338d187 100644 --- a/updates/fofa/request.py +++ b/updates/fofa/request.py @@ -53,12 +53,13 @@ async def get_channels_by_fofa(urls=None, multicast=False, callback=None): proxy = None open_proxy = config.getboolean("Settings", "open_proxy") open_driver = config.getboolean("Settings", "open_driver") + open_sort = config.getboolean("Settings", "open_sort") if open_proxy: test_url = fofa_urls[0][0] if multicast else fofa_urls[0] proxy = await get_proxy(test_url, best=True, with_test=True) def process_fofa_channels(fofa_info): - nonlocal proxy, fofa_urls_len, open_driver + nonlocal proxy, fofa_urls_len, open_driver, open_sort fofa_url = fofa_info[0] if multicast else fofa_info results = defaultdict(lambda: defaultdict(list)) try: @@ -88,7 +89,8 @@ def process_fofa_channels(fofa_info): else: with ThreadPoolExecutor(max_workers=100) as executor: futures = [ - executor.submit(process_fofa_json_url, url) for url in urls + executor.submit(process_fofa_json_url, url, open_sort) + for url in urls ] for future in futures: results = merge_objects(results, future.result()) @@ -120,7 +122,7 @@ def process_fofa_channels(fofa_info): return fofa_results -def process_fofa_json_url(url): +def process_fofa_json_url(url, open_sort): """ Process the FOFA json url """ @@ -141,7 +143,11 @@ def process_fofa_json_url(url): item_name = format_channel_name(item.get("name")) item_url = item.get("url").strip() if item_name and item_url: - total_url = f"{url}{item_url}$cache:{url}" + total_url = ( + f"{url}{item_url}$cache:{url}" + if open_sort + else f"{url}{item_url}" + ) if item_name not in channels: channels[item_name] = [(total_url, None, None)] else: diff --git a/updates/hotel/request.py b/updates/hotel/request.py index b46d8e22d9..a8974403a3 100644 --- a/updates/hotel/request.py +++ b/updates/hotel/request.py @@ -178,8 +178,9 @@ def process_region_by_hotel(region): for result in search_region_result.values() for url, _, _ in result ] + open_sort = config.getboolean("Settings", "open_sort") channels = await get_channels_by_subscribe_urls( - urls, hotel=True, retry=False, error_print=False, with_cache=True + urls, hotel=True, retry=False, error_print=False, with_cache=open_sort ) if not open_driver: close_session() diff --git a/utils/channel.py b/utils/channel.py index 06a7b69ce5..ab2f04bb7f 100644 --- a/utils/channel.py +++ b/utils/channel.py @@ -234,10 +234,15 @@ def get_channel_multicast_result(result, search_result): Get the channel multicast info result by result and search result """ info_result = {} + open_sort = config.getboolean("Settings", "open_sort") for name, result_obj in result.items(): info_list = [ ( - f"http://{url}/rtp/{ip}$cache:{result_region}_{result_type}", + ( + f"http://{url}/rtp/{ip}$cache:{result_region}_{result_type}" + if open_sort + else f"http://{url}/rtp/{ip}" + ), date, resolution, ) diff --git a/utils/speed.py b/utils/speed.py index 8beba4cd18..a7b0a531e8 100644 --- a/utils/speed.py +++ b/utils/speed.py @@ -127,10 +127,10 @@ async def get_speed_by_info(url_info, ffmpeg, semaphore, callback=None): url, cache_info = url.split("$", 1) if "cache:" in cache_info: cache_key = cache_info.replace("cache:", "") - if cache_key in speed_cache: - return tuple(url_info), speed_cache[cache_key] url = quote(url, safe=":/?&=$[]") url_info[0] = url + if cache_key in speed_cache: + return (tuple(url_info), speed_cache[cache_key]) try: if ".m3u8" not in url and ffmpeg: speed = await check_stream_speed(url_info) From 64777fcdd49fdecefc492749e7c6972706ca0b66 Mon Sep 17 00:00:00 2001 From: "guorong.zheng" <360996299@qq.com> Date: Thu, 5 Sep 2024 17:14:33 +0800 Subject: [PATCH 2/2] chore:fafa region --- updates/fofa/request.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/updates/fofa/request.py b/updates/fofa/request.py index 4d3338d187..165fc502e1 100644 --- a/updates/fofa/request.py +++ b/updates/fofa/request.py @@ -20,15 +20,15 @@ def get_fofa_urls_from_region_list(): """ Get the FOFA url from region """ - # region_list = config.get("Settings", "hotel_region_list").split(",") + region_list = config.get("Settings", "hotel_region_list").split(",") urls = [] region_url = getattr(fofa_map, "region_url") - # if "all" in region_list or "ALL" in region_list or "全部" in region_list: - urls = [url for url_list in region_url.values() for url in url_list if url] - # else: - # for region in region_list: - # if region in region_url: - # urls.append(region_url[region]) + if "all" in region_list or "ALL" in region_list or "全部" in region_list: + urls = [url for url_list in region_url.values() for url in url_list if url] + else: + for region in region_list: + if region in region_url: + urls.append(region_url[region]) return urls