Merge pull request #545 from qingchoulove/main

fix: Issue #539, the url may not be a valid url

kubespider/api/values.py

@@ -81,7 +81,7 @@ class Resource(Extra):
     Resource, used to describe the resource to be downloaded, result of the source provider
     """
 
-    def __init__(self, url: str, path: str,
+    def __init__(self, *, url: str, path: str,
                  link_type: str = None, file_type: str = None, uid: str = None,
                  **kwargs):
         super().__init__(**kwargs)

kubespider/source_provider/btbtt12_disposable_source_provider/provider.py

@@ -56,9 +56,10 @@ def is_webhook_enable(self) -> bool:
 
     def should_handle(self, event: Event) -> bool:
         parse_url = urlparse(event.source)
+        # Issue #539, the url may not be a valid url
         if not parse_url:
             return False
-        if parse_url.hostname.endswith('btbtt12.com') and 'attach-dialog-fid' in parse_url.path:
+        if parse_url.hostname and parse_url.hostname.endswith('btbtt12.com') and 'attach-dialog-fid' in parse_url.path:
             logging.info('%s belongs to Btbtt12DisposableSourceProvider', event.source)
             return True
         return False

kubespider/source_provider/magic_source_provider/provider.py

@@ -64,7 +64,11 @@ def is_webhook_enable(self) -> bool:
 
     def should_handle(self, event: Event) -> bool:
         data_source_url = event.source
-        if urlparse(data_source_url).hostname in self.handle_host:
+        # Issue #539, the url may not be a valid url
+        parse_url = urlparse(data_source_url)
+        if not parse_url:
+            return False
+        if parse_url.hostname in self.handle_host:
             logging.info('%s belongs to %s', data_source_url, self.provider_name)
             return True
         return False

kubespider/source_provider/youtube_source_provider/provider.py

@@ -53,6 +53,9 @@ def is_webhook_enable(self) -> bool:
 
     def should_handle(self, event: Event) -> bool:
         parse_url = urlparse(event.source)
+        # Issue #539, the url may not be a valid url
+        if not parse_url:
+            return False
         if parse_url.hostname in ('www.youtube.com', 'youtube.com', 'm.youtube.com', 'youtu.be'):
             logging.info('%s belongs to youtube_source_provider', event.source)
             return True