maintenance: simplify regexes (#56)

adbar · Aug 31, 2023 · ac6589e · ac6589e
1 parent 8292100
commit ac6589e
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 6 deletions.
diff --git a/courlan/clean.py b/courlan/clean.py
@@ -23,6 +23,7 @@
 SELECTION = re.compile(
     r'(https?://[^">&? ]+?)(?:https?://)|(?:https?://[^/]+?/[^/]+?[&?]u(rl)?=)(https?://[^"> ]+)'
 )
+
 MIDDLE_URL = re.compile(r"https?://.+?(https?://.+?)(?:https?://|$)")
 NETLOC_RE = re.compile(r"(?<=\w):(?:80|443)")
 

diff --git a/courlan/core.py b/courlan/core.py
@@ -39,7 +39,7 @@
 
 FIND_LINKS_REGEX = re.compile(r"<a [^<>]+?>", re.I)
 HREFLANG_REGEX = re.compile(r'hreflang=["\']?([a-z-]+)', re.I)
-LINK_REGEX = re.compile(r'href=["\']?([^ ]+?)(["\']|[ >])', re.I)
+LINK_REGEX = re.compile(r'href=["\']?([^ ]+?)(["\' >])', re.I)
 
 
 def check_url(

diff --git a/courlan/filters.py b/courlan/filters.py
@@ -23,6 +23,7 @@
 # domain/host names
 UNSUITABLE_DOMAIN = re.compile(r"[?=`$;,]|:$|\.(.|[0-9]+|[^.]{25,})$")
 
+
 # content filters
 SITE_STRUCTURE = re.compile(
     # wordpress
@@ -41,7 +42,7 @@
     re.IGNORECASE,
 )  # (?=[&?])
 ADULT_AND_VIDEOS = re.compile(
-    r"[/_-](?:bild\-?kontakte|fick|gangbang|incest|live-?chat|live-?cams?|porno?|sexyeroti[ck]|sexcam|swinger|xxx)\b",
+    r"[/_-](?:bild-?kontakte|fick|gangbang|incest|live-?cams?|live-?chat|porno?|sexcam|sexyeroti[ck]|swinger|x{3})\b",
     re.IGNORECASE,
 )
 

diff --git a/courlan/urlutils.py b/courlan/urlutils.py
@@ -12,18 +12,18 @@
 
 
 DOMAIN_REGEX = re.compile(
-    r"(?:http|ftp)s?://"  # protocols
+    r"(?:(?:f|ht)tp)s?://"  # protocols
     r"(?:[^/?#]{,63}\.)?"  # subdomain, www, etc.
     r"([^/?#.]{4,63}\.[^/?#]{2,63}|"  # domain and extension
-    r"[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|"  # IPv4
+    r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|"  # IPv4
     r"[0-9a-f:]{16,})"  # IPv6
     r"(?:/|$)"  # slash or end of string
 )
 NO_EXTENSION_REGEX = re.compile(r"(^[^.]+)")
-STRIP_DOMAIN_REGEX = re.compile(r"^.+?:.*?@|(?<=[^0-9]):[0-9]+")
+STRIP_DOMAIN_REGEX = re.compile(r"^.+?:.*?@|(?<=\D):\d+")
 CLEAN_FLD_REGEX = re.compile(r"^www[0-9]*\.")
 INNER_SLASH_REGEX = re.compile(r"(.+/)+")
-FEED_WHITELIST_REGEX = re.compile(r"(feedburner|feedproxy)", re.I)
+FEED_WHITELIST_REGEX = re.compile(r"(?:feed(?:burner|proxy))", re.I)
 
 
 @lru_cache(maxsize=1024)