prepare new version

HISTORY.md

@@ -1,6 +1,13 @@
 ## History / Changelog
 
 
+### 0.2.2
+
+- English and German language filters
+- Function to detect external links
+- Support for domain blacklisting 
+
+
 ### 0.2.1
 
 - Less aggressive strict filters

courlan/clean.py

@@ -65,7 +65,7 @@ def clean_query(parsed_url, strict=False, language=None):
                 if language == 'de' and found_lang not in TARGET_LANG_DE:
                     logging.debug('bad lang: %s %s %s', language, qelem, found_lang)
                     raise ValueError
-                elif language == 'en' and found_lang not in TARGET_LANG_EN:
+                if language == 'en' and found_lang not in TARGET_LANG_EN:
                     logging.debug('bad lang: %s %s %s', language, qelem, found_lang)
                     raise ValueError
             # insert

courlan/core.py

@@ -166,4 +166,3 @@ def is_external(url, reference, ignore_suffix=True):
     if domain != ref_domain:
         return True
     return False
-

courlan/filters.py

@@ -33,15 +33,15 @@ def extension_filter(component):
 
 
 def lang_filter(url, language):
-   '''Heuristic targeting internationalization'''
-   if language is not None and language in ('de', 'en'):
-       match = URL_LANG_FILTER.match(url)
-       if match:
-           if language == 'de' and match.group(1) not in ('de', 'deu'):
-               return False
-           elif language == 'en' and match.group(1) not in ('en', 'eng'):
-               return False
-   return True
+    '''Heuristic targeting internationalization'''
+    if language is not None and language in ('de', 'en'):
+        match = URL_LANG_FILTER.match(url)
+        if match:
+            if language == 'de' and match.group(1) not in ('de', 'deu'):
+                return False
+            if language == 'en' and match.group(1) not in ('en', 'eng'):
+                return False
+    return True
 
 
 def spam_filter(url):

setup.py

@@ -20,7 +20,7 @@ def readme():
 
 setup(
     name='courlan',
-    version='0.2.1',
+    version='0.2.2',
     description='Clean, filter, normalize, and sample URLs',
     long_description=readme(),
     classifiers=[
@@ -59,6 +59,7 @@ def readme():
     include_package_data=True,
     python_requires='>=3.4',
     install_requires=[
+        'requests',
         'tldextract',
     ],
     #extras_require=extras,