add openapi support, delay bs4 load to speed up everything, improve r…

…equests imports

CloudFlare/api_decode_from_openapi.py

@@ -0,0 +1,63 @@
+""" API from OpenAPI for Cloudflare API"""
+
+import sys
+import re
+import datetime
+import json
+
+API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
+
+match_identifier = re.compile('\{[A-Za-z0-9_]*\}')
+
+def do_path(cmd, info):
+    """ do_path() """
+
+    cmds = []
+
+    if cmd[0] != '/':
+        cmd = '/' + cmd  # make sure there's a leading /
+
+    cmd = match_identifier.sub(':id', cmd) 
+    if cmd[-4:] == '/:id':
+        cmd = cmd[:-4]
+    if cmd[-4:] == '/:id':
+        cmd = cmd[:-4]
+
+    for action in info:
+        action = action.upper()
+        if action == '' or action not in API_TYPES:
+            continue
+        deprecated = False
+        deprecated_date = ''
+        deprecated_already = False
+        v = {'action': action, 'cmd': cmd, 'deprecated': deprecated, 'deprecated_date': deprecated_date, 'deprecated_already': deprecated_already}
+        cmds.append(v)
+    return cmds
+
+def api_decode_from_openapi(content):
+    """ API decode from OpenAPI for Cloudflare API"""
+
+    try:
+        j = json.loads(content)
+    except Exception as e:
+        sys.stderr.write("OpenAPI json format issue: %s\n" % (e))
+        return None
+
+    try:
+        components = j['components']
+        info = j['info']
+        openapi = j['openapi']
+        paths = j['paths']
+        servers = ['servers']
+    except Exception as e:
+        sys.stderr.write("OpenAPI json format structure: %s\n" % (e))
+        return None
+
+    all_cmds = []
+    for path in paths:
+        if path[0] != '/':
+            sys.stderr.write("OpenAPI invalid path: %s\n" % (path))
+            continue
+        all_cmds += do_path(path, paths[path])
+
+    return sorted(all_cmds, key=lambda v: v['cmd'])

CloudFlare/api_decode_from_web.py

@@ -3,10 +3,37 @@
 import sys
 import datetime
 
-from bs4 import BeautifulSoup, Comment
-
 API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
 
+class mybs4():
+    """ mybs4 """
+
+    _BeautifulSoup = None
+    _Comment = None
+
+    def __init__(self):
+        """ __init__ """
+        pass
+
+    def _available(self):
+        """ _available() """
+        if not mybs4._BeautifulSoup:
+            try:
+                from bs4 import BeautifulSoup, Comment
+                mybs4._BeautifulSoup = BeautifulSoup
+                mybs4._Comment = Comment
+                self.Comment = mybs4._Comment
+            except ImportError:
+                return False
+        return True
+
+    def BeautifulSoup(self, content, parser):
+        """ BeautifulSoup() """
+        self._available()
+        return mybs4._BeautifulSoup(content, parser)
+
+my_bs4 = mybs4()
+
 def do_section(section):
     """ API extras for Cloudflare API"""
 
@@ -42,7 +69,7 @@ def do_section(section):
     for tag2 in section.find_all('pre'):
         cmd = []
         for child in tag2.children:
-            if isinstance(child, Comment):
+            if isinstance(child, my_bs4.Comment):
                 # remove <!-- react-text ... -> parts
                 continue
             cmd.append(str(child).strip())
@@ -61,7 +88,7 @@ def do_section(section):
 def api_decode_from_web(content):
     """ API extras for Cloudflare API"""
 
-    soup = BeautifulSoup(content, 'html.parser')
+    soup = my_bs4.BeautifulSoup(content, 'html.parser')
 
     for child in soup.find_all('p'):
         t = child.get_text()

CloudFlare/cloudflare.py

@@ -3,7 +3,7 @@
 
 import json
 import keyword
-import requests
+from requests import RequestException as requests_RequestException, ConnectionError as requests_ConnectionError, exceptions as requests_exceptions, codes as requests_codes
 
 from .network import CFnetwork
 from .logging_helper import CFlogger
@@ -12,6 +12,7 @@
 from .api_v4 import api_v4
 from .api_extras import api_extras
 from .api_decode_from_web import api_decode_from_web
+from .api_decode_from_openapi import api_decode_from_openapi
 from .exceptions import CloudFlareError, CloudFlareAPIError, CloudFlareInternalError
 
 BASE_URL = 'https://api.cloudflare.com/client/v4'
@@ -199,15 +200,15 @@ def _call_network(self, method, headers, parts, identifiers, params, data, files
 
             try:
                 response = self.network(method, url, headers, params, data, files)
-            except requests.RequestException as e:
+            except requests_RequestException as e:
                 if self.logger:
                     self.logger.debug('Call: requests exception! "%s"', e)
                 raise CloudFlareAPIError(0, e)
-            except requests.ConnectionError as e:
+            except requests_ConnectionError as e:
                 if self.logger:
                     self.logger.debug('Call: requests connection exception! "%s"', e)
                 raise CloudFlareAPIError(0, 'connection error')
-            except requests.exceptions.Timeout as e:
+            except requests_exceptions.Timeout as e:
                 if self.logger:
                     self.logger.debug('Call: requests timeout exception! "%s"', e)
                 raise CloudFlareAPIError(0, 'connection timeout')
@@ -297,7 +298,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                 except ValueError:
                     if response_data == '':
                         # This should really be 'null' but it isn't. Even then, it's wrong!
-                        if response_code == requests.codes.ok:
+                        if response_code == requests_codes.ok:
                             # 200 ok
                             response_data = {'success': True,
                                              'result': None}
@@ -320,7 +321,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                                 self.logger.debug('Response data not JSON: %r', response_data)
                             raise CloudFlareAPIError(0, 'JSON parse failed - report to Cloudflare.')
 
-                if response_code == requests.codes.ok:
+                if response_code == requests_codes.ok:
                     # 200 ok - so nothing needs to be done
                     pass
                 else:
@@ -329,7 +330,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                     pass
             elif response_type == 'application/octet-stream' and isinstance(response_data, (int, float)):
                 # It's binary data
-                if response_code == requests.codes.ok:
+                if response_code == requests_codes.ok:
                     # 200 ok
                     response_data = {'success': True,
                                      'result': response_data}
@@ -345,7 +346,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                 try:
                     response_data = json.loads(response_data)
                     if not isinstance(response_data, (dict)) or 'success' not in response_data:
-                        if response_code == requests.codes.ok:
+                        if response_code == requests_codes.ok:
                             # 200 ok
                             response_data = {'success': True,
                                              'result': response_data}
@@ -357,7 +358,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                 except ValueError:
                     # So it wasn't JSON - moving on as if it's text!
                     # A single value is returned (vs an array or object)
-                    if response_code == requests.codes.ok:
+                    if response_code == requests_codes.ok:
                         # 200 ok
                         response_data = {'success': True, 'result': response_data}
                     else:
@@ -377,7 +378,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                 except ValueError:
                     # So it wasn't JSON - moving on as if it's text!
                     # A single value is returned (vs an array or object)
-                    if response_code == requests.codes.ok:
+                    if response_code == requests_codes.ok:
                         # 200 ok
                         response_data = {'success': True, 'result': response_data}
                     else:
@@ -387,7 +388,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                                          'result': response_data}
             elif response_type in ['text/javascript', 'application/javascript']:
                 # used by Cloudflare workers
-                if response_code == requests.codes.ok:
+                if response_code == requests_codes.ok:
                     # 200 ok
                     response_data = {'success': True,
                                      'result': str(response_data)}
@@ -398,7 +399,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                                      'result': str(response_data)}
             elif response_type == 'text/html':
                 # used by media for preview
-                if response_code == requests.codes.ok:
+                if response_code == requests_codes.ok:
                     # 200 ok
                     response_data = {'success': True,
                                      'result': str(response_data)}
@@ -411,7 +412,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
             else:
                 # Assuming nothing - but continuing anyway
                 # A single value is returned (vs an array or object)
-                if response_code == requests.codes.ok:
+                if response_code == requests_codes.ok:
                     # 200 ok
                     response_data = {'success': True,
                                      'result': str(response_data)}
@@ -535,7 +536,15 @@ def api_from_web(self):
 
             # base url isn't enough; we need less
             url = '/'.join(self.base_url.split('/')[0:3])
+            return self._read_from_web(url)
 
+        def api_from_openapi(self, url):
+            """ Cloudflare v4 API"""
+
+            return self._read_from_web(url)
+
+        def _read_from_web(self, url):
+            """ Cloudflare v4 API"""
             try:
                 if self.logger:
                     self.logger.debug('Call: doit!')
@@ -894,6 +903,11 @@ def api_from_web(self):
 
         return api_decode_from_web(self._base.api_from_web())
 
+    def api_from_openapi(self, url):
+        """ Cloudflare v4 API"""
+
+        return api_decode_from_openapi(self._base.api_from_openapi(url))
+
     def __init__(self, email=None, key=None, token=None, certtoken=None, debug=False, raw=False, use_sessions=True, profile=None, base_url=None):
         """ Cloudflare v4 API"""