Merge pull request #9 from WPSemantix/task/improve-queries-runtime-performance

Task/improve queries runtime performance

Author: DorEliyahu176

pyproject.toml

@@ -46,11 +46,6 @@ classifiers = [
   "Development Status :: 5 - Production/Stable",
   "Intended Audience :: Developers",
   "Topic :: Software Development :: Build Tools",
-  "License :: OSI Approved :: MIT License",
-  "License :: OSI Approved :: Apache Software License",
-  "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
-  "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-  "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
   "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
@@ -59,14 +54,15 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-  "future==1.0.0",
-  "python-dateutil==2.9.0",
+  "future>=0.18.2",
+  "python-dateutil>=2.8.2",
   "ldap3",
-  "thrift_sasl==0.4.3",
+  "thrift>=0.16.0",
+  "thrift_sasl>=0.4.3",
   "pure-sasl>=0.6.2",
   "sqlalchemy>=1.4.36,<2.0.0",
-  "requests_kerberos==0.15.0",
-  "pyhive==0.7.0"
+  "requests_kerberos>=0.12.0",
+  "pyhive>=0.7.0"
 ]
 
 [project.urls]

pytimbr_sqla/__init__.py

@@ -1,3 +1,63 @@
 """Timbr Python SQLAlchemy connector."""
 
-__version__ = "2.0.0"
+__version__ = "2.0.1"
+
+try:
+  from thrift.transport import THttpClient
+  def timbr_flush(self):
+    if self.__http_response is not None:
+        self.__http_response.close()
+        self.__http_response = None
+
+    if not self.isOpen():
+        self.open()
+
+    # Pull data out of buffer
+    data = self.__wbuf.getvalue()
+    self.__wbuf = BytesIO()
+
+    # HTTP request
+    if self.using_proxy() and self.scheme == "http":
+        # need full URL of real host for HTTP proxy here (HTTPS uses CONNECT tunnel)
+        self.__http.putrequest('POST', "http://%s:%s%s" %
+                               (self.realhost, self.realport, self.path))
+    else:
+        self.__http.putrequest('POST', self.path)
+
+    # Write headers
+    self.__http.putheader('Content-Type', 'application/x-thrift')
+    self.__http.putheader('Content-Length', str(len(data)))
+    if self.using_proxy() and self.scheme == "http" and self.proxy_auth is not None:
+        self.__http.putheader("Proxy-Authorization", self.proxy_auth)
+
+    if not self.__custom_headers or 'User-Agent' not in self.__custom_headers:
+        user_agent = 'Python/THttpClient'
+        script = os.path.basename(sys.argv[0])
+        if script:
+            user_agent = '%s (%s)' % (user_agent, urllib.parse.quote(script))
+        self.__http.putheader('User-Agent', user_agent)
+
+    if self.__custom_headers:
+        for key, val in six.iteritems(self.__custom_headers):
+            self.__http.putheader(key, val)
+
+    # Saves the cookie sent by the server in the previous response.
+    # HTTPConnection.putheader can only be called after a request has been
+    # started, and before it's been sent.
+    if self.headers and 'Set-Cookie' in self.headers:
+        self.__http.putheader('Cookie', self.headers['Set-Cookie'])
+
+    self.__http.endheaders()
+
+    # Write payload
+    self.__http.send(data)
+
+    # Get reply to flush the request
+    self.__http_response = self.__http.getresponse()
+    self.code = self.__http_response.status
+    self.message = self.__http_response.reason
+    self.headers = self.__http_response.msg
+
+  THttpClient.flush = timbr_flush
+except:
+    pass

pytimbr_sqla/hive.py

@@ -366,7 +366,7 @@ class Cursor(common.DBAPICursor):
     visible by other cursors or connections.
     """
 
-    def __init__(self, connection, arraysize=1000):
+    def __init__(self, connection, arraysize=10000):
         self._operationHandle = None
         super(Cursor, self).__init__()
         self._arraysize = arraysize
@@ -391,7 +391,7 @@ def arraysize(self):
     @arraysize.setter
     def arraysize(self, value):
         """Array size cannot be None, and should be an integer"""
-        default_arraysize = 1000
+        default_arraysize = 10000
         try:
             self._arraysize = int(value) or default_arraysize
         except TypeError:

test/requirements.txt

@@ -172,9 +172,12 @@ def write(self, buf):
         self.__wbuf.write(buf)
 
     def flush(self):
-        if self.isOpen():
-            self.close()
-        self.open()
+        if self.__http_response is not None:
+            self.__http_response.close()
+            self.__http_response = None
+
+        if not self.isOpen():
+            self.open()
 
         # Pull data out of buffer
         data = self.__wbuf.getvalue()