From 5e2d451243623a64b536deb1bd48b4cdf44c37a4 Mon Sep 17 00:00:00 2001 From: davidcolangelo Date: Mon, 14 Apr 2025 10:40:51 -0400 Subject: [PATCH 1/6] - adding support for a shared reddis cache --- .bumpversion.cfg | 2 +- CONTRIBUTORS.rst | 2 +- src/zeep/cache.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 54a015a9..e080fe81 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.3.1 +current_version = 4.3.2 commit = true tag = true tag_name = {new_version} diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 1f02e449..1cc84055 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -51,4 +51,4 @@ Contributors * Zoltan Benedek * Øyvind Heddeland Instefjord * Pol Sanlorenzo - +* David Colangelo diff --git a/src/zeep/cache.py b/src/zeep/cache.py index 3cc734a7..8bd70e21 100644 --- a/src/zeep/cache.py +++ b/src/zeep/cache.py @@ -6,6 +6,7 @@ import threading from contextlib import contextmanager from typing import Dict, Tuple, Union +import redis import platformdirs import pytz @@ -163,6 +164,42 @@ def get(self, url): return self._decode_data(data) logger.debug("Cache MISS for %s", url) +class ReddisCache(Base): + """Cache contents via a redis database + - This is helpful if you make zeep calls from a pool of servers that need to share a common cache + """ + + def __init__(self, redis_host, password, db=0, port=6379, timeout=3600): + self._timeout = timeout + self._redis_host = redis_host + + self._redis_client = redis.StrictRedis( + host=redis_host, + port=port, + password=password, + db=db + ) + + def add(self, url, content): + logger.debug("Caching contents of %s", url) + # Remove the cached key + self._redis_client.delete(url) + # add the new cache response for the url + self._redis_client.set(url, value={ + 'time': datetime.datetime.now(datetime.timezone.utc), + 'value': content + }) + + def get(self, url): + cached_value = self._redis_client.get(url) + if not _is_expired(cached_value['time'], self._timeout): + logger.debug("Cache HIT for %s", url) + return cached_value['value'] + else: + logger.debug("Cache MISS for %s", url) + return None + + def _is_expired(value, timeout): """Return boolean if the value is expired""" From 2554e6632ab261b2d1358df7665b10edfc9f66d5 Mon Sep 17 00:00:00 2001 From: davidcolangelo Date: Mon, 14 Apr 2025 11:26:52 -0400 Subject: [PATCH 2/6] - fixing redis name --- pyproject.toml | 3 ++- src/zeep/cache.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c151100a..bc31bfbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "zeep" -version = "4.3.1" +version = "4.3.2" description = "A Python SOAP client" readme = "README.md" license = { text = "MIT" } @@ -28,6 +28,7 @@ dependencies = [ "requests-toolbelt>=0.7.1", "requests-file>=1.5.1", "pytz", + "redis" ] [project.urls] diff --git a/src/zeep/cache.py b/src/zeep/cache.py index 8bd70e21..630d91e9 100644 --- a/src/zeep/cache.py +++ b/src/zeep/cache.py @@ -164,7 +164,7 @@ def get(self, url): return self._decode_data(data) logger.debug("Cache MISS for %s", url) -class ReddisCache(Base): +class RedisCache(Base): """Cache contents via a redis database - This is helpful if you make zeep calls from a pool of servers that need to share a common cache """ From 67d08c029a3cbede3881f5c9f226041bc08c5fb9 Mon Sep 17 00:00:00 2001 From: davidcolangelo Date: Tue, 15 Apr 2025 13:31:05 -0400 Subject: [PATCH 3/6] - adding documentation for cache loading - allowing more external configuration of the cache object --- docs/transport.rst | 25 +++++++++++++++++++++++++ src/zeep/cache.py | 11 +++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/transport.rst b/docs/transport.rst index 7479d34d..942ae8d0 100644 --- a/docs/transport.rst +++ b/docs/transport.rst @@ -141,6 +141,31 @@ Another option is to use the InMemoryCache backend. It internally uses a global dict to store urls with the corresponding content. +If you run your servers in a pool you may wish to share your WSDL cache across multiple servers if they are making +similar calls. To do this you can offload the cache to a shared redis instance by setting a redis cache as follows: + +.. code-block:: python + from zeep import Client + from zeep.transports import Transport + from zeep.cache import RedisCache + + cache = RedisCache( + redis_host="127.0.0.1", + password="APasswordYouLike", + timeout=60 + ) + + transport = Transport( + cache=cache, + ) + + client = Client( + 'http://www.webservicex.net/ConvertSpeed.asmx?WSDL', + transport=transport) + + + + HTTP Authentication ------------------- While some providers incorporate security features in the header of a SOAP message, diff --git a/src/zeep/cache.py b/src/zeep/cache.py index 630d91e9..966f001b 100644 --- a/src/zeep/cache.py +++ b/src/zeep/cache.py @@ -169,7 +169,7 @@ class RedisCache(Base): - This is helpful if you make zeep calls from a pool of servers that need to share a common cache """ - def __init__(self, redis_host, password, db=0, port=6379, timeout=3600): + def __init__(self, redis_host, password, port=6379, timeout=3600, health_check_interval=10, socket_timeout=5, retry_on_timeout=True, single_connection_client=True): self._timeout = timeout self._redis_host = redis_host @@ -177,7 +177,10 @@ def __init__(self, redis_host, password, db=0, port=6379, timeout=3600): host=redis_host, port=port, password=password, - db=db + health_check_interval=health_check_interval, + socket_timeout=socket_timeout, + retry_on_timeout=retry_on_timeout, + single_connection_client = single_connection_client ) def add(self, url, content): @@ -192,9 +195,9 @@ def add(self, url, content): def get(self, url): cached_value = self._redis_client.get(url) - if not _is_expired(cached_value['time'], self._timeout): + if cached_value is not None and not _is_expired(cached_value['time'], self._timeout): logger.debug("Cache HIT for %s", url) - return cached_value['value'] + return cached_value.get('value', None) else: logger.debug("Cache MISS for %s", url) return None From 15a00ecaa0e12946a290bdc1f84018bc3f0f131a Mon Sep 17 00:00:00 2001 From: davidcolangelo Date: Wed, 16 Apr 2025 11:11:06 -0400 Subject: [PATCH 4/6] - make sure its a string for redis - protect against cache failures --- src/zeep/cache.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/zeep/cache.py b/src/zeep/cache.py index 966f001b..ce84a8c6 100644 --- a/src/zeep/cache.py +++ b/src/zeep/cache.py @@ -7,6 +7,7 @@ from contextlib import contextmanager from typing import Dict, Tuple, Union import redis +import json import platformdirs import pytz @@ -187,14 +188,30 @@ def add(self, url, content): logger.debug("Caching contents of %s", url) # Remove the cached key self._redis_client.delete(url) - # add the new cache response for the url - self._redis_client.set(url, value={ - 'time': datetime.datetime.now(datetime.timezone.utc), - 'value': content - }) + + try: + # Stringify the data + data = json.dumps({ + 'time': datetime.datetime.now(datetime.timezone.utc), + 'value': content + }) + + # add the new cache response for the url + self._redis_client.set(url, value=data) + except Exception as e: + logger.debug("Could not cache contents of %s", url) + logger.debug(e) def get(self, url): - cached_value = self._redis_client.get(url) + + try: + cached_value = json.loads(self._redis_client.get(url)) + except Exception as e: + logger.debug("Could extract from cache contents of %s", url) + logger.debug(e) + # if we cant decode it just return none + return None + if cached_value is not None and not _is_expired(cached_value['time'], self._timeout): logger.debug("Cache HIT for %s", url) return cached_value.get('value', None) From 9c08bd3b823381bb78876627ab24717e8f0c6255 Mon Sep 17 00:00:00 2001 From: davidcolangelo Date: Wed, 16 Apr 2025 12:49:03 -0400 Subject: [PATCH 5/6] - fixing encoding and adding some logging to debug issues --- src/zeep/cache.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/zeep/cache.py b/src/zeep/cache.py index ce84a8c6..7132ee40 100644 --- a/src/zeep/cache.py +++ b/src/zeep/cache.py @@ -190,10 +190,10 @@ def add(self, url, content): self._redis_client.delete(url) try: - # Stringify the data + # Stringify the data and add the time so we know when it was written data = json.dumps({ - 'time': datetime.datetime.now(datetime.timezone.utc), - 'value': content + 'time': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'value': base64.b64encode(content).decode('utf-8') }) # add the new cache response for the url @@ -205,16 +205,25 @@ def add(self, url, content): def get(self, url): try: - cached_value = json.loads(self._redis_client.get(url)) + value = self._redis_client.get(url) + if value is None: + logger.debug("Cache MISS for %s", url) + return None + + cached_value = json.loads(value) except Exception as e: - logger.debug("Could extract from cache contents of %s", url) + logger.debug("Could not extract from cache contents of %s", url) logger.debug(e) # if we cant decode it just return none return None - if cached_value is not None and not _is_expired(cached_value['time'], self._timeout): + if cached_value is not None and not _is_expired(datetime.datetime.fromisoformat(cached_value['time']), self._timeout): logger.debug("Cache HIT for %s", url) - return cached_value.get('value', None) + value = cached_value.get('value', None) + if value is not None: + return base64.b64decode(value) + else: + return None else: logger.debug("Cache MISS for %s", url) return None From 6356725590eab01fe20e581401120429335b6c5a Mon Sep 17 00:00:00 2001 From: davidcolangelo Date: Thu, 17 Apr 2025 15:16:07 -0400 Subject: [PATCH 6/6] - affix the redis version due to latest CVE found --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bc31bfbd..54016fde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "requests-toolbelt>=0.7.1", "requests-file>=1.5.1", "pytz", - "redis" + "redis>=5.2.1" ] [project.urls]