For testing, retry server errors from the container multiple times

tests/test_convert_chromium_html.py

@@ -14,6 +14,7 @@
 from tests.conftest import SAMPLE_DIR
 from tests.conftest import SAVE_DIR
 from tests.conftest import SAVE_OUTPUTS
+from tests.utils import call_run_with_server_error_handling
 from tests.utils import verify_stream_contains
 
 
@@ -22,7 +23,7 @@ def test_basic_convert(self, client: GotenbergClient):
         test_file = SAMPLE_DIR / "basic.html"
 
         with client.chromium.html_to_pdf() as route:
-            resp = route.index(test_file).run()
+            resp = call_run_with_server_error_handling(route.index(test_file))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -36,7 +37,7 @@ def test_convert_with_header_footer(self, client: GotenbergClient):
         footer_file = SAMPLE_DIR / "footer.html"
 
         with client.chromium.html_to_pdf() as route:
-            resp = route.index(test_file).header(header_file).footer(footer_file).run()
+            resp = call_run_with_server_error_handling(route.index(test_file).header(header_file).footer(footer_file))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -49,7 +50,9 @@ def test_convert_additional_files(self, client: GotenbergClient):
         style = SAMPLE_DIR / "style.css"
 
         with client.chromium.html_to_pdf() as route:
-            resp = route.index(test_file).resource(img).resource(font).resource(style).run()
+            resp = call_run_with_server_error_handling(
+                route.index(test_file).resource(img).resource(font).resource(style),
+            )
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -66,7 +69,7 @@ def test_convert_pdfa_1a_format(self, client: GotenbergClient, gt_format: PdfAFo
         test_file = SAMPLE_DIR / "basic.html"
 
         with client.chromium.html_to_pdf() as route:
-            resp = route.index(test_file).pdf_format(gt_format).run()
+            resp = call_run_with_server_error_handling(route.index(test_file).pdf_format(gt_format))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers

tests/test_convert_chromium_markdown.py

@@ -2,6 +2,7 @@
 
 from gotenberg_client._client import GotenbergClient
 from tests.conftest import SAMPLE_DIR
+from tests.utils import call_run_with_server_error_handling
 
 
 class TestConvertChromiumUrlRoute:
@@ -12,7 +13,9 @@ def test_basic_convert(self, client: GotenbergClient):
         font = SAMPLE_DIR / "font.woff"
         style = SAMPLE_DIR / "style.css"
         with client.chromium.markdown_to_pdf() as route:
-            resp = route.index(index).markdown_files(md_files).resources([img, font]).resource(style).run()
+            resp = call_run_with_server_error_handling(
+                route.index(index).markdown_files(md_files).resources([img, font]).resource(style),
+            )
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers

tests/test_convert_chromium_url.py

@@ -6,18 +6,23 @@
 
 from gotenberg_client._client import GotenbergClient
 from gotenberg_client._convert.chromium import EmulatedMediaType
+from tests.utils import call_run_with_server_error_handling
 from tests.utils import verify_stream_contains
 
 
 class TestConvertChromiumUrlRoute:
     def test_basic_convert(self, client: GotenbergClient):
         with client.chromium.url_to_pdf() as route:
-            resp = route.url("https://en.wikipedia.org/wiki/William_Edward_Sanders").run()
+            resp = call_run_with_server_error_handling(
+                route.url("https://en.wikipedia.org/wiki/William_Edward_Sanders"),
+            )
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
         assert resp.headers["Content-Type"] == "application/pdf"
 
+
+class TestConvertChromiumUrlMocked:
     @pytest.mark.parametrize(
         ("emulation"),
         [EmulatedMediaType.Screen, EmulatedMediaType.Print],

tests/test_convert_libre_office.py

@@ -12,13 +12,14 @@
 from tests.conftest import SAMPLE_DIR
 from tests.conftest import SAVE_DIR
 from tests.conftest import SAVE_OUTPUTS
+from tests.utils import call_run_with_server_error_handling
 
 
 class TestLibreOfficeConvert:
     def test_libre_office_convert_docx_format(self, client: GotenbergClient):
         test_file = SAMPLE_DIR / "sample.docx"
         with client.libre_office.to_pdf() as route:
-            resp = route.convert(test_file).run()
+            resp = call_run_with_server_error_handling(route.convert(test_file))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -30,7 +31,7 @@ def test_libre_office_convert_docx_format(self, client: GotenbergClient):
     def test_libre_office_convert_odt_format(self, client: GotenbergClient):
         test_file = SAMPLE_DIR / "sample.odt"
         with client.libre_office.to_pdf() as route:
-            resp = route.convert(test_file).run()
+            resp = call_run_with_server_error_handling(route.convert(test_file))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -42,7 +43,7 @@ def test_libre_office_convert_odt_format(self, client: GotenbergClient):
     def test_libre_office_convert_xlsx_format(self, client: GotenbergClient):
         test_file = SAMPLE_DIR / "sample.xlsx"
         with client.libre_office.to_pdf() as route:
-            resp = route.convert(test_file).run()
+            resp = call_run_with_server_error_handling(route.convert(test_file))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -54,7 +55,7 @@ def test_libre_office_convert_xlsx_format(self, client: GotenbergClient):
     def test_libre_office_convert_ods_format(self, client: GotenbergClient):
         test_file = SAMPLE_DIR / "sample.ods"
         with client.libre_office.to_pdf() as route:
-            resp = route.convert(test_file).run()
+            resp = call_run_with_server_error_handling(route.convert(test_file))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -65,7 +66,9 @@ def test_libre_office_convert_ods_format(self, client: GotenbergClient):
 
     def test_libre_office_convert_multiples_format(self, client: GotenbergClient):
         with client.libre_office.to_pdf() as route:
-            resp = route.convert_files([SAMPLE_DIR / "sample.docx", SAMPLE_DIR / "sample.odt"]).no_merge().run()
+            resp = call_run_with_server_error_handling(
+                route.convert_files([SAMPLE_DIR / "sample.docx", SAMPLE_DIR / "sample.odt"]).no_merge(),
+            )
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -76,7 +79,9 @@ def test_libre_office_convert_multiples_format(self, client: GotenbergClient):
 
     def test_libre_office_convert_multiples_format_merged(self, client: GotenbergClient):
         with client.libre_office.to_pdf() as route:
-            resp = route.convert_files([SAMPLE_DIR / "sample.docx", SAMPLE_DIR / "sample.odt"]).merge().run()
+            resp = call_run_with_server_error_handling(
+                route.convert_files([SAMPLE_DIR / "sample.docx", SAMPLE_DIR / "sample.odt"]).merge(),
+            )
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -89,7 +94,9 @@ def test_libre_office_convert_std_lib_mime(self, client: GotenbergClient):
         with patch("gotenberg_client._utils.guess_mime_type") as mocked_guess_mime_type:
             mocked_guess_mime_type.side_effect = guess_mime_type_stdlib
             with client.libre_office.to_pdf() as route:
-                resp = route.convert_files([SAMPLE_DIR / "sample.docx", SAMPLE_DIR / "sample.odt"]).no_merge().run()
+                resp = call_run_with_server_error_handling(
+                    route.convert_files([SAMPLE_DIR / "sample.docx", SAMPLE_DIR / "sample.odt"]).no_merge(),
+                )
 
             assert resp.status_code == codes.OK
             assert "Content-Type" in resp.headers
@@ -110,7 +117,7 @@ def test_libre_office_convert_xlsx_format_pdfa(
     ):
         test_file = SAMPLE_DIR / "sample.xlsx"
         with client.libre_office.to_pdf() as route:
-            resp = route.convert(test_file).pdf_format(gt_format).run()
+            resp = call_run_with_server_error_handling(route.convert(test_file).pdf_format(gt_format))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers

tests/test_convert_pdf_a.py

@@ -10,6 +10,7 @@
 from tests.conftest import SAMPLE_DIR
 from tests.conftest import SAVE_DIR
 from tests.conftest import SAVE_OUTPUTS
+from tests.utils import call_run_with_server_error_handling
 
 
 class TestPdfAConvert:
@@ -25,7 +26,7 @@ def test_pdf_a_single_file(
     ):
         test_file = SAMPLE_DIR / "sample1.pdf"
         with client.pdf_a.to_pdfa() as route:
-            resp = route.convert(test_file).pdf_format(gt_format).run()
+            resp = call_run_with_server_error_handling(route.convert(test_file).pdf_format(gt_format))
 
         assert resp.status_code == codes.OK
         assert "Content-Type" in resp.headers
@@ -52,7 +53,9 @@ def test_pdf_a_multiple_file(
             other_test_file = Path(temp_dir) / "sample2.pdf"
             other_test_file.write_bytes(test_file.read_bytes())
             with client.pdf_a.to_pdfa() as route:
-                resp = route.convert_files([test_file, other_test_file]).pdf_format(gt_format).run()
+                resp = call_run_with_server_error_handling(
+                    route.convert_files([test_file, other_test_file]).pdf_format(gt_format),
+                )
 
                 assert resp.status_code == codes.OK
                 assert "Content-Type" in resp.headers

tests/utils.py

@@ -1,7 +1,14 @@
+import time
+import warnings
+
+from httpx import HTTPStatusError
+from httpx import Response
 from httpx._multipart import DataField
 from httpx._multipart import FileField
 from httpx._multipart import MultipartStream
 
+from gotenberg_client._base import BaseRoute
+
 
 def verify_stream_contains(key: str, value: str, stream: MultipartStream):
     for item in stream.fields:
@@ -13,3 +20,43 @@ def verify_stream_contains(key: str, value: str, stream: MultipartStream):
 
     msg = f'Key "{key}" with value "{value}" not found in stream'
     raise AssertionError(msg)
+
+
+def call_run_with_server_error_handling(route: BaseRoute) -> Response:
+    """
+    For whatever reason, the images started during the test pipeline like to
+    segfault sometimes, crash and otherwise fail randomly, when run with the
+    exact files that usually pass.
+
+    So, this function will retry the given method/function up to 3 times, with larger backoff
+    periods between each attempt, in hopes the issue resolves itself during
+    one attempt to parse.
+
+    This will wait the following:
+        - Attempt 1 - 20s following failure
+        - Attempt 2 - 40s following failure
+        - Attempt 3 - 80s following failure
+        - Attempt 4 - 160s
+        - Attempt 5 - 320s
+
+    """
+    result = None
+    succeeded = False
+    retry_time = 20.0
+    retry_count = 0
+    max_retry_count = 5
+
+    while retry_count < max_retry_count and not succeeded:
+        try:
+            return route.run()
+        except HTTPStatusError as e:  # pragma: no cover
+            warnings.warn(f"HTTP error: {e}", stacklevel=1)
+        except Exception as e:  # pragma: no cover
+            warnings.warn(f"Unexpected error: {e}", stacklevel=1)
+
+        retry_count = retry_count + 1
+
+        time.sleep(retry_time)
+        retry_time = retry_time * 2.0
+
+    return result