increased the timeout ramp (#1)

plugins/loaders/target-jsonl--andyh1203.lock

@@ -0,0 +1,34 @@
+{
+  "plugin_type": "loaders",
+  "name": "target-jsonl",
+  "namespace": "target_jsonl",
+  "variant": "andyh1203",
+  "label": "JSON Lines (JSONL)",
+  "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203",
+  "repo": "https://github.com/andyh1203/target-jsonl",
+  "pip_url": "target-jsonl",
+  "description": "JSONL loader",
+  "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png",
+  "settings": [
+    {
+      "name": "destination_path",
+      "kind": "string",
+      "value": "output",
+      "label": "Destination Path",
+      "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n"
+    },
+    {
+      "name": "do_timestamp_file",
+      "kind": "boolean",
+      "value": false,
+      "label": "Include Timestamp in File Names",
+      "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n"
+    },
+    {
+      "name": "custom_name",
+      "kind": "string",
+      "label": "Custom File Name Override",
+      "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n"
+    }
+  ]
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tap-salesforce-connect"
-version = "0.1.0"
+version = "0.1.1"
 description = "`tap-salesforce-connect` is a Singer tap for SalesforceConnect, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["Josh Lloyd"]

tap_salesforce_connect/client.py

@@ -5,8 +5,9 @@
 import json
 import sys
 from pathlib import Path
-from typing import Any, Callable, Iterable
+from typing import Any, Callable, Generator, Iterable
 
+import backoff
 import requests
 from singer_sdk.helpers.jsonpath import extract_jsonpath
 from singer_sdk.streams import RESTStream
@@ -108,3 +109,25 @@ def post_process(self, row: dict, context: dict | None = None) -> dict | None:
             else:
                 stringified_row[k] = v
         return stringified_row
+
+    # there is a one-hour rate limit on the SalesforceConnect API
+    def backoff_wait_generator(self) -> Generator[float, None, None]:
+        """Use wait generator for the backoff decorator on request failure.
+
+        See for options:
+        https://github.com/litl/backoff/blob/master/backoff/_wait_gen.py
+
+        And see for examples: `Code Samples <../code_samples.html#custom-backoff>`_
+
+        Returns:
+            The wait generator
+        """
+        return backoff.expo(factor=3)  # type: ignore # ignore 'Returning Any'
+
+    def backoff_max_tries(self) -> int:
+        """Provide the number of attempts before giving up when retrying requests.
+
+        Returns:
+            Number of max retries.
+        """
+        return 20

tap_salesforce_connect/tap.py

@@ -60,7 +60,7 @@ def discover_streams(self) -> list[streams.SalesforceConnectStream]:
             streams.TopicFeedStream(self),
             streams.UsersStream(self),
             streams.TopicsStream(self),
-            streams.CommentsStream(self)
+            streams.CommentsStream(self),
         ]