diff --git a/plugins/loaders/target-jsonl--andyh1203.lock b/plugins/loaders/target-jsonl--andyh1203.lock new file mode 100644 index 0000000..5825fc4 --- /dev/null +++ b/plugins/loaders/target-jsonl--andyh1203.lock @@ -0,0 +1,34 @@ +{ + "plugin_type": "loaders", + "name": "target-jsonl", + "namespace": "target_jsonl", + "variant": "andyh1203", + "label": "JSON Lines (JSONL)", + "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203", + "repo": "https://github.com/andyh1203/target-jsonl", + "pip_url": "target-jsonl", + "description": "JSONL loader", + "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png", + "settings": [ + { + "name": "destination_path", + "kind": "string", + "value": "output", + "label": "Destination Path", + "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n" + }, + { + "name": "do_timestamp_file", + "kind": "boolean", + "value": false, + "label": "Include Timestamp in File Names", + "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n" + }, + { + "name": "custom_name", + "kind": "string", + "label": "Custom File Name Override", + "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n" + } + ] +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b4b1b4a..fd89c88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tap-salesforce-connect" -version = "0.1.0" +version = "0.1.1" description = "`tap-salesforce-connect` is a Singer tap for SalesforceConnect, built with the Meltano Singer SDK." readme = "README.md" authors = ["Josh Lloyd"] diff --git a/tap_salesforce_connect/client.py b/tap_salesforce_connect/client.py index ce06780..9d700f8 100644 --- a/tap_salesforce_connect/client.py +++ b/tap_salesforce_connect/client.py @@ -5,8 +5,9 @@ import json import sys from pathlib import Path -from typing import Any, Callable, Iterable +from typing import Any, Callable, Generator, Iterable +import backoff import requests from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import RESTStream @@ -108,3 +109,25 @@ def post_process(self, row: dict, context: dict | None = None) -> dict | None: else: stringified_row[k] = v return stringified_row + + # there is a one-hour rate limit on the SalesforceConnect API + def backoff_wait_generator(self) -> Generator[float, None, None]: + """Use wait generator for the backoff decorator on request failure. + + See for options: + https://github.com/litl/backoff/blob/master/backoff/_wait_gen.py + + And see for examples: `Code Samples <../code_samples.html#custom-backoff>`_ + + Returns: + The wait generator + """ + return backoff.expo(factor=3) # type: ignore # ignore 'Returning Any' + + def backoff_max_tries(self) -> int: + """Provide the number of attempts before giving up when retrying requests. + + Returns: + Number of max retries. + """ + return 20 diff --git a/tap_salesforce_connect/tap.py b/tap_salesforce_connect/tap.py index 4174086..7b4c98d 100644 --- a/tap_salesforce_connect/tap.py +++ b/tap_salesforce_connect/tap.py @@ -60,7 +60,7 @@ def discover_streams(self) -> list[streams.SalesforceConnectStream]: streams.TopicFeedStream(self), streams.UsersStream(self), streams.TopicsStream(self), - streams.CommentsStream(self) + streams.CommentsStream(self), ]