From 0b4b2c8f716389e973268d9bf6c612997bfcc460 Mon Sep 17 00:00:00 2001 From: Josh lloyd Date: Wed, 3 Apr 2024 09:56:15 -0600 Subject: [PATCH] fixed accounts stream --- plugins/loaders/target-jsonl--andyh1203.lock | 34 +++++++++++++++ tap_gainsightpx/streams.py | 46 ++++++++++---------- tap_gainsightpx/tests/test_core.py | 4 ++ 3 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 plugins/loaders/target-jsonl--andyh1203.lock diff --git a/plugins/loaders/target-jsonl--andyh1203.lock b/plugins/loaders/target-jsonl--andyh1203.lock new file mode 100644 index 0000000..5825fc4 --- /dev/null +++ b/plugins/loaders/target-jsonl--andyh1203.lock @@ -0,0 +1,34 @@ +{ + "plugin_type": "loaders", + "name": "target-jsonl", + "namespace": "target_jsonl", + "variant": "andyh1203", + "label": "JSON Lines (JSONL)", + "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203", + "repo": "https://github.com/andyh1203/target-jsonl", + "pip_url": "target-jsonl", + "description": "JSONL loader", + "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png", + "settings": [ + { + "name": "destination_path", + "kind": "string", + "value": "output", + "label": "Destination Path", + "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n" + }, + { + "name": "do_timestamp_file", + "kind": "boolean", + "value": false, + "label": "Include Timestamp in File Names", + "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n" + }, + { + "name": "custom_name", + "kind": "string", + "label": "Custom File Name Override", + "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n" + } + ] +} \ No newline at end of file diff --git a/tap_gainsightpx/streams.py b/tap_gainsightpx/streams.py index 5cd30ef..6f53033 100644 --- a/tap_gainsightpx/streams.py +++ b/tap_gainsightpx/streams.py @@ -18,40 +18,38 @@ class AccountsStream(GainsightPXStream): primary_keys = ["id"] replication_key = "lastModifiedDate" schema = th.PropertiesList( - th.Property("id", th.StringType), - th.Property("name", th.StringType), - th.Property("trackedSubscriptionId", th.StringType), - th.Property("sfdcId", th.StringType), - th.Property("lastSeenDate", th.IntegerType), + th.Property("createDate", th.IntegerType), + th.Property("customAttributes", th.ObjectType()), th.Property("dunsNumber", th.StringType), + th.Property("id", th.StringType), th.Property("industry", th.StringType), - th.Property("numberOfEmployees", th.IntegerType), - th.Property("sicCode", th.StringType), - th.Property("website", th.StringType), - th.Property("naicsCode", th.StringType), - th.Property("plan", th.StringType), + th.Property("lastModifiedDate", th.IntegerType), + th.Property("lastSeenDate", th.IntegerType), th.Property("location", th.ObjectType()), + th.Property("naicsCode", th.StringType), + th.Property("name", th.StringType), + th.Property("numberOfEmployees", th.IntegerType), th.Property("numberOfUsers", th.IntegerType), - th.Property("propertyKeys", th.ArrayType(th.StringType)), - th.Property("createDate", th.IntegerType), - th.Property("lastModifiedDate", th.IntegerType), - th.Property("customAttributes", th.ObjectType()), th.Property("parentGroupId", th.StringType), + th.Property("plan", th.StringType), + th.Property("propertyKeys", th.ArrayType(th.StringType)), + th.Property("sfdcId", th.StringType), + th.Property("sicCode", th.StringType), + th.Property("trackedSubscriptionId", th.StringType), + th.Property("website", th.StringType), ).to_dict() - def add_more_url_params( - self, params: dict, next_page_token: Optional[Any] + def get_url_params( + self, context: Optional[dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: - """Add more params specific to the stream.""" + """Return a dictionary of values to be used in URL parameterization.""" + params: dict = {"pageSize": 100} + + if self.replication_key: + params["sort"] = self.replication_key if next_page_token: - params["scrollId"] = next_page_token + params["scrollID"] = next_page_token - # todo: enable replication key method - # bookmarks = self.stream_state['bookmarks'] - # replication_key_value = bookmarks[self.name]['replication_key_value'] - # params["filter"] = ";".join([ - # f"{self.replication_key}>{replication_key_value}", - # ]) return params diff --git a/tap_gainsightpx/tests/test_core.py b/tap_gainsightpx/tests/test_core.py index 36ad1b7..101c33d 100644 --- a/tap_gainsightpx/tests/test_core.py +++ b/tap_gainsightpx/tests/test_core.py @@ -53,6 +53,10 @@ def test_standard_tap_tests(requests_mock): "https://api.example.com/v1/feature?pageSize=200", json=json_resp(), ) + requests_mock.get( + "https://api.example.com/v1/accounts?pageSize=100&sort=lastModifiedDate", + json=json_resp(), + ) requests_mock.get( "https://api.example.com/v1/events/pageView?pageSize=500&sort=date&filter=" "date%3E%3D2022-01-01T00%3A00%3A00Z%3B"