From 8209536f8236b2f68ebfd11b7749f5cc669fbb6d Mon Sep 17 00:00:00 2001 From: Paul Blankley <31221512+pblankley@users.noreply.github.com> Date: Thu, 13 Aug 2020 16:44:57 -0400 Subject: [PATCH 01/11] Fix json.dumps() error --- tap_klaviyo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_klaviyo/__init__.py b/tap_klaviyo/__init__.py index d724331..d8824e0 100644 --- a/tap_klaviyo/__init__.py +++ b/tap_klaviyo/__init__.py @@ -141,7 +141,7 @@ def discover(api_key): def do_discover(api_key): - print(json.dumps(discover(api_key), sys.stdout, indent=2)) + print(json.dumps(discover(api_key), indent=2)) def main(): From 807d51423f550dac9ef74c5d5fc284f659f1ef1d Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Wed, 26 Aug 2020 21:28:33 -0400 Subject: [PATCH 02/11] add campaign params to schemas --- tap_klaviyo/schemas/bounce.json | 18 +++++++++++++++++- tap_klaviyo/schemas/click.json | 18 +++++++++++++++++- tap_klaviyo/schemas/dropped_email.json | 12 +++++++++++- tap_klaviyo/schemas/mark_as_spam.json | 12 +++++++++++- tap_klaviyo/schemas/open.json | 12 +++++++++++- tap_klaviyo/schemas/receive.json | 12 +++++++++++- tap_klaviyo/schemas/unsubscribe.json | 12 +++++++++++- 7 files changed, 89 insertions(+), 7 deletions(-) diff --git a/tap_klaviyo/schemas/bounce.json b/tap_klaviyo/schemas/bounce.json index 0bf24d7..4718504 100644 --- a/tap_klaviyo/schemas/bounce.json +++ b/tap_klaviyo/schemas/bounce.json @@ -24,13 +24,29 @@ }, "event_properties": { "type": "object", - "properties": {} + "properties": { + "event_id": { + "type": "string" + }, + "Campaign Name": { + "type": "string" + }, + "Subject": { + "type": "string" + } + } }, "person": { "type": "object", "properties": { "email": { "type": "string" + }, + "timezone": { + "type": "string" + }, + "country": { + "type": "string" } } } diff --git a/tap_klaviyo/schemas/click.json b/tap_klaviyo/schemas/click.json index 0bf24d7..40b98c8 100644 --- a/tap_klaviyo/schemas/click.json +++ b/tap_klaviyo/schemas/click.json @@ -24,13 +24,29 @@ }, "event_properties": { "type": "object", - "properties": {} + "properties": { + "event_id": { + "type": "string" + }, + "Campaign Name": { + "type": "string" + }, + "Subject": { + "type": "string" + } + } }, "person": { "type": "object", "properties": { "email": { "type": "string" + }, + "timezone": { + "type": "string" + }, + "country": { + "type": "string" } } } diff --git a/tap_klaviyo/schemas/dropped_email.json b/tap_klaviyo/schemas/dropped_email.json index ae772b3..8ef3184 100644 --- a/tap_klaviyo/schemas/dropped_email.json +++ b/tap_klaviyo/schemas/dropped_email.json @@ -24,7 +24,17 @@ }, "event_properties": { "type": "object", - "properties": {} + "properties": { + "event_id": { + "type": "string" + }, + "Campaign Name": { + "type": "string" + }, + "Subject": { + "type": "string" + } + } }, "person": { "type": "object", diff --git a/tap_klaviyo/schemas/mark_as_spam.json b/tap_klaviyo/schemas/mark_as_spam.json index 0bf24d7..39a91bb 100644 --- a/tap_klaviyo/schemas/mark_as_spam.json +++ b/tap_klaviyo/schemas/mark_as_spam.json @@ -24,7 +24,17 @@ }, "event_properties": { "type": "object", - "properties": {} + "properties": { + "event_id": { + "type": "string" + }, + "Campaign Name": { + "type": "string" + }, + "Subject": { + "type": "string" + } + } }, "person": { "type": "object", diff --git a/tap_klaviyo/schemas/open.json b/tap_klaviyo/schemas/open.json index 0bf24d7..39a91bb 100644 --- a/tap_klaviyo/schemas/open.json +++ b/tap_klaviyo/schemas/open.json @@ -24,7 +24,17 @@ }, "event_properties": { "type": "object", - "properties": {} + "properties": { + "event_id": { + "type": "string" + }, + "Campaign Name": { + "type": "string" + }, + "Subject": { + "type": "string" + } + } }, "person": { "type": "object", diff --git a/tap_klaviyo/schemas/receive.json b/tap_klaviyo/schemas/receive.json index 0bf24d7..39a91bb 100644 --- a/tap_klaviyo/schemas/receive.json +++ b/tap_klaviyo/schemas/receive.json @@ -24,7 +24,17 @@ }, "event_properties": { "type": "object", - "properties": {} + "properties": { + "event_id": { + "type": "string" + }, + "Campaign Name": { + "type": "string" + }, + "Subject": { + "type": "string" + } + } }, "person": { "type": "object", diff --git a/tap_klaviyo/schemas/unsubscribe.json b/tap_klaviyo/schemas/unsubscribe.json index 0bf24d7..39a91bb 100644 --- a/tap_klaviyo/schemas/unsubscribe.json +++ b/tap_klaviyo/schemas/unsubscribe.json @@ -24,7 +24,17 @@ }, "event_properties": { "type": "object", - "properties": {} + "properties": { + "event_id": { + "type": "string" + }, + "Campaign Name": { + "type": "string" + }, + "Subject": { + "type": "string" + } + } }, "person": { "type": "object", From 49b84dd355e6f410392c1d8bd94c3b358e4e8a3c Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Wed, 26 Aug 2020 22:21:43 -0400 Subject: [PATCH 03/11] modify schemas and add logging to utils --- .gitignore | 3 ++- tap_klaviyo/schemas/dropped_email.json | 12 +----------- tap_klaviyo/schemas/mark_as_spam.json | 12 +----------- tap_klaviyo/schemas/unsubscribe.json | 14 ++------------ tap_klaviyo/utils.py | 4 +++- 5 files changed, 9 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 6832776..5982ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ - +.env *.egg-info/ */__pycache__/ +venv/ diff --git a/tap_klaviyo/schemas/dropped_email.json b/tap_klaviyo/schemas/dropped_email.json index 8ef3184..ae772b3 100644 --- a/tap_klaviyo/schemas/dropped_email.json +++ b/tap_klaviyo/schemas/dropped_email.json @@ -24,17 +24,7 @@ }, "event_properties": { "type": "object", - "properties": { - "event_id": { - "type": "string" - }, - "Campaign Name": { - "type": "string" - }, - "Subject": { - "type": "string" - } - } + "properties": {} }, "person": { "type": "object", diff --git a/tap_klaviyo/schemas/mark_as_spam.json b/tap_klaviyo/schemas/mark_as_spam.json index 39a91bb..0bf24d7 100644 --- a/tap_klaviyo/schemas/mark_as_spam.json +++ b/tap_klaviyo/schemas/mark_as_spam.json @@ -24,17 +24,7 @@ }, "event_properties": { "type": "object", - "properties": { - "event_id": { - "type": "string" - }, - "Campaign Name": { - "type": "string" - }, - "Subject": { - "type": "string" - } - } + "properties": {} }, "person": { "type": "object", diff --git a/tap_klaviyo/schemas/unsubscribe.json b/tap_klaviyo/schemas/unsubscribe.json index 39a91bb..f7e8fd5 100644 --- a/tap_klaviyo/schemas/unsubscribe.json +++ b/tap_klaviyo/schemas/unsubscribe.json @@ -19,22 +19,12 @@ "statistic_id": { "type": "string" }, - "id": { + "iddd": { "type": "string" }, "event_properties": { "type": "object", - "properties": { - "event_id": { - "type": "string" - }, - "Campaign Name": { - "type": "string" - }, - "Subject": { - "type": "string" - } - } + "properties": {} }, "person": { "type": "object", diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index 655355b..a8126c8 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -96,7 +96,9 @@ def get_incremental_pull(stream, endpoint, state, api_key, start_date): if events: counter.increment(len(events)) - + + print('getting events') + print(events) singer.write_records(stream['stream'], events) update_state(state, stream['stream'], get_latest_event_time(events)) From 82a3eb5c162b161f740a863a44d78f2ea566a109 Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Wed, 26 Aug 2020 22:25:38 -0400 Subject: [PATCH 04/11] logging in utils --- tap_klaviyo/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index a8126c8..5b6c6e8 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -97,8 +97,8 @@ def get_incremental_pull(stream, endpoint, state, api_key, start_date): if events: counter.increment(len(events)) - print('getting events') - print(events) + logger.info('getting events') + logger.info(events) singer.write_records(stream['stream'], events) update_state(state, stream['stream'], get_latest_event_time(events)) From 106a266ebb378b7aaf1fff6f76c11c4fff88acc1 Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Thu, 27 Aug 2020 07:54:15 -0400 Subject: [PATCH 05/11] update event id prop --- tap_klaviyo/schemas/bounce.json | 2 +- tap_klaviyo/schemas/click.json | 2 +- tap_klaviyo/schemas/open.json | 2 +- tap_klaviyo/schemas/receive.json | 2 +- tap_klaviyo/utils.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tap_klaviyo/schemas/bounce.json b/tap_klaviyo/schemas/bounce.json index 4718504..cd559b1 100644 --- a/tap_klaviyo/schemas/bounce.json +++ b/tap_klaviyo/schemas/bounce.json @@ -25,7 +25,7 @@ "event_properties": { "type": "object", "properties": { - "event_id": { + "$event_id": { "type": "string" }, "Campaign Name": { diff --git a/tap_klaviyo/schemas/click.json b/tap_klaviyo/schemas/click.json index 40b98c8..78fd88c 100644 --- a/tap_klaviyo/schemas/click.json +++ b/tap_klaviyo/schemas/click.json @@ -25,7 +25,7 @@ "event_properties": { "type": "object", "properties": { - "event_id": { + "$event_id": { "type": "string" }, "Campaign Name": { diff --git a/tap_klaviyo/schemas/open.json b/tap_klaviyo/schemas/open.json index 39a91bb..1ffddd6 100644 --- a/tap_klaviyo/schemas/open.json +++ b/tap_klaviyo/schemas/open.json @@ -25,7 +25,7 @@ "event_properties": { "type": "object", "properties": { - "event_id": { + "$event_id": { "type": "string" }, "Campaign Name": { diff --git a/tap_klaviyo/schemas/receive.json b/tap_klaviyo/schemas/receive.json index 39a91bb..1ffddd6 100644 --- a/tap_klaviyo/schemas/receive.json +++ b/tap_klaviyo/schemas/receive.json @@ -25,7 +25,7 @@ "event_properties": { "type": "object", "properties": { - "event_id": { + "$event_id": { "type": "string" }, "Campaign Name": { diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index 5b6c6e8..8ca8555 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -97,8 +97,6 @@ def get_incremental_pull(stream, endpoint, state, api_key, start_date): if events: counter.increment(len(events)) - logger.info('getting events') - logger.info(events) singer.write_records(stream['stream'], events) update_state(state, stream['stream'], get_latest_event_time(events)) @@ -112,6 +110,8 @@ def get_full_pulls(resource, endpoint, api_key): for response in get_all_pages(resource['stream'], endpoint, api_key): records = response.json().get('data') + logger.info('getting events') + logger.info(records) counter.increment(len(records)) singer.write_records(resource['stream'], records) From 70b7a0cd5130dc37872dd199cf2d5398b1a2d88b Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Thu, 27 Aug 2020 08:56:35 -0400 Subject: [PATCH 06/11] update schemas --- tap_klaviyo/schemas/bounce.json | 6 +++--- tap_klaviyo/schemas/click.json | 14 ++++---------- tap_klaviyo/schemas/open.json | 6 +++--- tap_klaviyo/schemas/receive.json | 6 +++--- tap_klaviyo/schemas/subscribe_list.json | 3 --- tap_klaviyo/schemas/unsubscribe.json | 2 +- tap_klaviyo/utils.py | 2 ++ 7 files changed, 16 insertions(+), 23 deletions(-) diff --git a/tap_klaviyo/schemas/bounce.json b/tap_klaviyo/schemas/bounce.json index cd559b1..da28dba 100644 --- a/tap_klaviyo/schemas/bounce.json +++ b/tap_klaviyo/schemas/bounce.json @@ -25,14 +25,14 @@ "event_properties": { "type": "object", "properties": { - "$event_id": { - "type": "string" - }, "Campaign Name": { "type": "string" }, "Subject": { "type": "string" + }, + "$event_id": { + "type": "string" } } }, diff --git a/tap_klaviyo/schemas/click.json b/tap_klaviyo/schemas/click.json index 78fd88c..42fc369 100644 --- a/tap_klaviyo/schemas/click.json +++ b/tap_klaviyo/schemas/click.json @@ -24,15 +24,15 @@ }, "event_properties": { "type": "object", - "properties": { - "$event_id": { - "type": "string" - }, + "properties": { "Campaign Name": { "type": "string" }, "Subject": { "type": "string" + }, + "$event_id": { + "type": "string" } } }, @@ -41,12 +41,6 @@ "properties": { "email": { "type": "string" - }, - "timezone": { - "type": "string" - }, - "country": { - "type": "string" } } } diff --git a/tap_klaviyo/schemas/open.json b/tap_klaviyo/schemas/open.json index 1ffddd6..42fc369 100644 --- a/tap_klaviyo/schemas/open.json +++ b/tap_klaviyo/schemas/open.json @@ -25,14 +25,14 @@ "event_properties": { "type": "object", "properties": { - "$event_id": { - "type": "string" - }, "Campaign Name": { "type": "string" }, "Subject": { "type": "string" + }, + "$event_id": { + "type": "string" } } }, diff --git a/tap_klaviyo/schemas/receive.json b/tap_klaviyo/schemas/receive.json index 1ffddd6..42fc369 100644 --- a/tap_klaviyo/schemas/receive.json +++ b/tap_klaviyo/schemas/receive.json @@ -25,14 +25,14 @@ "event_properties": { "type": "object", "properties": { - "$event_id": { - "type": "string" - }, "Campaign Name": { "type": "string" }, "Subject": { "type": "string" + }, + "$event_id": { + "type": "string" } } }, diff --git a/tap_klaviyo/schemas/subscribe_list.json b/tap_klaviyo/schemas/subscribe_list.json index 008a357..d405b8b 100644 --- a/tap_klaviyo/schemas/subscribe_list.json +++ b/tap_klaviyo/schemas/subscribe_list.json @@ -27,9 +27,6 @@ "properties": { "List": { "type": "string" - }, - "$event_id": { - "type": "string" } } }, diff --git a/tap_klaviyo/schemas/unsubscribe.json b/tap_klaviyo/schemas/unsubscribe.json index f7e8fd5..0bf24d7 100644 --- a/tap_klaviyo/schemas/unsubscribe.json +++ b/tap_klaviyo/schemas/unsubscribe.json @@ -19,7 +19,7 @@ "statistic_id": { "type": "string" }, - "iddd": { + "id": { "type": "string" }, "event_properties": { diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index 8ca8555..0de5f2e 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -97,6 +97,8 @@ def get_incremental_pull(stream, endpoint, state, api_key, start_date): if events: counter.increment(len(events)) + logger.info('getting events') + logger.info(events) singer.write_records(stream['stream'], events) update_state(state, stream['stream'], get_latest_event_time(events)) From 9dd3d0c41da4066cfb5b2df229ed779223bd067f Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Thu, 27 Aug 2020 09:01:14 -0400 Subject: [PATCH 07/11] add timezone and country to schemas --- tap_klaviyo/schemas/bounce.json | 4 ++-- tap_klaviyo/schemas/click.json | 6 ++++++ tap_klaviyo/schemas/open.json | 6 ++++++ tap_klaviyo/schemas/receive.json | 6 ++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tap_klaviyo/schemas/bounce.json b/tap_klaviyo/schemas/bounce.json index da28dba..bd5d057 100644 --- a/tap_klaviyo/schemas/bounce.json +++ b/tap_klaviyo/schemas/bounce.json @@ -42,10 +42,10 @@ "email": { "type": "string" }, - "timezone": { + "$timezone": { "type": "string" }, - "country": { + "$country": { "type": "string" } } diff --git a/tap_klaviyo/schemas/click.json b/tap_klaviyo/schemas/click.json index 42fc369..bd5d057 100644 --- a/tap_klaviyo/schemas/click.json +++ b/tap_klaviyo/schemas/click.json @@ -41,6 +41,12 @@ "properties": { "email": { "type": "string" + }, + "$timezone": { + "type": "string" + }, + "$country": { + "type": "string" } } } diff --git a/tap_klaviyo/schemas/open.json b/tap_klaviyo/schemas/open.json index 42fc369..bd5d057 100644 --- a/tap_klaviyo/schemas/open.json +++ b/tap_klaviyo/schemas/open.json @@ -41,6 +41,12 @@ "properties": { "email": { "type": "string" + }, + "$timezone": { + "type": "string" + }, + "$country": { + "type": "string" } } } diff --git a/tap_klaviyo/schemas/receive.json b/tap_klaviyo/schemas/receive.json index 42fc369..bd5d057 100644 --- a/tap_klaviyo/schemas/receive.json +++ b/tap_klaviyo/schemas/receive.json @@ -41,6 +41,12 @@ "properties": { "email": { "type": "string" + }, + "$timezone": { + "type": "string" + }, + "$country": { + "type": "string" } } } From b11375c27eb6c6f529873ad07908a1c76e22b282 Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Thu, 27 Aug 2020 09:01:58 -0400 Subject: [PATCH 08/11] get rid of log events --- tap_klaviyo/utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index 0de5f2e..d6679e0 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -97,8 +97,6 @@ def get_incremental_pull(stream, endpoint, state, api_key, start_date): if events: counter.increment(len(events)) - logger.info('getting events') - logger.info(events) singer.write_records(stream['stream'], events) update_state(state, stream['stream'], get_latest_event_time(events)) @@ -111,9 +109,7 @@ def get_full_pulls(resource, endpoint, api_key): with metrics.record_counter(resource['stream']) as counter: for response in get_all_pages(resource['stream'], endpoint, api_key): records = response.json().get('data') - - logger.info('getting events') - logger.info(records) + counter.increment(len(records)) singer.write_records(resource['stream'], records) From e66fdd4bccb3d84b56de69d7f10caae9e14d523c Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Wed, 14 Oct 2020 07:24:43 -0600 Subject: [PATCH 09/11] fix whitespace and remove .env from gitignore --- .gitignore | 1 - tap_klaviyo/utils.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 5982ae7..c0769be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.env *.egg-info/ */__pycache__/ venv/ diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index d6679e0..655355b 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -96,7 +96,7 @@ def get_incremental_pull(stream, endpoint, state, api_key, start_date): if events: counter.increment(len(events)) - + singer.write_records(stream['stream'], events) update_state(state, stream['stream'], get_latest_event_time(events)) @@ -109,7 +109,7 @@ def get_full_pulls(resource, endpoint, api_key): with metrics.record_counter(resource['stream']) as counter: for response in get_all_pages(resource['stream'], endpoint, api_key): records = response.json().get('data') - + counter.increment(len(records)) singer.write_records(resource['stream'], records) From f74c26cd40374526838a9e015b36ff59a0d48b32 Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Wed, 14 Oct 2020 08:07:36 -0600 Subject: [PATCH 10/11] add event_id back to subscribe list --- tap_klaviyo/schemas/subscribe_list.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tap_klaviyo/schemas/subscribe_list.json b/tap_klaviyo/schemas/subscribe_list.json index d405b8b..210a3e1 100644 --- a/tap_klaviyo/schemas/subscribe_list.json +++ b/tap_klaviyo/schemas/subscribe_list.json @@ -27,6 +27,9 @@ "properties": { "List": { "type": "string" + }, + "$event_id": { + "type": "string" } } }, @@ -39,4 +42,4 @@ } } } -} \ No newline at end of file +} From a5d3efe9a60bf3bdfea34f3d72fee4a2e4320225 Mon Sep 17 00:00:00 2001 From: pblankley <31221512+pblankley@users.noreply.github.com> Date: Wed, 14 Oct 2020 08:20:56 -0600 Subject: [PATCH 11/11] fix whitespace in json --- tap_klaviyo/schemas/subscribe_list.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tap_klaviyo/schemas/subscribe_list.json b/tap_klaviyo/schemas/subscribe_list.json index 210a3e1..b082aab 100644 --- a/tap_klaviyo/schemas/subscribe_list.json +++ b/tap_klaviyo/schemas/subscribe_list.json @@ -27,9 +27,9 @@ "properties": { "List": { "type": "string" - }, - "$event_id": { - "type": "string" + }, + "$event_id": { + "type": "string" } } },