From bf20fac8db286a322d8a4bc3c1ce0434d81d1859 Mon Sep 17 00:00:00 2001 From: Golda Velez Date: Tue, 28 May 2024 16:02:31 -0700 Subject: [PATCH] on conflict do nothing --- logs-to-tsdb/backfill/get-all-days.py | 4 ++-- logs-to-tsdb/kinesis-lambda.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/logs-to-tsdb/backfill/get-all-days.py b/logs-to-tsdb/backfill/get-all-days.py index c756523..8d837ef 100644 --- a/logs-to-tsdb/backfill/get-all-days.py +++ b/logs-to-tsdb/backfill/get-all-days.py @@ -36,8 +36,8 @@ def get_query_results(query_id, filename): def main(): log_group = "/ecs/ceramic-prod-cas" - start_date_str = "2024-05-22 10:00:00" - end_date_str = "2024-05-22 23:59:00" + start_date_str = "2024-05-22 23:59:00" + end_date_str = "2024-05-23 08:00:00" start_date = datetime.strptime(start_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) end_date = datetime.strptime(end_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) diff --git a/logs-to-tsdb/kinesis-lambda.py b/logs-to-tsdb/kinesis-lambda.py index dafcfdf..39abb7d 100644 --- a/logs-to-tsdb/kinesis-lambda.py +++ b/logs-to-tsdb/kinesis-lambda.py @@ -53,7 +53,7 @@ def handler(event, context): batched.append((ts, log_data['cid'], log_data['did'], log_data.get('model'), log_data.get('family'), log_data.get('stream'), log_data.get('origin'), log_data.get('cacao'), log_data.get('cap_cid'))) # Insert logs into TimescaleDB - insert_query = "INSERT INTO cas_log_data (timestamp, cid, did, model, family, stream, origin, cacao, cap_cid) VALUES %s" + insert_query = "INSERT INTO cas_log_data (timestamp, cid, did, model, family, stream, origin, cacao, cap_cid) VALUES %s on CONFLICT DO NOTHING" if batched: print("Inserting {} rows".format(len(batched)))