Merge pull request #3 from 3box/fix/update-backfill

update backfill for no did
3box · Jun 28, 2024 · f175897 · f175897
2 parents 6d7a8fa + 2a1da5f
commit f175897
Show file tree

Hide file tree

Showing 6 changed files with 10 additions and 34 deletions.
diff --git a/logs-to-tsdb/backfill/get-all-days.py b/logs-to-tsdb/backfill/get-all-days.py
@@ -36,8 +36,8 @@ def get_query_results(query_id, filename):
 def main():
     log_group = "/ecs/ceramic-prod-cas"
 
-    start_date_str = "2024-05-22 23:59:00"
-    end_date_str = "2024-05-23 08:00:00"
+    start_date_str = "2024-06-17 23:59:00"
+    end_date_str = "2024-06-26 00:00:00"
     start_date = datetime.strptime(start_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
     end_date = datetime.strptime(end_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
 

diff --git a/logs-to-tsdb/backfill/munge2.py b/logs-to-tsdb/backfill/munge2.py
@@ -79,5 +79,10 @@
     newl['awslogs']['data'] = subprocess.run(['base64', '{}.gz'.format(fname)], capture_output=True, text=True).stdout.strip()
 
     with open('ready_{}.json'.format(logid), 'w') as f:
-       json.dump(newl, f)
+        json.dump(newl, f)
+#    os.system('python-lambda-local -f handler ../logs-lambda.py ready_{}.json --environment DB_HOST={} DB_PASSWORD={} --timeout 20'.format(
+#       logid,
+#       os.environ['DB_HOST'],
+#       os.environ['DB_PASSWORD']
+#    ))
     os.system('python-lambda-local -f handler ../logs-lambda.py ready_{}.json --timeout 20'.format(logid)) 
diff --git a/logs-to-tsdb/logs-lambda.py b/logs-to-tsdb/logs-lambda.py
@@ -49,10 +49,10 @@ def handler(event, context):
 
         data = json.loads(match.group(1))
 
-        batched.append((ts, data['cid'], data['did'], data.get('model'), data.get('family'), data.get('stream'), data.get('origin'), data.get('cacao'), data.get('cap_cid')))
+        batched.append((ts, data['cid'], data.get('did'), data.get('model'), data.get('family'), data.get('stream'), data.get('origin'), data.get('cacao'), data.get('cap_cid')))
 
     # Insert logs into TimescaleDB
-    insert_query = "INSERT INTO cas_log_data (timestamp, cid, did, model, family, stream, origin, cacao, cap_cid) VALUES %s"
+    insert_query = "INSERT INTO cas_log_data (timestamp, cid, did, model, family, stream, origin, cacao, cap_cid) VALUES %s on CONFLICT DO NOTHING"
 
     print("Inserting {} rows including {}".format(len(batched), batched[0]))
 

diff --git a/logs-to-tsdb/sample.json b/logs-to-tsdb/sample.json
diff --git a/logs-to-tsdb/sample_base64.txt b/logs-to-tsdb/sample_base64.txt
diff --git a/logs-to-tsdb/sample_test.json b/logs-to-tsdb/sample_test.json