From 2a1da5f6c2d81b6b2550fee3e7fa28235d0c6388 Mon Sep 17 00:00:00 2001 From: Golda Velez Date: Fri, 28 Jun 2024 06:53:28 -0700 Subject: [PATCH] update backfill for no did --- logs-to-tsdb/backfill/get-all-days.py | 4 ++-- logs-to-tsdb/backfill/munge2.py | 7 ++++++- logs-to-tsdb/logs-lambda.py | 4 ++-- logs-to-tsdb/sample.json | 23 ----------------------- logs-to-tsdb/sample_base64.txt | 1 - logs-to-tsdb/sample_test.json | 5 ----- 6 files changed, 10 insertions(+), 34 deletions(-) delete mode 100644 logs-to-tsdb/sample.json delete mode 100644 logs-to-tsdb/sample_base64.txt delete mode 100644 logs-to-tsdb/sample_test.json diff --git a/logs-to-tsdb/backfill/get-all-days.py b/logs-to-tsdb/backfill/get-all-days.py index 8d837ef..b55cab0 100644 --- a/logs-to-tsdb/backfill/get-all-days.py +++ b/logs-to-tsdb/backfill/get-all-days.py @@ -36,8 +36,8 @@ def get_query_results(query_id, filename): def main(): log_group = "/ecs/ceramic-prod-cas" - start_date_str = "2024-05-22 23:59:00" - end_date_str = "2024-05-23 08:00:00" + start_date_str = "2024-06-17 23:59:00" + end_date_str = "2024-06-26 00:00:00" start_date = datetime.strptime(start_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) end_date = datetime.strptime(end_date_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) diff --git a/logs-to-tsdb/backfill/munge2.py b/logs-to-tsdb/backfill/munge2.py index 5e4416c..c8a6acc 100644 --- a/logs-to-tsdb/backfill/munge2.py +++ b/logs-to-tsdb/backfill/munge2.py @@ -79,5 +79,10 @@ newl['awslogs']['data'] = subprocess.run(['base64', '{}.gz'.format(fname)], capture_output=True, text=True).stdout.strip() with open('ready_{}.json'.format(logid), 'w') as f: - json.dump(newl, f) + json.dump(newl, f) +# os.system('python-lambda-local -f handler ../logs-lambda.py ready_{}.json --environment DB_HOST={} DB_PASSWORD={} --timeout 20'.format( +# logid, +# os.environ['DB_HOST'], +# os.environ['DB_PASSWORD'] +# )) os.system('python-lambda-local -f handler ../logs-lambda.py ready_{}.json --timeout 20'.format(logid)) diff --git a/logs-to-tsdb/logs-lambda.py b/logs-to-tsdb/logs-lambda.py index 6f53e11..042274c 100644 --- a/logs-to-tsdb/logs-lambda.py +++ b/logs-to-tsdb/logs-lambda.py @@ -49,10 +49,10 @@ def handler(event, context): data = json.loads(match.group(1)) - batched.append((ts, data['cid'], data['did'], data.get('model'), data.get('family'), data.get('stream'), data.get('origin'), data.get('cacao'), data.get('cap_cid'))) + batched.append((ts, data['cid'], data.get('did'), data.get('model'), data.get('family'), data.get('stream'), data.get('origin'), data.get('cacao'), data.get('cap_cid'))) # Insert logs into TimescaleDB - insert_query = "INSERT INTO cas_log_data (timestamp, cid, did, model, family, stream, origin, cacao, cap_cid) VALUES %s" + insert_query = "INSERT INTO cas_log_data (timestamp, cid, did, model, family, stream, origin, cacao, cap_cid) VALUES %s on CONFLICT DO NOTHING" print("Inserting {} rows including {}".format(len(batched), batched[0])) diff --git a/logs-to-tsdb/sample.json b/logs-to-tsdb/sample.json deleted file mode 100644 index 67b34b9..0000000 --- a/logs-to-tsdb/sample.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "messageType": "DATA_MESSAGE", - "owner": "123456789012", - "logGroup": "/ecs/ceramic-prod-cas", - "logStream": "cas_api/cas_api/1e9ab363f8d540af9fd9b19e676deeba", - "subscriptionFilters": ["PLACEHOLDER"], - "logEvents": [ - { - "logStreamName": "cas_api/cas_api/1e9ab363f8d540af9fd9b19e676deeba", - "timestamp": 1692743115313, - "message": "[2023-08-22T22:25:15.313Z] INFO: 'Anchor request received: {\"cid\":\"bagcqceraxkwya2kwxvu7uarwtcnh2yaxngzuf3stzljp3l7fzo4eofcq6npa\",\"did\":\"did:pkh:eip155:1:0x6ec5e7162282aafdb7be56d459d9c40f63531fff\",\"schema\":{},\"family\":\"orbis\",\"stream\":\"kjzl6cwe1jw149xdpvb91bupbi15wt3nhunh7byg644q6n61qhczer4x21bb9em\",\"origin\":\"157.245.135.141\"}'", - "ingestionTime": 1692743119504, - "eventId": "37749432901162153688680743539049382392309708367980658733" - }, - { - "logStreamName": "cas_api/cas_api/1e9ab363f8d540af9fd9b19e676deeba", - "timestamp": 1692743284784, - "message": "[2023-08-22T22:28:04.784Z] INFO: 'Anchor request received: {\"cid\":\"bagcqcera2k62dnsxotfo7sk6kxtrhrxbekn4z6etuorcmkrutl6fpfhfosrq\",\"did\":\"did:pkh:eip155:1:0xB7532ba5a52872658715Bf0eC05958Cec85E4Db6\",\"schema\":{},\"family\":\"orbis\",\"stream\":\"kjzl6cwe1jw14aiyzjubq0ejjxy26kdqaigujgautf8075d23779w3erp1jn92b\",\"origin\":\"157.245.135.141\"}'", - "ingestionTime": 1692743289504, - "eventId": "37749436680491743228863978163766422951382467794372395043" - } - ] -} diff --git a/logs-to-tsdb/sample_base64.txt b/logs-to-tsdb/sample_base64.txt deleted file mode 100644 index 385863b..0000000 --- a/logs-to-tsdb/sample_base64.txt +++ /dev/null @@ -1 +0,0 @@ -H4sICCk65WQAA3NhbXBsZS5qc29uAMWSy27bOBSG93kKQ5tuchHvpHZu4l6A3tBkNXVQkNShRMuWZEqKZQd591JOmymymBl0CpSAQIo85+fhf777k9ks2UDX6QJu9i0k2Sy5mt/Mv75fXF/PXy+S0ymg2dUQpiOECWVcSJUi/Hi0borXoRna6fQCbHdhIeiNt2dtaPIzq7unsOs+gN5McXH3q279xY8ZgdKGcOJkzmiqnXK5MkgBFzwHMPpRohtMZ4Nve9/Ur/y6h9BFsS/Jp3fzy8Wbj++uFp+T2x+XLe6g7o/nceNx3D+tpvF3RR/0Bn65qp8Eex997PVmsgJxhQUlCDGCyLO4725PV37BKSZnqTzD+AbjDLMMsfOY8dft7O2HVx+z2Yt5bcsmzAJshygeZwv+DvJsdr9MrM+XSbZMjC7sdrJ9rHZ7javdeDeIQYddb+sS7/VYF4fBka4/rFctWQt3aCg0zm553eplcrpM8u9Kcc7aqszAt4jFYrJ05GAZCMQxllhrlxthgPGcMpUrS1PHSXyic+6o09kSNlEyu3+Ify5ysN4fhZtgfPcYcvT8uFmtDmtud4BWO0TVmLd3RiEztMYjtutJXQ51Kcy+4JTGUjnalvYAgY4YGaNgc5Rrgi98fZRDTJxjys4RiR9Fy+ThxfMW+bqILkZ+bvyx509tUiylz2JhIuhtPrWJCEEVJThSH42IPeVScpnGTEZUShWRmChMUiVSSbhQMuVMCkKSJ8WH0z+JIZZUyOfv+wcMZZbS85jxixjiiuO87samd43oKl6NfSjDaKCq6YFDPzTBbqow9GvuWle6pgvbf8PwpWAEG800w1LgyV7EXroULlOmmLwEK9mCXhn+fzHUfn9YDWabwmo17jGv8q32xbAq9NC72HKW40iD2hEILVrVCpvfhyGW/wlDHtGjCk0JOHJIlJCIE8E5xVgxFGGkPJZIiYhURsGfMZxWtycPJ98AWeENI/cFAAA= diff --git a/logs-to-tsdb/sample_test.json b/logs-to-tsdb/sample_test.json deleted file mode 100644 index 621f87f..0000000 --- a/logs-to-tsdb/sample_test.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "awslogs": { - "data": "H4sICCk65WQAA3NhbXBsZS5qc29uAMWSy27bOBSG93kKQ5tuchHvpHZu4l6A3tBkNXVQkNShRMuWZEqKZQd591JOmymymBl0CpSAQIo85+fhf777k9ks2UDX6QJu9i0k2Sy5mt/Mv75fXF/PXy+S0ymg2dUQpiOECWVcSJUi/Hi0borXoRna6fQCbHdhIeiNt2dtaPIzq7unsOs+gN5McXH3q279xY8ZgdKGcOJkzmiqnXK5MkgBFzwHMPpRohtMZ4Nve9/Ur/y6h9BFsS/Jp3fzy8Wbj++uFp+T2x+XLe6g7o/nceNx3D+tpvF3RR/0Bn65qp8Eex997PVmsgJxhQUlCDGCyLO4725PV37BKSZnqTzD+AbjDLMMsfOY8dft7O2HVx+z2Yt5bcsmzAJshygeZwv+DvJsdr9MrM+XSbZMjC7sdrJ9rHZ7javdeDeIQYddb+sS7/VYF4fBka4/rFctWQt3aCg0zm553eplcrpM8u9Kcc7aqszAt4jFYrJ05GAZCMQxllhrlxthgPGcMpUrS1PHSXyic+6o09kSNlEyu3+Ify5ysN4fhZtgfPcYcvT8uFmtDmtud4BWO0TVmLd3RiEztMYjtutJXQ51Kcy+4JTGUjnalvYAgY4YGaNgc5Rrgi98fZRDTJxjys4RiR9Fy+ThxfMW+bqILkZ+bvyx509tUiylz2JhIuhtPrWJCEEVJThSH42IPeVScpnGTEZUShWRmChMUiVSSbhQMuVMCkKSJ8WH0z+JIZZUyOfv+wcMZZbS85jxixjiiuO87samd43oKl6NfSjDaKCq6YFDPzTBbqow9GvuWle6pgvbf8PwpWAEG800w1LgyV7EXroULlOmmLwEK9mCXhn+fzHUfn9YDWabwmo17jGv8q32xbAq9NC72HKW40iD2hEILVrVCpvfhyGW/wlDHtGjCk0JOHJIlJCIE8E5xVgxFGGkPJZIiYhURsGfMZxWtycPJ98AWeENI/cFAAA=" - } -}