Skip to content

Commit

Permalink
added benchmarks for datetime parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
BuzzCutNorman committed Nov 17, 2023
1 parent d04d39c commit ff4e02a
Showing 1 changed file with 61 additions and 0 deletions.
61 changes: 61 additions & 0 deletions tests/core/sinks/test_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import datetime

import pytest

from tests.conftest import BatchSinkMock, TargetMock


Expand Down Expand Up @@ -39,3 +41,62 @@ def test_validate_record():
)
assert updated_record["missing_datetime"] == "2021-01-01T00:00:00+00:00"
assert updated_record["invalid_datetime"] == "9999-12-31 23:59:59.999999"


@pytest.fixture
def bench_sink() -> BatchSinkMock:
target = TargetMock()
return BatchSinkMock(
target,
"users",
{
"type": "object",
"properties": {
"id": {"type": "integer"},
"created_at": {"type": "string", "format": "date-time"},
"updated_at": {"type": "string", "format": "date-time"},
"deleted_at": {"type": "string", "format": "date-time"},
},
},
["id"],
)


@pytest.fixture
def bench_record():
return {
"id": 1,
"created_at": "2021-01-01T00:08:00-07:00",
"updated_at": "2022-01-02T00:09:00-07:00",
"deleted_at": "2023-01-03T00:10:00.0000",
}


def test_bench_parse_timestamps_in_record(benchmark, bench_sink, bench_record):
"""Run benchmark tests using the "repositories" stream."""
record_size_scale = 10000

sink: BatchSinkMock = bench_sink
record: dict = bench_record

def run_parse_timestamps_in_record():
for _ in range(record_size_scale):
_ = sink._parse_timestamps_in_record(
record, sink.schema, sink.datetime_error_treatment
)

benchmark(run_parse_timestamps_in_record)


def test_bench_validate_and_parse(benchmark, bench_sink, bench_record):
"""Run benchmark tests using the "repositories" stream."""
record_size_scale = 10000

sink: BatchSinkMock = bench_sink
record: dict = bench_record

def run_validate_and_parse():
for _ in range(record_size_scale):
_ = sink._validate_and_parse(record)

benchmark(run_validate_and_parse)

0 comments on commit ff4e02a

Please sign in to comment.