Skip to content

Commit

Permalink
test: Add benchmark for SQL tap discovery
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon committed Dec 3, 2024
1 parent 894eed3 commit b323742
Showing 1 changed file with 41 additions and 0 deletions.
41 changes: 41 additions & 0 deletions tests/core/test_connector_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
from singer_sdk.exceptions import ConfigValidationError

if t.TYPE_CHECKING:
from pathlib import Path

from sqlalchemy.engine import Engine


Expand Down Expand Up @@ -688,3 +690,42 @@ def handle_raw_string(self, schema):
}
result = json_schema_to_sql.to_sql_type(image_type)
assert isinstance(result, sa.types.LargeBinary)


def test_bench_discovery(benchmark, tmp_path: Path):
def _discover_catalog(connector):
connector.discover_catalog_entries()

number_of_tables = 500
number_of_views = 500
number_of_columns = 15
db_path = tmp_path / "foo.db"
engine = sa.create_engine(f"sqlite:///{db_path}")

columns_fragment = ",".join(f"col_{i} VARCHAR" for i in range(number_of_columns))

# Seed a large number of tables
table_ddl = f"""
CREATE TABLE table_{{n}} (
id INTEGER NOT NULL,
{columns_fragment},
PRIMARY KEY (id)
);
"""

# Seed a large number of views
view_ddl = """
CREATE VIEW view_{n} AS
SELECT * FROM table_{n};
"""

with engine.connect() as conn:
for i in range(number_of_tables):
conn.execute(sa.text(table_ddl.format(n=i)))

for i in range(number_of_views):
conn.execute(sa.text(view_ddl.format(n=i)))

connector = SQLConnector(config={"sqlalchemy_url": f"sqlite:///{db_path}"})

benchmark(_discover_catalog, connector)

0 comments on commit b323742

Please sign in to comment.