feat: add utils amd main module

main.py

@@ -1,12 +1,17 @@
-from config import logger
-from scraper.engine import Engine
+from config import logger, settings
+from scraper import Engine
+from transformer import Agent
+from utils import create_inserter_objects
 
 
 def main():
     logger.info("Initializing Application")
     engine = Engine()
     df = engine.fetch()
-    print(df)
+    df_transformed = Agent(df).transform()
+    inserter = create_inserter_objects()
+    result = inserter.insert(df_transformed, settings.OUTPUT_TABLE)
+    print(result)
     return
 
 

requirements.txt

@@ -2,3 +2,5 @@ pandas==2.0.3
 python-decouple==3.8
 requests==2.31.0
 lxml==4.9.3
+SQLAlchemy==2.0.22
+pyodbc==5.0.1

utils/__init__.py

@@ -0,0 +1 @@
+from .db_helper import create_inserter_objects

utils/db_helper.py

@@ -0,0 +1,17 @@
+from config import settings
+from database.mssql import MSSQLDatabaseConnection, PandasSQLDataInserter
+
+
+def create_inserter_objects() -> PandasSQLDataInserter:
+    db_connection = MSSQLDatabaseConnection(
+        settings.MSSQL_SERVER,
+        settings.MSSQL_DATABASE,
+        settings.MSSQL_USERNAME,
+        settings.MSSQL_PASSWORD,
+    )
+
+    data_inserter = PandasSQLDataInserter(
+        db_connection, max_retries=settings.INSERTER_MAX_RETRIES
+    )
+
+    return data_inserter