Dev (#23)

* docs: update readme * chore: mypy linting * chore: sourcery lint * fix: import shorthand skipping backends fail * chore: lint list_backends * docs: info about snowsql config * docs: add url for snowflake con * chore: up version --------- Co-authored-by: sfc-gh-kmason <[email protected]> Co-authored-by: Tyler White <[email protected]>
learning-the-computers · Aug 23, 2023 · bcdabd5 · bcdabd5
1 parent 1b4173d
commit bcdabd5
Show file tree

Hide file tree

Showing 6 changed files with 96 additions and 42 deletions.
diff --git a/dot_connect/__init__.py b/dot_connect/__init__.py
@@ -9,6 +9,33 @@
 
     load_dotenv(override=True)
 
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import aws  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import azure  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import impala  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import mssql  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import mysql  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import postgres  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import pyspark  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import snowflake  # noqa: F401
+
+with contextlib.suppress(ImportError):
+    from dot_connect.backends import snowpark  # noqa: F401
+
 
 def read_json(file: str) -> Dict:
     """
@@ -91,9 +118,7 @@ def read_ini_conf_cfg(file: str) -> Dict:
 
     config = configparser.ConfigParser()
     config.read(file)
-    connection_parameters = {s: dict(config.items(s)) for s in config.sections()}
-
-    return connection_parameters
+    return {s: dict(config.items(s)) for s in config.sections()}
 
 
 def parse_file(file: str, *args) -> Union[Dict[Any, Any], str]:
@@ -108,19 +133,18 @@ def parse_file(file: str, *args) -> Union[Dict[Any, Any], str]:
         file: Absolute or relative path to file.
 
     """
-    if isinstance(file, str):
-        supported_types = (".json", ".yaml", ".yml", ".ini", ".cfg", ".conf")
-        if file.endswith(".json"):
-            connection_parameters = read_json(file)
-        elif file.endswith((".yml", ".yaml")):
-            connection_parameters = read_yaml(file)
-        elif file.endswith((".ini", ".cfg", ".conf")):
-            connection_parameters = read_ini_conf_cfg(file)
-        else:
-            return f"File must be of type {supported_types}."
-    else:
+    if not isinstance(file, str):
         raise TypeError("File argument must be string.")
 
+    if file.endswith(".json"):
+        connection_parameters = read_json(file)
+    elif file.endswith((".yml", ".yaml")):
+        connection_parameters = read_yaml(file)
+    elif file.endswith((".ini", ".cfg", ".conf")):
+        connection_parameters = read_ini_conf_cfg(file)
+    else:
+        supported_types = (".json", ".yaml", ".yml", ".ini", ".cfg", ".conf")
+        return f"File must be of type {supported_types}."
     if args:  # Traverse any nested keys passed
         for k in args:
             if k in connection_parameters:
@@ -166,5 +190,19 @@ def load_config(prefix: str, file: Optional[str] = None, *args):
         }
 
     if file:
-        creds = parse_file(file, *args)
-        return creds
+        return parse_file(file, *args)
+
+
+def list_backends() -> list:
+    """List compatible dot-connect backends."""
+    return [
+        "aws",
+        "azure",
+        "impala",
+        "mssql",
+        "mysql",
+        "postgres",
+        "pyspark",
+        "snowflake",
+        "snowpark",
+    ]
diff --git a/dot_connect/backends/__init__.py b/dot_connect/backends/__init__.py
@@ -90,9 +90,7 @@ def read_ini_conf_cfg(file: str) -> Dict:
 
     config = configparser.ConfigParser()
     config.read(file)
-    connection_parameters = {s: dict(config.items(s)) for s in config.sections()}
-
-    return connection_parameters
+    return {s: dict(config.items(s)) for s in config.sections()}
 
 
 def parse_file(file: str, *args) -> Union[Dict[Any, Any], str]:
@@ -107,19 +105,18 @@ def parse_file(file: str, *args) -> Union[Dict[Any, Any], str]:
         file: Absolute or relative path to file.
 
     """
-    if isinstance(file, str):
-        supported_types = (".json", ".yaml", ".yml", ".ini", ".cfg", ".conf")
-        if file.endswith(".json"):
-            connection_parameters = read_json(file)
-        elif file.endswith((".yml", ".yaml")):
-            connection_parameters = read_yaml(file)
-        elif file.endswith((".ini", ".cfg", ".conf")):
-            connection_parameters = read_ini_conf_cfg(file)
-        else:
-            return f"File must be of type {supported_types}."
-    else:
+    if not isinstance(file, str):
         raise TypeError("File argument must be string.")
 
+    if file.endswith(".json"):
+        connection_parameters = read_json(file)
+    elif file.endswith((".yml", ".yaml")):
+        connection_parameters = read_yaml(file)
+    elif file.endswith((".ini", ".cfg", ".conf")):
+        connection_parameters = read_ini_conf_cfg(file)
+    else:
+        supported_types = (".json", ".yaml", ".yml", ".ini", ".cfg", ".conf")
+        return f"File must be of type {supported_types}."
     if args:  # Traverse any nested keys passed
         for k in args:
             if k in connection_parameters:
@@ -165,5 +162,4 @@ def load_config(prefix: str, file: Optional[str] = None, *args):
         }
 
     if file:
-        creds = parse_file(file, *args)
-        return creds
+        return parse_file(file, *args)
diff --git a/dot_connect/backends/pyspark.py b/dot_connect/backends/pyspark.py
@@ -39,10 +39,8 @@ def connect(**kwargs):
 
     config.update(**kwargs)
 
-    spark = (
+    return (
         SparkSession.builder.appName(config.get("appName", "MyApp"))
         .master(config.get("master", "local"))
         .getOrCreate()
     )
-
-    return spark
diff --git a/dot_connect/backends/snowflake.py b/dot_connect/backends/snowflake.py
@@ -1,14 +1,18 @@
 """Utility module to handle Snowflake-related configurations."""
 
 import os
+from typing import Dict
 
 from dot_connect.backends import load_config
 
 
-def load_snowsql_config():
+def load_snowsql_config() -> Dict[str, Dict[str, str]]:
     """
     Load configuration from the ~/.snowsql/config file.
 
+    More information about this specification can be found at
+    https://docs.snowflake.com/en/user-guide/snowsql-config#snowsql-config-file.
+
     Returns:
         dict: A dictionary containing the SnowSQL configurations. Each section
               in the config will be a key in the dictionary, and its value will
@@ -38,13 +42,24 @@ def load_snowsql_config():
     return read_ini_conf_cfg(config_path)
 
 
-def format_url(**kwargs):
-    """Format SQLAlchemy-compatible URL for Snowflake."""
+def format_url(**kwargs) -> str:
+    """
+    Format SQLAlchemy-compatible URL for Snowflake.
+
+    More information about this format specification can be
+    found at
+    https://docs.snowflake.com/en/developer-guide/python-connector/sqlalchemy#connection-parameters
+    """
     from snowflake.sqlalchemy import URL
 
     connection_params = kwargs
-    connection_params["schema"] = connection_params.get("database").split("/")[1]
-    connection_params["database"] = connection_params.get("database").split("/")[0]
+
+    database_value = connection_params.get("database")
+    if not database_value:
+        raise ValueError("Expected 'database' key in connection_params.")
+
+    connection_params["schema"] = database_value.split("/")[1]
+    connection_params["database"] = database_value.split("/")[0]
 
     return URL(**connection_params)
 
@@ -59,6 +74,10 @@ def connect(**kwargs):
     passed to the function. The resulting configuration is used to establish
     the Snowflake connection.
 
+    More information about using the Snowflake Python Connector API can be
+    found at
+    https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api
+
     Args:
         **kwargs: Additional keyword arguments to customize the connection
                   parameters. These arguments will be used to update the default

diff --git a/dot_connect/backends/snowpark.py b/dot_connect/backends/snowpark.py
@@ -1,5 +1,4 @@
 """Utility module to handle Snowflake-related configurations."""
-
 from dot_connect.backends import load_config
 
 
@@ -13,6 +12,9 @@ def connect(**kwargs):
     arguments passed to the function. The resulting configuration is used to
     establish the Snowflake connection via Snowpark.
 
+    More information regarding creating Snowpark sessions can be found at
+    https://docs.snowflake.com/en/developer-guide/snowpark/python/creating-session
+
     Args:
         **kwargs: Additional keyword arguments to customize the connection
                   parameters. These arguments will be used to update the default

diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = dot_connect
-version = 0.3.3
+version = 0.3.31
 description = Improve your workflow efficiency by connecting to databases and cloud systems effortlessly.
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -50,5 +50,6 @@ pyspark =
     pyspark>=3.4.1
 snowflake =
     snowflake-connector-python>=3.0.4
+    snowflake-sqlalchemy>=1.4.7
 snowpark =
     snowflake-snowpark-python>=1.6.1