Profiler CLI (#1623)

<!-- REMOVE IRRELEVANT COMMENTS BEFORE CREATING A PULL REQUEST -->
## Changes
<!-- Summary of your changes that are easy to understand. Add
screenshots when necessary, they're helpful to illustrate the before and
after state -->
### What does this PR do? 
This introduces the intial version of CLI
### Relevant implementation details

### Caveats/things to watch out for when reviewing:

### Linked issues
<!-- DOC: Link issue with a keyword: close, closes, closed, fix, fixes,
fixed, resolve, resolves, resolved. See
https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword
-->

``` shell
~ ❯ databricks labs lakebridge execute-database-profiler --help                                                                                                                                                 26s
Profile the source system database

Usage:
  databricks labs lakebridge execute-database-profiler [flags]

Flags:
  -h, --help                 help for execute-database-profiler
      --source-tech string   (Optional) The technology/platform of the sources to Profile

Global Flags:
      --debug            enable debug logging
  -o, --output type      output type: text or json (default text)
  -p, --profile string   ~/.databrickscfg profile
  -t, --target string    bundle target to use (if applicable)

```

``` shell
~ ❯ databricks labs lakebridge execute-database-profiler --source-tech mssql
12:24:59    ERROR [d.labs.lakebridge] Only following source system is supported ['synapse']
12:24:59    ERROR [d.l.lakebridge.execute-database-profiler] ValueError: Invalid source technology mssql


~ ❯ databricks labs lakebridge execute-database-profiler --source-tech Synapse
[UPGRADE ADVISED] Newer lakebridge version was released 12 days ago. Please run `databricks labs upgrade lakebridge` to upgrade: feature/profiler_entry_point -> v0.10.11
12:25:15    ERROR [d.l.lakebridge.execute-database-profiler] ValueError: Connection details not found. Please run `databricks labs lakebridge configure-database-profiler` to set up connection details for synapse.
~ ❯

```

``` shell
~ ❯ databricks labs lakebridge execute-database-profiler
Select the source technology
[0] synapse
Enter a number between 0 and 0: 0
12:32:50    ERROR [d.l.lakebridge.execute-database-profiler] ValueError: Connection details not found. Please run `databricks labs lakebridge configure-database-profiler` to set up connection details for synapse.

```
### Functionality

- [ ] added relevant user documentation
- [x] added new CLI command
- [ ] modified existing command: `databricks labs remorph ...`
- [ ] ... +add your own

### Tests
<!-- How is this tested? Please see the checklist below and also
describe any other relevant tests -->

- [x] manually tested
- [x] added unit tests
- [x] added integration tests

---------

Co-authored-by: Andrew Snare <[email protected]>
Co-authored-by: Guenia Izquierdo <[email protected]>
Co-authored-by: Will Girten <[email protected]>

Author: 4 people

labs.yml

@@ -73,7 +73,7 @@ commands:
     description: Reconcile source and target data residing on Databricks using aggregated metrics
 
   - name: configure-database-profiler
-    description: Configure database profiler
+    description: (Experimental) Configure database profiler
 
   - name: create-profiler-dashboard
     description: (Experimental) Upload the profiler results as a Databricks dashboard.
@@ -115,3 +115,9 @@ commands:
 
   - name: configure-reconcile
     description: Configure 'reconcile' dependencies
+
+  - name: execute-database-profiler
+    description: (Experimental) Profile the source system database
+    flags:
+      - name: source-tech
+        description: (Optional) The technology/platform of the sources to Profile

src/databricks/labs/lakebridge/assessments/_constants.py

@@ -8,7 +8,7 @@
 }
 
 # TODO modify this PLATFORM_TO_SOURCE_TECHNOLOGY.keys() once all platforms are supported
-PROFILER_SOURCE_SYSTEM = ["mssql", "synapse"]
+PROFILER_SOURCE_SYSTEM = ["synapse"]
 
 
 # This flag indicates whether a connector is required for the source system when pipeline is trigger

src/databricks/labs/lakebridge/cli.py

@@ -21,10 +21,12 @@
 
 
 from databricks.labs.lakebridge.assessments.configure_assessment import create_assessment_configurator
-from databricks.labs.lakebridge.assessments import PROFILER_SOURCE_SYSTEM
+from databricks.labs.lakebridge.assessments import PROFILER_SOURCE_SYSTEM, PRODUCT_NAME
+from databricks.labs.lakebridge.assessments.profiler import Profiler
 
 from databricks.labs.lakebridge.config import TranspileConfig, LSPConfigOptionV1
 from databricks.labs.lakebridge.contexts.application import ApplicationContext
+from databricks.labs.lakebridge.connections.credential_manager import cred_file
 from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
 from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
 from databricks.labs.lakebridge.install import installer
@@ -711,18 +713,19 @@ def configure_secrets(*, w: WorkspaceClient) -> None:
     recon_conf.prompt_and_save_connection_details()
 
 
-@lakebridge.command(is_unauthenticated=True)
-def configure_database_profiler() -> None:
-    """[Experimental] Install the lakebridge Assessment package"""
-    prompts = Prompts()
-
-    # Prompt for source system
-    source_system = str(
-        prompts.choice("Please select the source system you want to configure", PROFILER_SOURCE_SYSTEM)
-    ).lower()
+@lakebridge.command
+def configure_database_profiler(w: WorkspaceClient) -> None:
+    """[Experimental] Installs and runs the Lakebridge Assessment package for database profiling"""
+    ctx = ApplicationContext(w)
+    ctx.add_user_agent_extra("cmd", "configure-profiler")
+    prompts = ctx.prompts
+    source_tech = prompts.choice("Select the source technology", PROFILER_SOURCE_SYSTEM).lower()
+    ctx.add_user_agent_extra("profiler_source_tech", make_alphanum_or_semver(source_tech))
+    user = ctx.current_user
+    logger.debug(f"User: {user}")
 
     # Create appropriate assessment configurator
-    assessment = create_assessment_configurator(source_system=source_system, product_name="lakebridge", prompts=prompts)
+    assessment = create_assessment_configurator(source_system=source_tech, product_name="lakebridge", prompts=prompts)
     assessment.run()
 
 
@@ -960,6 +963,36 @@ def llm_transpile(
     )
 
 
+@lakebridge.command()
+def execute_database_profiler(w: WorkspaceClient, source_tech: str | None = None) -> None:
+    """Execute the Profiler Extraction for the given source technology"""
+    ctx = ApplicationContext(w)
+    ctx.add_user_agent_extra("cmd", "execute-profiler")
+    prompts = ctx.prompts
+    if source_tech is None:
+        source_tech = prompts.choice("Select the source technology", PROFILER_SOURCE_SYSTEM)
+    source_tech = source_tech.lower()
+
+    if source_tech not in PROFILER_SOURCE_SYSTEM:
+        logger.error(f"Only the following source systems are supported: {PROFILER_SOURCE_SYSTEM}")
+        raise_validation_exception(f"Invalid source technology {source_tech}")
+
+    ctx.add_user_agent_extra("profiler_source_tech", make_alphanum_or_semver(source_tech))
+    user = ctx.current_user
+    logger.debug(f"User: {user}")
+    # check if cred_file is present which has the connection details before running the profiler
+    file = cred_file(PRODUCT_NAME)
+    if not file.exists():
+        raise_validation_exception(
+            f"Connection details not found. Please run `databricks labs lakebridge configure-database-profiler` "
+            f"to set up connection details for {source_tech}."
+        )
+    profiler = Profiler.create(source_tech)
+
+    # TODO: Add extractor logic to ApplicationContext instead of creating inside the Profiler class
+    profiler.profile()
+
+
 @lakebridge.command()
 def create_profiler_dashboard(
     *,