Merge branch 'main' into feature/odsc-65115

codeloop · web-flow · commit fa5f53d03063 · 2025-02-04T10:06:34.000+05:30
diff --git a/ads/aqua/app.py b/ads/aqua/app.py
@@ -4,6 +4,7 @@
 
 import json
 import os
+import traceback
 from dataclasses import fields
 from typing import Dict, Union
 
@@ -23,7 +24,7 @@
 from ads.aqua.constants import UNKNOWN
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
-from ads.common.utils import extract_region
+from ads.common.utils import extract_region, is_path_exists
 from ads.config import (
     AQUA_TELEMETRY_BUCKET,
     AQUA_TELEMETRY_BUCKET_NS,
@@ -296,33 +297,44 @@ def get_config(self, model_id: str, config_file_name: str) -> Dict:
             raise AquaRuntimeError(f"Target model {oci_model.id} is not Aqua model.")
 
         config = {}
-        artifact_path = get_artifact_path(oci_model.custom_metadata_list)
+        # if the current model has a service model tag, then
+        if Tags.AQUA_SERVICE_MODEL_TAG in oci_model.freeform_tags:
+            base_model_ocid = oci_model.freeform_tags[Tags.AQUA_SERVICE_MODEL_TAG]
+            logger.info(
+                f"Base model found for the model: {oci_model.id}. "
+                f"Loading {config_file_name} for base model {base_model_ocid}."
+            )
+            base_model = self.ds_client.get_model(base_model_ocid).data
+            artifact_path = get_artifact_path(base_model.custom_metadata_list)
+            config_path = f"{os.path.dirname(artifact_path)}/config/"
+        else:
+            logger.info(f"Loading {config_file_name} for model {oci_model.id}...")
+            artifact_path = get_artifact_path(oci_model.custom_metadata_list)
+            config_path = f"{artifact_path.rstrip('/')}/config/"
+
         if not artifact_path:
             logger.debug(
                 f"Failed to get artifact path from custom metadata for the model: {model_id}"
             )
             return config
 
-        try:
-            config_path = f"{os.path.dirname(artifact_path)}/config/"
-            config = load_config(
-                config_path,
-                config_file_name=config_file_name,
-            )
-        except Exception:
-            # todo: temp fix for issue related to config load for byom models, update logic to choose the right path
+        config_file_path = f"{config_path}{config_file_name}"
+        if is_path_exists(config_file_path):
             try:
-                config_path = f"{artifact_path.rstrip('/')}/config/"
                 config = load_config(
                     config_path,
                     config_file_name=config_file_name,
                 )
             except Exception:
-                pass
+                logger.debug(
+                    f"Error loading the {config_file_name} at path {config_path}.\n"
+                    f"{traceback.format_exc()}"
+                )
 
         if not config:
-            logger.error(
-                f"{config_file_name} is not available for the model: {model_id}. Check if the custom metadata has the artifact path set."
+            logger.debug(
+                f"{config_file_name} is not available for the model: {model_id}. "
+                f"Check if the custom metadata has the artifact path set."
             )
             return config
 
diff --git a/ads/aqua/extension/model_handler.py b/ads/aqua/extension/model_handler.py
@@ -133,7 +133,7 @@ def post(self, *args, **kwargs):  # noqa: ARG002
         )
         local_dir = input_data.get("local_dir")
         cleanup_model_cache = (
-            str(input_data.get("cleanup_model_cache", "true")).lower() == "true"
+            str(input_data.get("cleanup_model_cache", "false")).lower() == "true"
         )
         inference_container_uri = input_data.get("inference_container_uri")
         allow_patterns = input_data.get("allow_patterns")
diff --git a/ads/aqua/finetuning/entities.py b/ads/aqua/finetuning/entities.py
@@ -122,6 +122,8 @@ class CreateFineTuningDetails(Serializable):
         The log group id for fine tuning job infrastructure.
     log_id: (str, optional). Defaults to `None`.
         The log id for fine tuning job infrastructure.
+    watch_logs: (bool, optional). Defaults to `False`.
+        The flag to watch the job run logs when a fine-tuning job is created.
     force_overwrite: (bool, optional). Defaults to `False`.
         Whether to force overwrite the existing file in object storage.
     freeform_tags: (dict, optional)
@@ -148,6 +150,7 @@ class CreateFineTuningDetails(Serializable):
     subnet_id: Optional[str] = None
     log_id: Optional[str] = None
     log_group_id: Optional[str] = None
+    watch_logs: Optional[bool] = False
     force_overwrite: Optional[bool] = False
     freeform_tags: Optional[dict] = None
     defined_tags: Optional[dict] = None
diff --git a/ads/aqua/finetuning/finetuning.py b/ads/aqua/finetuning/finetuning.py
@@ -4,6 +4,8 @@
 
 import json
 import os
+import time
+import traceback
 from typing import Dict
 
 from oci.data_science.models import (
@@ -149,6 +151,15 @@ def create(
                 f"Logging is required for fine tuning if replica is larger than {DEFAULT_FT_REPLICA}."
             )
 
+        if create_fine_tuning_details.watch_logs and not (
+            create_fine_tuning_details.log_id
+            and create_fine_tuning_details.log_group_id
+        ):
+            raise AquaValueError(
+                "Logging is required for fine tuning if watch_logs is set to True. "
+                "Please provide log_id and log_group_id with the request parameters."
+            )
+
         ft_parameters = self._get_finetuning_params(
             create_fine_tuning_details.ft_parameters
         )
@@ -422,6 +433,20 @@ def create(
             value=source.display_name,
         )
 
+        if create_fine_tuning_details.watch_logs:
+            logger.info(
+                f"Watching fine-tuning job run logs for {ft_job_run.id}. Press Ctrl+C to stop watching logs.\n"
+            )
+            try:
+                ft_job_run.watch()
+            except KeyboardInterrupt:
+                logger.info(f"\nStopped watching logs for {ft_job_run.id}.\n")
+                time.sleep(1)
+            except Exception:
+                logger.debug(
+                    f"Something unexpected occurred while watching logs.\n{traceback.format_exc()}"
+                )
+
         return AquaFineTuningSummary(
             id=ft_model.id,
             name=ft_model.display_name,
diff --git a/ads/aqua/model/entities.py b/ads/aqua/model/entities.py
@@ -283,7 +283,7 @@ class ImportModelDetails(CLIBuilderMixin):
     os_path: str
     download_from_hf: Optional[bool] = True
     local_dir: Optional[str] = None
-    cleanup_model_cache: Optional[bool] = True
+    cleanup_model_cache: Optional[bool] = False
     inference_container: Optional[str] = None
     finetuning_container: Optional[str] = None
     compartment_id: Optional[str] = None
diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py
@@ -29,7 +29,6 @@
     LifecycleStatus,
     _build_resource_identifier,
     cleanup_local_hf_model_artifact,
-    copy_model_config,
     create_word_icon,
     generate_tei_cmd_var,
     get_artifact_path,
@@ -969,24 +968,6 @@ def _create_model_catalog_entry(
                 )
                 tags[Tags.LICENSE] = validation_result.tags.get(Tags.LICENSE, UNKNOWN)
 
-        try:
-            # If verified model already has a artifact json, use that.
-            artifact_path = metadata.get(MODEL_BY_REFERENCE_OSS_PATH_KEY).value
-            logger.info(
-                f"Found model artifact in the service bucket. "
-                f"Using artifact from service bucket instead of {os_path}."
-            )
-
-            # todo: implement generic copy_folder method
-            # copy model config from artifact path to user bucket
-            copy_model_config(
-                artifact_path=artifact_path, os_path=os_path, auth=default_signer()
-            )
-        except Exception:
-            logger.debug(
-                f"Proceeding with model registration without copying model config files at {os_path}. "
-                f"Default configuration will be used for deployment and fine-tuning."
-            )
         # Set artifact location to user bucket, and replace existing key if present.
         metadata.add(
             key=MODEL_BY_REFERENCE_OSS_PATH_KEY,
diff --git a/ads/cli.py b/ads/cli.py
@@ -1,14 +1,15 @@
 #!/usr/bin/env python
-
 # Copyright (c) 2021, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+import json
 import logging
 import sys
 import traceback
 import uuid
 
 import fire
+from pydantic import BaseModel
 
 from ads.common import logger
 
@@ -84,7 +85,13 @@ def serialize(data):
         The string representation of each dataclass object.
     """
     if isinstance(data, list):
-        [print(str(item)) for item in data]
+        for item in data:
+            if isinstance(item, BaseModel):
+                print(json.dumps(item.dict(), indent=4))
+            else:
+                print(str(item))
+    elif isinstance(data, BaseModel):
+        print(json.dumps(data.dict(), indent=4))
     else:
         print(str(data))
 
diff --git a/tests/unitary/with_extras/aqua/test_config.py b/tests/unitary/with_extras/aqua/test_config.py
@@ -1,13 +1,17 @@
 #!/usr/bin/env python
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json
 import os
-from unittest.mock import patch
+import pytest
+from unittest.mock import patch, MagicMock
+
+import oci.data_science.models
 
 from ads.aqua.common.entities import ContainerSpec
 from ads.aqua.config.config import get_evaluation_service_config
+from ads.aqua.app import AquaApp
 
 
 class TestConfig:
@@ -37,3 +41,63 @@ def test_evaluation_service_config(self, mock_get_container_config):
             test_result.to_dict()
             == expected_result[ContainerSpec.CONTAINER_SPEC]["test_container"]
         )
+
+    @pytest.mark.parametrize(
+        "custom_metadata",
+        [
+            {
+                "category": "Other",
+                "description": "test_desc",
+                "key": "artifact_location",
+                "value": "artifact_location",
+            },
+            {},
+        ],
+    )
+    @pytest.mark.parametrize("verified_model", [True, False])
+    @pytest.mark.parametrize("path_exists", [True, False])
+    @patch("ads.aqua.app.load_config")
+    def test_load_config(
+        self, mock_load_config, custom_metadata, verified_model, path_exists
+    ):
+        mock_load_config.return_value = {"config_key": "config_value"}
+        service_model_tag = (
+            {"aqua_service_model": "aqua_service_model_id"} if verified_model else {}
+        )
+
+        self.app = AquaApp()
+
+        model = {
+            "id": "mock_id",
+            "lifecycle_details": "mock_lifecycle_details",
+            "lifecycle_state": "mock_lifecycle_state",
+            "project_id": "mock_project_id",
+            "freeform_tags": {
+                **{
+                    "OCI_AQUA": "",
+                },
+                **service_model_tag,
+            },
+            "custom_metadata_list": [
+                oci.data_science.models.Metadata(**custom_metadata)
+            ],
+        }
+
+        self.app.ds_client.get_model = MagicMock(
+            return_value=oci.response.Response(
+                status=200,
+                request=MagicMock(),
+                headers=MagicMock(),
+                data=oci.data_science.models.Model(**model),
+            )
+        )
+        with patch("ads.aqua.app.is_path_exists", return_value=path_exists):
+            result = self.app.get_config(
+                model_id="test_model_id", config_file_name="test_config_file_name"
+            )
+            if not path_exists:
+                assert result == {}
+            if not custom_metadata:
+                assert result == {}
+            if path_exists and custom_metadata:
+                assert result == {"config_key": "config_value"}
diff --git a/tests/unitary/with_extras/aqua/test_finetuning.py b/tests/unitary/with_extras/aqua/test_finetuning.py
@@ -22,6 +22,7 @@
 from ads.aqua.finetuning.constants import FineTuneCustomMetadata
 from ads.aqua.finetuning.entities import AquaFineTuningParams
 from ads.jobs.ads_job import Job
+from ads.jobs.builders.infrastructure.dsc_job import DataScienceJobRun
 from ads.model.datascience_model import DataScienceModel
 from ads.model.model_metadata import ModelCustomMetadata
 from ads.aqua.common.errors import AquaValueError
@@ -49,6 +50,12 @@ def tearDownClass(cls):
         reload(ads.aqua)
         reload(ads.aqua.finetuning.finetuning)
 
+    @parameterized.expand(
+        [
+            ("watch_logs", True),
+            ("watch_logs", False),
+        ]
+    )
     @patch.object(Job, "run")
     @patch("ads.jobs.ads_job.Job.name", new_callable=PropertyMock)
     @patch("ads.jobs.ads_job.Job.id", new_callable=PropertyMock)
@@ -60,6 +67,8 @@ def tearDownClass(cls):
     @patch.object(AquaApp, "get_source")
     def test_create_fine_tuning(
         self,
+        mock_watch_logs,
+        mock_watch_logs_called,
         mock_get_source,
         mock_mvs_create,
         mock_ds_model_create,
@@ -117,6 +126,7 @@ def test_create_fine_tuning(
         ft_job_run.id = "test_ft_job_run_id"
         ft_job_run.lifecycle_details = "Job run artifact execution in progress."
         ft_job_run.lifecycle_state = "IN_PROGRESS"
+        ft_job_run.watch = MagicMock()
         mock_job_run.return_value = ft_job_run
 
         self.app.ds_client.update_model = MagicMock()
@@ -144,7 +154,20 @@ def test_create_fine_tuning(
             defined_tags=ft_model_defined_tags,
         )
 
-        aqua_ft_summary = self.app.create(**create_aqua_ft_details)
+        inputs = {
+            **create_aqua_ft_details,
+            **{
+                mock_watch_logs: mock_watch_logs_called,
+                "log_id": "test_log_id",
+                "log_group_id": "test_log_group_id",
+            },
+        }
+        aqua_ft_summary = self.app.create(**inputs)
+
+        if mock_watch_logs_called:
+            ft_job_run.watch.assert_called()
+        else:
+            ft_job_run.watch.assert_not_called()
 
         assert aqua_ft_summary.to_dict() == {
             "console_url": f"https://cloud.oracle.com/data-science/models/{ft_model.id}?region={self.app.region}",
diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py
diff --git a/tests/unitary/with_extras/aqua/test_model_handler.py b/tests/unitary/with_extras/aqua/test_model_handler.py

Original file line number	Diff line number	Diff line change
`@@ -133,7 +133,7 @@ def post(self, args, *kwargs): # noqa: ARG002`
`133`	`133`	`)`
`134`	`134`	`local_dir = input_data.get("local_dir")`
`135`	`135`	`cleanup_model_cache = (`
`136`		`- str(input_data.get("cleanup_model_cache", "true")).lower() == "true"`
	`136`	`+ str(input_data.get("cleanup_model_cache", "false")).lower() == "true"`
`137`	`137`	`)`
`138`	`138`	`inference_container_uri = input_data.get("inference_container_uri")`
`139`	`139`	`allow_patterns = input_data.get("allow_patterns")`