evaluator fix

docs/examples/roboflow.md

@@ -7,7 +7,9 @@ python3 examples/roboflow_example.py --eval_data_path <path_to_eval_data> \
 --roboflow_workspace_url <workspace_url> \
 --roboflow_project_url <project_url> \
 --roboflow_model_version <model_version>
+--model_type:<model-type>
 ```
+`model_type` accepts values for the type of model, as shown in [How to evaluate with CVevals](https://blog.roboflow.com/evaluate-computer-vision-models-cvevals/#step-3-run-the-evaluation)
 
 ## Compare Roboflow Model Versions
 

evaluations/evaluator.py

@@ -56,19 +56,35 @@ def __init__(
         merged_data = {}
 
         for key in predictions.keys():
-            gt = ground_truth[key.split("/")[-1]]
+            gt = ground_truth.get(key, [])
 
-            gts = [list(item) for item in gt.xyxy]
+            # Key is the absolute path of the image, if splited and taken only filename
+            # it leads to a KeyError Exception because ground_truth has absolute paths
+            # gt = ground_truth[key.split("/")[-1]]
 
-            for idx, item in enumerate(gt.class_id):
-                gts[idx].append(item)
+            if gt:
+                gt = gt["ground_truth"]
+            else:
+                gt = []
 
             merged_data[key] = {
-                "ground_truth": gts,
+                "ground_truth": gt,
                 "predictions": predictions[key]["predictions"],
                 "filename": key,
             }
 
+            # gt does not have an attribute "xyxy", but it exist inside the "predictions" object
+            # gts = [list(item) for item in gt.xyxy]
+            #
+            # for idx, item in enumerate(gt.class_id):
+            #     gts[idx].append(item)
+            #
+            # merged_data[key] = {
+            #     "ground_truth": gts,
+            #     "predictions": predictions[key]["predictions"],
+            #     "filename": key,
+            # }
+
         self.data = merged_data
 
     def eval_model_predictions(self) -> dict:

examples/roboflow_example.py

@@ -22,30 +22,35 @@
 parser.add_argument(
     "--roboflow_model_version", type=int, required=True, help="Roboflow model version"
 )
+parser.add_argument(
+    "--model_type", type=str, required=True, help="Roboflow model type"
+)
 
 args = parser.parse_args()
 
 EVAL_DATA_PATH = args.eval_data_path
 ROBOFLOW_WORKSPACE_URL = args.roboflow_workspace_url
 ROBOFLOW_PROJECT_URL = args.roboflow_project_url
 ROBOFLOW_MODEL_VERSION = args.roboflow_model_version
+ROBOFLOW_MODEL_TYPE = args.model_type
 
 class_names, data, model = RoboflowDataLoader(
     workspace_url=ROBOFLOW_WORKSPACE_URL,
     project_url=ROBOFLOW_PROJECT_URL,
     project_version=ROBOFLOW_MODEL_VERSION,
     image_files=EVAL_DATA_PATH,
+    model_type=ROBOFLOW_MODEL_TYPE
 ).download_dataset()
 
 predictions = RoboflowPredictionsDataLoader(
     model=model,
-    model_type="object-detection",
+    model_type=ROBOFLOW_MODEL_TYPE,
     image_files=EVAL_DATA_PATH,
     class_names=class_names,
 ).process_files()
 
 evaluator = RoboflowEvaluator(
-    ground_truth=data, predictions=predictions, class_names=class_names, mode="batch"
+    ground_truth=data, predictions=predictions, class_names=class_names, mode="batch", model_type=ROBOFLOW_MODEL_TYPE
 )
 
 cf = evaluator.eval_model_predictions()