Iconography for model eval types

app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx

@@ -1,4 +1,3 @@
-import _ from "lodash";
 import { Dialog } from "@fiftyone/components";
 import { editingFieldAtom, view } from "@fiftyone/state";
 import {
@@ -48,6 +47,9 @@ import EvaluationNotes from "./EvaluationNotes";
 import EvaluationPlot from "./EvaluationPlot";
 import Status from "./Status";
 import { formatValue, getNumericDifference, useTriggerEvent } from "./utils";
+import EvaluationIcon from "./EvaluationIcon";
+import { ConcreteEvaluationType } from "./Types";
+import get from "lodash/get";
 
 const KEY_COLOR = "#ff6d04";
 const COMPARE_KEY_COLOR = "#03a9f4";
@@ -128,12 +130,12 @@ export default function Evaluation(props: EvaluationProps) {
     compareEvaluationMaskTargets,
   ]);
   const compareKeys = useMemo(() => {
-    const keys: string[] = [];
+    const keys: Record<string, string>[] = [];
     const evaluations = data?.evaluations || [];
     for (const evaluation of evaluations) {
-      const { key } = evaluation;
+      const { key, type, method } = evaluation;
       if (key !== name) {
-        keys.push(key);
+        keys.push({ key, type, method });
       }
     }
     return keys;
@@ -523,7 +525,7 @@ export default function Evaluation(props: EvaluationProps) {
   return (
     <Stack spacing={2} sx={{ p: 2 }}>
       <Stack direction="row" sx={{ justifyContent: "space-between" }}>
-        <Stack direction="row" spacing={1} sx={{ alignItems: "center" }}>
+        <Stack direction="row" spacing={0} sx={{ alignItems: "center" }}>
           <IconButton
             onClick={() => {
               navigateBack();
@@ -532,6 +534,7 @@ export default function Evaluation(props: EvaluationProps) {
           >
             <ArrowBack />
           </IconButton>
+          <EvaluationIcon type={evaluationType} method={evaluationMethod} />
           <Typography>{name}</Typography>
         </Stack>
         <Stack direction="row" spacing={2} sx={{ alignItems: "center" }}>
@@ -601,6 +604,10 @@ export default function Evaluation(props: EvaluationProps) {
                 height: 28,
                 width: "100%",
                 background: theme.palette.background.card,
+                "& .MuiOutlinedInput-input": {
+                  display: "flex",
+                  alignItems: "center",
+                },
               }}
               defaultValue={compareKey}
               onChange={(e) => {
@@ -627,9 +634,13 @@ export default function Evaluation(props: EvaluationProps) {
                 ) : null
               }
             >
-              {compareKeys.map((key) => {
+              {compareKeys.map(({ key, type, method }) => {
                 return (
-                  <MenuItem value={key} key={key}>
+                  <MenuItem value={key} key={key} sx={{ p: 0 }}>
+                    <EvaluationIcon
+                      type={type as ConcreteEvaluationType}
+                      method={method}
+                    />
                     <Typography>{key}</Typography>
                   </MenuItem>
                 );
@@ -1803,10 +1814,10 @@ type SummaryRow = {
 
 function formatCustomMetricRows(evaluationMetrics, comparisonMetrics) {
   const results = [] as SummaryRow[];
-  const customMetrics = (_.get(evaluationMetrics, "custom_metrics", null) ||
+  const customMetrics = (get(evaluationMetrics, "custom_metrics", null) ||
     {}) as CustomMetrics;
   for (const [operatorUri, customMetric] of Object.entries(customMetrics)) {
-    const compareValue = _.get(
+    const compareValue = get(
       comparisonMetrics,
       `custom_metrics.${operatorUri}.value`,
       null

app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationIcon.tsx

@@ -0,0 +1,59 @@
+import _, { capitalize } from "lodash";
+
+import { IconButton } from "@mui/material";
+import React from "react";
+
+import CallSplitOutlinedIcon from "@mui/icons-material/CallSplitOutlined";
+import CrisisAlertOutlinedIcon from "@mui/icons-material/CrisisAlertOutlined";
+import Layers from "@mui/icons-material/Layers";
+import PieChartOutlinedIcon from "@mui/icons-material/PieChartOutlined";
+import ShowChartOutlinedIcon from "@mui/icons-material/ShowChartOutlined";
+
+import TooltipProvider from "../TooltipProvider";
+import { ConcreteEvaluationType } from "./Types";
+
+interface Props {
+  type: ConcreteEvaluationType;
+  method?: string;
+}
+
+export default function EvaluationIcon(props: Props) {
+  const { type, method } = props;
+
+  let evalIcon = <Layers />;
+  if (type === "classification" && method === "binary") {
+    evalIcon = <CallSplitOutlinedIcon />;
+  } else if (type === "classification" && method !== "binary") {
+    evalIcon = <Layers />;
+  } else if (type === "detection") {
+    evalIcon = <CrisisAlertOutlinedIcon />;
+  } else if (type === "segmentation") {
+    evalIcon = <PieChartOutlinedIcon />;
+  } else if (type === "regression") {
+    evalIcon = <ShowChartOutlinedIcon />;
+  }
+
+  return (
+    <TooltipProvider
+      title={
+        <span style={{ fontSize: "1rem", fontWeight: "normal" }}>
+          Evaluation type: {capitalize(type)}
+        </span>
+      }
+      arrow
+      placement="bottom"
+    >
+      <IconButton
+        size="small"
+        sx={{
+          color: "#FFC48B",
+          "&:hover": {
+            backgroundColor: "transparent",
+          },
+        }}
+      >
+        {evalIcon}
+      </IconButton>
+    </TooltipProvider>
+  );
+}

app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Overview.tsx

@@ -1,16 +1,15 @@
 import { LoadingDots } from "@fiftyone/components";
-import {
-  Card,
-  CardActionArea,
-  Chip,
-  Stack,
-  Typography,
-  useTheme,
-} from "@mui/material";
+import { Card, CardActionArea, Chip, Stack, Typography } from "@mui/material";
 import React from "react";
 import Evaluate from "./Evaluate";
 import EvaluationNotes from "./EvaluationNotes";
 import Status from "./Status";
+import {
+  ConcreteEvaluationType,
+  EvaluationCardProps,
+  OverviewProps,
+} from "./Types";
+import EvaluationIcon from "./EvaluationIcon";
 
 export default function Overview(props: OverviewProps) {
   const {
@@ -40,7 +39,7 @@ export default function Overview(props: OverviewProps) {
         />
       </Stack>
       {evaluations.map((evaluation) => {
-        const { key, id } = evaluation;
+        const { key, id, type, method } = evaluation;
         const status = statuses[id] || "needs_review";
         const note = notes[id];
         return (
@@ -50,18 +49,22 @@ export default function Overview(props: OverviewProps) {
             id={id}
             status={status}
             note={note}
+            type={type}
+            method={method}
             onSelect={onSelect}
           />
         );
       })}
       {pending_evaluations.map((evaluation) => {
-        const { eval_key } = evaluation;
+        const { eval_key, type, method } = evaluation;
         return (
           <EvaluationCard
             key={eval_key}
             eval_key={eval_key}
             pending
             onSelect={onSelect}
+            type={type}
+            method={method}
           />
         );
       })}
@@ -70,8 +73,7 @@ export default function Overview(props: OverviewProps) {
 }
 
 function EvaluationCard(props: EvaluationCardProps) {
-  const { pending, onSelect, eval_key, note, status, id } = props;
-  const theme = useTheme();
+  const { pending, onSelect, eval_key, note, status, id, type, method } = props;
 
   return (
     <CardActionArea key={eval_key} disabled={pending}>
@@ -81,10 +83,20 @@ function EvaluationCard(props: EvaluationCardProps) {
           onSelect(eval_key, id);
         }}
       >
-        <Stack direction="row" justifyContent="space-between">
-          <Typography sx={{ fontSize: 16, fontWeight: 600 }}>
-            {eval_key}
-          </Typography>
+        <Stack
+          direction="row"
+          justifyContent="space-between"
+          alignItems="center"
+        >
+          <Stack direction="row" alignItems="center" spacing={0.5}>
+            <EvaluationIcon
+              type={type as ConcreteEvaluationType}
+              method={method}
+            />
+            <Typography sx={{ fontSize: 16, fontWeight: 600 }}>
+              {eval_key}
+            </Typography>
+          </Stack>
           {pending && (
             <Chip
               variant="filled"
@@ -104,34 +116,3 @@ function EvaluationCard(props: EvaluationCardProps) {
     </CardActionArea>
   );
 }
-
-type OverviewProps = {
-  evaluations: EvaluationType[];
-  onSelect: (key: string, id: string) => void;
-  onEvaluate: () => void;
-  statuses?: Record<string, string>;
-  notes?: Record<string, string>;
-  permissions?: Record<string, boolean>;
-  pending_evaluations: PendingEvaluationType[];
-};
-
-type EvaluationType = {
-  key: string;
-  id: string;
-  description: string;
-  status: string;
-};
-
-type PendingEvaluationType = {
-  eval_key: string;
-  doc_id?: string;
-};
-
-type EvaluationCardProps = {
-  eval_key: string;
-  id?: string;
-  note?: string;
-  onSelect: OverviewProps["onSelect"];
-  pending?: boolean;
-  status?: string;
-};

app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Types.tsx

@@ -0,0 +1,42 @@
+export type OverviewProps = {
+  evaluations: EvaluationType[];
+  onSelect: (key: string, id: string) => void;
+  onEvaluate: () => void;
+  statuses?: Record<string, string>;
+  notes?: Record<string, string>;
+  permissions?: Record<string, boolean>;
+  pending_evaluations: PendingEvaluationType[];
+};
+
+export type EvaluationType = {
+  key: string;
+  id: string;
+  type: string;
+  description: string;
+  status: string;
+  method?: string;
+};
+
+export type PendingEvaluationType = {
+  eval_key: string;
+  type: string;
+  doc_id?: string;
+  method?: string;
+};
+
+export type EvaluationCardProps = {
+  eval_key: string;
+  type: string;
+  method?: string;
+  id?: string;
+  note?: string;
+  onSelect: OverviewProps["onSelect"];
+  pending?: boolean;
+  status?: string;
+};
+
+export type ConcreteEvaluationType =
+  | "classification"
+  | "detection"
+  | "segmentation"
+  | "regression";

app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx

@@ -30,9 +30,11 @@ export default function NativeModelEvaluationView(props) {
     pending_evaluations = [],
   } = data;
   const computedEvaluations = useMemo(() => {
-    return evaluations.map(({ key, id }) => ({
+    return evaluations.map(({ key, id, type, method }) => ({
       key,
       id,
+      type,
+      method,
       description: "The description for evaluation " + key,
       status: "reviewed",
     }));

plugins/panels/model_evaluation/__init__.py

@@ -89,6 +89,10 @@ def on_load(self, ctx):
                 evaluation = {
                     "key": key,
                     "id": self.get_evaluation_id(ctx.dataset, key),
+                    "type": ctx.dataset.get_evaluation_info(key).config.type,
+                    "method": ctx.dataset.get_evaluation_info(
+                        key
+                    ).config.method,
                 }
                 evaluations.append(evaluation)
         ctx.panel.set_state("evaluations", evaluations)
@@ -427,8 +431,15 @@ def load_pending_evaluations(self, ctx, skip_update=False):
                     pending
                 )
         for key in eval_keys:
+            conf = ctx.dataset.get_evaluation_info(key).config
             if not self.has_evaluation_results(ctx.dataset, key):
-                pending_evaluations.append({"eval_key": key})
+                pending_evaluations.append(
+                    {
+                        "eval_key": key,
+                        "type": conf.type,
+                        "method": conf.method,
+                    }
+                )
         if update_store:
             pending_evaluations_in_store[
                 dataset_id