Visualize discovery scores (#134)

* make discovery call possible

* fmt

* fix typo in filename

* paint colors 🧑‍🎨

* add description of color_by usage

* hacky show image tooltip

* hacky but better show only image

* Make tooltip more beautifulerer

* fmt

* improvements on tooltip and sample code

* additional improvements from review, thanks @Rendez

* fmt

* remove extra comma

package.json

@@ -28,6 +28,7 @@
     "autocomplete-openapi": "^0.1.1",
     "axios": "^1.3.4",
     "chart.js": "^4.3.0",
+    "chroma-js": "^2.4.2",
     "jsonc-parser": "^3.2.0",
     "lodash": "^4.17.21",
     "monaco-editor": "^0.44.0",

src/components/FilterEditorWindow/config/RequestFromCode.js

@@ -0,0 +1,167 @@
+import axios from 'axios';
+
+export async function requestFromCode(text, collectionName) {
+  const data = codeParse(text);
+  if (data.error) {
+    return data;
+  }
+  // Sending request
+  const colorBy = data.reqBody.color_by;
+  if (colorBy?.payload) {
+    return await actionFromCode(collectionName, data, 'scroll');
+  }
+  if (colorBy?.discover_score) {
+    return discoverFromCode(collectionName, data);
+  }
+  return await actionFromCode(collectionName, data, 'scroll');
+}
+
+async function actionFromCode(collectionName, data, action) {
+  try {
+    const response = await axios({
+      method: 'POST',
+      url: `collections/${collectionName}/points/${action || 'scroll'}`,
+      data: data.reqBody,
+    });
+    response.data.color_by = data.reqBody.color_by;
+    response.data.vector_name = data.reqBody.vector_name;
+    return {
+      data: response.data,
+      error: null,
+    };
+  } catch (err) {
+    return {
+      data: null,
+      error: err.response?.data?.status ? err.response?.data?.status : err,
+    };
+  }
+}
+
+async function discoverFromCode(collectionName, data) {
+  // Do 20/80 split. 20% of the points will be returned with the query
+  // and 80 % will be returned with random sampling
+  const queryLimit = Math.floor(data.reqBody.limit * 0.2);
+  const randomLimit = data.reqBody.limit - queryLimit;
+  data.reqBody.limit = queryLimit;
+  data.reqBody.with_payload = true;
+
+  const queryResponse = await actionFromCode(collectionName, data, 'discover');
+  if (queryResponse.error) {
+    return {
+      data: null,
+      error: queryResponse.error,
+    };
+  }
+
+  // Add tag to know which points were returned by the query
+  queryResponse.data.result = queryResponse.data.result.map((point) => ({
+    ...point,
+    from_query: true,
+  }));
+
+  // Get "random" points ids.
+  // There is no sampling endpoint in Qdrant yet, so for now we just scroll excluding the previous results
+  const idsToExclude = queryResponse.data.result.map((point) => point.id);
+
+  const originalFilter = data.reqBody.filter;
+  const mustNotFilter = [{ has_id: idsToExclude }];
+  data.reqBody.filter = originalFilter || {};
+  data.reqBody.filter.must_not = mustNotFilter.concat(data.reqBody.filter.must_not ?? []);
+
+  data.reqBody.limit = randomLimit;
+  const randomResponse = await actionFromCode(collectionName, data, 'scroll');
+  if (randomResponse.error) {
+    return {
+      data: null,
+      error: randomResponse.error,
+    };
+  }
+
+  // Then score these random points
+  const idsToInclude = randomResponse.data.result.points.map((point) => point.id);
+  const mustFilter = [{ has_id: idsToInclude }];
+  data.reqBody.filter = originalFilter || {};
+  data.reqBody.filter.must = mustFilter.concat(data.reqBody.filter.must || []);
+
+  const scoredRandomResponse = await actionFromCode(collectionName, data, 'discover');
+  if (scoredRandomResponse.error) {
+    return {
+      data: null,
+      error: scoredRandomResponse.error,
+    };
+  }
+
+  // Concat both results
+  const points = queryResponse.data.result.concat(scoredRandomResponse.data.result);
+
+  return {
+    data: {
+      ...queryResponse.data,
+      result: {
+        points: points,
+      },
+    },
+    error: null,
+  };
+}
+
+export function codeParse(codeText) {
+  let reqBody = {};
+
+  // Parse JSON
+  if (codeText) {
+    try {
+      reqBody = JSON.parse(codeText);
+    } catch (e) {
+      return {
+        reqBody: codeText,
+        error: 'Fix the position brackets to run & check the json',
+      };
+    }
+  }
+
+  // Validate color_by
+  if (reqBody.color_by) {
+    const colorBy = reqBody.color_by;
+
+    if (typeof colorBy === 'string') {
+      // Parse into payload variant
+      reqBody.color_by = {
+        payload: colorBy,
+      };
+    } else {
+      // Check we only have one of the options: payload, or discover_score
+      const options = [colorBy.payload, colorBy.discover_score];
+      const optionsCount = options.filter((option) => option).length;
+      if (optionsCount !== 1) {
+        return {
+          reqBody: reqBody,
+          error: '`color_by`: Only one of `payload`, or `discover_score` can be used',
+        };
+      }
+
+      // Put search arguments in main request body
+      if (colorBy.discover_score) {
+        reqBody = {
+          ...reqBody,
+          ...colorBy.discover_score,
+        };
+      }
+    }
+  }
+
+  // Set with_vector name
+  if (reqBody.vector_name) {
+    reqBody.with_vector = [reqBody.vector_name];
+    return {
+      reqBody: reqBody,
+      error: null,
+    };
+  } else if (!reqBody.vector_name) {
+    reqBody.with_vector = true;
+    return {
+      reqBody: reqBody,
+      error: null,
+    };
+  }
+}

src/components/FilterEditorWindow/index.jsx

@@ -5,7 +5,7 @@ import { useParams } from 'react-router-dom';
 import { useClient } from '../../context/client-context';
 import { useTheme } from '@mui/material/styles';
 import { autocomplete } from './config/Autocomplete';
-import { requestFromCode } from './config/RequesFromCode';
+import { requestFromCode } from './config/RequestFromCode';
 import './editor.css';
 import EditorCommon from '../EditorCommon';
 

src/components/VisualizeChart/ImageTooltip.jsx

@@ -0,0 +1,100 @@
+import { toFont } from 'chart.js/helpers';
+import React from 'react';
+import { createRoot } from 'react-dom/client';
+import { flushSync } from 'react-dom';
+
+const DEFAULT_BORDER_COLOR = '#333333';
+
+export function imageTooltip(context) {
+  // Tooltip Element
+  let tooltipEl = document.getElementById('chartjs-tooltip');
+
+  // Create element on first render
+  if (!tooltipEl) {
+    tooltipEl = document.createElement('div');
+    tooltipEl.id = 'chartjs-tooltip';
+    tooltipEl.appendChild(document.createElement('table'));
+    document.body.appendChild(tooltipEl);
+  }
+
+  // Hide if no tooltip
+  const tooltipModel = context.tooltip;
+  if (tooltipModel.opacity === 0) {
+    tooltipEl.style.opacity = 0;
+    return;
+  }
+
+  // Set caret Position
+  tooltipEl.classList.remove('above', 'below', 'no-transform');
+  if (tooltipModel.yAlign) {
+    tooltipEl.classList.add(tooltipModel.yAlign);
+  } else {
+    tooltipEl.classList.add('no-transform');
+  }
+
+  // Set content
+  if (tooltipModel.body) {
+    const bodyLines = tooltipModel.body[0].lines;
+
+    const imageSrc = tooltipModel.dataPoints[0].dataset.data[tooltipModel.dataPoints[0].dataIndex].point.payload?.image;
+
+    const borderColor = tooltipModel.labelColors[0]?.borderColor || DEFAULT_BORDER_COLOR;
+
+    const child = (
+      <div
+        style={{
+          display: 'flex',
+          flexDirection: 'column',
+          width: 'auto',
+        }}
+      >
+        {imageSrc && (
+          <img
+            src={imageSrc}
+            style={{
+              width: '250px',
+              height: 'auto',
+              objectFit: 'cover',
+              border: '3px solid ' + borderColor,
+              borderRadius: '10px 10px 0px 0px',
+            }}
+          />
+        )}
+        <div
+          style={{
+            display: 'flex',
+            flexDirection: 'column',
+            backgroundColor: '#333333',
+            borderRadius: '0px 0px 10px 10px',
+            padding: '5px',
+          }}
+        >
+          {bodyLines.map((line, i) => (
+            <span key={i} style={{ color: 'white' }}>
+              {line}
+            </span>
+          ))}
+        </div>
+      </div>
+    );
+
+    // Render html to insert in tooltip
+    const tableRoot = tooltipEl.querySelector('table');
+    const root = createRoot(tableRoot);
+    flushSync(() => {
+      root.render(child);
+    });
+  }
+
+  const position = context.chart.canvas.getBoundingClientRect();
+  const bodyFont = toFont(tooltipModel.options.bodyFont);
+
+  // Display, position, and set styles for font
+  tooltipEl.style.opacity = 1;
+  tooltipEl.style.position = 'absolute';
+  tooltipEl.style.left = position.left + window.scrollX + tooltipModel.caretX + 'px';
+  tooltipEl.style.top = position.top + window.scrollY + tooltipModel.caretY + 'px';
+  tooltipEl.style.font = bodyFont.string;
+  tooltipEl.style.padding = tooltipModel.padding + 'px ' + tooltipModel.padding + 'px';
+  tooltipEl.style.pointerEvents = 'none';
+}