Skip to content

Commit

Permalink
Expanding graph (#197)
Browse files Browse the repository at this point in the history
* added a new tab and basic graph visualisation

* highlight last hovered node

* point/node preview

* graph wrapper size

* small fixes

* incorporated editor to the graph page

* refactoring

* formatted code

* make graph correctly handle re-render

* error handelling

* actual error type

* format

* add short manual

* adjust default size

* separate autocompletion

---------

Co-authored-by: generall <[email protected]>
  • Loading branch information
trean and generall authored Jul 28, 2024
1 parent 9b3162c commit 7b2ab8b
Show file tree
Hide file tree
Showing 16 changed files with 1,108 additions and 130 deletions.
521 changes: 514 additions & 7 deletions package-lock.json

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"@mui/icons-material": "^5.11.11",
"@mui/material": "^5.11.15",
"@mui/x-data-grid": "^6.0.4",
"@qdrant/js-client-rest": "^1.8.1",
"@qdrant/js-client-rest": "^1.10.0",
"@saehrimnir/druidjs": "^0.6.3",
"@testing-library/jest-dom": "^5.16.5",
"@testing-library/react": "^13.4.0",
Expand All @@ -29,6 +29,7 @@
"axios": "^1.6.7",
"chart.js": "^4.3.0",
"chroma-js": "^2.4.2",
"force-graph": "^1.43.5",
"jose": "^5.2.3",
"jsonc-parser": "^3.2.0",
"lodash": "^4.17.21",
Expand Down
41 changes: 4 additions & 37 deletions src/components/FilterEditorWindow/config/Autocomplete.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { OpenapiAutocomplete } from 'autocomplete-openapi/src/autocomplete';

export const autocomplete = async (monaco, qdrantClient, collectionName) => {
export const autocomplete = async (monaco, qdrantClient, collectionName, customRequestSchema) => {
const response = await fetch(import.meta.env.BASE_URL + './openapi.json');
const openapi = await response.json();

Expand All @@ -15,41 +15,8 @@ export const autocomplete = async (monaco, qdrantClient, collectionName) => {
} catch (e) {
console.error(e);
}
const FilterRequest = {
description: 'Filter request',
type: 'object',
properties: {
limit: {
description: 'Page size. Default: 10',
type: 'integer',
format: 'uint',
minimum: 1,
nullable: true,
},
filter: {
description: 'Look only for points which satisfies this conditions. If not provided - all points.',
anyOf: [
{
$ref: '#/components/schemas/Filter',
},
{
nullable: true,
},
],
},
vector_name: {
description: 'Vector field name',
type: 'string',
enum: vectorNames,
},
color_by: {
description: 'Color points by this field',
type: 'string',
nullable: true,
},
},
};
openapi.components.schemas.FilterRequest = FilterRequest;

openapi.components.schemas.CustomRequest = customRequestSchema(vectorNames);

const autocomplete = new OpenapiAutocomplete(openapi, []);

Expand Down Expand Up @@ -79,7 +46,7 @@ export const autocomplete = async (monaco, qdrantClient, collectionName) => {

const requestBody = requestBodyLines.join('\n');

let suggestions = autocomplete.completeRequestBodyByDataRef('#/components/schemas/FilterRequest', requestBody);
let suggestions = autocomplete.completeRequestBodyByDataRef('#/components/schemas/CustomRequest', requestBody);
suggestions = suggestions.map((s) => {
return {
label: s,
Expand Down
100 changes: 48 additions & 52 deletions src/components/FilterEditorWindow/config/RequestFromCode.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,54 @@
import axios from 'axios';
import { bigIntJSON } from '../../../common/bigIntJSON';

export async function requestFromCode(text, collectionName) {
const data = codeParse(text);
if (data.error) {
return data;
function parseDataToRequest(reqBody) {
// Validate color_by
if (reqBody.color_by) {
const colorBy = reqBody.color_by;

if (typeof colorBy === 'string') {
// Parse into payload variant
reqBody.color_by = {
payload: colorBy,
};
} else {
// Check we only have one of the options: payload, or discover_score
const options = [colorBy.payload, colorBy.discover_score];
const optionsCount = options.filter((option) => option).length;
if (optionsCount !== 1) {
return {
reqBody: reqBody,
error: '`color_by`: Only one of `payload`, or `discover_score` can be used',
};
}

// Put search arguments in main request body
if (colorBy.discover_score) {
reqBody = {
...reqBody,
...colorBy.discover_score,
};
}
}
}

// Set with_vector name
if (reqBody.vector_name) {
reqBody.with_vector = [reqBody.vector_name];
return {
reqBody: reqBody,
error: null,
};
} else if (!reqBody.vector_name) {
reqBody.with_vector = true;
return {
reqBody: reqBody,
error: null,
};
}
}
export async function requestFromCode(dataRaw, collectionName) {
const data = parseDataToRequest(dataRaw);
// Sending request
const colorBy = data.reqBody.color_by;
if (colorBy?.payload) {
Expand Down Expand Up @@ -108,62 +151,15 @@ async function discoverFromCode(collectionName, data) {
}

export function codeParse(codeText) {
let reqBody = {};

// Parse JSON
if (codeText) {
try {
reqBody = bigIntJSON.parse(codeText);
return bigIntJSON.parse(codeText);
} catch (e) {
return {
reqBody: codeText,
error: 'Fix the position brackets to run & check the json',
};
}
}

// Validate color_by
if (reqBody.color_by) {
const colorBy = reqBody.color_by;

if (typeof colorBy === 'string') {
// Parse into payload variant
reqBody.color_by = {
payload: colorBy,
};
} else {
// Check we only have one of the options: payload, or discover_score
const options = [colorBy.payload, colorBy.discover_score];
const optionsCount = options.filter((option) => option).length;
if (optionsCount !== 1) {
return {
reqBody: reqBody,
error: '`color_by`: Only one of `payload`, or `discover_score` can be used',
};
}

// Put search arguments in main request body
if (colorBy.discover_score) {
reqBody = {
...reqBody,
...colorBy.discover_score,
};
}
}
}

// Set with_vector name
if (reqBody.vector_name) {
reqBody.with_vector = [reqBody.vector_name];
return {
reqBody: reqBody,
error: null,
};
} else if (!reqBody.vector_name) {
reqBody.with_vector = true;
return {
reqBody: reqBody,
error: null,
};
}
}
28 changes: 19 additions & 9 deletions src/components/FilterEditorWindow/index.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,17 @@ import { useParams } from 'react-router-dom';
import { useClient } from '../../context/client-context';
import { useTheme } from '@mui/material/styles';
import { autocomplete } from './config/Autocomplete';
import { requestFromCode } from './config/RequestFromCode';
import { useSnackbar } from 'notistack';
import { codeParse } from './config/RequestFromCode';
import './editor.css';
import EditorCommon from '../EditorCommon';

const CodeEditorWindow = ({ onChange, code, onChangeResult }) => {
const CodeEditorWindow = ({ onChange, code, onChangeResult, customRequestSchema }) => {
const { enqueueSnackbar } = useSnackbar();
const editorRef = useRef(null);
const lensesRef = useRef(null);
const autocompleteRef = useRef(null);
const { collectionName } = useParams();

const { client: qdrantClient } = useClient();

let runBtnCommandId = null;
Expand All @@ -29,16 +30,25 @@ const CodeEditorWindow = ({ onChange, code, onChangeResult }) => {
[]
);

function onRun(codeText) {
const data = codeParse(codeText);
if (data.error) {
enqueueSnackbar(`Visualization Unsuccessful, error: ${JSON.stringify(data.error)}`, {
variant: 'error',
});
return data;
}
onChangeResult(data, collectionName);
}

function handleEditorDidMount(editor, monaco) {
editorRef.current = editor;
let decorations = [];

runBtnCommandId = editor.addCommand(
0,
async (_ctx, ...args) => {
const data = args[0];
const result = await requestFromCode(data, collectionName);
onChangeResult(result);
onRun(args[0]);
},
''
);
Expand Down Expand Up @@ -74,14 +84,13 @@ const CodeEditorWindow = ({ onChange, code, onChangeResult }) => {
);
editor.addCommand(monaco.KeyMod.CtrlCmd + monaco.KeyCode.Enter, async () => {
const data = selectedCodeBlock.blockText;
const result = await requestFromCode(data, collectionName);
onChangeResult(result);
onRun(data);
});
}
});
}
function handleEditorWillMount(monaco) {
autocomplete(monaco, qdrantClient, collectionName).then((autocomplete) => {
autocomplete(monaco, qdrantClient, collectionName, customRequestSchema).then((autocomplete) => {
autocompleteRef.current = monaco.languages.registerCompletionItemProvider('custom-language', autocomplete);
});
}
Expand All @@ -107,5 +116,6 @@ CodeEditorWindow.propTypes = {
onChange: PropTypes.func.isRequired,
code: PropTypes.string.isRequired,
onChangeResult: PropTypes.func.isRequired,
customRequestSchema: PropTypes.func.isRequired,
};
export default CodeEditorWindow;
100 changes: 100 additions & 0 deletions src/components/GraphVisualisation/GraphVisualisation.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import React, { useEffect, useRef } from 'react';
import PropTypes from 'prop-types';
import { deduplicatePoints, getSimilarPoints, initGraph } from '../../lib/graph-visualization-helpers';
import ForceGraph from 'force-graph';
import { useClient } from '../../context/client-context';
import { useSnackbar } from 'notistack';

const GraphVisualisation = ({ initNode, options, onDataDisplay, wrapperRef }) => {
const graphRef = useRef(null);
const { client: qdrantClient } = useClient();
const { enqueueSnackbar } = useSnackbar();
const NODE_R = 4;
let highlightedNode = null;

const handleNodeClick = async (node) => {
node.clicked = true;
const { nodes, links } = graphRef.current.graphData();
const pointId = node.id;

let similarPoints = [];
try {
similarPoints = await getSimilarPoints(qdrantClient, {
collectionName: options.collectionName,
pointId,
limit: options.limit,
filter: options.filter,
using: options.using,
});
} catch (e) {
enqueueSnackbar(e.message, { variant: 'error' });
return;
}

graphRef.current.graphData({
nodes: [...nodes, ...deduplicatePoints(nodes, similarPoints)],
links: [...links, ...similarPoints.map((point) => ({ source: pointId, target: point.id }))],
});
};

useEffect(() => {
const elem = document.getElementById('graph');
// eslint-disable-next-line new-cap
graphRef.current = ForceGraph()(elem)
.nodeColor((node) => (node.clicked ? '#e94' : '#2cb'))
.onNodeHover((node) => {
if (!node) {
elem.style.cursor = 'default';
return;
}
node.aa = 1;
elem.style.cursor = 'pointer';
highlightedNode = node;
onDataDisplay(node);
})
.autoPauseRedraw(false)
.nodeCanvasObjectMode((node) => (node?.id === highlightedNode?.id ? 'before' : undefined))
.nodeCanvasObject((node, ctx) => {
if (!node) return;
// add ring for last hovered nodes
ctx.beginPath();
ctx.arc(node.x, node.y, NODE_R * 1.4, 0, 2 * Math.PI, false);
ctx.fillStyle = node.id === highlightedNode?.id ? '#817' : 'transparent';
ctx.fill();
})
.linkColor(() => '#a6a6a6');
}, [initNode, options]);

useEffect(() => {
graphRef.current.width(wrapperRef?.clientWidth).height(wrapperRef?.clientHeight);
}, [wrapperRef, initNode, options]);

useEffect(() => {
const initNewGraph = async () => {
const graphData = await initGraph(qdrantClient, {
...options,
initNode,
});
if (graphRef.current && options) {
const initialActiveNode = graphData.nodes[0];
onDataDisplay(initialActiveNode);
highlightedNode = initialActiveNode;
graphRef.current.graphData(graphData).linkDirectionalArrowLength(3).onNodeClick(handleNodeClick);
}
};
initNewGraph().catch((e) => {
enqueueSnackbar(JSON.stringify(e.getActualType()), { variant: 'error' });
});
}, [initNode, options]);

return <div id="graph"></div>;
};

GraphVisualisation.propTypes = {
initNode: PropTypes.object,
options: PropTypes.object.isRequired,
onDataDisplay: PropTypes.func.isRequired,
wrapperRef: PropTypes.object,
};

export default GraphVisualisation;
Loading

0 comments on commit 7b2ab8b

Please sign in to comment.