diff --git a/Makefile b/Makefile index be59729f0..445e36b5b 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ pkg_src = visyn_core black = black --line-length 140 $(pkg_src) setup.py pyright = pyright $(pkg_src) setup.py -ruff = ruff $(pkg_src) setup.py --line-length 140 --select E,W,F,N,I,C,B,UP,PT,SIM,RUF --ignore E501,C901,B008 +ruff = ruff check $(pkg_src) setup.py --line-length 140 --select E,W,F,N,I,C,B,UP,PT,SIM,RUF --ignore E501,C901,B008 .PHONY: start ## Start the development server start: @@ -26,7 +26,7 @@ check-format: .PHONY: lint ## Run flake8 and pyright lint: - $(ruff) --format=github + $(ruff) --output-format=github $(pyright) .PHONY: test ## Run tests diff --git a/package.json b/package.json index 6b0e91b6e..110a3a0a6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "visyn_core", "description": "Core repository for datavisyn applications.", - "version": "9.2.2", + "version": "10.0.0", "author": { "name": "datavisyn GmbH", "email": "contact@datavisyn.io", @@ -114,7 +114,7 @@ "react-plotly.js": "^2.5.1", "react-spring": "^9.7.1", "use-deep-compare-effect": "^1.8.0", - "visyn_scripts": "^8.0.0" + "visyn_scripts": "^9.0.0" }, "devDependencies": { "chromatic": "^6.19.9", diff --git a/requirements.txt b/requirements.txt index ec0e4e869..99ded8da4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,31 +1,30 @@ # a2wsgi==1.6.0 # This WSIGMiddleware is not compatible with starlette_context -alembic==1.11.1 -cachetools==5.3.2 -fastapi==0.101.1 -Flask[async]>=2.1.0,<=2.2.2 +alembic==1.13.1 +cachetools==5.3.3 +fastapi==0.110.1 json-cfg==0.4.2 openpyxl==3.1.2 -opentelemetry-api==1.19.0 -opentelemetry-exporter-otlp==1.19.0 -opentelemetry-exporter-prometheus==1.12.0rc1 -opentelemetry-instrumentation-fastapi==0.40b0 -opentelemetry-instrumentation-httpx==0.40b0 -opentelemetry-instrumentation-logging==0.40b0 -opentelemetry-instrumentation-requests==0.40b0 -opentelemetry-instrumentation-sqlalchemy==0.40b0 -opentelemetry-instrumentation-system-metrics==0.40b0 -opentelemetry-sdk==1.19.0 -psycopg==3.1.9 -psycopg2==2.9.6 -pydantic==1.10.11 +opentelemetry-api==1.24.0 +opentelemetry-exporter-otlp==1.24.0 +opentelemetry-exporter-prometheus==0.45b0 +opentelemetry-instrumentation-fastapi==0.45b0 +opentelemetry-instrumentation-httpx==0.45b0 +opentelemetry-instrumentation-logging==0.45b0 +opentelemetry-instrumentation-requests==0.45b0 +opentelemetry-instrumentation-sqlalchemy==0.45b0 +opentelemetry-instrumentation-system-metrics==0.45b0 +opentelemetry-sdk==1.24.0 +psycopg==3.1.18 +psycopg2==2.9.9 +pydantic==1.10.14 pyjwt[crypto]==2.8.0 -pytest-postgresql==5.0.0 +pytest-postgresql==6.0.0 python-dateutil==2.8.2 -python-multipart==0.0.6 +python-multipart==0.0.9 requests==2.31.0 SQLAlchemy>=1.4.40,<=1.4.49 starlette-context==0.3.6 -uvicorn[standard]==0.23.1 +uvicorn[standard]==0.29.0 # Extras from fastapi[all], which we can't install because it requires pydantic v2: https://github.com/tiangolo/fastapi/blob/f7e3559bd5997f831fb9b02bef9c767a50facbc3/pyproject.toml#L57-L67 httpx>=0.23.0 jinja2>=2.11.2 @@ -33,5 +32,3 @@ itsdangerous>=1.1.0 pyyaml>=5.3.1 ujson>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0 orjson>=3.2.1 -# pin werkzeug version because the new major version 3.0.0 breaks our applications on Oct 2 2023; the version can be removed later again when the other packages support v3.0.0 -werkzeug==2.3.7 diff --git a/requirements_dev.txt b/requirements_dev.txt index 499776345..17a1739ef 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,5 @@ -black~=22.12.0 -pyright==1.1.308 -pytest-runner~=6.0.0 -pytest~=7.2.0 -ruff==0.0.218 +black~=24.3.0 +pyright==1.1.356 +pytest-runner~=6.0.1 +pytest~=8.1.1 +ruff==0.3.5 diff --git a/src/idtype/IDTypeManager.ts b/src/idtype/IDTypeManager.ts index 22a993909..decff4211 100644 --- a/src/idtype/IDTypeManager.ts +++ b/src/idtype/IDTypeManager.ts @@ -113,7 +113,7 @@ export class IDTypeManager { */ public searchMapping = (idType: IDType, pattern: string, toIDType: string | IDType, limit = 10): Promise<{ match: string; to: string }[]> => { const target = this.resolveIdType(toIDType); - return appContext.getAPIJSON(`/idtype/${idType.id}/${target.id}/search`, { q: pattern, limit }); + return appContext.getAPIJSON(`/idtype/${idType.id}/${target.id}/search/`, { q: pattern, limit }); }; /** @@ -140,7 +140,7 @@ export class IDTypeManager { if (idType.id === target.id) { return names; } - return IDType.chooseRequestMethod(`/idtype/${idType.id}/${target.id}`, { q: names, mode: 'first' }); + return IDType.chooseRequestMethod(`/idtype/${idType.id}/${target.id}/`, { q: names, mode: 'first' }); }; public mapNameToName = async (idType: IDType, names: string[], toIDtype: IDTypeLike): Promise => { @@ -149,7 +149,7 @@ export class IDTypeManager { // if(idType.id === target.id) { // return names.map((name) => [name]); // } - return IDType.chooseRequestMethod(`/idtype/${idType.id}/${target.id}`, { q: names }); + return IDType.chooseRequestMethod(`/idtype/${idType.id}/${target.id}/`, { q: names }); }; public findMappablePlugins = (target: IDType, all: IPluginDesc[]) => { diff --git a/src/ranking/overrides/DatavisynTaggle.tsx b/src/ranking/overrides/DatavisynTaggle.tsx index c9710b9eb..a35f02560 100644 --- a/src/ranking/overrides/DatavisynTaggle.tsx +++ b/src/ranking/overrides/DatavisynTaggle.tsx @@ -1,4 +1,4 @@ -import { IRankingDump, Ranking, Taggle, DataProvider, LocalDataProvider, TaggleRenderer } from 'lineupjs'; +import { Column, DataProvider, IRankingDump, LocalDataProvider, Ranking, Taggle, TaggleRenderer } from 'lineupjs'; import castArray from 'lodash/castArray'; import { IScoreColumnDesc, IScoreResult } from '../score/interfaces'; @@ -37,8 +37,11 @@ export class DatavisynTaggle extends * Creates a score column in the supplied ranking. Uses the default ranking if none is supplied. * * @param desc The score description + * @param options Options for the score creation (optional) */ - createScoreColumn(desc: IScoreResult, ranking = this.ranking) { + createScoreColumn(desc: IScoreResult, options?: { ranking?: Ranking; insertAfter?: Column }) { + const ranking = options?.ranking || this.ranking; + if (!ranking) { throw new Error('No ranking found'); } @@ -55,6 +58,9 @@ export class DatavisynTaggle extends const col = this.data.create(colDesc); + if (options?.insertAfter) { + return ranking.insertAfter(col, options.insertAfter); + } return ranking.push(col); }); } diff --git a/src/security/LoginUtils.ts b/src/security/LoginUtils.ts index ed4e2ed76..d9316b0e3 100644 --- a/src/security/LoginUtils.ts +++ b/src/security/LoginUtils.ts @@ -1,5 +1,4 @@ -import { appContext } from '../base/AppContext'; -import { userSession, UserSession } from './UserSession'; +import { userSession } from './UserSession'; import { IUser, IUserStore } from './interfaces'; import { Ajax } from '../base/ajax'; @@ -13,7 +12,7 @@ export class LoginUtils { */ static login(username: string, password: string) { userSession.reset(); - const r = Ajax.send('/login', { username, password }, 'post').then((user) => { + const r = Ajax.send('/api/login', { username, password }, 'post').then((user) => { userSession.login(user); return user; }); @@ -31,7 +30,7 @@ export class LoginUtils { * @return {Promise} when done also from the server side */ static logout(): Promise { - return Ajax.send('/logout', {}, 'post') + return Ajax.send('/api/logout', {}, 'post') .then((r) => { userSession.logout(r); }) @@ -43,7 +42,7 @@ export class LoginUtils { } static loggedInAs(): Promise { - return Ajax.send('/loggedinas', {}, 'POST').then((user: string | IUser) => { + return Ajax.send('/api/loggedinas', {}, 'POST').then((user: string | IUser) => { if (typeof user !== 'string' && user.name) { return user; } diff --git a/src/utils/XlsxUtils.ts b/src/utils/XlsxUtils.ts index 9035e11d2..03d03893c 100644 --- a/src/utils/XlsxUtils.ts +++ b/src/utils/XlsxUtils.ts @@ -21,21 +21,21 @@ export class XlsxUtils { const data = new FormData(); data.set('file', file); - return appContext.sendAPI('/tdp/xlsx/to_json', data, 'POST'); + return appContext.sendAPI('/xlsx/to_json/', data, 'POST'); } static xlsx2jsonArray(file: File): Promise { const data = new FormData(); data.set('file', file); - return appContext.sendAPI('/tdp/xlsx/to_json_array', data, 'POST'); + return appContext.sendAPI('/xlsx/to_json_array/', data, 'POST'); } static json2xlsx(file: IXLSXJSONFile): Promise { - return Ajax.send(appContext.api2absURL('/tdp/xlsx/from_json'), file, 'POST', 'blob', 'application/json'); + return Ajax.send(appContext.api2absURL('/xlsx/from_json/'), file, 'POST', 'blob', 'application/json'); } static jsonArray2xlsx(file: any[][]): Promise { - return Ajax.send(appContext.api2absURL('/tdp/xlsx/from_json_array'), file, 'POST', 'blob', 'application/json'); + return Ajax.send(appContext.api2absURL('/xlsx/from_json_array/'), file, 'POST', 'blob', 'application/json'); } } diff --git a/src/vis/sidebar/FilterButtons.tsx b/src/vis/sidebar/FilterButtons.tsx index 3ddafef47..527f42c6c 100644 --- a/src/vis/sidebar/FilterButtons.tsx +++ b/src/vis/sidebar/FilterButtons.tsx @@ -8,21 +8,21 @@ interface FilterButtonsProps { export function FilterButtons({ callback }: FilterButtonsProps) { return ( - + - + - + - + diff --git a/src/vis/sidebar/MultiSelect.tsx b/src/vis/sidebar/MultiSelect.tsx index ce2535e78..66ec7a957 100644 --- a/src/vis/sidebar/MultiSelect.tsx +++ b/src/vis/sidebar/MultiSelect.tsx @@ -1,4 +1,4 @@ -import { CloseButton, Combobox, Input, Pill, PillsInput, Stack, Tooltip, useCombobox, Text, Group } from '@mantine/core'; +import { CloseButton, Combobox, Input, Pill, PillsInput, Stack, Tooltip, useCombobox, Text, Group, ScrollArea } from '@mantine/core'; import * as React from 'react'; import { ColumnInfo, EColumnTypes, VisColumn } from '../interfaces'; @@ -26,9 +26,9 @@ export function MultiSelect({ callback(currentSelected.filter((s) => s.id !== id)); }; - const handleValueSelect = (name: string) => { - const itemToAdd = filteredColumns.find((c) => c.info.name === name); - currentSelected.find((c) => c.name === name) ? handleValueRemove(itemToAdd.info.id) : callback([...currentSelected, itemToAdd.info]); + const handleValueSelect = (id: string) => { + const itemToAdd = filteredColumns.find((c) => c.info.id === id); + currentSelected.find((c) => c.id === id) ? handleValueRemove(itemToAdd.info.id) : callback([...currentSelected, itemToAdd.info]); }; const handleValueRemoveAll = () => { @@ -39,7 +39,7 @@ export function MultiSelect({ .filter((c) => !currentSelected.map((s) => s.id).includes(c.info.id)) .map((item) => { return ( - + - {options.length === 0 ? All options selected : options} + + + {options.length === 0 ? All options selected : options} + + ); diff --git a/src/vis/stories/Iris.stories.tsx b/src/vis/stories/Iris.stories.tsx index 66b8df58c..5f5fbf8dd 100644 --- a/src/vis/stories/Iris.stories.tsx +++ b/src/vis/stories/Iris.stories.tsx @@ -22,7 +22,7 @@ const Template: ComponentStory = (args) => { return (
- + {}} columns={columns} selected={selection} selectionCallback={setSelection} />
); diff --git a/src/vis/stories/Vis/Bar/BarRandom.stories.tsx b/src/vis/stories/Vis/Bar/BarRandom.stories.tsx index da050a9be..a7ca4d5b9 100644 --- a/src/vis/stories/Vis/Bar/BarRandom.stories.tsx +++ b/src/vis/stories/Vis/Bar/BarRandom.stories.tsx @@ -123,7 +123,7 @@ const Template: ComponentStory = (args) => {
- + {}} columns={columns} />
diff --git a/src/vis/stories/Vis/Correlation/CorrelationIris.stories.tsx b/src/vis/stories/Vis/Correlation/CorrelationIris.stories.tsx index 58a40691d..8b7799a0e 100644 --- a/src/vis/stories/Vis/Correlation/CorrelationIris.stories.tsx +++ b/src/vis/stories/Vis/Correlation/CorrelationIris.stories.tsx @@ -26,7 +26,7 @@ const Template: ComponentStory = (args) => {
- + {}} columns={columns} selected={selection} selectionCallback={setSelection} />
diff --git a/src/vis/stories/Vis/Heatmap/HeatmapRandom.stories.tsx b/src/vis/stories/Vis/Heatmap/HeatmapRandom.stories.tsx index 26d7dad43..7c33c3dfb 100644 --- a/src/vis/stories/Vis/Heatmap/HeatmapRandom.stories.tsx +++ b/src/vis/stories/Vis/Heatmap/HeatmapRandom.stories.tsx @@ -129,7 +129,7 @@ const Template: ComponentStory = (args) => {
- + {}} selected={selected} selectionCallback={setSelected} columns={columns} />
diff --git a/src/vis/stories/Vis/Hexbin/HexbinRandom.stories.tsx b/src/vis/stories/Vis/Hexbin/HexbinRandom.stories.tsx index 485167000..dddeb84f6 100644 --- a/src/vis/stories/Vis/Hexbin/HexbinRandom.stories.tsx +++ b/src/vis/stories/Vis/Hexbin/HexbinRandom.stories.tsx @@ -100,7 +100,7 @@ const Template: ComponentStory = (args) => {
- + {}} columns={columns} />
diff --git a/src/vis/stories/Vis/Raincloud/RaincloudRandom.stories.tsx b/src/vis/stories/Vis/Raincloud/RaincloudRandom.stories.tsx index 7eee5d333..9584e166c 100644 --- a/src/vis/stories/Vis/Raincloud/RaincloudRandom.stories.tsx +++ b/src/vis/stories/Vis/Raincloud/RaincloudRandom.stories.tsx @@ -132,7 +132,7 @@ const Template: ComponentStory = (args) => {
- + {}} columns={columns} selected={selected} selectionCallback={setSelected} />
diff --git a/src/vis/stories/Vis/Scatter/ScatterRandom.stories.tsx b/src/vis/stories/Vis/Scatter/ScatterRandom.stories.tsx index 91e59eedf..7ada2bf84 100644 --- a/src/vis/stories/Vis/Scatter/ScatterRandom.stories.tsx +++ b/src/vis/stories/Vis/Scatter/ScatterRandom.stories.tsx @@ -111,7 +111,7 @@ const Template: ComponentStory = (args) => {
- + {}} selected={selected} selectionCallback={setSelected} columns={columns} />
diff --git a/src/vis/stories/Vis/Violin/ViolinIris.stories.tsx b/src/vis/stories/Vis/Violin/ViolinIris.stories.tsx index 72329b34c..ba96d6654 100644 --- a/src/vis/stories/Vis/Violin/ViolinIris.stories.tsx +++ b/src/vis/stories/Vis/Violin/ViolinIris.stories.tsx @@ -21,7 +21,7 @@ const Template: ComponentStory = (args) => {
- + {}} columns={columns} />
diff --git a/src/vis/violin/ViolinVis.tsx b/src/vis/violin/ViolinVis.tsx index 458d02c11..f2011d388 100644 --- a/src/vis/violin/ViolinVis.tsx +++ b/src/vis/violin/ViolinVis.tsx @@ -8,43 +8,16 @@ import { Plotly } from '../../plotly/full'; import { InvalidCols } from '../general'; import { beautifyLayout } from '../general/layoutUtils'; import { ICommonVisProps } from '../interfaces'; -import { createViolinTraces } from './utils'; import { IViolinConfig } from './interfaces'; +import { createViolinTraces } from './utils'; export function ViolinVis({ config, columns, scales, dimensions, selectedList, selectedMap, selectionCallback }: ICommonVisProps) { const { value: traces, status: traceStatus, error: traceError } = useAsync(createViolinTraces, [columns, config, scales, selectedList, selectedMap]); - const [clearTimeoutValue, setClearTimeoutValue] = useState(null); const id = useMemo(() => uniqueId('ViolinVis'), []); const [layout, setLayout] = useState>(null); - // Filter out null values from traces as null values cause the tooltip to not show up - const filteredTraces = useMemo(() => { - if (!traces) return null; - const indexWithNull = traces.plots?.map( - (plot) => (plot?.data.y as PlotlyTypes.Datum[])?.reduce((acc: number[], curr, i) => (curr === null ? [...acc, i] : acc), []) as number[], - ); - const filtered = { - ...traces, - plots: traces?.plots?.map((p, p_index) => { - return { - ...p, - data: { - ...p.data, - y: (p.data?.y as PlotlyTypes.Datum[])?.filter((v, i) => !indexWithNull[p_index].includes(i)), - x: (p.data?.x as PlotlyTypes.Datum[])?.filter((v, i) => !indexWithNull[p_index].includes(i)), - ids: p.data?.ids?.filter((v, i) => !indexWithNull[p_index].includes(i)), - transforms: p.data?.transforms?.map( - (t) => (t.groups as unknown[])?.filter((v, i) => !indexWithNull[p_index].includes(i)) as Partial, - ), - }, - }; - }), - }; - return filtered; - }, [traces]); - const onClick = (e: (Readonly & { event: MouseEvent }) | null) => { if (!e || !e.points || !e.points[0]) { selectionCallback([]); @@ -87,22 +60,12 @@ export function ViolinVis({ config, columns, scales, dimensions, selectedList, s if (plotDiv) { // NOTE: @dv-usama-ansari: This is a hack to update the plotly plots on resize. // The `setTimeout` is used to pass the resize function to the next event loop, so that the plotly plots are rendered first. - const n = setTimeout(() => Plotly.Plots.resize(plotDiv)); - setClearTimeoutValue(n); + setTimeout(() => Plotly.Plots.resize(plotDiv)); } }, [id, dimensions, traces]); - // NOTE: @dv-usama-ansari: Clear the timeout on unmount. useEffect(() => { - return () => { - if (clearTimeoutValue) { - clearTimeout(clearTimeoutValue); - } - }; - }, [clearTimeoutValue]); - - useEffect(() => { - if (!filteredTraces) { + if (!traces) { return; } @@ -122,13 +85,14 @@ export function ViolinVis({ config, columns, scales, dimensions, selectedList, s family: 'Roboto, sans-serif', }, clickmode: 'event+select', + dragmode: false, // Disables zoom (makes no sense in violin plots) autosize: true, - grid: { rows: filteredTraces.rows, columns: filteredTraces.cols, xgap: 0.3, pattern: 'independent' }, + grid: { rows: traces.rows, columns: traces.cols, xgap: 0.3, pattern: 'independent' }, shapes: [], }; - setLayout((prev) => ({ ...prev, ...beautifyLayout(filteredTraces, innerLayout, prev, true) })); - }, [filteredTraces]); + setLayout((prev) => ({ ...prev, ...beautifyLayout(traces, innerLayout, prev, true) })); + }, [traces]); return ( - {traceStatus === 'success' && layout && filteredTraces?.plots.length > 0 ? ( + {traceStatus === 'success' && layout && traces?.plots.length > 0 ? ( p.data), ...filteredTraces.legendPlots.map((p) => p.data)]} + data={[...traces.plots.map((p) => p.data), ...traces.legendPlots.map((p) => p.data)]} layout={layout} config={{ responsive: true, displayModeBar: false }} useResizeHandler @@ -163,7 +127,7 @@ export function ViolinVis({ config, columns, scales, dimensions, selectedList, s }} /> ) : traceStatus !== 'pending' && traceStatus !== 'idle' && layout ? ( - + ) : null} ); diff --git a/src/vis/violin/utils.ts b/src/vis/violin/utils.ts index 4c6cb29e2..36ecfa35e 100644 --- a/src/vis/violin/utils.ts +++ b/src/vis/violin/utils.ts @@ -54,9 +54,10 @@ export async function createViolinTraces( // if we onl have numerical columns, add them individually. if (catColValues.length === 0) { for (const numCurr of numColValues) { + const y = numCurr.resolvedValues.map((v) => v.val); plots.push({ data: { - y: numCurr.resolvedValues.map((v) => v.val), + y, ids: numCurr.resolvedValues.map((v) => v.id), xaxis: plotCounter === 1 ? 'x' : `x${plotCounter}`, yaxis: plotCounter === 1 ? 'y' : `y${plotCounter}`, @@ -88,11 +89,15 @@ export async function createViolinTraces( for (const numCurr of numColValues) { for (const catCurr of catColValues) { + const y = numCurr.resolvedValues.map((v) => v.val); + // Null values in categorical columns would break the plot --> replace with 'missing' + const categoriesWithMissing = catCurr.resolvedValues?.map((v) => ({ ...v, val: v.val || 'missing' })); + const x = categoriesWithMissing.map((v) => v.val); plots.push({ data: { - x: catCurr.resolvedValues.map((v) => v.val), - ids: catCurr.resolvedValues.map((v) => v.id), - y: numCurr.resolvedValues.map((v) => v.val), + x, + y, + ids: categoriesWithMissing.map((v) => v.id), xaxis: plotCounter === 1 ? 'x' : `x${plotCounter}`, yaxis: plotCounter === 1 ? 'y' : `y${plotCounter}`, type: 'violin', @@ -112,14 +117,14 @@ export async function createViolinTraces( transforms: [ { type: 'groupby', - groups: catCurr.resolvedValues.map((v) => v.val) as string[], - styles: [...new Set(catCurr.resolvedValues.map((v) => v.val) as string[])].map((c) => { + groups: x as string[], + styles: [...new Set(x as string[])].map((c) => { return { target: c, value: { line: { color: - selectedList.length !== 0 && catCurr.resolvedValues.filter((val) => val.val === c).find((val) => selectedMap[val.id]) + selectedList.length !== 0 && categoriesWithMissing.filter((val) => val.val === c).find((val) => selectedMap[val.id]) ? SELECT_COLOR : '#878E95', }, diff --git a/visyn_core/__init__.py b/visyn_core/__init__.py index 0262b9a45..722dfdd08 100644 --- a/visyn_core/__init__.py +++ b/visyn_core/__init__.py @@ -23,17 +23,11 @@ def init_app(self, app: FastAPI): def register(self, registry: RegHelper): # phovea_server - registry.append( - "namespace", - "caleydo-idtype", - "visyn_core.id_mapping.idtype_api", - {"namespace": "/api/idtype", "factory": "create_idtype"}, - ) + registry.append_router("caleydo-idtype", "visyn_core.id_mapping.idtype_api", {}) # General routers - registry.append("namespace", "visyn_core_main", "visyn_core.server.mainapp", {"namespace": "/app"}) registry.append_router("visyn_plugin_router", "visyn_core.plugin.router", {}) - registry.append("namespace", "visyn_xlsx2json", "visyn_core.xlsx", {"namespace": "/api/tdp/xlsx"}) + registry.append_router("visyn_xlsx_router", "visyn_core.xlsx", {}) # DB migration plugins registry.append( @@ -42,12 +36,6 @@ def register(self, registry: RegHelper): "visyn_core.dbmigration.manager", {"factory": "create_migration_command"}, ) - registry.append( - "namespace", - "db-migration-api", - "visyn_core.dbmigration.router", - {"factory": "create_migration_api", "namespace": "/api/tdp/db-migration"}, - ) # Security plugins registry.append( diff --git a/visyn_core/dbmanager.py b/visyn_core/dbmanager.py index f2e68e001..a9670cc50 100644 --- a/visyn_core/dbmanager.py +++ b/visyn_core/dbmanager.py @@ -30,9 +30,9 @@ def init_app(self, app: FastAPI): if not connector.dburl: connector.dburl = config["dburl"] if not connector.statement_timeout: - connector.statement_timeout = config.get("statement_timeout", None) + connector.statement_timeout = config.get("statement_timeout") if not connector.statement_timeout_query: - connector.statement_timeout_query = config.get("statement_timeout_query", None) + connector.statement_timeout_query = config.get("statement_timeout_query") if not connector.dburl: _log.critical( "no db url defined for %s at config key %s - is your configuration up to date?", diff --git a/visyn_core/dbmigration/env.py b/visyn_core/dbmigration/env.py index a6a3b1da6..23061d52d 100644 --- a/visyn_core/dbmigration/env.py +++ b/visyn_core/dbmigration/env.py @@ -1,5 +1,6 @@ from alembic import context from sqlalchemy import engine_from_config, pool +from sqlalchemy.engine import Engine # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -38,7 +39,7 @@ def run_migrations_online(): if version_table_schema: additional_configuration["version_table_schema"] = version_table_schema - connectable = engine_from_config( + connectable: Engine = engine_from_config( config.get_section(config.config_ini_section), # type: ignore prefix="sqlalchemy.", poolclass=pool.NullPool, diff --git a/visyn_core/dbmigration/manager.py b/visyn_core/dbmigration/manager.py index 64484c2c5..676626d28 100644 --- a/visyn_core/dbmigration/manager.py +++ b/visyn_core/dbmigration/manager.py @@ -180,8 +180,8 @@ def init_app(self, app: FastAPI, plugins: list[AExtensionDesc] | None = None): version_table_schema = config.get("versionTableSchema") or (p.versionTableSchema if hasattr(p, "versionTableSchema") else None) auto_upgrade = ( config.get("autoUpgrade") - if type(config.get("autoUpgrade")) == bool - else (p.autoUpgrade if hasattr(p, "autoUpgrade") and type(p.autoUpgrade) == bool else auto_upgrade_default) + if isinstance(config.get("autoUpgrade"), bool) + else (p.autoUpgrade if hasattr(p, "autoUpgrade") and isinstance(p.autoUpgrade, bool) else auto_upgrade_default) ) # Validate the plugin description diff --git a/visyn_core/dbmigration/router.py b/visyn_core/dbmigration/router.py deleted file mode 100644 index 7d5b2228f..000000000 --- a/visyn_core/dbmigration/router.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging - -from flask import Flask, abort, jsonify - -from .. import manager -from ..security import login_required -from .manager import DBMigration - -__author__ = "Datavisyn" -_log = logging.getLogger(__name__) - -app = Flask(__name__) - - -def _get_migration_by_id(id: str) -> DBMigration: - if id not in manager.db_migration: - abort(404, f"No migration with id {id} found") - return manager.db_migration[id] - - -@app.route("/") -@login_required -def list_migrations(): - return jsonify([migration.id for migration in manager.db_migration.migrations]), 200 - - -@app.route("/") -@login_required -def list_migration(id): - return jsonify(_get_migration_by_id(id).id), 200 - - -def create_migration_api(): - return app diff --git a/visyn_core/dbview.py b/visyn_core/dbview.py index 431dbccac..5da167201 100644 --- a/visyn_core/dbview.py +++ b/visyn_core/dbview.py @@ -514,12 +514,10 @@ def add_common_queries( .idtype(idtype) .table(table) .query( - """ - SELECT {id}, {{column}} AS text + f""" + SELECT {id_query}, {{column}} AS text FROM {table} WHERE LOWER({{column}}) LIKE :query - ORDER BY {{column}} ASC""".format( - id=id_query, table=table - ) + ORDER BY {{column}} ASC""" ) .replace("column", columns) .call(call_function) @@ -533,11 +531,9 @@ def add_common_queries( .idtype(idtype) .table(table) .query( - """ - SELECT {id}, {name} AS text - FROM {table}""".format( - id=id_query, table=table, name=name_column - ) + f""" + SELECT {id_query}, {name_column} AS text + FROM {table}""" ) .call(call_function) .call(inject_where) @@ -548,15 +544,13 @@ def add_common_queries( queries[prefix + "_unique"] = ( DBViewBuilder("lookup") .query( - """ + f""" SELECT d as id, d as text FROM ( SELECT distinct {{column}} AS d FROM {table} WHERE LOWER({{column}}) LIKE :query ) as t - ORDER BY d ASC""".format( - table=table - ) + ORDER BY d ASC""" ) .replace("column", columns) .call(limit_offset) @@ -567,11 +561,9 @@ def add_common_queries( queries[prefix + "_unique_all"] = ( DBViewBuilder("helper") .query( - """ + f""" SELECT distinct {{column}} AS text - FROM {table} ORDER BY {{column}} ASC """.format( - table=table - ) + FROM {table} ORDER BY {{column}} ASC """ ) .replace("column", columns) .build() @@ -625,7 +617,7 @@ def create_engine(self, config) -> Engine: "pool_pre_ping": True, } engine_options.update(config.get("engine", {})) - return sqlalchemy.create_engine(self.dburl, **engine_options) + return sqlalchemy.create_engine(self.dburl, **engine_options) # type: ignore def create_sessionmaker(self, engine) -> sessionmaker: return sessionmaker(bind=engine) diff --git a/visyn_core/id_mapping/idtype_api.py b/visyn_core/id_mapping/idtype_api.py index c6da0e575..0da579390 100644 --- a/visyn_core/id_mapping/idtype_api.py +++ b/visyn_core/id_mapping/idtype_api.py @@ -1,10 +1,12 @@ import logging +from typing import Literal -from flask import Flask, abort, jsonify, request +from fastapi import APIRouter +from pydantic import BaseModel from .. import manager -app_idtype = Flask(__name__) +idtype_router = APIRouter(prefix="/api/idtype", tags=["idtype"]) _log = logging.getLogger(__name__) @@ -15,59 +17,64 @@ def to_plural(s): return s + "s" -@app_idtype.route("/") -def _list_idtypes(): - tmp = {} - # TODO: We probably don't want to have these idtypes as "all" idtypes - # for d in list_datasets(): - # for idtype in d.to_idtype_descriptions(): - # tmp[idtype["id"]] = idtype +class IdType(BaseModel): + id: str + name: str + names: list[str] - # also include the known elements from the mapping graph - for idtype_id in manager.id_mapping.known_idtypes(): - tmp[idtype_id] = {"id": idtype_id, "name": idtype_id, "names": to_plural(idtype_id)} - return jsonify(list(tmp.values())) +class IdTypeMappingRequest(BaseModel): + q: list[str] + mode: Literal["all", "first"] = "all" -@app_idtype.route("//") -def _maps_to(idtype): - target_id_types = manager.id_mapping.maps_to(idtype) - return jsonify(target_id_types) +class IdTypeMappingSearchRequest(BaseModel): + q: str + limit: int | None = 10 -@app_idtype.route("//", methods=["GET", "POST"]) -def _mapping_to(idtype, to_idtype): - return _do_mapping(idtype, to_idtype) +class IdTypeMappingSearchResponse(BaseModel): + match: str + to: str -@app_idtype.route("///search") -def _mapping_to_search(idtype, to_idtype): - query = request.args.get("q", None) - max_results = int(request.args.get("limit", 10)) # type: ignore - if hasattr(manager.id_mapping, "search"): - return jsonify(manager.id_mapping.search(idtype, to_idtype, query, max_results)) - return jsonify([]) + +@idtype_router.get("/", response_model=list[IdType]) +def list_idtypes(): + # TODO: We probably don't want to have these idtypes as "all" idtypes + # for d in list_datasets(): + # for idtype in d.to_idtype_descriptions(): + # tmp[idtype["id"]] = idtype + + # also include the known elements from the mapping graph + return [IdType(id=idtype_id, name=idtype_id, names=to_plural(idtype_id)) for idtype_id in manager.id_mapping.known_idtypes()] -def _do_mapping(idtype, to_idtype): - args = request.values - first_only = args.get("mode", "all") == "first" +@idtype_router.get("/{idtype}/", response_model=list[IdType]) +def maps_to(idtype: str): + return manager.id_mapping.maps_to(idtype) - if "q" in args: - names = args["q"].split(",") - elif "q[]" in args: - names = args.getlist("q[]") - else: - abort(400) - return +@idtype_router.api_route("/{idtype}/{to_idtype}/", methods=["GET", "POST"], response_model=list[str]) +def mapping_to(body: IdTypeMappingRequest, idtype: str, to_idtype: str): + first_only = body.mode == "first" + + names = body.q mapped_list = manager.id_mapping(idtype, to_idtype, names) if first_only: mapped_list = [None if a is None or len(a) == 0 else a[0] for a in mapped_list] - return jsonify(mapped_list) + return mapped_list + + +@idtype_router.get("/{idtype}/{to_idtype}/search/", response_model=list[IdTypeMappingSearchResponse]) +def mapping_to_search(body: IdTypeMappingSearchRequest, idtype, to_idtype): + query = body.q + max_results = body.limit + if hasattr(manager.id_mapping, "search"): + return manager.id_mapping.search(idtype, to_idtype, query, max_results) + return [] -def create_idtype(): - return app_idtype +def create(): + return idtype_router diff --git a/visyn_core/id_mapping/manager.py b/visyn_core/id_mapping/manager.py index bca48ecd9..9b79c84e1 100644 --- a/visyn_core/id_mapping/manager.py +++ b/visyn_core/id_mapping/manager.py @@ -17,7 +17,7 @@ def __init__(self, providers): self.mappers = {} self.paths = {} graph = {} - for (from_idtype, to_idtype, mapper) in providers: + for from_idtype, to_idtype, mapper in providers: # generate mapper mapping from_mappings = self.mappers.get(from_idtype, {}) self.mappers[from_idtype] = from_mappings diff --git a/visyn_core/middleware/close_web_sessions_middleware.py b/visyn_core/middleware/close_web_sessions_middleware.py index 3998ba552..980c40042 100644 --- a/visyn_core/middleware/close_web_sessions_middleware.py +++ b/visyn_core/middleware/close_web_sessions_middleware.py @@ -1,6 +1,6 @@ import contextlib -from fastapi import FastAPI +from starlette.types import ASGIApp from .request_context_plugin import get_request @@ -8,8 +8,8 @@ # Use basic ASGI middleware instead of BaseHTTPMiddleware as it is significantly faster: https://github.com/tiangolo/fastapi/issues/2696#issuecomment-768224643 # Raw middlewares are actually quite complex: https://github.com/encode/starlette/blob/048643adc21e75b668567fc6bcdd3650b89044ea/starlette/middleware/errors.py#L147 class CloseWebSessionsMiddleware: - def __init__(self, app: FastAPI): - self.app: FastAPI = app + def __init__(self, app: ASGIApp): + self.app: ASGIApp = app async def __call__(self, scope, receive, send): if scope["type"] != "http": diff --git a/visyn_core/middleware/exception_handler_middleware.py b/visyn_core/middleware/exception_handler_middleware.py index aa1aff7ec..20052d326 100644 --- a/visyn_core/middleware/exception_handler_middleware.py +++ b/visyn_core/middleware/exception_handler_middleware.py @@ -1,8 +1,8 @@ import logging -from fastapi import FastAPI, HTTPException +from fastapi import HTTPException from fastapi.exception_handlers import http_exception_handler -from starlette.types import Message +from starlette.types import ASGIApp, Message from ..server.utils import detail_from_exception @@ -10,8 +10,8 @@ # Use basic ASGI middleware instead of BaseHTTPMiddleware as it is significantly faster: https://github.com/tiangolo/fastapi/issues/2696#issuecomment-768224643 # Raw middlewares are actually quite complex: https://github.com/encode/starlette/blob/048643adc21e75b668567fc6bcdd3650b89044ea/starlette/middleware/errors.py#L147 class ExceptionHandlerMiddleware: - def __init__(self, app: FastAPI): - self.app: FastAPI = app + def __init__(self, app: ASGIApp): + self.app: ASGIApp = app async def __call__(self, scope, receive, send): if scope["type"] != "http": diff --git a/visyn_core/rdkit/models.py b/visyn_core/rdkit/models.py index 770cc6db9..88354dbce 100644 --- a/visyn_core/rdkit/models.py +++ b/visyn_core/rdkit/models.py @@ -1,3 +1,5 @@ +from typing import ClassVar + from pydantic import BaseModel from rdkit.Chem import Mol, MolFromSmarts, MolFromSmiles # type: ignore from starlette.responses import Response @@ -6,7 +8,7 @@ class SmilesMolecule(str): """We can't directly extend mol, as this would break swagger""" - parsers = [MolFromSmiles] + parsers: ClassVar = [MolFromSmiles] _mol: Mol @property @@ -32,7 +34,7 @@ def validate(cls, value: str | None) -> "SmilesMolecule": class SmilesSmartsMolecule(SmilesMolecule): """Try parings smiles first, then smarts""" - parsers = [MolFromSmiles, MolFromSmarts] + parsers: ClassVar = [MolFromSmiles, MolFromSmarts] class SvgResponse(Response): diff --git a/visyn_core/security/jwt_router.py b/visyn_core/security/jwt_router.py index b609ff962..848a8b59f 100644 --- a/visyn_core/security/jwt_router.py +++ b/visyn_core/security/jwt_router.py @@ -12,7 +12,7 @@ from .model import Token _log = logging.getLogger(__name__) -jwt_router = APIRouter(tags=["Security"]) +jwt_router = APIRouter(prefix="/api", tags=["Security"]) # TODO: Use schema to allow auto-doc of endpoint # from fastapi.security import OAuth2PasswordBearer @@ -86,7 +86,7 @@ def loggedinas(request: Request): return user_to_dict(user, access_token=user.access_token) if user else '"not_yet_logged_in"' -@jwt_router.get("/api/security/stores") +@jwt_router.get("/security/stores") def stores(request: Request) -> list[SecurityStoreResponse]: """Returns a list of activated security stores. Can be used to infer the details of the shown login menu.""" return [ diff --git a/visyn_core/server/mainapp.py b/visyn_core/server/mainapp.py index 1291de474..baea1d1a1 100644 --- a/visyn_core/server/mainapp.py +++ b/visyn_core/server/mainapp.py @@ -1,117 +1,14 @@ import logging import os -import re -from flask import Flask, send_from_directory -from werkzeug.security import safe_join - -from .. import manager +from fastapi import APIRouter _log = logging.getLogger(__name__) - -black_list = re.compile(r"(.*\.(py|pyc|gitignore|gitattributes)|(\w+)/((config|package)\.json|_deploy/.*))") -public_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "public")) - - -def _is_on_black_list(path): - # print 'check',path,black_list.match(path) is not None - return black_list.match(path) is not None - - -def _deliver_production(path): - # print path - if path.endswith("/"): - path += "index.html" - if _is_on_black_list(path): - return "This page does not exist", 404 - # serve public - return send_from_directory(public_dir, path) - - -def _deliver(path): - # print path - if path.endswith("/"): - path += "index.html" - if _is_on_black_list(path): - return "This page does not exist", 404 - - # serve public - if os.path.exists(safe_join(public_dir, path)): # type: ignore - return send_from_directory(public_dir, path) - - # check all plugins - elems = path.split("/") - if len(elems) > 0: - plugin_id = elems[0] - elems[0] = "build" - from .. import manager - - plugin = next((p for p in manager.registry.plugins if p.id == plugin_id), None) - if plugin: - dpath = safe_join(plugin.folder, "/".join(elems)) # type: ignore - if os.path.exists(dpath): # type: ignore - # send_static_file will guess the correct MIME type - # print 'sending',dpath - return send_from_directory(plugin.folder, "/".join(elems)) # type: ignore - - return "This page does not exist", 404 - - -def _generate_index(): - text = [ - """ - - Caleydo Web Apps - - -
-

Caleydo Web Apps

-
- - - """ - ) - return "\n".join(text) +mainapp_router = APIRouter(tags=["MainApp"]) +@mainapp_router.get("/buildInfo.json") def build_info(): from codecs import open @@ -139,17 +36,13 @@ def build_info(): # health check for docker-compose, kubernetes +@mainapp_router.api_route("/api/health", methods=["GET", "HEAD"]) async def health(): return "ok" -def create(): - # check initialization - app = Flask(__name__) - if manager.settings.is_development_mode: - app.add_url_rule("/", "index", _generate_index) - app.add_url_rule("/index.html", "index", _generate_index) - app.add_url_rule("/", "deliver", _deliver) - else: - app.add_url_rule("/", "deliver", _deliver_production) - return app +# TODO: Remove this endpoint after everyone switched to it. +@mainapp_router.api_route("/health", methods=["GET", "HEAD"]) +async def deprecated_health(): + _log.warn("Using deprecated /health endpoint. Consider switching to /api/health.") + return "ok" diff --git a/visyn_core/server/utils.py b/visyn_core/server/utils.py index b2c40e371..eff3efc34 100644 --- a/visyn_core/server/utils.py +++ b/visyn_core/server/utils.py @@ -3,43 +3,58 @@ import time import traceback -from flask import Flask, jsonify -from werkzeug.exceptions import HTTPException +from fastapi import HTTPException from .. import manager +# Flask is using exceptions from Werkzeug, which are not compatible with FastAPI's HTTPException +# Only import and use FlaskHTTPException if Flask is available +try: + from werkzeug.exceptions import HTTPException as FlaskHTTPException # type: ignore +except ImportError: + FlaskHTTPException = None + _log = logging.getLogger(__name__) -def init_legacy_app(app: Flask): - """ - initializes an application by setting common properties and options - :param app: - :param is_default_app: - :return: - """ - if hasattr(app, "got_first_request") and app.got_first_request: - return +init_legacy_app = None +try: + # Flask is an optional dependency and must be added to the requirements for legacy apps. + from flask import Flask, jsonify # type: ignore + + def _init_legacy_app(app: Flask): + """ + initializes an application by setting common properties and options + :param app: + :param is_default_app: + :return: + """ + if hasattr(app, "got_first_request") and app.got_first_request: + return + + if hasattr(app, "debug"): + # TODO: Evaluate if this should be set to manager.settings.is_development_mode + app.debug = False - if hasattr(app, "debug"): - # TODO: Evaluate if this should be set to manager.settings.is_development_mode - app.debug = False + if manager.settings.visyn_core: + app.config["SECRET_KEY"] = manager.settings.secret_key - if manager.settings.visyn_core: - app.config["SECRET_KEY"] = manager.settings.secret_key + @app.errorhandler(FlaskHTTPException) + @app.errorhandler(Exception) # type: ignore + async def handle_exception(e): + """Handles Flask exceptions by returning the same JSON response as FastAPI#HTTPException would.""" + _log.exception(repr(e)) + # Extract status information if a Flask#HTTPException is given, otherwise return 500 with exception information + status_code = e.code if FlaskHTTPException and isinstance(e, FlaskHTTPException) else 500 + detail = detail_from_exception(e) + # Exact same response as the one from FastAPI#HTTPException. + return jsonify({"detail": detail or http.HTTPStatus(status_code).phrase}), status_code - @app.errorhandler(HTTPException) - @app.errorhandler(Exception) # type: ignore - async def handle_exception(e): - """Handles Flask exceptions by returning the same JSON response as FastAPI#HTTPException would.""" - _log.exception(repr(e)) - # Extract status information if a Flask#HTTPException is given, otherwise return 500 with exception information - status_code = e.code if isinstance(e, HTTPException) else 500 - detail = detail_from_exception(e) - # Exact same response as the one from FastAPI#HTTPException. - return jsonify({"detail": detail or http.HTTPStatus(status_code).phrase}), status_code + return app - return app + init_legacy_app = _init_legacy_app +except ImportError: + pass def load_after_server_started_hooks(): @@ -73,6 +88,8 @@ def detail_from_exception(e: Exception) -> str | None: ) # Exception specific returns if isinstance(e, HTTPException): - return e.description + return e.detail + if FlaskHTTPException and isinstance(e, FlaskHTTPException): + return e.description # type: ignore # Fallback to the string representation of the exception return repr(e) diff --git a/visyn_core/server/visyn_server.py b/visyn_core/server/visyn_server.py index cb99a1d84..1b8971805 100644 --- a/visyn_core/server/visyn_server.py +++ b/visyn_core/server/visyn_server.py @@ -56,7 +56,7 @@ def create_visyn_server( # Filter out the metrics endpoint from the access log class EndpointFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: - return "GET /metrics" and "GET /health" not in record.getMessage() + return "GET /api/metrics" and "GET /api/health" and "GET /metrics" and "GET /health" not in record.getMessage() logging.getLogger("uvicorn.access").addFilter(EndpointFilter()) @@ -69,7 +69,7 @@ def filter(self, record: logging.LogRecord) -> bool: plugins = load_all_plugins() # With all the plugins, load the corresponding configuration files and create a new model based on the global settings, with all plugin models as sub-models [plugin_config_files, plugin_settings_models] = get_config_from_plugins(plugins) - visyn_server_settings = create_model("VisynServerSettings", __base__=GlobalSettings, **plugin_settings_models) + visyn_server_settings = create_model("VisynServerSettings", __base__=GlobalSettings, **plugin_settings_models) # type: ignore # Patch the global settings by instantiating the new settings model with the global config, all config.json(s), and pydantic models manager.settings = visyn_server_settings(**deep_update(*plugin_config_files, workspace_config)) @@ -140,10 +140,8 @@ def filter(self, record: logging.LogRecord) -> bool: _log.info(f"Registering {len(namespace_plugins)} legacy namespace(s) via WSGIMiddleware") for p in namespace_plugins: namespace = p.namespace # type: ignore - sub_app = p.load().factory() init_legacy_app(sub_app) - app.mount(namespace, WSGIMiddleware(sub_app)) # Load all FastAPI apis @@ -153,11 +151,9 @@ def filter(self, record: logging.LogRecord) -> bool: for p in router_plugins: app.include_router(p.load().factory()) - # TODO: Check mainapp.py what it does and transfer them here. Currently, we cannot mount a flask app at root, such that the flask app is now mounted at /app/ - from .mainapp import build_info, health + from .mainapp import mainapp_router - app.add_api_route("/health", health, methods=["GET", "HEAD"]) # type: ignore - app.add_api_route("/api/buildInfo.json", build_info) # type: ignore + app.include_router(mainapp_router) @app.on_event("startup") async def change_anyio_total_tokens(): diff --git a/visyn_core/settings/router.py b/visyn_core/settings/router.py index d6d5d3f05..a9665c126 100644 --- a/visyn_core/settings/router.py +++ b/visyn_core/settings/router.py @@ -5,6 +5,7 @@ router = APIRouter(tags=["Configuration"], prefix="/api/tdp/config", dependencies=[Depends(get_current_user)]) + # TODO: Move to tdp_core @router.get("/{path:path}") def get_config_path(path: str): diff --git a/visyn_core/telemetry/__init__.py b/visyn_core/telemetry/__init__.py index 191c5a0d8..208c552c3 100644 --- a/visyn_core/telemetry/__init__.py +++ b/visyn_core/telemetry/__init__.py @@ -56,9 +56,11 @@ def filter(self, record: logging.LogRecord) -> bool: PeriodicExportingMetricReader( exporter=OTLPMetricExporter( # If we are using the global exporter settings, append the metrics path - endpoint=_append_metrics_path(exporter_settings.endpoint) - if exporter_settings == global_exporter_settings - else exporter_settings.endpoint, + endpoint=( + _append_metrics_path(exporter_settings.endpoint) + if exporter_settings == global_exporter_settings + else exporter_settings.endpoint + ), headers=exporter_settings.headers, timeout=exporter_settings.timeout, **exporter_settings.kwargs, @@ -89,13 +91,21 @@ def shutdown_meter_event(): class CustomMetricsResponse(Response): media_type = CONTENT_TYPE_LATEST - @app.get("/metrics", tags=["Telemetry"], response_class=CustomMetricsResponse) + @app.get("/api/metrics", tags=["Telemetry"], response_class=CustomMetricsResponse) def prometheus_metrics(): """ Prometheus metrics endpoint. Is not required as we are pushing metrics using OTLPMetricExporter, but can be used for debugging purposes. """ return CustomMetricsResponse(generate_latest(REGISTRY), headers={"Content-Type": CONTENT_TYPE_LATEST}) + @app.get("/metrics", tags=["Telemetry"], response_class=CustomMetricsResponse) + def deprecated_prometheus_metrics(): + """ + Deprecated: Consider using /api/metrics instead. + """ + _log.warn("Using deprecated /metrics endpoint. Consider switching to /api/metrics.") + return CustomMetricsResponse(generate_latest(REGISTRY), headers={"Content-Type": CONTENT_TYPE_LATEST}) + tracer_provider: TracerProvider | None = None if settings.traces.enabled: _log.info("Enabling OpenTelemetry traces") @@ -112,9 +122,11 @@ def prometheus_metrics(): BatchSpanProcessor( OTLPSpanExporter( # If we are using the global exporter settings, append the traces path - endpoint=_append_trace_path(exporter_settings.endpoint) - if exporter_settings == global_exporter_settings - else exporter_settings.endpoint, + endpoint=( + _append_trace_path(exporter_settings.endpoint) + if exporter_settings == global_exporter_settings + else exporter_settings.endpoint + ), headers=exporter_settings.headers, timeout=exporter_settings.timeout, **exporter_settings.kwargs, @@ -155,9 +167,11 @@ def shutdown_tracer_event(): BatchLogRecordProcessor( OTLPLogExporter( # If we are using the global exporter settings, append the logs path - endpoint=_append_logs_path(exporter_settings.endpoint) - if exporter_settings == global_exporter_settings - else exporter_settings.endpoint, + endpoint=( + _append_logs_path(exporter_settings.endpoint) + if exporter_settings == global_exporter_settings + else exporter_settings.endpoint + ), headers=exporter_settings.headers, timeout=exporter_settings.timeout, **exporter_settings.kwargs, diff --git a/visyn_core/tests/fixtures/postgres_db.py b/visyn_core/tests/fixtures/postgres_db.py index 6dcd17c63..30e32f9d2 100644 --- a/visyn_core/tests/fixtures/postgres_db.py +++ b/visyn_core/tests/fixtures/postgres_db.py @@ -16,7 +16,7 @@ class PostgreSQLExecutorWithUrl(PostgreSQLExecutor): def postgres_db(postgresql_proc) -> Generator[PostgreSQLExecutorWithUrl, None, None]: d = postgresql_proc d.url = f"postgresql://{d.user}:{d.password}@{d.host}:{d.port}/{d.dbname}" - janitor = DatabaseJanitor(d.user, d.host, d.port, d.dbname, d.version, d.password) + janitor = DatabaseJanitor(user=d.user, host=d.host, port=d.port, dbname=d.dbname, version=d.version, password=d.password) janitor.init() # import this ONCE in your conftest.py, not in each test module yield d janitor.drop() diff --git a/visyn_core/tests/test_main_app.py b/visyn_core/tests/test_main_app.py index 70dfe8e3b..4571a743a 100644 --- a/visyn_core/tests/test_main_app.py +++ b/visyn_core/tests/test_main_app.py @@ -1,21 +1,9 @@ def test_health(client): - response = client.get("/health") + response = client.get("/api/health") assert response.status_code == 200 assert response.json() == "ok" -def test_buildinfo_json(client): - response = client.get("/api/buildInfo.json") - assert response.status_code == 200 - build_info = response.json() - # Check for the main build name - assert build_info["name"] == "visyn_core" - # Check if plugins are returned as list - assert isinstance(build_info["plugins"], list) - # Check if dependencies are returned (i.e. look for FastAPI) - assert next(d for d in build_info["dependencies"] if d.lower().startswith("fastapi")) - - def test_idtype(client): response = client.get("/api/idtype/") assert response.status_code == 200 diff --git a/visyn_core/tests/test_security_login.py b/visyn_core/tests/test_security_login.py index e6edc7dc7..0eaa3b687 100644 --- a/visyn_core/tests/test_security_login.py +++ b/visyn_core/tests/test_security_login.py @@ -9,13 +9,13 @@ def test_api_key(client: TestClient): - assert client.get("/loggedinas", headers={"apiKey": "invalid_user:password"}).json() == '"not_yet_logged_in"' - assert client.get("/loggedinas", headers={"apiKey": "admin:admin"}).json()["name"] == "admin" + assert client.get("/api/loggedinas", headers={"apiKey": "invalid_user:password"}).json() == '"not_yet_logged_in"' + assert client.get("/api/loggedinas", headers={"apiKey": "admin:admin"}).json()["name"] == "admin" def test_basic_authorization(client: TestClient): - assert client.get("/loggedinas", auth=("invalid_user", "password")).json() == '"not_yet_logged_in"' - assert client.get("/loggedinas", auth=("admin", "admin")).json()["name"] == "admin" + assert client.get("/api/loggedinas", auth=("invalid_user", "password")).json() == '"not_yet_logged_in"' + assert client.get("/api/loggedinas", auth=("admin", "admin")).json()["name"] == "admin" def test_jwt_login(client: TestClient): @@ -32,12 +32,12 @@ def claims_loader_2(user: User): assert stores == [{"id": "DummyStore", "ui": "DefaultLoginForm", "configuration": {}}] # Check if we are actually not logged in - response = client.get("/loggedinas") + response = client.get("/api/loggedinas") assert response.status_code == 200 assert response.json() == '"not_yet_logged_in"' # Login with the dummy user - response = client.post("/login", data={"username": "admin", "password": "admin"}) + response = client.post("/api/login", data={"username": "admin", "password": "admin"}) assert response.status_code == 200 user: dict = response.json() assert user["name"] == "admin" @@ -46,7 +46,7 @@ def claims_loader_2(user: User): assert user["payload"]["username"] == "admin" # Check if we are logged in and get the same response as from the login - response = client.get("/loggedinas") + response = client.get("/api/loggedinas") assert response.status_code == 200 assert user == response.json() assert ( @@ -58,7 +58,7 @@ def claims_loader_2(user: User): manager.settings.jwt_refresh_if_expiring_in_seconds = manager.settings.jwt_expire_in_seconds + 5 # Check if we are still logged in and get the same response as the refresh happens *after* the request - assert user == client.get("/loggedinas").json() + assert user == client.get("/api/loggedinas").json() assert ( client.cookies.get(manager.settings.jwt_access_cookie_name) != user["access_token"] ) # Access token is different in response and cookies @@ -67,7 +67,7 @@ def claims_loader_2(user: User): manager.settings.jwt_refresh_if_expiring_in_seconds = original_jwt_refresh_if_expiring_in_seconds # Check if we are logged in and get a different response as the cookie was auto-refreshed in the last request - refreshed_user = client.get("/loggedinas").json() + refreshed_user = client.get("/api/loggedinas").json() assert user["name"] == refreshed_user["name"] # Same user assert user["access_token"] != refreshed_user["access_token"] # But different token assert user["payload"]["exp"] < refreshed_user["payload"]["exp"] # With longer expiry date @@ -76,18 +76,18 @@ def claims_loader_2(user: User): ) # Access token is equal in new response and cookies # Logout - response = client.post("/logout") + response = client.post("/api/logout") assert response.status_code == 200 # Check if we are actually not logged in anymore - response = client.get("/loggedinas") + response = client.get("/api/loggedinas") assert response.status_code == 200 assert response.json() == '"not_yet_logged_in"' def test_jwt_token_location(client: TestClient): # Login to set a cookie - response = client.post("/login", data={"username": "admin", "password": "admin"}) + response = client.post("/api/login", data={"username": "admin", "password": "admin"}) assert response.status_code == 200 access_token = response.json()["access_token"] @@ -95,25 +95,25 @@ def test_jwt_token_location(client: TestClient): manager.settings.jwt_token_location = [] # Does not work even though both header and cookies are passed - response = client.get("/loggedinas", headers={"Authorization": f"Bearer {access_token}"}) + response = client.get("/api/loggedinas", headers={"Authorization": f"Bearer {access_token}"}) assert response.json() == '"not_yet_logged_in"' # Allow headers manager.settings.jwt_token_location = ["headers"] # Does not work as only headers are accepted - response = client.get("/loggedinas") + response = client.get("/api/loggedinas") assert response.json() == '"not_yet_logged_in"' # Does work as header is passed - response = client.get("/loggedinas", headers={"Authorization": f"Bearer {access_token}"}) + response = client.get("/api/loggedinas", headers={"Authorization": f"Bearer {access_token}"}) assert response.json() == '"not_yet_logged_in"' # Allow cookies manager.settings.jwt_token_location = ["cookies"] # Does work even without header - response = client.get("/loggedinas") + response = client.get("/api/loggedinas") assert response.json() != '"not_yet_logged_in"' @@ -123,7 +123,7 @@ def test_alb_security_store(client: TestClient): manager.settings.visyn_core.security.store.alb_security_store.email_token_field = ["field1", "field2", "email"] manager.settings.visyn_core.security.store.alb_security_store.decode_options = {"verify_signature": False} manager.settings.visyn_core.security.store.alb_security_store.cookie_name = "TestCookie" - manager.settings.visyn_core.security.store.alb_security_store.signout_url = "http://localhost/logout" + manager.settings.visyn_core.security.store.alb_security_store.signout_url = "http://localhost/api/logout" store = create_alb_security_store() assert store is not None @@ -141,26 +141,26 @@ def test_alb_security_store(client: TestClient): } # Check loggedinas with a JWT - response = client.get("/loggedinas", headers=headers) + response = client.get("/api/loggedinas", headers=headers) assert response.status_code == 200 assert response.json() != '"not_yet_logged_in"' assert response.json()["name"] == "admin@localhost" # Logout and check if we get the correct redirect url - response = client.post("/logout", headers=headers) + response = client.post("/api/logout", headers=headers) assert response.status_code == 200 - assert response.json()["redirect"] == "http://localhost/logout" + assert response.json()["redirect"] == "http://localhost/api/logout" # Test if we are not logged in if we use invalid fields store.email_token_fields = ["field1", "field2"] - assert client.get("/loggedinas", headers=headers).json() == '"not_yet_logged_in"' + assert client.get("/api/loggedinas", headers=headers).json() == '"not_yet_logged_in"' def test_oauth2_security_store(client: TestClient): # Add some basic configuration manager.settings.visyn_core.security.store.oauth2_security_store.enable = True manager.settings.visyn_core.security.store.oauth2_security_store.cookie_name = "TestCookie" - manager.settings.visyn_core.security.store.oauth2_security_store.signout_url = "http://localhost/logout" + manager.settings.visyn_core.security.store.oauth2_security_store.signout_url = "http://localhost/api/logout" store = create_oauth2_security_store() assert store is not None @@ -176,15 +176,15 @@ def test_oauth2_security_store(client: TestClient): } # Check loggedinas with a JWT - response = client.get("/loggedinas", headers=headers) + response = client.get("/api/loggedinas", headers=headers) assert response.status_code == 200 assert response.json() != '"not_yet_logged_in"' assert response.json()["name"] == "admin@localhost" # Logout and check if we get the correct redirect url - response = client.post("/logout", headers=headers) + response = client.post("/api/logout", headers=headers) assert response.status_code == 200 - assert response.json()["redirect"] == "http://localhost/logout" + assert response.json()["redirect"] == "http://localhost/api/logout" def test_no_security_store(client: TestClient): @@ -198,7 +198,7 @@ def test_no_security_store(client: TestClient): manager.security.user_stores = [store] - user_info = client.get("/loggedinas").json() + user_info = client.get("/api/loggedinas").json() assert user_info != '"not_yet_logged_in"' assert user_info["name"] == "test_name" assert user_info["roles"] == ["test_role"] @@ -214,7 +214,7 @@ def on_user_loaded_increment(user: User): assert counter == 0 - client.get("/loggedinas", auth=("admin", "admin")) + client.get("/api/loggedinas", auth=("admin", "admin")) assert counter == 1 @@ -223,6 +223,6 @@ def on_user_loaded_decrement(user: User): nonlocal counter counter -= 1 - client.get("/loggedinas", auth=("admin", "admin")) + client.get("/api/loggedinas", auth=("admin", "admin")) assert counter == 1 diff --git a/visyn_core/tests/test_telemetry.py b/visyn_core/tests/test_telemetry.py index 5e953f09b..5351de03e 100644 --- a/visyn_core/tests/test_telemetry.py +++ b/visyn_core/tests/test_telemetry.py @@ -23,9 +23,9 @@ ) def test_fastapi_metrics(client: TestClient): # Trigger a request - client.get("/health") + client.get("/api/health") - metrics_text = client.get("/metrics").text + metrics_text = client.get("/api/metrics").text parsed = {m.name: m for m in text_string_to_metric_families(metrics_text)} # Check for app info @@ -36,20 +36,20 @@ def test_fastapi_metrics(client: TestClient): # Check for request counts fastapi_requests_metric = parsed["fastapi_requests_1"] # TODO: Why _1? assert len(fastapi_requests_metric.samples) == 2 - assert fastapi_requests_metric.samples[0].labels["path"] == "/health" + assert fastapi_requests_metric.samples[0].labels["path"] == "/api/health" assert fastapi_requests_metric.samples[0].value == 1 - assert fastapi_requests_metric.samples[1].labels["path"] == "/metrics" + assert fastapi_requests_metric.samples[1].labels["path"] == "/api/metrics" assert fastapi_requests_metric.samples[1].value == 1 # Trigger it again - client.get("/health") - metrics_text = client.get("/metrics").text + client.get("/api/health") + metrics_text = client.get("/api/metrics").text parsed = {m.name: m for m in text_string_to_metric_families(metrics_text)} # And check for increased counts fastapi_requests_metric = parsed["fastapi_requests_1"] # TODO: Why _1? assert len(fastapi_requests_metric.samples) == 2 - assert fastapi_requests_metric.samples[0].labels["path"] == "/health" + assert fastapi_requests_metric.samples[0].labels["path"] == "/api/health" assert fastapi_requests_metric.samples[0].value == 2 - assert fastapi_requests_metric.samples[1].labels["path"] == "/metrics" + assert fastapi_requests_metric.samples[1].labels["path"] == "/api/metrics" assert fastapi_requests_metric.samples[1].value == 2 diff --git a/visyn_core/tests/test_xlsx.py b/visyn_core/tests/test_xlsx.py new file mode 100644 index 000000000..838d9a2c7 --- /dev/null +++ b/visyn_core/tests/test_xlsx.py @@ -0,0 +1,48 @@ +from starlette.testclient import TestClient + +table_defintion_json = { + "sheets": [ + { + "title": "TestTable", + "columns": [{"name": "Col1", "type": "string"}, {"name": "Col2", "type": "float"}, {"name": "Col3", "type": "boolean"}], + "rows": [{"Col1": "test1", "Col2": 0.5, "Col3": False}, {"Col1": "test2", "Col2": None, "Col3": True}], + } + ] +} + +# this is the xlsx representation of the table_defintion_json above +xlsx_from_json_result = b"PK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cXFZ\xc1\x0c\x82\x00\x00\x00\xb1\x00\x00\x00\x10\x00\x00\x00docProps/app.xmlM\x8eM\x0b\xc20\x10D\xffJ\xe9\xddnU\xf0 1 \xd4\xa3\xe0\xc9{H76\x90dCv\x85\xfc|S\xc1\x8f\xdb<\xde0\x8c\xba\x15\xcaX\xc4#w5\x86\xc4\xa7~\x11\xc9G\x00\xb6\x0bF\xc3C\xd3\xa9\x19G%\x1aiX\x1e@\xcey\x8b\x13\xd9g\xc4$\xb0\x1b\xc7\x03`\x15L3\xce\x9b\xfc\x1d\xec\xb5:\xe7\x1c\xbc5\xe2)\xe9\xab\xb7\x85\x98\x9ct\x97j1(\xf8\x97k\xf3\x8e\x85\xd7\xbc\x1f\xb6o\xf9a\x05\xbf\x93\xfa\x05PK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cX\xddR\xb3\x7f\xea\x00\x00\x00\xcb\x01\x00\x00\x11\x00\x00\x00docProps/core.xml\xa5\x91\xc1j\xc30\x0c\x86_\xa5\xe4\x9e(N\xcb(\xc6\xf5ec\xa7\x16\x06+l\xecfd\xb5\r\x8b\x13ck$}\xfb9Y\x9bnl\xb7\x1d\xad\xff\xd3'\t+\xf4\x12\xbb@O\xa1\xf3\x14\xb8\xa6\xb8\x18\\\xd3F\x89~\x93\x9d\x98\xbd\x04\x88x\"gb\x91\x886\x85\x87.8\xc3\xe9\x19\x8e\xe0\r\xbe\x9b#AU\x96w\xe0\x88\x8d5l`\x14\xe6~6f\x17\xa5\xc5Y\xe9?B3\t,\x025\xe4\xa8\xe5\x08\xa2\x10pc\x99\x82\x8b\x7f6L\xc9L\x0e\xb1\x9e\xa9\xbe\xef\x8b~9qi#\x01\xaf\xbb\xed\xf3\xb4|^\xb7\x91M\x8b\x94ieQb \xc3]\xd0\xe3E\xfe<4\n\xbe\x15\xd5e\xf6W\x81\xec\"M\x90|\xf6\xb4\xc9\xae\xc9\xcb\xf2\xfea\xff\x98\xe9\xaa\xacVy\xb9\xcaE\xb5\x17B\x96k)\xd6o\xa3\xebG\xffM\xe8:[\x1f\xea\x7f\x18\xaf\x02\xad\xe0\xd7\xbf\xe9OPK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cX\x99\\\x9c#\x10\x06\x00\x00\x9c'\x00\x00\x13\x00\x00\x00xl/theme/theme1.xml\xedZ[s\xda8\x14~\xef\xaf\xd0xg\xf6m\x0b\xc66\x81\xb6\xb4\x13siv\xdb\xb4\x99\x84\xedN\x1f\x85\x11X\x8dlyd\x91\x84\x7f\xbfG6\x10\xcb\x96\r\xed\x92M\xba\x9b<\x04,\xe9\xfb\xceEG\xe7\xe88y\xf3\xee.b\xe8\x86\x88\x94\xf2x`\xd9/\xdb\xd6\xbb\xb7/\xde\xe0W2$\x11A0\x19\xa7\xaf\xf0\xc0\n\xa5L^\xb5Zi\x00\xc38}\xc9\x13\x12\xc3\xdc\x82\x8b\x08Kx\x14\xcb\xd6\\\xe0[\x1a/#\xd6\xea\xb4\xdb\xddV\x84il\xa1\x18Gd`}^,h@\xd0TQZo_ \xb4\xe5\x1f3\xf8\x15\xcbT\x8de\xa3\x01\x13WA&\xb9\x88\xb4\xf2\xf9l\xc5\xfc\xda\xde>e\xcf\xe9:\x1d2\x81n0\x1bX \x7f\xceo\xa7\xe4NZ\x88\xe1T\xc2\xc4\xc0jg?Vk\xc7\xd1\xd2H\x80\x82\xc9}\x94\x05\xbaI\xf6\xa3\xd3\x15\x082\r;:\x9dX\xcev|\xf6\xc4\xed\x9f\x8c\xca\xdat4m\x1a\xe0\xe3\xf1x8\xb6\xcb\xd2\x8bp\x1c\x04\xe0Q\xbb\x9e\xc2\x9d\xf4l\xbf\xa4A\t\xb4\xa3i\xd0d\xd8\xf6\xda\xae\x91\xa6\xaa\x8dSO\xd3\xf7}\xdf\xeb\x9bh\x9c\n\x8d[O\xd3kw\xdd\xd3\x8e\x89\xc6\xad\xd0x\r\xbe\xf1O\x87\xc3\xae\x89\xc6\xab\xd0t\xebi&'\xfd\xaek\xa4\xe9\x16hBF\xe3\xebz\x12\x15\xb5\xe5@\xd3 \x00Xpv\xd6\xcc\xd2\x03\x96^)\xfau\x94\x1a\xd9\x1d\xbb\xddA\\\xf0X\xee9\x89\x11\xfe\xc6\xc5\x04\xd6i\xd2\x19\x964Fr\x9d\x90\x05\x0e\x007\xc4\xd1LP|\xafA\xb6\x8a\xe0\xc2\x92\xd2\\\x90\xd6\xcf)\xb5P\x1a\x08\x9a\xc8\x81\xf5G\x82!\xc5\xdc\xaf\xfd\xf5\x97\xbb\xc9\xa43z\x9d}:\xcek\x94\x7fi\xab\x01\xa7\xed\xbb\x9b\xcf\x93\xfcs\xe8\xe4\x9f\xa7\x93\xd7MB\xcep\xbc,\t\xf1\xfb#[a\x87'n;\x13r:\x1cgB|\xcf\xf6\xf6\x91\xa5%2\xcf\xef\xf9\n\xebNb\x196\x1ar\xb5\x16\x81\xb6q\xa9\x84`Z\x12\xc6\xd1xN\xd2\xb4\x11\xfcY\xac5\x93>`\xc8\xec\xcd\x91u\xce\xd6\x91\x0e\x11\x92^7B>b\xce\x8b\x90\x11\xbf\x1e\x868J\x9a\xed\xa2qX\x04\xfd\x9e^\xc3I\xc1\xe8\x82\xcbf\xfd\xb8~\x86\xd53l,\x8e\xf7G\xd4\x17J\xe4\x0f&\xa7?\xe924\x07\xa3\x9aY\t\xbd\x84Vj\x9f\xaa\x874>\xa8\x1e2\n\x05\xf1\xb9\x1e>\xe5zx\n7\x96\xc6\xbcP\xae\x82{\x01\xff\xd1\xda7\xc2\xab\xf8\x82\xc09\x7f.}\xcf\xa5\xef\xb9\xf4=\xa1\xd2\xb77#}g\xc1\xd3\x8b[\xdeFn[\xc4\xfb\xae1\xda\xd74.(cWr\xcd\xc8\xc7T\xaf\x93)\xd89\x9f\xc0\xec\xfdh>\x9e\xf1\xed\xfa\xd9$\x84\xaf\x9aY-#\x16\x90K\x81\xb3A$\xb8\xfc\x8b\xca\xf0*\xc4\t\xe8d[%\t\xcbT\xd3e7\x8a\x12\x9eB\x1bn\xe9S\xf5J\x95\xd7\xe5\xaf\xb9(\xb8<[\xe4\xe9\xaf\xa1t>,\xcf\xf9<_\xe7\xb4\xcd\x0b3C\xb7rK\xea\xb6\x94\xbe\xb5&8J\xf4\xb1\xccpN\x1e\xcb\x0c;g<\x92\x1d\xb6w\xa0\x1d5\xfb\xf6]v\xe4#\xa50S\x97C\xb8\x1aB\xbe\x03m\xba\x9d\xdc:8\x9e\x98\x91\xb9\n\xd3R\x90o\xc3\xf9\xe9\xc5x\x1a\xe29\xd9\x04\xb9}\x98Wm\xe7\xd8\xd1\xd1\xfb\xe7\xc1Q\xb0\xa3\xef<\x96\x1d\xc7\x88\xf2\xa2!\xee\xa1\x86\x98\xcf\xc3C\x87y{_\x98g\x95\xc6P4\x14ml\xac$,F\xb7`\xb8\xd7\xf1,\x14\xe0d`-\xa0\x07\x83\xafQ\x02\xf2RU`1[\xc6\x03+\x90\xa2|L\x8cE\xe8p\xe7\x97\\_\xe3\xd1\x92\xe3\xdb\xa6e\xb5n\xaf)w\x19m\"R9\xc2i\x98\x13g\xab\xca\xdee\xb1\xc1U\x1d\xcfU[\xf2\xb0\xbej=\xb4\x15N\xcf\xfeY\xad\xc8\x9f\x0c\x11N\x16\x0b\x12Hc\x94\x17\xa6J\xa2\xf3\x19S\xbe\xe7+I\xc4U8\xbfE3\xb6\x12\x97\x18\xbc\xe3\xe6\xc7qNS\xb8\x12v\xb6\x0f\x022\xb9\xbb9\xa9ze1g\xa6\xf2\xdf-\x0c\t,[\x88Y\x12\xe2M]\xed\xd5\xe7\x9b\x9c\xaez\"v\xfa\x97w\xc1`\xf2\xfdp\xc9G\x0f\xe5;\xe7_\xf4]C\xae~\xf6\xdd\xe3\xfan\x93;HL\x9cy\xc5\x11\x01tE\x02#\x95\x1c\x06\x16\x172\xe4P\xee\x92\x90\x06\x13\x01\xcd\x94\xc9D\xf0\x02\x82d\xa6\x1c\x80\x98\xfa\x0b\xbd\xf2\x0c\xb9)\x15\xce\xad>9\x7fE,\x83\x86N^\xd2%\x12\x14\x8a\xb0\x0c\x05!\x17r\xe3\xef\xef\x93jw\x8c\xd7\xfa,\x81m\x84T2d\xd5\x17\xcaC\x89\xc1=3rC\xd8T%\xf3\xae\xda&\x0b\x85\xdb\xe2T\xcd\xbb\x1a\xbe&`K\xc3zn\x9d-'\xff\xdb^\xd4=\xb4\x17=F\xf3\xa3\x99\xe0\x1e\xb3\x87s\x9bz\xb8\xc2E\xac\xffX\xd6\x1e\xf92\xdf9p\xdb:\xde\x03^\xe6\x13,C\xa4~\xc1}\x8a\x8a\x80\x11\xabb\xbe\xba\xafO\xf9%\x9c;\xb4{\xf1\x81 \x9b\xfc\xd6\xdb\xa4\xf6\xdd\xe0\x0c|\xd4\xabZ\xa5d+\x11?K\x07|\x1f\x92\x06c\x8c[\xf44_\x8f\x14b\xad\xa6\xb1\xad\xc6\xda1\x0cy\x80X\xf3\x0c\xa1f8\xdf\x87E\x9a\x1a3\xd5\x8b\xac9\x8d\noA\xd5@\xe5?\xdb\xd4\rh\xf6\r4\x1c\x91\x05^1\x99\xb66\xa3\xe4N\n<\xdc\xfe\xef\r\xb0\xc2\xc4\x8e\xe1\xed\x8b\xbf\x01PK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cX\x04q\xaa\xb1n\x01\x00\x00F\x03\x00\x00\x18\x00\x00\x00xl/worksheets/sheet1.xml\x85S\xddN\x830\x14~\x15\xc2\x03\xac\x0c25\x06H\xdc\x8c\xd1\x0b\x93e\x8bz]\xe0\x00\xcdZ\x8a\xed\xd9\xd0\xb7\xb7-[\xc3\xcctW\x9c\x9f~?\xa7\xa7\xa4\x83T;\xdd\x02`\xf0%x\xa7\xb3\xb0E\xec\xef\t\xd1e\x0b\x82\xea\x99\xec\xa13\x9dZ*A\xd1\xa4\xaa!\xbaW@+\x07\x12\x9c\xc4QtC\x04e]\x98\xa7\xae\xb6Vy*\xf7\xc8Y\x07k\x15\xe8\xbd\x10T}/\x81\xcb!\x0b\xe7\xe1\xa9\xb0aM\x8b\xae@\xf2\xb4\xa7\rl\x01\xdfz\x030)\xf1<.xg0\xe8I\x1cX\xcf\x85\x94;\x9b\xbcTY\x18Yi\xe0P\"\x93]@\xcd\xe7\x00+\xe0<\x0b\x1f\xac\xe0\xa7\x82z\x0c=\xb7EN\xe3\x13\xfd\x93\x1b\xd3\xb8(\xa8\x86\x95\xe4\x1f\xac\xc26\x0b\xef\xc2\xa0\x82\x9a\xee9n\xe4\xf0\x0cG\xeb\x0bG\xe8p\x8f\x14i\x9e*9\x04\xca\xce\x94\xa7\xa5\r\x9c\xba\x9b\xd1\x1cg\x9d\xbd\x91-*\xd3eF\x0fs\xc3?O\t\x1a\x1f6'\xe5\x11\xb5\xbc\x8a\x8a/\xa0VWQ\xc99\x8a\x18\xb7\xder\xec-\xc7\x7f\x10 h\xbc\xe8v\x04\xd8\xed\x1f\xf2h\xb6H\xc9a\xeaj\xec\x16c\xd7\xf7\xce\xb4\x13\xaf\x9d\xfc\xa3}q\xe6d\xc2>\xff\xc5N&\x8b\xb1\xef\xeb\x95\xaa\x86u:\xe0P\x1bL4\xbb5\xebS\xe3&\xc7\x04e\xef\xee\xaf\x90\x88R\xb8\xb05\xef\x1c\x94=`\xfa\xb5\x94\xe8\x13\xfb\x92\xfc\xaf\x93\xff\x00PK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cX\x18\xb4\x1cWd\x02\x00\x00\xc0\n\x00\x00\r\x00\x00\x00xl/styles.xml\xddV\xdb\x8e\x9b0\x10\xfd\x15\xe4\x0f(KPQ\xa8B\x1e\x8a\x14\xa9R[\xad\xb4\xfb\xd0W\x13Lb\xc9\x17j\xcc*\xe9\xd7\xd7c;\x81\xecfX\xb5\xeaSA\x81\x999>s\xf3Xd3\xd8\xb3`OG\xc6lr\x92B\r\x159Z\xdb\x7fJ\xd3a\x7fd\x92\x0e\x1ft\xcf\x94C:m$\xb5N5\x87t\xe8\r\xa3\xed\x00$)\xd2\xd5\xc3C\x91J\xca\x15\xd9n\xd4(w\xd2\x0e\xc9^\x8f\xcaV\xe4\x81$\xe9v\xd3i5\x99V$\x18\xdcZ*Y\xf2BEEj*xcxXL%\x17\xe7`_y\xcb^\x0bm\x12\xeb\xb2a\x15\xc9\xbci\xf8\x15\x16dQ\x85T\xa3/\xc9\x956\xde\x9a\x860\xe1\xd9D\xc2\x1c\xf1\xaf\xc1\xad\xe0B\xdc\xe6\xe7\x0c\xdbMO\xadeF\xed\x9c\x12H\xde\xfa\x16\x8b\xf2\xf3\xb9w\xf9\x1d\x0c=g\xab\x8fd\xc6\xf0/\x17\xa6\xd1\xa6e\xe6\x1a(#\x17\xd3v#Xg\x81a\xf8\xe1\xe8\x05\xab{x5\xdaZ-Aj9=hEC&\x17Z\x14\x9c\xef=\x13\xe2\tv\xf2Gw\x13\xe0\xd4%aK\xbe\xb4~7\xa0\xe2\x8b\xe8\xb2\x8abp\x13\x15\x080w\x17\x9c\xcf\xfc\xae\xfe\xceo\xcf_\xb4\xfd<\xba\x82\x94\xd7\x7f\x8e\xda\xb2G\xc3:~\xf2\xfa\xa9\x9b\x12\xc0\xdcg\xff\xc6}\x1a+\x9a\xf5\xed\xa6kWk\x023Z\x91\xef0\xfbb\xe6\xa3\x19\xb9\xb0\\E\xed\xc8\xdb\x96\xa9\xb7\xcds\xfe-m\xdc\xe9\xba\t\xe0V\xb5\xac\xa3\xa3\xb0\xcfW\xb0\"\x93\xfc\x8d\xb5|\x94\xe5u\xd5#\x14\x16WM\xf2W\x98\x94\xac\x98\x0e\x88\x0b\xc6U\xcbN\xac\xad\xa3j\x0e\x8d\x17\x13'\xb8\xb0\xf1\xf2\x8c\xd7\xd0\xce_\x08\x84\xb2\x02\x88@\x00\xa2\xb1\xd04PV\xe0\xa1\xb1\xfe\xc7\xba\xd6x]\x01D3\\\xdf\x87\xd68k\x8d\xb3\x02\xef.T\xfb\x1b\x8d\x85\xb0Jw!%\x97e\x9e\x17\x05\xda\xde\xba\xbe\x9fF\x8d\xf6\xb0(\xe0\x878D3\x04\x0e\x1a\x0b\xa2\xfdi\xe7\x17\x06`al\xde\x99\rt\x97\x17\xc7\x06-yaD\xd1\x92\x17:\x0f\x10\xd2C\xe0\x94%2\x00h,\xe0\xa0\x9b\x82N\x14$\x81\xc4\x82QCXy\x0e\xfb\x8cf\x88\x1e\xf3\x05\xa8,Q\x08\x86\x14\x99\xde\xa2\xc0\x1aU\xc0\x8d\xec\x17z\x88\xf2\xbc,\x11\x08@$\x8d\xa4\xaf\xbeg\xe9\xe5;\x97N\xffY\xb7\xbf\x01PK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cX\xb7G\xeb\x8a\xc0\x00\x00\x00\x16\x02\x00\x00\x0b\x00\x00\x00_rels/.rels\x9d\x92Kn\x021\x0c@\xaf\x12e_L\xa9\xc4\x021\xac\xd8\xb0C\x88\x0b\xb8\x89\xe7\xa3\x99\xc4\x91c\xc4\xf4\xf6\x8d\xd8\xc0 h\x11K\xff\x9e\x9e-\xaf\x0f4\xa0v\x1cs\xdb\xa5l\xc60\xc4\\\xd9V5\xad\x00\xb2k)`\x9eq\xa2X*5K@-\xa14\x90\xd0\xf5\xd8\x10,\xe6\xf3%\xc8-\xc3n\xd6\xb7Ls\xfcI\xf4\n\x91\xeb\xbas\xb4ew\n\x14\xf5\x01\xf8\xae\xc3\x9a#JCZ\xd9q\x803K\xff\xcd\xdc\xcf\n\xd4\x9a\x9d\xaf\xac\xec\xfc\xa75\xf0\xa6\xcc\xf3\xf5 \x90\xa2GEp,\xf4\x91\xa4L\x8bv\x94\xaf>\x9e\xdd\xbe\xa4\xf3\xa5cb\xb4x\xdf\xe8\xff\xf3\xd0\xa8\x14=\xf9\xbf\x9d0\xa5\x89\xd2\xd7E\t&o\xb0\xf9\x05PK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cXg\xc5t\x873\x01\x00\x00,\x02\x00\x00\x0f\x00\x00\x00xl/workbook.xml\x8d\x90\xd1N\xc30\x0cE\x7f\xa5\xca\x07\xd0n\x82IL+/L\xc0$\x04\x88M{O[w\xb5\x96\xc4\x95\xe3m\xb0\xaf'I)L\xe2\x85\xa7\xc4\xd7\xd6\xf1\xbd^\x9c\x88\xf7\x15\xd1>\xfb\xb0\xc6\xf99\x97\xaa\x13\xe9\xe7y\xee\xeb\x0e\xac\xf6W\xd4\x83\x0b\xbd\x96\xd8j\t%\xefrj[\xacaI\xf5\xc1\x82\x93|Z\x14\xb3\x9c\xc1hAr\xbe\xc3\xde\xab\x81\xf6\x1f\x96\xef\x19t\xe3;\x00\xb1f@Y\x8dN\xdd-Fgo\x9c\xe5\x97\x15\t\xd4qST\xa3\xb2E8\xf9\xdf\x81XfG\xf4X\xa1A\xf9,U\xfa\x1bP\x99E\x87\x16\xcf\xd0\x94\xaaP\x99\xef\xe8\xf4D\x8cgr\xa2\xcd\xbaf2\xa6T\x93\xa1\xb1\x05\x16\xac\xff\xc8\xebhs\xa3+\x9f\x14\xd1\xd5{\xcc\\\xaaY\x11\x80-\xb2\x974\x91\xf8:\x98\xae\x8d!\x92\xbcj\x06K\x12X\x17\x01y\x8e\xa1\xc1\xab\xe6\x9b:\xa2\x1ah\xd1A\xf3\x12X>6B\xb0:\\5>\x894\xbd\xbe\x99\xdc\x86\x00\x07c\xee\x83\xf6\xea\x9eI7?\xde\xc6\xc3\xde}\x01PK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cX3\xeb\xe3\xba\xad\x00\x00\x00\xfb\x01\x00\x00\x1a\x00\x00\x00xl/_rels/workbook.xml.rels\xb5\x91=\x0e\x830\x0c\x85\xaf\x12\xe5\x00\x18\xa8\xd4\xa1\x02\xa6.\xac\x15\x17\x88\x82\xf9\x11\x81D\xb1\xab\xc2\xed\x1b\xc1\x00H\x1d\xba0Y\xcf\x96\xbf\xf7dg/4\x8a{;Q\xd7;\x12\xf3h&\xcae\xc7\xec\x1e\x00\xa4;\x1c\x15E\xd6\xe1\x14&\x8d\xf5\xa3\xe2 }\x0bN\xe9A\xb5\x08i\x1c\xdf\xc1\x1f\x19\xb2\xc8\x8eLQ-\x0e\xff!\xda\xa6\xe95>\xad~\x8f8\xf1\x0f0|\xac\x1f\xa8Cd)*\xe5[\xe4\\\xc2l\xf66\xc1Z\x92(\x90\xa5(\xeb\\\xfa\xb2N\xa4\x80\xcb\x12\x11/\x06i\x8f\xb3\xe9\x93\x7fz\xa5?\x87]\xdc\xedW\xb95\xcfG\xb8\xad!\xe0\xf4\xeb\xe2\x0bPK\x03\x04\x14\x00\x00\x00\x08\x00\ti\x8cX\x9b\x86B\x84\x1b\x01\x00\x00\xd7\x03\x00\x00\x13\x00\x00\x00[Content_Types].xml\xad\x93\xcfN\xc30\x0c\xc6_\xa5\xeauj38p@\xeb.\x8c+\xec\xc0\x0b\x84\xc4]\xa3\xe6\x9fboto\x8f\xdb\xb2J\xa0\xb1\r\x95K\xa3\xc6\xf6\xf7s\xfc%\xab\xb7c\x04\xcc:g=VyC\x14\x1f\x85@\xd5\x80\x93X\x86\x08\x9e#uHN\x12\xff\xa6\x9d\x88R\xb5r\x07\xe2~\xb9|\x10*x\x02O\x05\xf5\x1a\xf9z\xb5\x81Z\xee-e\xcf\x1do\xa3\t\xbe\xca\x13X\xcc\xb3\xa71\xb1gU\xb9\x8c\xd1\x1a%\x89\xe3\xe2\xe0\xf5\x0fJ\xf1E(\xb9r\xc8\xc1\xc6D\\pB\x9e\x89\xb3\x88!\xf4+\xe1T\xf8z\x80\x94\x8c\x86l+\x13\xbdH\xc7i\xa2\xb3\x02\xe9h\x01\xcb\xcb\x1ag\xba\x0cum\x14\xe8\xa0\xf6\x8eKJ\x8c\t\xa4\xc6\x06\x80\x9c-G\xd1\xc5\x154\xf1\x90a\xfc\xde\xcdn`\x90\xb9H\xe4\xd4m\n\x11\xd9\xb5\x04\x7f\xe7\x9dl\xe9\xab\x8b\xc8B\x90\xc8\\9\xe4\x84d\xed\xd9'\x84\xdeq\r\xfaV8O\xf8#\xa4v\xf0\x04\xc5\xb0\xcc\x1f\xf3w\x9f'\xfd[\x1ay\x0f\xa1\xfd\xef{\xd6\xaf\xa5\x93\xc6O\r\x88\xe1=\xaf?\x01PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cXFZ\xc1\x0c\x82\x00\x00\x00\xb1\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00\x00docProps/app.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cX\xddR\xb3\x7f\xea\x00\x00\x00\xcb\x01\x00\x00\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\xb0\x00\x00\x00docProps/core.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cX\x99\\\x9c#\x10\x06\x00\x00\x9c'\x00\x00\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\xc9\x01\x00\x00xl/theme/theme1.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cX\x04q\xaa\xb1n\x01\x00\x00F\x03\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x81\n\x08\x00\x00xl/worksheets/sheet1.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cX\x18\xb4\x1cWd\x02\x00\x00\xc0\n\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\xae\t\x00\x00xl/styles.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cX\xb7G\xeb\x8a\xc0\x00\x00\x00\x16\x02\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01=\x0c\x00\x00_rels/.relsPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cXg\xc5t\x873\x01\x00\x00,\x02\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01&\r\x00\x00xl/workbook.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cX3\xeb\xe3\xba\xad\x00\x00\x00\xfb\x01\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x86\x0e\x00\x00xl/_rels/workbook.xml.relsPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\ti\x8cX\x9b\x86B\x84\x1b\x01\x00\x00\xd7\x03\x00\x00\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01k\x0f\x00\x00[Content_Types].xmlPK\x05\x06\x00\x00\x00\x00\t\x00\t\x00>\x02\x00\x00\xb7\x10\x00\x00\x00\x00" + + +table_definition_json_array = [["Col1", "Col2", "Col3"], ["test1", 0.5, False], ["test2", None, True]] + +# this is the xlsx representation of the table_definition_json_array above +xlsx_from_json_array_result = b"PK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cXFZ\xc1\x0c\x82\x00\x00\x00\xb1\x00\x00\x00\x10\x00\x00\x00docProps/app.xmlM\x8eM\x0b\xc20\x10D\xffJ\xe9\xddnU\xf0 1 \xd4\xa3\xe0\xc9{H76\x90dCv\x85\xfc|S\xc1\x8f\xdb<\xde0\x8c\xba\x15\xcaX\xc4#w5\x86\xc4\xa7~\x11\xc9G\x00\xb6\x0bF\xc3C\xd3\xa9\x19G%\x1aiX\x1e@\xcey\x8b\x13\xd9g\xc4$\xb0\x1b\xc7\x03`\x15L3\xce\x9b\xfc\x1d\xec\xb5:\xe7\x1c\xbc5\xe2)\xe9\xab\xb7\x85\x98\x9ct\x97j1(\xf8\x97k\xf3\x8e\x85\xd7\xbc\x1f\xb6o\xf9a\x05\xbf\x93\xfa\x05PK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cXh\xedm\xda\xea\x00\x00\x00\xcb\x01\x00\x00\x11\x00\x00\x00docProps/core.xml\xa5\x91MO\xc30\x0c\x86\xff\xca\x94{\xeb\xa4\x1b\x1c\xa2\xae\x97\xa1\x9d\x86\x84\xc4$\x10\xb7(\xf1\xb6\x88\xe6C\x89Q\xbb\x7fO[\xb6\x0e\x047\x8e\xf1\xfb\xf8\xb1\xad\xd4:J\x1d\x12>\xa5\x101\x91\xc5\xbc\xe8]\xeb\xb3\xd4q\xcdNDQ\x02d}B\xa7r9\x10~\x08\x0f!9E\xc33\x1d!*\xfd\xae\x8e\x08\x15\xe7\xf7\xe0\x90\x94Q\xa4`\x14\x16q6\xb2\x8b\xd2\xe8Y\x19?R;\t\x8c\x06l\xd1\xa1\xa7\x0c\xa2\x14pc\t\x93\xcb\x7f6L\xc9L\xf6\xd9\xceT\xd7ue\xb7\x9c\xb8a#\x01\xaf\x8f\xbb\xe7i\xf9\xc2\xfaL\xcakdMm\xb4\xd4\t\x15\x85\xd4\x8c\x17\xc5s\xdf\xd6\xf0\xadX_f\x7f\x15\xd0,\x86\t\x92\xce\x11\xd7\xec\x9a\xbc,7\x0f\xfb-k*^\xad\n\xbe*D\xb5\x17BV\\\xde\xf1\xb7\xd1\xf5\xa3\xff&t\xc1\xd8\x83\xfd\x87\xf1*hj\xf8\xf5o\xcd'PK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cX\x99\\\x9c#\x10\x06\x00\x00\x9c'\x00\x00\x13\x00\x00\x00xl/theme/theme1.xml\xedZ[s\xda8\x14~\xef\xaf\xd0xg\xf6m\x0b\xc66\x81\xb6\xb4\x13siv\xdb\xb4\x99\x84\xedN\x1f\x85\x11X\x8dlyd\x91\x84\x7f\xbfG6\x10\xcb\x96\r\xed\x92M\xba\x9b<\x04,\xe9\xfb\xceEG\xe7\xe88y\xf3\xee.b\xe8\x86\x88\x94\xf2x`\xd9/\xdb\xd6\xbb\xb7/\xde\xe0W2$\x11A0\x19\xa7\xaf\xf0\xc0\n\xa5L^\xb5Zi\x00\xc38}\xc9\x13\x12\xc3\xdc\x82\x8b\x08Kx\x14\xcb\xd6\\\xe0[\x1a/#\xd6\xea\xb4\xdb\xddV\x84il\xa1\x18Gd`}^,h@\xd0TQZo_ \xb4\xe5\x1f3\xf8\x15\xcbT\x8de\xa3\x01\x13WA&\xb9\x88\xb4\xf2\xf9l\xc5\xfc\xda\xde>e\xcf\xe9:\x1d2\x81n0\x1bX \x7f\xceo\xa7\xe4NZ\x88\xe1T\xc2\xc4\xc0jg?Vk\xc7\xd1\xd2H\x80\x82\xc9}\x94\x05\xbaI\xf6\xa3\xd3\x15\x082\r;:\x9dX\xcev|\xf6\xc4\xed\x9f\x8c\xca\xdat4m\x1a\xe0\xe3\xf1x8\xb6\xcb\xd2\x8bp\x1c\x04\xe0Q\xbb\x9e\xc2\x9d\xf4l\xbf\xa4A\t\xb4\xa3i\xd0d\xd8\xf6\xda\xae\x91\xa6\xaa\x8dSO\xd3\xf7}\xdf\xeb\x9bh\x9c\n\x8d[O\xd3kw\xdd\xd3\x8e\x89\xc6\xad\xd0x\r\xbe\xf1O\x87\xc3\xae\x89\xc6\xab\xd0t\xebi&'\xfd\xaek\xa4\xe9\x16hBF\xe3\xebz\x12\x15\xb5\xe5@\xd3 \x00Xpv\xd6\xcc\xd2\x03\x96^)\xfau\x94\x1a\xd9\x1d\xbb\xddA\\\xf0X\xee9\x89\x11\xfe\xc6\xc5\x04\xd6i\xd2\x19\x964Fr\x9d\x90\x05\x0e\x007\xc4\xd1LP|\xafA\xb6\x8a\xe0\xc2\x92\xd2\\\x90\xd6\xcf)\xb5P\x1a\x08\x9a\xc8\x81\xf5G\x82!\xc5\xdc\xaf\xfd\xf5\x97\xbb\xc9\xa43z\x9d}:\xcek\x94\x7fi\xab\x01\xa7\xed\xbb\x9b\xcf\x93\xfcs\xe8\xe4\x9f\xa7\x93\xd7MB\xcep\xbc,\t\xf1\xfb#[a\x87'n;\x13r:\x1cgB|\xcf\xf6\xf6\x91\xa5%2\xcf\xef\xf9\n\xebNb\x196\x1ar\xb5\x16\x81\xb6q\xa9\x84`Z\x12\xc6\xd1xN\xd2\xb4\x11\xfcY\xac5\x93>`\xc8\xec\xcd\x91u\xce\xd6\x91\x0e\x11\x92^7B>b\xce\x8b\x90\x11\xbf\x1e\x868J\x9a\xed\xa2qX\x04\xfd\x9e^\xc3I\xc1\xe8\x82\xcbf\xfd\xb8~\x86\xd53l,\x8e\xf7G\xd4\x17J\xe4\x0f&\xa7?\xe924\x07\xa3\x9aY\t\xbd\x84Vj\x9f\xaa\x874>\xa8\x1e2\n\x05\xf1\xb9\x1e>\xe5zx\n7\x96\xc6\xbcP\xae\x82{\x01\xff\xd1\xda7\xc2\xab\xf8\x82\xc09\x7f.}\xcf\xa5\xef\xb9\xf4=\xa1\xd2\xb77#}g\xc1\xd3\x8b[\xdeFn[\xc4\xfb\xae1\xda\xd74.(cWr\xcd\xc8\xc7T\xaf\x93)\xd89\x9f\xc0\xec\xfdh>\x9e\xf1\xed\xfa\xd9$\x84\xaf\x9aY-#\x16\x90K\x81\xb3A$\xb8\xfc\x8b\xca\xf0*\xc4\t\xe8d[%\t\xcbT\xd3e7\x8a\x12\x9eB\x1bn\xe9S\xf5J\x95\xd7\xe5\xaf\xb9(\xb8<[\xe4\xe9\xaf\xa1t>,\xcf\xf9<_\xe7\xb4\xcd\x0b3C\xb7rK\xea\xb6\x94\xbe\xb5&8J\xf4\xb1\xccpN\x1e\xcb\x0c;g<\x92\x1d\xb6w\xa0\x1d5\xfb\xf6]v\xe4#\xa50S\x97C\xb8\x1aB\xbe\x03m\xba\x9d\xdc:8\x9e\x98\x91\xb9\n\xd3R\x90o\xc3\xf9\xe9\xc5x\x1a\xe29\xd9\x04\xb9}\x98Wm\xe7\xd8\xd1\xd1\xfb\xe7\xc1Q\xb0\xa3\xef<\x96\x1d\xc7\x88\xf2\xa2!\xee\xa1\x86\x98\xcf\xc3C\x87y{_\x98g\x95\xc6P4\x14ml\xac$,F\xb7`\xb8\xd7\xf1,\x14\xe0d`-\xa0\x07\x83\xafQ\x02\xf2RU`1[\xc6\x03+\x90\xa2|L\x8cE\xe8p\xe7\x97\\_\xe3\xd1\x92\xe3\xdb\xa6e\xb5n\xaf)w\x19m\"R9\xc2i\x98\x13g\xab\xca\xdee\xb1\xc1U\x1d\xcfU[\xf2\xb0\xbej=\xb4\x15N\xcf\xfeY\xad\xc8\x9f\x0c\x11N\x16\x0b\x12Hc\x94\x17\xa6J\xa2\xf3\x19S\xbe\xe7+I\xc4U8\xbfE3\xb6\x12\x97\x18\xbc\xe3\xe6\xc7qNS\xb8\x12v\xb6\x0f\x022\xb9\xbb9\xa9ze1g\xa6\xf2\xdf-\x0c\t,[\x88Y\x12\xe2M]\xed\xd5\xe7\x9b\x9c\xaez\"v\xfa\x97w\xc1`\xf2\xfdp\xc9G\x0f\xe5;\xe7_\xf4]C\xae~\xf6\xdd\xe3\xfan\x93;HL\x9cy\xc5\x11\x01tE\x02#\x95\x1c\x06\x16\x172\xe4P\xee\x92\x90\x06\x13\x01\xcd\x94\xc9D\xf0\x02\x82d\xa6\x1c\x80\x98\xfa\x0b\xbd\xf2\x0c\xb9)\x15\xce\xad>9\x7fE,\x83\x86N^\xd2%\x12\x14\x8a\xb0\x0c\x05!\x17r\xe3\xef\xef\x93jw\x8c\xd7\xfa,\x81m\x84T2d\xd5\x17\xcaC\x89\xc1=3rC\xd8T%\xf3\xae\xda&\x0b\x85\xdb\xe2T\xcd\xbb\x1a\xbe&`K\xc3zn\x9d-'\xff\xdb^\xd4=\xb4\x17=F\xf3\xa3\x99\xe0\x1e\xb3\x87s\x9bz\xb8\xc2E\xac\xffX\xd6\x1e\xf92\xdf9p\xdb:\xde\x03^\xe6\x13,C\xa4~\xc1}\x8a\x8a\x80\x11\xabb\xbe\xba\xafO\xf9%\x9c;\xb4{\xf1\x81 \x9b\xfc\xd6\xdb\xa4\xf6\xdd\xe0\x0c|\xd4\xabZ\xa5d+\x11?K\x07|\x1f\x92\x06c\x8c[\xf44_\x8f\x14b\xad\xa6\xb1\xad\xc6\xda1\x0cy\x80X\xf3\x0c\xa1f8\xdf\x87E\x9a\x1a3\xd5\x8b\xac9\x8d\noA\xd5@\xe5?\xdb\xd4\rh\xf6\r4\x1c\x91\x05^1\x99\xb66\xa3\xe4N\n<\xdc\xfe\xef\r\xb0\xc2\xc4\x8e\xe1\xed\x8b\xbf\x01PK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cX\t\xf8\xfa\x7fm\x01\x00\x004\x03\x00\x00\x18\x00\x00\x00xl/worksheets/sheet1.xml}S]O\xc20\x14\xfd+K\x7f\x00\x1d[Pc\xb6%\x821\xfa`B \xeas7\xee\xb6\x86v\x9d\xed\x85\xe9\xbf\xb7\xed\xb0\x19\x06y\xe2~\x9c{\xce\xe9\xee%\x1b\x94\xde\x9b\x16\x00\xa3/):\x93\x93\x16\xb1\xbf\xa7\xd4T-Hff\xaa\x87\xcevj\xa5%C\x9b\xea\x86\x9a^\x03\xdb\xf9!)h\x12\xc77T2\xde\x91\"\xf3\xb5\xb5.2u@\xc1;X\xeb\xc8\x1c\xa4d\xfa{\tB\r9\x99\x93\xdf\xc2\x867-\xfa\x02-\xb2\x9e5\xb0\x05|\xeb\xed\x80Mi\xe0\xf1\xc1;\x87\xc1L\xe2\xc8y.\x95\xda\xbb\xe4e\x97\x93\xd8I\x83\x80\n\xb9\xea\"f\x7f\x8e\xb0\x02!r\xf2\xe0\x04?5\xd4c\x18\xb8\xdd\xe44\xfe\xa5\x7f\xf2\xcf\xb4.Jf`\xa5\xc4\x07\xdfa\x9b\x93;\x12\xed\xa0f\x07\x81\x1b5<\xc3\xc9\xfa\xc2\x13\xfa\xb9G\x86\xac\xc8\xb4\x1a\"\xed\xdeTd\x95\x0b\x9c\xa4\x05\xf2\xce}\x8b-j[\xe7V\t\x0b\xcb<\xcf(Z\x07.\xa7\xd5\t\xbf\xbc\x82O.\xe0WW\xf0\xe99\x9eZo\xc1`\x12\x0c&\xff\x10 \x18\xbc\xe8p\x1cp\xbb>\x16\xf1l\x91\xd1\xe3\xd4\xcf\xd8-\xc7n\xe8\x9di\xa7A;\xbd\xa2}\xf1\xb5\xe9\x84}\xfe\x87\x9dN\xd6\xe0\xae\xe9\x95\xe9\x86w&\x12P\xdb\x99xvk\x97\xa5\xc7\xbd\x8d\t\xaa\xde__\xa9\x10\x95\xf4ak\xaf\x1a\xb4\x03\xd8~\xad\x14\x86\xc4\xddM\xf8\xa3\x14?PK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cX\xd2\x05\xf1FR\x02\x00\x00G\n\x00\x00\r\x00\x00\x00xl/styles.xml\xddV\xdb\x8a\xdb0\x10\xfd\x15\xe3\x0f\xa8\x93\x98\x9a\xb8$y\xa8!Ph\xcb\xc2\xeeC_\xe5XN\x04\xba\xb8\xb2\xbc$\xfd\xfaj$\xe7\xb6\x9b\xe3R\xfaV\x9b\xe0\x999:3g\xa41\xce\xaaw'\xc9\x9f\x0f\x9c\xbb\xe4\xa8\xa4\xee\xd7\xe9\xc1\xb9\xeeS\x96\xf5\xbb\x03W\xac\xff`:\xae=\xd2\x1a\xab\x98\xf3\xae\xddg}g9kz\")\x99-f\xb3\"SL\xe8t\xb3\xd2\x83\xda*\xd7';3h\xb7Ngi\x92mV\xad\xd1\xd7\xd0<\x8d\x01\xbf\x96)\x9e\xbc2\xb9N+&EmE\\\xcc\x94\x90\xa7\x18_\x84\xc8\xceHc\x13\xe7\xd5p\xa2S\xa8\xff\x15\x17\xccG\x97\xa4\x8e\xb9\x94\xd0\xc6\x86h\x16\xcb\x84G\xef\x13\x0b)/*\x16i\x0clV\x1ds\x8e[\xbd\xf5N$\x85\xe8{l\xb4_N\x9dW\xb1\xb7\xec4_|Lo\x18\xe1\xe1\xcb\xd4\xc66\xdc\xde\xb5\x1bC\x9b\x95\xe4\xad#\x86\x15\xfbC0\x9c\xe9\xe8Q\x1b\xe7\x8c\"\xab\x11lo4\x8bJ\xce\xb4\xd1\xf0\xb9w\\\xcag:\xaf\x1f\xed]\x81c\x9b\xc4\x8d\xff\xd2\x84=\xa7\x8e\xcf\xa6W5\x9a1\xcd\xe8P\x81\xdbt1\xf9\xbf\xe7\xed\xc4\xabq\x9f\x07\xdf\x90\x0e\xfe\xcf\xc18\xfedy+\x8e\xc1?\xb6o\x04\\j\x07%w\xe5/\xd1\x84Fe\x9d~\xa7\x11\x9479\xeaAH'\xf4\xe8\x1dD\xd3p\xfd\xbe;\x9f\xdf\xb1\xda\x0f\xf9]\x01\xbf\xaa\xe1-\x1b\xa4{\xb9\x80\xeb\xf4j\x7f\xe3\x8d\x18TyY\xf5D\x8d\x8d\xab\xae\xf6W:\xcayq\x9dS_L\xe8\x86\x1fyS\x8d\xae\xdd\xd7\xc1L\xbc\xe1\xcb\x8eW`\xbc\x85\xb6\xe1\x02\x10dE\x10@\x04\xc2ZP\x06dE\x1e\xac\xf5?\xf6\xb5\xc4}E\x10*\\>\x86\x96\x98\xb5\xc4\xac\xc8{\x08U\xe1\x86\xb5\x00\xab\xf4\x17h\xb9,\xf3\xbc(\xe0\xf6V\xd5c\x19\x15\xdc\xc3\xa2\xa0\x1fH\x08\x15\x12\x07\xd6\xa2j\x7f\xbb\xf3\x13\x03016\x7f\x98\rx\xca\x93c\x03[\x9e\x18Q\xd8\xf2\xc4\xce\x13\x04\xf6\x908e\t\x06\x00\xd6\"\x0e<\x148Q$\x02\xd4\xa2Q\x03\xac<\xa7s\x86\n\xe1k>\x01\x95%\x84hH\xc1\xf4\x16\x05\xda\xa8\x82np^\xf0%\xca\xf3\xb2\x04\x10\x81@F\x9eC\x88^\xd8\t\x08\xca !\x10\xca\xf3\xf8!}\xf3=\xcb\xce\xdf\xb9\xec\xfa\xd7q\xf3\x1bPK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cX\xb7G\xeb\x8a\xc0\x00\x00\x00\x16\x02\x00\x00\x0b\x00\x00\x00_rels/.rels\x9d\x92Kn\x021\x0c@\xaf\x12e_L\xa9\xc4\x021\xac\xd8\xb0C\x88\x0b\xb8\x89\xe7\xa3\x99\xc4\x91c\xc4\xf4\xf6\x8d\xd8\xc0 h\x11K\xff\x9e\x9e-\xaf\x0f4\xa0v\x1cs\xdb\xa5l\xc60\xc4\\\xd9V5\xad\x00\xb2k)`\x9eq\xa2X*5K@-\xa14\x90\xd0\xf5\xd8\x10,\xe6\xf3%\xc8-\xc3n\xd6\xb7Ls\xfcI\xf4\n\x91\xeb\xbas\xb4ew\n\x14\xf5\x01\xf8\xae\xc3\x9a#JCZ\xd9q\x803K\xff\xcd\xdc\xcf\n\xd4\x9a\x9d\xaf\xac\xec\xfc\xa75\xf0\xa6\xcc\xf3\xf5 \x90\xa2GEp,\xf4\x91\xa4L\x8bv\x94\xaf>\x9e\xdd\xbe\xa4\xf3\xa5cb\xb4x\xdf\xe8\xff\xf3\xd0\xa8\x14=\xf9\xbf\x9d0\xa5\x89\xd2\xd7E\t&o\xb0\xf9\x05PK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cX\xe4\xb0k\xee0\x01\x00\x00(\x02\x00\x00\x0f\x00\x00\x00xl/workbook.xml\x8d\x90\xd1N\xc30\x0cE\x7f\xa5\xca\x07\xd0n\x82IL\xeb^\x98\x80I\x08\x10C{\xcfZw\xb5\x96\xc4\x95\xe3n\xb0\xaf'I)L\xe2\x85'\xc7\xd7\xd6\xc9\xbd^\x9c\x88\x0f;\xa2C\xf6a\x8d\xf3s.U+\xd2\xcd\xf3\xdcW-X\xed\xaf\xa8\x03\x17f\r\xb1\xd5\x12Z\xde\xe7\xd44X\xc1\x8a\xaa\xde\x82\x93|Z\x14\xb3\x9c\xc1hAr\xbe\xc5\xce\xab\x81\xf6\x1f\x96\xef\x18t\xed[\x00\xb1f@Y\x8dN-\x17\xa3\xb3W\xce\xf2\xcb\x8e\x04\xaa\xf8ST\xa3\xb2E8\xf9\xdf\x85\xd8fG\xf4\xb8C\x83\xf2Y\xaa\xf46\xa02\x8b\x0e-\x9e\xa1.U\xa12\xdf\xd2\xe9\x91\x18\xcf\xe4D\x9bM\xc5dL\xa9&\xc3`\x0b,X\xfd\x917\xd1\xe6\xbb\xde\xf9\xa4\x88\xde\xbd\xc5\xcc\xa5\x9a\x15\x01\xd8 {I\x1b\x89\xaf\x83\xc9#\x84\xe5\xa1\xeb\x85\xee\xd1\x08\xf0J\x0b<0\xf5\x1d\xba}\xc2\x84\x18\xf9E\x8et\x8a\xb1fN[(U\xa2F\x0b\xa1\xac\xeb\xc1\x8e\x04\xceE8\x9ec\x18\xf0\xba\xfe&\x8e\x98\x1a\x1atP?\x07\x8e\x8f\x83\x10\xaa\n\x17\x8d%\x91\xa6\xd77\x93\xdb`\xbe7\xe6.h/\xee\x89t\xfd\xe3k<\xea\xf2\x0bPK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cX3\xeb\xe3\xba\xad\x00\x00\x00\xfb\x01\x00\x00\x1a\x00\x00\x00xl/_rels/workbook.xml.rels\xb5\x91=\x0e\x830\x0c\x85\xaf\x12\xe5\x00\x18\xa8\xd4\xa1\x02\xa6.\xac\x15\x17\x88\x82\xf9\x11\x81D\xb1\xab\xc2\xed\x1b\xc1\x00H\x1d\xba0Y\xcf\x96\xbf\xf7dg/4\x8a{;Q\xd7;\x12\xf3h&\xcae\xc7\xec\x1e\x00\xa4;\x1c\x15E\xd6\xe1\x14&\x8d\xf5\xa3\xe2 }\x0bN\xe9A\xb5\x08i\x1c\xdf\xc1\x1f\x19\xb2\xc8\x8eLQ-\x0e\xff!\xda\xa6\xe95>\xad~\x8f8\xf1\x0f0|\xac\x1f\xa8Cd)*\xe5[\xe4\\\xc2l\xf66\xc1Z\x92(\x90\xa5(\xeb\\\xfa\xb2N\xa4\x80\xcb\x12\x11/\x06i\x8f\xb3\xe9\x93\x7fz\xa5?\x87]\xdc\xedW\xb95\xcfG\xb8\xad!\xe0\xf4\xeb\xe2\x0bPK\x03\x04\x14\x00\x00\x00\x08\x00\x99j\x8cX\x9b\x86B\x84\x1b\x01\x00\x00\xd7\x03\x00\x00\x13\x00\x00\x00[Content_Types].xml\xad\x93\xcfN\xc30\x0c\xc6_\xa5\xeauj38p@\xeb.\x8c+\xec\xc0\x0b\x84\xc4]\xa3\xe6\x9fboto\x8f\xdb\xb2J\xa0\xb1\r\x95K\xa3\xc6\xf6\xf7s\xfc%\xab\xb7c\x04\xcc:g=VyC\x14\x1f\x85@\xd5\x80\x93X\x86\x08\x9e#uHN\x12\xff\xa6\x9d\x88R\xb5r\x07\xe2~\xb9|\x10*x\x02O\x05\xf5\x1a\xf9z\xb5\x81Z\xee-e\xcf\x1do\xa3\t\xbe\xca\x13X\xcc\xb3\xa71\xb1gU\xb9\x8c\xd1\x1a%\x89\xe3\xe2\xe0\xf5\x0fJ\xf1E(\xb9r\xc8\xc1\xc6D\\pB\x9e\x89\xb3\x88!\xf4+\xe1T\xf8z\x80\x94\x8c\x86l+\x13\xbdH\xc7i\xa2\xb3\x02\xe9h\x01\xcb\xcb\x1ag\xba\x0cum\x14\xe8\xa0\xf6\x8eKJ\x8c\t\xa4\xc6\x06\x80\x9c-G\xd1\xc5\x154\xf1\x90a\xfc\xde\xcdn`\x90\xb9H\xe4\xd4m\n\x11\xd9\xb5\x04\x7f\xe7\x9dl\xe9\xab\x8b\xc8B\x90\xc8\\9\xe4\x84d\xed\xd9'\x84\xdeq\r\xfaV8O\xf8#\xa4v\xf0\x04\xc5\xb0\xcc\x1f\xf3w\x9f'\xfd[\x1ay\x0f\xa1\xfd\xef{\xd6\xaf\xa5\x93\xc6O\r\x88\xe1=\xaf?\x01PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cXFZ\xc1\x0c\x82\x00\x00\x00\xb1\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00\x00docProps/app.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cXh\xedm\xda\xea\x00\x00\x00\xcb\x01\x00\x00\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\xb0\x00\x00\x00docProps/core.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cX\x99\\\x9c#\x10\x06\x00\x00\x9c'\x00\x00\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\xc9\x01\x00\x00xl/theme/theme1.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cX\t\xf8\xfa\x7fm\x01\x00\x004\x03\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x81\n\x08\x00\x00xl/worksheets/sheet1.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cX\xd2\x05\xf1FR\x02\x00\x00G\n\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\xad\t\x00\x00xl/styles.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cX\xb7G\xeb\x8a\xc0\x00\x00\x00\x16\x02\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01*\x0c\x00\x00_rels/.relsPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cX\xe4\xb0k\xee0\x01\x00\x00(\x02\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x13\r\x00\x00xl/workbook.xmlPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cX3\xeb\xe3\xba\xad\x00\x00\x00\xfb\x01\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01p\x0e\x00\x00xl/_rels/workbook.xml.relsPK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00\x99j\x8cX\x9b\x86B\x84\x1b\x01\x00\x00\xd7\x03\x00\x00\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01U\x0f\x00\x00[Content_Types].xmlPK\x05\x06\x00\x00\x00\x00\t\x00\t\x00>\x02\x00\x00\xa1\x10\x00\x00\x00\x00" + + +def test_xlsx_from_json(client: TestClient): + res = client.post("/api/xlsx/from_json", json=table_defintion_json) + xlsx_file = res.content + assert res.status_code == 200 + assert isinstance(xlsx_file, (bytes, bytearray)) + + +def test_xlsx_to_json(client: TestClient): + res = client.post("/api/xlsx/to_json", files={"file": xlsx_from_json_result}) + assert res.status_code == 200 + # Result must be the same as the input for table download + assert res.json() == table_defintion_json + + +def test_xlsx_from_json_array(client: TestClient): + res = client.post("/api/xlsx/from_json_array", json=table_definition_json_array) + xlsx_file = res.content + assert res.status_code == 200 + assert isinstance(xlsx_file, (bytes, bytearray)) + + +def test_xlsx_to_json_array(client: TestClient): + res = client.post("/api/xlsx/to_json_array", files={"file": xlsx_from_json_array_result}) + assert res.status_code == 200 + # Result must be the same as the input for table download + assert res.json() == table_definition_json_array diff --git a/visyn_core/xlsx.py b/visyn_core/xlsx.py index fa5977433..a95d86877 100644 --- a/visyn_core/xlsx.py +++ b/visyn_core/xlsx.py @@ -1,19 +1,20 @@ import logging from datetime import datetime +from io import BytesIO from tempfile import NamedTemporaryFile +from typing import Annotated, Any import dateutil.parser -from flask import Flask, abort, jsonify, request -from flask.wrappers import Response +from fastapi import APIRouter, File, HTTPException, Response from openpyxl import Workbook, load_workbook from openpyxl.cell import WriteOnlyCell from openpyxl.styles import Font +from pydantic import BaseModel -_log = logging.getLogger(__name__) -app = Flask(__name__) - +router = APIRouter(prefix="/api/xlsx", tags=["xlsx"]) _types = {"b": "boolean", "s": "string"} +_log = logging.getLogger(__name__) def to_type(cell): @@ -37,20 +38,36 @@ def _convert_value(v): return v -@app.route("/to_json", methods=["POST"]) -def _xlsx2json(): - file = request.files.get("file") +class TableColumn(BaseModel): + name: str + type: str + + +CELL_CONTENT = str | int | float | bool | datetime | None + + +class TableSheet(BaseModel): + title: str + columns: list[TableColumn] + rows: list[dict[str, Any]] + + +class TableData(BaseModel): + sheets: list[TableSheet] + + +@router.post("/to_json/", response_model=TableData) +def xlsx2json(file: Annotated[bytes, File()]): if not file: - abort(403, "missing file") + raise HTTPException(status_code=403, detail="missing file") - wb = load_workbook(file, read_only=True, data_only=True) # type: ignore + wb = load_workbook(filename=BytesIO(file), read_only=True, data_only=True) # type: ignore - def convert_row(row, cols): + def convert_row(row, cols: list[TableColumn]) -> dict[str, CELL_CONTENT]: result = {} - for r, c in zip(cols, row, strict=False): - result[c["name"]] = _convert_value(r.value) - + for r, c in zip(row, cols, strict=False): + result[c.name] = _convert_value(r.value) return result def convert_sheet(ws): @@ -59,43 +76,39 @@ def convert_sheet(ws): ws_cols = next(ws_rows, []) ws_first_row = next(ws_rows, []) - cols = [{"name": h.value, "type": to_type(r)} for h, r in zip(ws_cols, ws_first_row, strict=False)] + cols = [TableColumn(name=h.value, type=to_type(r)) for h, r in zip(ws_cols, ws_first_row, strict=False)] rows = [] - rows.append(convert_row(cols, ws_first_row)) + rows.append(convert_row(ws_first_row, cols)) for row in ws_rows: - rows.append(str(convert_row(cols, row))) - - return {"title": ws.title, "columns": cols, "rows": rows} - - data = {"sheets": [convert_sheet(ws) for ws in wb.worksheets]} + rows.append(convert_row(row, cols)) + return TableSheet(title=ws.title, columns=cols, rows=rows) - return jsonify(data) + data = TableData(sheets=[convert_sheet(ws) for ws in wb.worksheets]) + return data -@app.route("/to_json_array", methods=["POST"]) -def _xlsx2json_array(): - file = request.files.get("file") +@router.post("/to_json_array/", response_model=list[list[Any]]) +def xlsx2json_array(file: Annotated[bytes, File()]): if not file: - abort(403, "missing file") + raise HTTPException(status_code=403, detail="missing file") - wb = load_workbook(file, read_only=True, data_only=True) # type: ignore + wb = load_workbook(filename=BytesIO(file), read_only=True, data_only=True) def convert_row(row): return [_convert_value(cell.value) for cell in row] if not wb.worksheets: - return jsonify([]) + return [] ws = wb.worksheets[0] rows = [convert_row(row) for row in ws.iter_rows()] - return jsonify(rows) + return rows -@app.route("/from_json", methods=["POST"]) -def _json2xlsx(): - data: dict = request.json # type: ignore +@router.post("/from_json/") +def json2xlsx(data: TableData): wb = Workbook(write_only=True) bold = Font(bold=True) @@ -127,27 +140,26 @@ def to_value(v, coltype): v = dateutil.parser.parse(v) return to_cell(v) - for sheet in data.get("sheets", []): - ws = wb.create_sheet(title=sheet["title"]) - cols = sheet["columns"] - ws.append(to_header(col["name"]) for col in cols) + for sheet in data.sheets: + ws = wb.create_sheet(title=sheet.title) + cols = sheet.columns + ws.append(to_header(col.name) for col in cols) - for row in sheet["rows"]: - ws.append(to_value(row.get(col["name"], None), col["type"]) for col in cols) + for row in sheet.rows: + ws.append(to_value(row.get(col.name, None), col.type) for col in cols) with NamedTemporaryFile() as tmp: wb.save(tmp.name) tmp.seek(0) s = tmp.read() return Response( - s, - mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + content=s, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) -@app.route("/from_json_array", methods=["POST"]) -def _json_array2xlsx(): - data: list = request.json # type: ignore +@router.post("/from_json_array/") +def json_array2xlsx(data: list[list[Any]]): wb = Workbook(write_only=True) ws = wb.create_sheet() @@ -159,8 +171,8 @@ def _json_array2xlsx(): tmp.seek(0) s = tmp.read() return Response( - s, - mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + content=s, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) @@ -168,4 +180,4 @@ def create(): """ entry point of this plugin """ - return app + return router