Skip to content

Commit 2493b8a

Browse files
nolanm1122nolanm1122
committed
Offload job processing to server attempt #2 
1 parent 8f60985 commit 2493b8a

File tree

3 files changed

+128
-111
lines changed

3 files changed

+128
-111
lines changed

‎src/oqlParse/oqlParse.js‎

Lines changed: 52 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@ function makeColumnIDsMap() {
99
const columns = [];
1010
for (const colKey in configs[key].columns) {
1111
columns.push(
12-
{name: configs[key].columns[colKey].displayName.toLowerCase(),
13-
id: configs[key].columns[colKey].id,
14-
entityId: configs[key].columns[colKey].entityId}
12+
{
13+
name: configs[key].columns[colKey].displayName.toLowerCase(),
14+
id: configs[key].columns[colKey].id,
15+
entityId: configs[key].columns[colKey].entityId
16+
}
1517
);
1618
}
1719
map[key] = columns;
@@ -25,7 +27,7 @@ function getEntityId(subjectEntity, colName) {
2527
if (!(subjectEntity in COLUMN_IDS_MAP)) {
2628
throw new Error(`${subjectEntity} is not a valid subjectEntity`);
2729
}
28-
for (const m of COLUMN_IDS_MAP[subjectEntity]) {
30+
for (const m of COLUMN_IDS_MAP[subjectEntity]) {
2931
if (colName === m.name || colName === m.value) return m.entityId;
3032
}
3133
return null;
@@ -80,7 +82,7 @@ function parseCondition(condition, subjectEntity) {
8082

8183
value = parsePrimitive(value);
8284
let entityId = getEntityId(subjectEntity, columnName);
83-
if (typeof value === "string" && entityId !== null && !value.includes(`${entityId}/`)) value = `${entityId}/${value}`;
85+
if (typeof value === "string" && entityId !== null && entityId !== "works" && !value.includes(`${entityId}/`)) value = `${entityId}/${value}`;
8486

8587
return {
8688
id: generateId(),
@@ -163,7 +165,7 @@ function parseFilters(oql) {
163165
let worksOperator = "and";
164166
let summarizeByOperator = "and";
165167

166-
const worksMatch = oql.match(/where (.+?)(;|$)/);
168+
const worksMatch = oql.match(/(?<!summarize by.*)where (.+?)(;|$)/);
167169
if (worksMatch) {
168170
const worksClause = worksMatch[1];
169171
if (worksClause.includes("(")) {
@@ -172,8 +174,7 @@ function parseFilters(oql) {
172174
condition.subjectEntity = 'works';
173175
}
174176
filters.push(...nestedConditions);
175-
}
176-
else if (worksClause.includes(" or ")) {
177+
} else if (worksClause.includes(" or ")) {
177178
worksOperator = "or";
178179
worksConditions.push(...worksClause.split(" or "));
179180
} else {
@@ -329,66 +330,68 @@ function oqlToQuery(oql) {
329330
}
330331

331332
function queryToOQL(query) {
332-
if (!query.filters || query.filters.length === 0) {
333-
return "get works";
334-
}
333+
let oql = "get works";
334+
335+
if (Array.isArray(query.filters) && query.filters.length > 0) {
336+
const worksFilters = query.filters.filter(filter => filter.subjectEntity === "works");
337+
if (worksFilters.length > 0) {
338+
oql += ` where ${generateFilters(worksFilters, "works")}`;
339+
}
340+
}
335341

336-
const rootFilter = findRootFilter(query.filters);
337-
let oql = `get ${rootFilter.subjectEntity} where `;
338-
oql += generateFilters(query.filters);
339342

340-
if (query.summarize_by) {
341-
oql += `; summarize by ${query.summarize_by}`;
343+
if (query.summarize_by) {
344+
oql += `; summarize by ${query.summarize_by}`;
342345

343-
const summaryFilters = query.filters.filter(filter => filter.subjectEntity === query.summarize_by);
344-
if (summaryFilters.length > 0) {
345-
oql += ` where ${generateFilters(summaryFilters)}`;
346+
const summaryFilters = (query.filters || []).filter(filter => filter.subjectEntity === query.summarize_by);
347+
if (summaryFilters.length > 0) {
348+
oql += ` where ${generateFilters(summaryFilters, query.summarize_by)}`;
349+
}
346350
}
347-
}
348351

349-
if (query.sort_by) {
350-
oql += `; sort by ${query.sort_by.column_id} ${query.sort_by.direction}`;
351-
}
352+
if (query.sort_by) {
353+
oql += `; sort by ${query.sort_by.column_id} ${query.sort_by.direction}`;
354+
}
352355

353-
if (query.return_columns && query.return_columns.length > 0) {
354-
oql += `; return ${query.return_columns.join(', ')}`;
355-
}
356+
if (query.return_columns && query.return_columns.length > 0) {
357+
oql += `; return ${query.return_columns.join(', ')}`;
358+
}
356359

357-
return oql;
360+
return oql;
358361
}
359362

360-
function findRootFilter(filters) {
361-
const childIds = new Set(filters.flatMap(filter => filter.children || []));
362-
return filters.find(filter => !childIds.has(filter.id));
363+
function findRootFilter(filters, subjEntity) {
364+
const childIds = new Set(filters.flatMap(filter => filter.children || []));
365+
return filters.find(filter => !childIds.has(filter.id) && filter.subjectEntity === subjEntity);
363366
}
364367

365-
function generateFilters(filters) {
366-
const rootFilter = findRootFilter(filters);
367-
return generateFilterString(rootFilter, filters);
368+
function generateFilters(filters, subjEntity) {
369+
const rootFilter = findRootFilter(filters, subjEntity);
370+
return generateFilterString(rootFilter, filters);
368371
}
369372

370373
function generateFilterString(filter, allFilters) {
371-
if (filter.type === 'leaf') {
372-
return `${filter.column_id} ${filter.operator || 'is'} ${filter.value}`;
373-
} else if (filter.type === 'branch') {
374-
const childFilters = filter.children.map(childId => {
375-
const childFilter = allFilters.find(f => f.id === childId);
376-
return generateFilterString(childFilter, allFilters);
377-
});
374+
if (filter.type === 'leaf') {
375+
return `${filter.column_id} ${filter.operator || 'is'} ${filter.value}`;
376+
} else if (filter.type === 'branch') {
377+
const childFilters = filter.children.map(childId => {
378+
const childFilter = allFilters.find(f => f.id === childId);
379+
return generateFilterString(childFilter, allFilters);
380+
});
378381

379-
const needsParentheses = filter.children.length > 1 &&
380-
allFilters.some(f => f.type === 'branch' && f.id !== filter.id && f.subjectEntity === filter.subjectEntity);
382+
const needsParentheses = filter.children.length > 1 &&
383+
allFilters.some(f => f.type === 'branch' && f.id !== filter.id && f.subjectEntity === filter.subjectEntity);
381384

382-
const joinedFilters = childFilters.join(` ${filter.operator} `);
383-
return needsParentheses ? `(${joinedFilters})` : joinedFilters;
384-
}
385+
const joinedFilters = childFilters.join(` ${filter.operator} `);
386+
return needsParentheses ? `(${joinedFilters})` : joinedFilters;
387+
}
385388
}
386389

387390
function formatValue(value) {
388-
if (typeof value === 'string') {
389-
return `"${value}"`;
390-
}
391-
return value;
391+
if (typeof value === 'string') {
392+
return `"${value}"`;
393+
}
394+
return value;
392395
}
393396

394397
export {

‎src/oqlParse/test.js‎

Lines changed: 71 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ class OQOTestRunner {
2121
this.tests = tests;
2222
this.onTestResultCb = onTestResultCb;
2323
this.serverUrl = 'https://api.openalex.org';
24-
this.queueId = null;
24+
// this.serverUrl = 'http://localhost:5000';
25+
this.jobId = null;
2526
}
2627

2728
expectedResults(tests, cases = ["oqlToQuery", "queryToOql", "natLang", "queryToSearch"]) {
@@ -237,7 +238,7 @@ class OQOTestRunner {
237238
}
238239
}
239240

240-
async startServerTests(cases) {
241+
async startServerTests(cases) {
241242
const serverTests = this.tests.flatMap(test => {
242243
const testId = objectMD5ShortUUID(test);
243244
const serverTestCases = [];
@@ -263,6 +264,9 @@ class OQOTestRunner {
263264

264265
const response = await fetch(`${this.serverUrl}/bulk_test`, {
265266
method: 'POST',
267+
headers: {
268+
'Content-Type': 'application/json',
269+
},
266270
body: JSON.stringify(serverTests),
267271
});
268272

@@ -271,66 +275,76 @@ class OQOTestRunner {
271275
}
272276

273277
const data = await response.json();
274-
this.queueId = data.queue_id;
278+
this.jobId = data.job_id;
275279
}
276280

277-
listenForResults() {
278-
return new Promise((resolve, reject) => {
279-
const eventSource = new EventSource(`${this.serverUrl}/stream/${this.queueId}`);
281+
async pollForResults() {
282+
const pollInterval = 5000; // 5 seconds
283+
const maxAttempts = 60; // 5 minutes total polling time
284+
let attempts = 0;
285+
286+
while (attempts < maxAttempts) {
287+
const response = await fetch(`${this.serverUrl}/job_status/${this.jobId}`,{
288+
method: 'GET',
289+
headers: {
290+
'Cache-Control': 'no-cache, no-store, must-revalidate',
291+
'Pragma': 'no-cache',
292+
'Expires': '0'
293+
}
294+
});
295+
if (!response.ok) {
296+
throw new Error(`HTTP error! status: ${response.status}`);
297+
}
280298

281-
eventSource.onmessage = (event) => {
282-
const data = JSON.parse(event.data);
283-
if (data.status === 'processing') {
284-
// Optionally handle processing status
285-
return;
286-
}
287-
if (data.status === 'all_completed') {
288-
eventSource.close();
289-
resolve();
290-
return;
291-
}
292-
if (data.status === 'error') {
293-
console.error(data.message);
294-
eventSource.close();
295-
reject(new Error(data.message));
296-
return;
297-
}
299+
const data = await response.json();
298300

299-
if (data.hasOwnProperty('id') && data.id !== null) {
300-
if (data.case === 'natLang') {
301-
const results = [];
302-
const test = this.tests.find(test => objectMD5ShortUUID(test) === data.id);
303-
for (const _result of data.results) {
304-
const result = OQOTestRunner.queriesEqual(_result.oqo, test.query, test.ignore ?? []);
305-
if (!result.equal) {
306-
result.expected = test.query;
307-
result.actual = _result.oqo;
308-
}
309-
results.push({
310-
"case": "natLang",
311-
prompt: _result.prompt,
312-
isPassing: result.equal,
313-
details: result,
314-
});
301+
if (data.is_completed) {
302+
return this.processResults(data.results);
303+
}
304+
305+
if (data.status === 'failed') {
306+
throw new Error(`Job failed: ${data.error || 'Unknown error'}`);
307+
}
308+
309+
await new Promise(resolve => setTimeout(resolve, pollInterval));
310+
attempts++;
311+
}
312+
313+
throw new Error('Polling timed out');
314+
}
315+
316+
processResults(results) {
317+
for (const result of results) {
318+
if (result.case === 'natLang') {
319+
const test = this.tests.find(test => objectMD5ShortUUID(test) === result.id);
320+
const subTests = result.results.map(subResult => {
321+
let comparisonResult = {equal: false};
322+
try {
323+
comparisonResult = OQOTestRunner.queriesEqual(subResult.oqo, test.query, test.ignore ?? []);
324+
} catch (e) {}
325+
return {
326+
case: "natLang",
327+
prompt: subResult.prompt,
328+
isPassing: comparisonResult.equal && !subResult.hasOwnProperty("error"),
329+
details: comparisonResult.equal ? {} : {
330+
expected: test.query,
331+
actual: subResult.oqo,
332+
...(subResult.hasOwnProperty("error") ? { error: subResult.error } : {}),
333+
...comparisonResult
315334
}
316-
this.onTestResultCb({
317-
"case": "natLang",
318-
id: data.id,
319-
isPassing: results.every((o) => o.isPassing),
320-
subTests: results
321-
});
322-
} else if (data.case === 'jsonToSearch') {
323-
this.onTestResultCb(data);
324-
}
325-
}
326-
};
335+
};
336+
});
327337

328-
eventSource.onerror = (error) => {
329-
console.error('EventSource failed:', error);
330-
eventSource.close();
331-
reject(error);
332-
};
333-
});
338+
this.onTestResultCb({
339+
case: "natLang",
340+
id: result.id,
341+
isPassing: subTests.every(st => st.isPassing),
342+
subTests: subTests
343+
});
344+
} else if (result.case === 'queryToSearch') {
345+
this.onTestResultCb(result);
346+
}
347+
}
334348
}
335349

336350
async runTests(cases = ["oqlToQuery", "queryToOql", "natLang", "queryToSearch"]) {
@@ -354,7 +368,7 @@ class OQOTestRunner {
354368
// Run server-side tests
355369
if (cases.includes("natLang") || cases.includes("queryToSearch")) {
356370
await this.startServerTests(cases);
357-
await this.listenForResults();
371+
await this.pollForResults();
358372
}
359373
}
360374
}

‎src/router/index.js‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -121,15 +121,15 @@ const routes = [
121121
component: OQOTestDetails,
122122
props: (route) => ({
123123
testId: route.params.id,
124-
autoRun: route.path.endsWith('/run')
124+
autoRun: route.query.run
125125
})
126126
},
127127
{
128-
path: '/tests/:id/run',
129-
component: OQOTestDetails,
128+
path: '/tests/tag/:tag',
129+
component: OQOTests,
130130
props: (route) => ({
131-
testId: route.params.id,
132-
autoRun: true
131+
initialTag: route.params.tag,
132+
autoRun: route.query.run
133133
})
134134
},
135135

0 commit comments

Comments
 (0)