diff --git a/engine/services/technicalComponentDirectory.js b/engine/services/technicalComponentDirectory.js index 2fe9f6f3..67181368 100644 --- a/engine/services/technicalComponentDirectory.js +++ b/engine/services/technicalComponentDirectory.js @@ -12,7 +12,8 @@ module.exports = { cacheNosql: require('../workspaceComponentExecutor/cacheNosql.js'), gouvFrInverseGeo: require('../workspaceComponentExecutor/gouvFrInverseGeo.js'), restApiGet: require('../workspaceComponentExecutor/restApiGet.js'), - restApiPost: require('../workspaceComponentExecutor/restApiPost.js'), + restApiPost: require('../workspaceComponentExecutor/httpProvider.js'), + httpProvider: require('../workspaceComponentExecutor/httpProvider.js'), // xmlToObject: require('./workspaceComponentExecutor/xmlToObject.js'), framcalcGetCsv: require('../workspaceComponentExecutor/framcalcGetCsv.js'), gouvFrGeoLocaliser: require('../workspaceComponentExecutor/gouvFrGeoLocaliser.js'), @@ -22,7 +23,8 @@ module.exports = { filter: require('../workspaceComponentExecutor/filter.js'), upload: require('../workspaceComponentExecutor/upload.js'), scrapper: require('../workspaceComponentExecutor/scrapper/scrapper.js'), - httpGet: require('../workspaceComponentExecutor/restGetFile.js'), + httpGet: require('../workspaceComponentExecutor/httpConsumerFile.js'), + httpConsumerFile: require('../workspaceComponentExecutor/httpConsumerFile.js'), sqlConnector: require('../workspaceComponentExecutor/sqlConnecteur.js'), mongoConnector: require('../workspaceComponentExecutor/MongoDB.js'), influxdbConnector: require('../workspaceComponentExecutor/influxdb.js'), @@ -33,7 +35,8 @@ module.exports = { valueFromPath: require('../workspaceComponentExecutor/valueFromPath.js'), unicity: require('../workspaceComponentExecutor/unicity.js'), propertiesMatrix: require('../workspaceComponentExecutor/propertiesMatrix.js'), - postConsumer: require('../workspaceComponentExecutor/postConsumer.js'), + postConsumer: require('../workspaceComponentExecutor/httpConsumer.js'), + httpConsumer: require('../workspaceComponentExecutor/httpConsumer.js'), keyToArray: require('../workspaceComponentExecutor/keyToArray.js'), sftpConsumer: require('../workspaceComponentExecutor/sftpConsumer.js'), flat: require('../workspaceComponentExecutor/flat.js'), diff --git a/engine/workspaceComponentExecutor/postConsumer.js b/engine/workspaceComponentExecutor/httpConsumer.js similarity index 99% rename from engine/workspaceComponentExecutor/postConsumer.js rename to engine/workspaceComponentExecutor/httpConsumer.js index 7510241d..a4f27a4d 100644 --- a/engine/workspaceComponentExecutor/postConsumer.js +++ b/engine/workspaceComponentExecutor/httpConsumer.js @@ -4,7 +4,7 @@ const fs = require('fs'); const https = require('https'); const fileLib = require('../../core/lib/file_lib.js') -class PostConsumer { +class HttpConsumer { constructor () { this.fetch = require('node-fetch'); this.stringReplacer = require('../utils/stringReplacer.js'); @@ -298,4 +298,4 @@ class PostConsumer { } } -module.exports = new PostConsumer() +module.exports = new HttpConsumer() diff --git a/engine/workspaceComponentExecutor/restGetFile.js b/engine/workspaceComponentExecutor/httpConsumerFile.js similarity index 97% rename from engine/workspaceComponentExecutor/restGetFile.js rename to engine/workspaceComponentExecutor/httpConsumerFile.js index 3e61b767..ae7cd505 100644 --- a/engine/workspaceComponentExecutor/restGetFile.js +++ b/engine/workspaceComponentExecutor/httpConsumerFile.js @@ -1,5 +1,5 @@ 'use strict'; -class HttpGet { +class HttpConsumerFile { constructor () { this.url = require('url') this.http = require('http') @@ -68,4 +68,4 @@ class HttpGet { return this.makeRequest('GET', data.specificData.url, data.specificData.contentType,flowdata, pullParams) } } -module.exports = new HttpGet() +module.exports = new HttpConsumerFile() diff --git a/engine/workspaceComponentExecutor/restApiPost.js b/engine/workspaceComponentExecutor/httpProvider.js similarity index 92% rename from engine/workspaceComponentExecutor/restApiPost.js rename to engine/workspaceComponentExecutor/httpProvider.js index 59c1c401..3c92453a 100644 --- a/engine/workspaceComponentExecutor/restApiPost.js +++ b/engine/workspaceComponentExecutor/httpProvider.js @@ -1,5 +1,5 @@ 'use strict' -class RestApiPost { +class HttpProvider { constructor () { this.stepNode = false this.workspace_component_lib = require('../../core/lib/workspace_component_lib') @@ -22,4 +22,4 @@ class RestApiPost { } } -module.exports = new RestApiPost() +module.exports = new HttpProvider() diff --git a/main/client/static/application.html b/main/client/static/application.html index 909d5f60..e21f97ea 100644 --- a/main/client/static/application.html +++ b/main/client/static/application.html @@ -107,14 +107,17 @@ + + + diff --git a/main/client/static/tag/editorComponents/dataFlow/http-consumer-editor.tag b/main/client/static/tag/editorComponents/dataFlow/http-consumer-editor.tag new file mode 100644 index 00000000..cc51e31c --- /dev/null +++ b/main/client/static/tag/editorComponents/dataFlow/http-consumer-editor.tag @@ -0,0 +1,168 @@ + + +
+ Aide +
+
{data.type}
+
+
+
+ +
{data.description}
+ +
+
+
+ + +
+ +
+
+ +
+ +
+
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ + + +
+ +
+ + +
+ +
+ +
+ +
+ + +
+
+ Ajouter + + +
+
+ + + + + + + + + + diff --git a/main/client/static/tag/editorComponents/dataFlow/http-provider-editor.tag b/main/client/static/tag/editorComponents/dataFlow/http-provider-editor.tag new file mode 100644 index 00000000..e4bed51c --- /dev/null +++ b/main/client/static/tag/editorComponents/dataFlow/http-provider-editor.tag @@ -0,0 +1,134 @@ + + +
+ Aide +
+ +
{data.type}
+
+
+
+ +
{data.description}
+
+
+
+ + +
+
{data._id}-
+ +
+ +
+ {window.location.origin +'/data/api/'+data.specificData.url} +
+ +
+ +
+ +
+ +
+ + + + diff --git a/main/client/static/tag/editorComponents/files/http-consumer-file-editor.tag b/main/client/static/tag/editorComponents/files/http-consumer-file-editor.tag new file mode 100644 index 00000000..c4babf17 --- /dev/null +++ b/main/client/static/tag/editorComponents/files/http-consumer-file-editor.tag @@ -0,0 +1,69 @@ + + +
+ Aide +
+ +
{data.type}
+
+
+
+ +
{data.description}
+
+
+
+ + + +
+ +
+ +
+ +
+ + + diff --git a/main/server/services/technicalComponentDirectory.js b/main/server/services/technicalComponentDirectory.js index 1007226c..a8e12141 100644 --- a/main/server/services/technicalComponentDirectory.js +++ b/main/server/services/technicalComponentDirectory.js @@ -13,6 +13,7 @@ module.exports = { gouvFrInverseGeo: require('../workspaceComponentInitialize/gouvFrInverseGeo.js'), restApiGet: require('../workspaceComponentInitialize/restApiGet.js'), restApiPost: require('../workspaceComponentInitialize/restApiPost.js'), + httpProvider: require('../workspaceComponentInitialize/httpProvider.js'), // xmlToObject: require('./workspaceComponentInitialize/xmlToObject.js'), framcalcGetCsv: require('../workspaceComponentInitialize/framcalcGetCsv.js'), gouvFrGeoLocaliser: require('../workspaceComponentInitialize/gouvFrGeoLocaliser.js'), @@ -23,6 +24,7 @@ module.exports = { upload: require('../workspaceComponentInitialize/upload.js'), scrapper: require('../workspaceComponentInitialize/scrapper.js'), httpGet: require('../workspaceComponentInitialize/restGetFile.js'), + httpConsumerFile: require('../workspaceComponentInitialize/httpConsumerFile.js'), sqlConnector: require('../workspaceComponentInitialize/sqlConnecteur.js'), mongoConnector: require('../workspaceComponentInitialize/MongoDB.js'), influxdbConnector: require('../workspaceComponentInitialize/influxdb.js'), @@ -34,6 +36,7 @@ module.exports = { unicity: require('../workspaceComponentInitialize/unicity.js'), propertiesMatrix: require('../workspaceComponentInitialize/propertiesMatrix.js'), postConsumer: require('../workspaceComponentInitialize/postConsumer.js'), + httpConsumer: require('../workspaceComponentInitialize/httpConsumer.js'), keyToArray: require('../workspaceComponentInitialize/keyToArray.js'), sftpConsumer: require('../workspaceComponentInitialize/sftpConsumer.js'), flat: require('../workspaceComponentInitialize/flat.js'), diff --git a/main/server/workspaceComponentInitialize/httpConsumer.js b/main/server/workspaceComponentInitialize/httpConsumer.js new file mode 100644 index 00000000..b84122bd --- /dev/null +++ b/main/server/workspaceComponentInitialize/httpConsumer.js @@ -0,0 +1,13 @@ +'use strict'; +class HttpConsumer { + constructor () { + this.type = 'HTTP consumer' + this.description = 'Appeler une API HTTP; Executer une requête HTTP.' + this.editor = 'http-consumer-editor' + this.graphIcon = 'Post_consumer.svg' + this.tags = [ + ] + } +} + +module.exports = new HttpConsumer() diff --git a/main/server/workspaceComponentInitialize/httpConsumerFile.js b/main/server/workspaceComponentInitialize/httpConsumerFile.js new file mode 100644 index 00000000..97210728 --- /dev/null +++ b/main/server/workspaceComponentInitialize/httpConsumerFile.js @@ -0,0 +1,14 @@ +'use strict'; +class HttpConsumerFile { + constructor () { + this.type = 'File consumer' + this.description = 'Interroger un fichier mis à disposition sur une API REST avec une requete GET.' + this.editor = 'http-consumer-file-editor' + this.graphIcon = 'File_consumer.svg' + this.tags = [ + 'http://semantic-bus.org/data/tags/inComponents', + 'http://semantic-bus.org/data/tags/fileComponents' + ] + } +} +module.exports = new HttpConsumerFile() diff --git a/main/server/workspaceComponentInitialize/httpProvider.js b/main/server/workspaceComponentInitialize/httpProvider.js new file mode 100644 index 00000000..fb5ff558 --- /dev/null +++ b/main/server/workspaceComponentInitialize/httpProvider.js @@ -0,0 +1,297 @@ +'use strict' + +const { v4: uuidv4 } = require('uuid'); +const MODE = 'AMQP' // MODE could be AMQP when all workflow will migrate over V1 +class HttpProvider { + constructor() { + this.type = 'HTTP provider' + this.description = `Mettre à disposition une API HTTP; Permettre à votre workflow d'être appelé par une requete HTTP.` + this.editor = 'http-provider-editor' + this.graphIcon = 'Post_provider.svg' + this.tags = [ + 'http://semantic-bus.org/data/tags/inComponents', + 'http://semantic-bus.org/data/tags/outComponents', + 'http://semantic-bus.org/data/tags/APIComponents' + ], + this.stepNode = false + this.workspace_component_lib = require('../../../core/lib/workspace_component_lib') + this.workspace_lib = require('../../../core/lib/workspace_lib') + this.fragment_lib = require('../../../core/lib/fragment_lib') + this.data2xml = require('data2xml'); + this.xmlJS = require('xml-js'); + this.dataTraitment = require('../../../core/dataTraitmentLibrary/index.js') + this.json2yaml = require('json2yaml') + this.request = require('request') + this.config = require('../../config.json') + + const { + pathToRegexp, + match, + parse, + compile + } = require("path-to-regexp"); + this.pathToRegexp = pathToRegexp; + this.pendingWork= {}; + this.amqpConnection; + } + + setAmqp(amqpConnection){ + // console.log('set AMQP') + this.amqpConnection=amqpConnection; + amqpConnection.consume('process-persist', (msg) => { + const messageObject = JSON.parse(msg.content.toString()) + const pendingWork = this.pendingWork[messageObject.tracerId||messageObject.processId] + if(pendingWork?.component == messageObject.componentId){ + pendingWork.frag = messageObject.frag; + } + + }, { + noAck: true + }) + + amqpConnection.consume('process-start', (msg) => { + const messageObject = JSON.parse(msg.content.toString()) + // console.log('messageObject',messageObject) + // console.log('process-start',messageObject.tracerId,this.id) + const pendingWork = this.pendingWork[messageObject.tracerId||messageObject._id] + if(pendingWork){ + pendingWork.process = messageObject._id; + } + }, { + noAck: true + }) + + amqpConnection.consume('process-error', (msg) => { + const messageObject = JSON.parse(msg.content.toString()) + const pendingWork = this.pendingWork[messageObject.tracerId||messageObject._id] + if(pendingWork){ + pendingWork.error = messageObject._id; + } + }, { + noAck: true + }) + + } + + initialise(router,engineTracer) { + + router.all('*', async (req, res, next) => { + + // console.log('pendingWork',this.pendingWork); + // console.log(req) + const urlRequiered = req.params[0].split('/')[1]; + const urlRequieredFull = req.params[0].replace('/', ''); + const query = req.query; + // console.log(); + let targetedComponent; + const regex = /([^-]*)-.*/g; + let componentId = regex.exec(urlRequiered)[1]; + let component; + try { + let component = await this.workspace_component_lib.get({ + _id: componentId, + }); + if (component != undefined && component.specificData.url != undefined) { + req.setTimeout(0); + let keys = [] + let regexp = this.pathToRegexp(component.specificData.url, keys); + //convert query url variable to query properties + if (regexp.test(urlRequieredFull)) { + let values = regexp.exec(urlRequieredFull); + + let valueIndex = 1; + for (let key of keys) { + let value = values[valueIndex] + query[key.name] = value + valueIndex++ + } + for (let queryKey in query) { + try { + query[queryKey] = JSON.parse(query[queryKey]) + } catch (e) { + } + } + } else { + // console.log('NO MATH!!'); + } + + // console.log('req.body',req.body); + + const worksapce = await this.workspace_lib.get_workspace_simple(component.workspaceId) + + const version = worksapce.engineVersion==undefined||worksapce.engineVersion=='default'?'v1':worksapce.engineVersion; + + // console.log('VERSION',version) + if (MODE=='HTTP'){ + // console.log('CALL Direct HTTP') + const versionUrl = `${this.config.engineUrl}/${version}/work-ask/${component._id}` + // console.log('versionUrl',this.config.engineUrl + versionUrl + component._id); + this.request.post(versionUrl, { + body: { + queryParams: { + query: req.query, + body: req.body, + headers: req.headers, + method :req.method + }, + pushData: req.body + }, + json: true + } + // eslint-disable-next-line handle-callback-err + , (err, data) => { + + try { + if (err) { + console.error("restpiIPost request error", err); + res.status(500).send(err) + } else { + if (data.statusCode != 200) { + res.status(500).send({ + engineResponse: data.body + }) + } else { + console.log('WORK response',data.body); + if(data.body.data){ + console.log(data.body); + this.sendResult(component, data.body.data, res) + }else { + // engineTracer.pendingProcess.push(data.body.processId); + this.pendingWork[data.body.processId]={component :component._id}; + let counter=0 + const intervalId = setInterval(async () => { + console.log(counter,data.body.processId) + if (this.pendingWork[data.body.processId].frag){ + clearInterval(intervalId); + // res.send(this.pendingWork[data.body.processId]); + const dataResponse = await this.fragment_lib.getWithResolutionByBranch(this.pendingWork[data.body.processId].frag); + console.log(dataResponse) + this.sendResult(component, dataResponse, res) + }else{ + // console.log('waiting'); + } + }, 100); + } + } + } + } catch (e) { + console.log('api error after engine call', e); + res.send(new Error(e.message)) + } + }); + }else if (MODE=='AMQP'){ + // console.log('CALL AMQP') + const tracerId = uuidv4(); + const workParams={ + tracerId , + id : component._id, + queryParams: { + query: req.query, + body: req.body, + headers: req.headers, + method :req.method + }, + // pushData: req.body + } + this.pendingWork[tracerId] = { + component :component._id + } + // console.log(this.amqpConnection) + this.amqpConnection.sendToQueue( + 'work-ask', + Buffer.from(JSON.stringify(workParams)), + null, + + (err, ok) => { + if (err !== null) { + console.error('Erreur lors de l\'envoi du message :', err); + res.status(500).send({ + error: 'AMQP server no connected' + }) + } else { + // console.log(`Message envoyé à la file `); + // res.send(workParams); + } + } + ) + // let counter=1; + const intervalId = setInterval(async () => { + if (this.pendingWork[tracerId].frag){ + clearInterval(intervalId); + const dataResponse = await this.fragment_lib.getWithResolutionByBranch(this.pendingWork[tracerId].frag); + this.sendResult(component, dataResponse, res) + + } else if (this.pendingWork[tracerId].error){ + clearInterval(intervalId); + res.status(500).send({ + error:'engine error' + }) + }else{ + // console.log('waiting'); + // counter++; + } + }, 100); + } + } else { + res.status(404).send('no API for this url'); + } + } catch (e) { + console.log(e); + res.status(404).send('no API for this url'); + } + }) + } + + sendResult(component, dataToSend, res) { + if (component.specificData != undefined) { // exception in previous promise + if (component.specificData.contentType != undefined && component.specificData.contentType != '') { + // console.log('contentType',component.specificData.contentType); + if (dataToSend == undefined) { + res.status(201).send() + } else if (component.specificData.contentType.search('application/vnd.ms-excel') != -1) { + res.setHeader('content-type', component.specificData.contentType) + this.dataTraitment.type.buildFile(undefined, JSON.stringify(dataToSend), undefined, true, component.specificData.contentType).then((result) => { + res.setHeader('Content-disposition', 'attachment; filename=' + component.specificData.url + '.xlsx') + res.send(result) + }) + } else if (component.specificData.contentType.search('rdf') != -1) { + + res.setHeader('content-type', component.specificData.contentType) + this.dataTraitment.type.buildFile(undefined, JSON.stringify(dataToSend), undefined, true, component.specificData.contentType).then((result) => { + res.setHeader('Content-disposition', 'attachment; filename=' + component.specificData.url + '.xml') + res.send(result) + }) + } else if (component.specificData.contentType.search('xml') != -1) { + res.setHeader('content-type', component.specificData.contentType) + let out = this.xmlJS.js2xml(dataToSend, { + compact: true, + ignoreComment: true, + spaces: 0 + }) + out = out.replace(/\0/g, '') + // console.log('xml out', out); + // console.log(Buffer.byteLength(out, 'utf8') + " bytes"); + res.send(out) + // res.end(); + } else if (component.specificData.contentType.search('yaml') != -1) { + res.setHeader('content-type', component.specificData.contentType) + res.send(this.json2yaml.stringify(dataToSend)) + } else if (component.specificData.contentType.search('json') != -1) { + res.setHeader('content-type', component.specificData.contentType) + var buf = Buffer.from(JSON.stringify(dataToSend)) + res.send(buf) + } else { + res.status(400).send('no supported content-type') + // res.send(new Error('no supported madiatype')) + // return ('type mime non géré') + } + } else { + console.log('ERROR content-type have to be set') + res.status(400).send(`content-type have to be set`) + // return ('type mime non géré') + } + } + } +} + +module.exports = new HttpProvider() diff --git a/main/server/workspaceComponentInitialize/postConsumer.js b/main/server/workspaceComponentInitialize/postConsumer.js index 3f12bbe9..e28eb8d8 100644 --- a/main/server/workspaceComponentInitialize/postConsumer.js +++ b/main/server/workspaceComponentInitialize/postConsumer.js @@ -5,11 +5,6 @@ class PostConsumer { this.description = 'Appeler une API HTTP; Executer une requête HTTP.' this.editor = 'post-consumer-editor' this.graphIcon = 'Post_consumer.svg' - this.tags = [ - 'http://semantic-bus.org/data/tags/outComponents', - 'http://semantic-bus.org/data/tags/inComponents', - 'http://semantic-bus.org/data/tags/APIComponents' - ] } } diff --git a/main/server/workspaceComponentInitialize/restApiPost.js b/main/server/workspaceComponentInitialize/restApiPost.js index c3cc754d..5c3435d7 100644 --- a/main/server/workspaceComponentInitialize/restApiPost.js +++ b/main/server/workspaceComponentInitialize/restApiPost.js @@ -8,12 +8,7 @@ class RestApiPost { this.description = `Mettre à disposition une API HTTP; Permettre à votre workflow d'être appelé par une requete HTTP.` this.editor = 'rest-api-post-editor' this.graphIcon = 'Post_provider.svg' - this.tags = [ - 'http://semantic-bus.org/data/tags/inComponents', - 'http://semantic-bus.org/data/tags/outComponents', - 'http://semantic-bus.org/data/tags/APIComponents' - ], - this.stepNode = false + this.stepNode = false this.workspace_component_lib = require('../../../core/lib/workspace_component_lib') this.workspace_lib = require('../../../core/lib/workspace_lib') this.fragment_lib = require('../../../core/lib/fragment_lib') diff --git a/main/server/workspaceComponentInitialize/restGetFile.js b/main/server/workspaceComponentInitialize/restGetFile.js index 24a28625..8b21ff58 100644 --- a/main/server/workspaceComponentInitialize/restGetFile.js +++ b/main/server/workspaceComponentInitialize/restGetFile.js @@ -5,10 +5,6 @@ class HttpGet { this.description = 'Interroger un fichier mis à disposition sur une API REST avec une requete GET.' this.editor = 'rest-get-editor' this.graphIcon = 'File_consumer.svg' - this.tags = [ - 'http://semantic-bus.org/data/tags/inComponents', - 'http://semantic-bus.org/data/tags/fileComponents' - ] } } module.exports = new HttpGet() diff --git a/main/server/workspaceComponentInitialize/sftpConsumer.js b/main/server/workspaceComponentInitialize/sftpConsumer.js index 74730816..332afc68 100644 --- a/main/server/workspaceComponentInitialize/sftpConsumer.js +++ b/main/server/workspaceComponentInitialize/sftpConsumer.js @@ -1,7 +1,7 @@ 'use strict'; class SftpConsumer { constructor () { - this.type = 'sftp consumer' + this.type = 'HTTP file consumer' this.description = 'Interroger un fichier mis à disposition sur un serveur FTP.' this.editor = 'sftp-consumer-editor' this.graphIcon = 'File_consumer.svg'