From d3bb6fc5c6abacc4557ed60913ccbf52eef7b15a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20B=C3=BChler?= Date: Thu, 28 Nov 2024 11:20:57 +0100 Subject: [PATCH 1/4] Remove old plugin runner endpoints from API Endpoints are superseded by plugin registry --- Dependencies.toml | 4 +-- main.bal | 92 ----------------------------------------------- types.bal | 45 ----------------------- 3 files changed, 2 insertions(+), 139 deletions(-) diff --git a/Dependencies.toml b/Dependencies.toml index c506477..4100ab1 100644 --- a/Dependencies.toml +++ b/Dependencies.toml @@ -64,7 +64,7 @@ modules = [ [[package]] org = "ballerina" name = "http" -version = "2.12.2" +version = "2.12.3" dependencies = [ {org = "ballerina", name = "auth"}, {org = "ballerina", name = "cache"}, @@ -274,7 +274,7 @@ modules = [ [[package]] org = "ballerina" name = "sql" -version = "1.14.1" +version = "1.14.2" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, diff --git a/main.bal b/main.bal index d95647c..c07b51b 100644 --- a/main.bal +++ b/main.bal @@ -310,102 +310,10 @@ service / on new http:Listener(serverPort) { return { '\@self: serverHost + "/", experiments: serverHost + "/experiments/", - pluginRunners: serverHost + "/pluginRunners/", tasks: serverHost + "/tasks/" }; } - # Get a list of configured plugin or plugin-runner endpoints. - # - # + return - the list of endpoints as a "ListResponse". - resource function get plugin\-endpoints() returns PluginEndpointsListResponse|http:InternalServerError { - int endpointCount; - database:PluginEndpointFull[] endpoints; - - transaction { - endpointCount = check database:getPluginEndpointsCount(); - endpoints = check database:getPluginEndpoints(); - check commit; - } on fail error err { - log:printError("Could not get plugin endpoints.", 'error = err, stackTrace = err.stackTrace()); - // if with return does not correctly narrow type for rest of function... this does. - http:InternalServerError resultErr = {body: "Something went wrong. Please try again later."}; - return resultErr; - } - - var result = from var endpoint in endpoints - select mapToPluginEndpointResponse(endpoint); - // FIXME load from database... - return { - '\@self: serverHost + "/plugin-endpoints", - items: result, - itemCount: endpointCount - }; - } - - # Add a new endpoint to the list of plugin(-runner) endpoints. - # - # + return - the created resource - resource function post plugin\-endpoints(@http:Payload PluginEndpointPost endpoint) returns PluginEndpointResponse|http:InternalServerError { - database:PluginEndpointFull result; - transaction { - result = check database:addPluginEndpoint(endpoint); - check commit; - } on fail error err { - log:printError("Could not add new plugin endpoint", 'error = err, stackTrace = err.stackTrace()); - return {body: "Something went wrong. Please try again later."}; - } - - return mapToPluginEndpointResponse(result); - } - - # Get a specific plugin(-runner) endpoint resource. - # - # + return - the endpoint resource - resource function get plugin\-endpoints/[int endpointId]() returns PluginEndpointResponse|http:InternalServerError { - database:PluginEndpointFull result; - transaction { - result = check database:getPluginEndpoint(endpointId); - check commit; - } on fail error err { - log:printError("Could not get plugin endpoint.", 'error = err, stackTrace = err.stackTrace()); - return {body: "Something went wrong. Please try again later."}; - } - - return mapToPluginEndpointResponse(result); - } - - # Update an existing plugin(-runner) endpoint resource. - # - # + return - the updated endpoint resource - resource function put plugin\-endpoints/[int endpointId](@http:Payload PluginEndpointPost endpoint) returns PluginEndpointResponse|http:InternalServerError { - database:PluginEndpointFull result; - transaction { - result = check database:editPluginEndpoint(endpointId, endpoint.'type); - check commit; - } on fail error err { - log:printError("Could not update plugin endpoint", 'error = err, stackTrace = err.stackTrace()); - return {body: "Something went wrong. Please try again later."}; - } - - return mapToPluginEndpointResponse(result); - } - - # Remove an existing plugin(-runner) endpoint resource. - # - # + return - an empty response with a 2xx http status code on success - resource function delete plugin\-endpoints/[int endpointId]() returns http:Ok|http:InternalServerError { - transaction { - check database:deletePluginEndpoint(endpointId); - check commit; - } on fail error err { - log:printError("Could not delete plugin endpoint", 'error = err, stackTrace = err.stackTrace()); - return {body: "Something went wrong. Please try again later."}; - } - - return {}; - } - //////////////////////////////////////////////////////////////////////////// // Experiments ///////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////// diff --git a/types.bal b/types.bal index 34bdb36..4a4ee17 100644 --- a/types.bal +++ b/types.bal @@ -36,58 +36,13 @@ public type ApiResponse record {| # The root api response # # + experiments - Url to the experiments collection resource -# + pluginRunners - Url to the plugin runners collection resource # + tasks - Url to the tasks collection resource public type RootResponse record {| *ApiResponse; string experiments; - string pluginRunners; string tasks; |}; -# Post payload to create new plugin endpoint resources. -# -# + url - the URL of the plugin endpoint -# + 'type - the type of the endpoint ("Plugin"|"PluginRunner") -public type PluginEndpointPost record {| - string url; - string 'type = "PluginRunner"; -|}; - -# A plugin endpoint resource. -# -# + endpointId - the id of the plugin endpoint -# + url - the URL of the plugin endpoint -# + 'type - the type of the endpoint ("Plugin"|"PluginRunner") -public type PluginEndpointResponse record {| - *ApiResponse; - int endpointId; - string url; - string 'type = "PluginRunner"; -|}; - -# The plugin endpoints list resource. -# -# + items - the plugin endpoint resources -# + itemCount - the total count of plugin endpoints -public type PluginEndpointsListResponse record {| - *ApiResponse; - PluginEndpointResponse[] items; - int itemCount; -|}; - -# Helper function to map from `PluginEndpointFull` database record to API record. -# -# + endpoint - the input database record -# + return - the mapped record -public isolated function mapToPluginEndpointResponse(database:PluginEndpointFull endpoint) returns PluginEndpointResponse { - return { - '\@self: string `${serverHost}/plugin-endpoints/${endpoint.id}`, - endpointId: endpoint.id, - url: endpoint.url, - 'type: endpoint.'type - }; -} //////////////////////////////////////////////////////////////////////////////// // Experiments ///////////////////////////////////////////////////////////////// From b05c9a4c4b208127ce521e1b6c4f3bdcee36703a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20B=C3=BChler?= Date: Thu, 28 Nov 2024 11:21:56 +0100 Subject: [PATCH 2/4] Fix timeline step page endpoint returning 404 even for first page --- main.bal | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/main.bal b/main.bal index c07b51b..2c56863 100644 --- a/main.bal +++ b/main.bal @@ -604,13 +604,20 @@ service / on new http:Listener(serverPort) { transaction { stepCount = check database:getTimelineStepCount(experimentId, plugin\-name, 'version, status, uncleared\-substep, result\-quality); - if (offset >= stepCount) { - // page is out of range! + if (stepCount == 0 && offset == 0) { + // empty first page check commit; - return {}; + return {'\@self: string `${serverHost}/experiments/${experimentId}/timeline`, items: [], itemCount: stepCount}; } else { - steps = check database:getTimelineStepList(experimentId, plugin\-name, 'version, status, uncleared\-substep, result\-quality, 'limit = item\-count, offset = offset, sort = intSort); - check commit; + // else is requred for type checker to be happy... + if (offset >= stepCount) { + // page is out of range! + check commit; + return {}; + } else { + steps = check database:getTimelineStepList(experimentId, plugin\-name, 'version, status, uncleared\-substep, result\-quality, 'limit = item\-count, offset = offset, sort = intSort); + check commit; + } } } on fail error err { From a8b269e65fb4dadf7675abf7b32d4eb36ea7238a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20B=C3=BChler?= Date: Thu, 28 Nov 2024 17:04:52 +0100 Subject: [PATCH 3/4] Add endpoint to get all data versions --- main.bal | 30 ++++++++++++++++++++++++++++++ modules/database/database.bal | 18 ++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/main.bal b/main.bal index 2c56863..ce0111f 100644 --- a/main.bal +++ b/main.bal @@ -508,6 +508,7 @@ service / on new http:Listener(serverPort) { # Get a specific experiment data resource. # # + experimentId - the id of the experiment + # + name - the name of the experiment data resource # + version - the version of the experiment data resource (optional, defaults to "latest") # + return - the experiment data resource resource function get experiments/[int experimentId]/data/[string name](string? 'version) returns ExperimentDataResponse|http:InternalServerError { @@ -529,6 +530,35 @@ service / on new http:Listener(serverPort) { return mapToExperimentDataResponse(data, producingStep, inputFor); } + # Get all versions of an experiment data resource. + # + # + experimentId - the id of the experiment + # + name - the name of the experiment data resource + # + return -all versions of the experiment data in a simple list + resource function get experiments/[int experimentId]/data/[string name]/versions() returns ExperimentDataListResponse|http:NotFound|http:InternalServerError { + database:ExperimentDataFull[] data = []; + + transaction { + data = check database:getAllDataVersions(experimentId, name); + check commit; + } on fail error err { + log:printError(string`Could not get versions of data with name ${name}.`, 'error = err, stackTrace = err.stackTrace()); + + // if with return does not correctly narrow type for rest of function... this does. + http:InternalServerError resultErr = {body: "Something went wrong. Please try again later."}; + return resultErr; + } + + if data.length() == 0 { + return {}; + } + + var dataList = from var d in data + select mapToExperimentDataResponse(d); + // TODO add query params to self URL + return {'\@self: string `${serverHost}/experiments/${experimentId}/data/${name}/versions/`, items: dataList, itemCount: dataList.length()}; + } + # Download the actual data behind the experiment data resource. # # + experimentId - the id of the experiment diff --git a/modules/database/database.bal b/modules/database/database.bal index 2a9d918..c8886bf 100644 --- a/modules/database/database.bal +++ b/modules/database/database.bal @@ -679,6 +679,24 @@ public isolated transactional function getData(int experimentId, string name, st return error(string `Experiment data with experimentId: ${experimentId}, name: ${name} and version: ${'version == () ? "latest" : 'version} was not found!`); } +public isolated transactional function getAllDataVersions(int experimentId, string name) returns ExperimentDataFull[]|error { + sql:ParameterizedQuery baseQuery = `SELECT dataId, experimentId, name, version, location, type, contentType + FROM ExperimentData WHERE experimentId=${experimentId} AND name=${name} ORDER BY version DESC;`; + + stream experimentData = experimentDB->query(baseQuery); + + ExperimentDataFull[]? experimentDataList = check from var data in experimentData + select data; + + check experimentData.close(); + + if experimentDataList != () { + return experimentDataList; + } + + return []; +} + public isolated transactional function getProducingStepOfData(int|ExperimentDataFull data) returns int|error { stream step; From 37b9cad81d9e83a7f8bbddb55cf4d3a6a13141a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20B=C3=BChler?= Date: Thu, 28 Nov 2024 17:07:00 +0100 Subject: [PATCH 4/4] Add endpoint to get related data and add header for related attribute metadata to download resource --- main.bal | 52 ++++++++++++++++++++++++++++ modules/database/database.bal | 64 +++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/main.bal b/main.bal index ce0111f..855d226 100644 --- a/main.bal +++ b/main.bal @@ -559,6 +559,45 @@ service / on new http:Listener(serverPort) { return {'\@self: string `${serverHost}/experiments/${experimentId}/data/${name}/versions/`, items: dataList, itemCount: dataList.length()}; } + # Get related experiment data resources. + # + # + experimentId - the id of the experiment + # + name - the name of the experiment data resource to look up related data for + # + 'version - the version of the experiment data resource to look up related data for (optional, defaults to "latest") + # + relation - the type of relation (i.e. "exact" data produced in the same timeline step, "pre" data produced in the same or any previous timeline step where a version of this data was produced, "post" same as "pre" but with future timeline steps, "any" combines "pre" and "post") + # + include\-self - if true, then different versions (including the requested version) of the data resource to look up related data for are included + # + data\-type - limit the related data to data with a matching data type + # + content\-type - limit the related data to data with a matching content type + # + return - the related data resources as a simple list + resource function get experiments/[int experimentId]/data/[string name]/related(string 'version="latest", "any"|"pre"|"post"|"exact" relation="exact", boolean include\-self=false, string? data\-type=(), string? content\-type=()) returns ExperimentDataListResponse|http:InternalServerError { + database:ExperimentDataFull[] data = []; + + transaction { + data = check database:getRelatedData(experimentId, name, 'version, relation, include\-self, data\-type, content\-type); + check commit; + } on fail error err { + log:printError(string`Could not get related data of data with name ${name} and version ${'version}.`, 'error = err, stackTrace = err.stackTrace()); + + // if with return does not correctly narrow type for rest of function... this does. + http:InternalServerError resultErr = {body: "Something went wrong. Please try again later."}; + return resultErr; + } + + var dataList = from var d in data + select mapToExperimentDataResponse(d); + var selfUrl = string `${serverHost}/experiments/${experimentId}/data/${name}/related/?version=${version}&relation=${relation}`; + if !(data\-type is ()) { + selfUrl = string `${selfUrl}&data-type=${data\-type}`; + } + if !(content\-type is ()) { + selfUrl = string `${selfUrl}&content-type=${content\-type}`; + } + if include\-self { + selfUrl = string `${selfUrl}&include-self=${include\-self}`; + } + return {'\@self: selfUrl, items: dataList, itemCount: dataList.length()}; + } + # Download the actual data behind the experiment data resource. # # + experimentId - the id of the experiment @@ -566,6 +605,7 @@ service / on new http:Listener(serverPort) { # + return - the data of the experiment data resource resource function get experiments/[int experimentId]/data/[string name]/download(string? 'version, http:Caller caller) returns error? { database:ExperimentDataFull data; + database:ExperimentDataFull[] relatedAttributeMetadata = []; http:Response resp = new; resp.addHeader("Access-Control-Allow-Origin", "*"); @@ -574,6 +614,12 @@ service / on new http:Listener(serverPort) { transaction { data = check database:getData(experimentId, name, 'version); + + string dataType = data.'type; + + if (dataType.startsWith("entity/") && !dataType.endsWith("/attribute-metadata") || dataType.startsWith("graph/")) { + relatedAttributeMetadata = check database:getRelatedData(experimentId, name, data.'version, "pre", dataType = "entity/attribute-metadata"); + } check commit; } on fail error err { log:printError("Could not get experiment data for download.", 'error = err, stackTrace = err.stackTrace()); @@ -585,6 +631,12 @@ service / on new http:Listener(serverPort) { return; } + if relatedAttributeMetadata.length() > 0 { + var attrMetadata = relatedAttributeMetadata[relatedAttributeMetadata.length()-1]; + var downloadLink = string`${serverHost}/experiments/${attrMetadata.experimentId}/data/${attrMetadata.name}/download?version=${attrMetadata.'version}`; + resp.addHeader("X-Attribute-Metadata", downloadLink); + } + resp.statusCode = http:STATUS_OK; var cType = data.contentType; if cType.startsWith("text/") || cType.startsWith("application/json") || cType.startsWith("application/X-lines+json") { diff --git a/modules/database/database.bal b/modules/database/database.bal index c8886bf..60bac41 100644 --- a/modules/database/database.bal +++ b/modules/database/database.bal @@ -697,6 +697,70 @@ public isolated transactional function getAllDataVersions(int experimentId, stri return []; } +public isolated transactional function getRelatedData(int experimentId, string name, string|int 'version, "pre"|"exact"|"post"|"any" relation, boolean includeSelf=false, string? dataType=(), string? contentType=()) returns ExperimentDataFull[]|error { + final var data = check getData(experimentId, name, 'version); + + // get all steps where this data object was created/changed + sql:ParameterizedQuery[] subQuery = [ + `SELECT stepId FROM StepData JOIN ExperimentData ON StepData.dataId = ExperimentData.dataId + WHERE ExperimentData.experimentId=${data.experimentId} AND ExperimentData.name=${data.name} ` + ]; + if (relation == "pre") { // include steps before the specific version + subQuery.push(` AND ExperimentData.version <= ${data.version} `); + } + if (relation == "exact") { // only the step of the specific version + subQuery.push(` AND ExperimentData.version = ${data.version} `); + } + if (relation == "post") { // include steps after the specific verion + subQuery.push(` AND ExperimentData.version >= ${data.version} `); + } + // default case "any", include steps for all versions + + + sql:ParameterizedQuery[] baseQuery = [ + `SELECT DISTINCT ExperimentData.dataId, experimentId, name, version, location, type, contentType + FROM ExperimentData JOIN StepData ON StepData.dataId = ExperimentData.dataId + WHERE experimentId=${experimentId} ` + ]; + baseQuery.push(` AND stepId IN ( `); + baseQuery.push(sql:queryConcat(...subQuery)); + baseQuery.push(` ) `); + + // exclude different versions and current version from results + if !includeSelf { + baseQuery.push(` AND name != ${data.name} `); + } + + // only include certain file types + if !(dataType is ()) { + var dataTypeFilter = mimetypeLikeToDbLikeString(dataType); + if dataTypeFilter != () { + baseQuery.push(` AND type LIKE ${dataTypeFilter} `); + } + } + if !(contentType is ()) { + var contentTypeFilter = mimetypeLikeToDbLikeString(contentType); + if contentTypeFilter != () { + baseQuery.push(` AND contentType LIKE ${contentTypeFilter} `); + } + } + + baseQuery.push(`;`); + + stream experimentData = experimentDB->query(sql:queryConcat(...baseQuery)); + + ExperimentDataFull[]? experimentDataList = check from var relatedData in experimentData + select relatedData; + + check experimentData.close(); + + if experimentDataList != () { + return experimentDataList; + } + + return []; +} + public isolated transactional function getProducingStepOfData(int|ExperimentDataFull data) returns int|error { stream step;