Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/spring clean #58

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Feature/spring clean #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d3bb6fc
Remove old plugin runner endpoints from API
buehlefs Nov 28, 2024
b05c9a4
Fix timeline step page endpoint returning 404 even for first page
buehlefs Nov 28, 2024
a8b269e
Add endpoint to get all data versions
buehlefs Nov 28, 2024
37b9cad
Add endpoint to get related data and add header for related attribute…
buehlefs Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Dependencies.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ modules = [
[[package]]
org = "ballerina"
name = "http"
version = "2.12.2"
version = "2.12.3"
dependencies = [
{org = "ballerina", name = "auth"},
{org = "ballerina", name = "cache"},
Expand Down Expand Up @@ -274,7 +274,7 @@ modules = [
[[package]]
org = "ballerina"
name = "sql"
version = "1.14.1"
version = "1.14.2"
dependencies = [
{org = "ballerina", name = "io"},
{org = "ballerina", name = "jballerina.java"},
Expand Down
191 changes: 94 additions & 97 deletions main.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,102 +310,10 @@ service / on new http:Listener(serverPort) {
return {
'\@self: serverHost + "/",
experiments: serverHost + "/experiments/",
pluginRunners: serverHost + "/pluginRunners/",
tasks: serverHost + "/tasks/"
};
}

# Get a list of configured plugin or plugin-runner endpoints.
#
# + return - the list of endpoints as a "ListResponse".
resource function get plugin\-endpoints() returns PluginEndpointsListResponse|http:InternalServerError {
int endpointCount;
database:PluginEndpointFull[] endpoints;

transaction {
endpointCount = check database:getPluginEndpointsCount();
endpoints = check database:getPluginEndpoints();
check commit;
} on fail error err {
log:printError("Could not get plugin endpoints.", 'error = err, stackTrace = err.stackTrace());
// if with return does not correctly narrow type for rest of function... this does.
http:InternalServerError resultErr = {body: "Something went wrong. Please try again later."};
return resultErr;
}

var result = from var endpoint in endpoints
select mapToPluginEndpointResponse(endpoint);
// FIXME load from database...
return {
'\@self: serverHost + "/plugin-endpoints",
items: result,
itemCount: endpointCount
};
}

# Add a new endpoint to the list of plugin(-runner) endpoints.
#
# + return - the created resource
resource function post plugin\-endpoints(@http:Payload PluginEndpointPost endpoint) returns PluginEndpointResponse|http:InternalServerError {
database:PluginEndpointFull result;
transaction {
result = check database:addPluginEndpoint(endpoint);
check commit;
} on fail error err {
log:printError("Could not add new plugin endpoint", 'error = err, stackTrace = err.stackTrace());
return <http:InternalServerError>{body: "Something went wrong. Please try again later."};
}

return mapToPluginEndpointResponse(result);
}

# Get a specific plugin(-runner) endpoint resource.
#
# + return - the endpoint resource
resource function get plugin\-endpoints/[int endpointId]() returns PluginEndpointResponse|http:InternalServerError {
database:PluginEndpointFull result;
transaction {
result = check database:getPluginEndpoint(endpointId);
check commit;
} on fail error err {
log:printError("Could not get plugin endpoint.", 'error = err, stackTrace = err.stackTrace());
return <http:InternalServerError>{body: "Something went wrong. Please try again later."};
}

return mapToPluginEndpointResponse(result);
}

# Update an existing plugin(-runner) endpoint resource.
#
# + return - the updated endpoint resource
resource function put plugin\-endpoints/[int endpointId](@http:Payload PluginEndpointPost endpoint) returns PluginEndpointResponse|http:InternalServerError {
database:PluginEndpointFull result;
transaction {
result = check database:editPluginEndpoint(endpointId, endpoint.'type);
check commit;
} on fail error err {
log:printError("Could not update plugin endpoint", 'error = err, stackTrace = err.stackTrace());
return <http:InternalServerError>{body: "Something went wrong. Please try again later."};
}

return mapToPluginEndpointResponse(result);
}

# Remove an existing plugin(-runner) endpoint resource.
#
# + return - an empty response with a 2xx http status code on success
resource function delete plugin\-endpoints/[int endpointId]() returns http:Ok|http:InternalServerError {
transaction {
check database:deletePluginEndpoint(endpointId);
check commit;
} on fail error err {
log:printError("Could not delete plugin endpoint", 'error = err, stackTrace = err.stackTrace());
return <http:InternalServerError>{body: "Something went wrong. Please try again later."};
}

return <http:Ok>{};
}

////////////////////////////////////////////////////////////////////////////
// Experiments /////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -600,6 +508,7 @@ service / on new http:Listener(serverPort) {
# Get a specific experiment data resource.
#
# + experimentId - the id of the experiment
# + name - the name of the experiment data resource
# + version - the version of the experiment data resource (optional, defaults to "latest")
# + return - the experiment data resource
resource function get experiments/[int experimentId]/data/[string name](string? 'version) returns ExperimentDataResponse|http:InternalServerError {
Expand All @@ -621,13 +530,82 @@ service / on new http:Listener(serverPort) {
return mapToExperimentDataResponse(data, producingStep, inputFor);
}

# Get all versions of an experiment data resource.
#
# + experimentId - the id of the experiment
# + name - the name of the experiment data resource
# + return -all versions of the experiment data in a simple list
resource function get experiments/[int experimentId]/data/[string name]/versions() returns ExperimentDataListResponse|http:NotFound|http:InternalServerError {
database:ExperimentDataFull[] data = [];

transaction {
data = check database:getAllDataVersions(experimentId, name);
check commit;
} on fail error err {
log:printError(string`Could not get versions of data with name ${name}.`, 'error = err, stackTrace = err.stackTrace());

// if with return does not correctly narrow type for rest of function... this does.
http:InternalServerError resultErr = {body: "Something went wrong. Please try again later."};
return resultErr;
}

if data.length() == 0 {
return <http:NotFound>{};
}

var dataList = from var d in data
select mapToExperimentDataResponse(d);
// TODO add query params to self URL
return {'\@self: string `${serverHost}/experiments/${experimentId}/data/${name}/versions/`, items: dataList, itemCount: dataList.length()};
}

# Get related experiment data resources.
#
# + experimentId - the id of the experiment
# + name - the name of the experiment data resource to look up related data for
# + 'version - the version of the experiment data resource to look up related data for (optional, defaults to "latest")
# + relation - the type of relation (i.e. "exact" data produced in the same timeline step, "pre" data produced in the same or any previous timeline step where a version of this data was produced, "post" same as "pre" but with future timeline steps, "any" combines "pre" and "post")
# + include\-self - if true, then different versions (including the requested version) of the data resource to look up related data for are included
# + data\-type - limit the related data to data with a matching data type
# + content\-type - limit the related data to data with a matching content type
# + return - the related data resources as a simple list
resource function get experiments/[int experimentId]/data/[string name]/related(string 'version="latest", "any"|"pre"|"post"|"exact" relation="exact", boolean include\-self=false, string? data\-type=(), string? content\-type=()) returns ExperimentDataListResponse|http:InternalServerError {
database:ExperimentDataFull[] data = [];

transaction {
data = check database:getRelatedData(experimentId, name, 'version, relation, include\-self, data\-type, content\-type);
check commit;
} on fail error err {
log:printError(string`Could not get related data of data with name ${name} and version ${'version}.`, 'error = err, stackTrace = err.stackTrace());

// if with return does not correctly narrow type for rest of function... this does.
http:InternalServerError resultErr = {body: "Something went wrong. Please try again later."};
return resultErr;
}

var dataList = from var d in data
select mapToExperimentDataResponse(d);
var selfUrl = string `${serverHost}/experiments/${experimentId}/data/${name}/related/?version=${version}&relation=${relation}`;
if !(data\-type is ()) {
selfUrl = string `${selfUrl}&data-type=${data\-type}`;
}
if !(content\-type is ()) {
selfUrl = string `${selfUrl}&content-type=${content\-type}`;
}
if include\-self {
selfUrl = string `${selfUrl}&include-self=${include\-self}`;
}
return {'\@self: selfUrl, items: dataList, itemCount: dataList.length()};
}

# Download the actual data behind the experiment data resource.
#
# + experimentId - the id of the experiment
# + version - the version of the experiment data resource (optional, defaults to "latest")
# + return - the data of the experiment data resource
resource function get experiments/[int experimentId]/data/[string name]/download(string? 'version, http:Caller caller) returns error? {
database:ExperimentDataFull data;
database:ExperimentDataFull[] relatedAttributeMetadata = [];

http:Response resp = new;
resp.addHeader("Access-Control-Allow-Origin", "*");
Expand All @@ -636,6 +614,12 @@ service / on new http:Listener(serverPort) {

transaction {
data = check database:getData(experimentId, name, 'version);

string dataType = data.'type;

if (dataType.startsWith("entity/") && !dataType.endsWith("/attribute-metadata") || dataType.startsWith("graph/")) {
relatedAttributeMetadata = check database:getRelatedData(experimentId, name, data.'version, "pre", dataType = "entity/attribute-metadata");
}
check commit;
} on fail error err {
log:printError("Could not get experiment data for download.", 'error = err, stackTrace = err.stackTrace());
Expand All @@ -647,6 +631,12 @@ service / on new http:Listener(serverPort) {
return;
}

if relatedAttributeMetadata.length() > 0 {
var attrMetadata = relatedAttributeMetadata[relatedAttributeMetadata.length()-1];
var downloadLink = string`${serverHost}/experiments/${attrMetadata.experimentId}/data/${attrMetadata.name}/download?version=${attrMetadata.'version}`;
resp.addHeader("X-Attribute-Metadata", downloadLink);
}

resp.statusCode = http:STATUS_OK;
var cType = data.contentType;
if cType.startsWith("text/") || cType.startsWith("application/json") || cType.startsWith("application/X-lines+json") {
Expand Down Expand Up @@ -696,13 +686,20 @@ service / on new http:Listener(serverPort) {
transaction {

stepCount = check database:getTimelineStepCount(experimentId, plugin\-name, 'version, status, uncleared\-substep, result\-quality);
if (offset >= stepCount) {
// page is out of range!
if (stepCount == 0 && offset == 0) {
// empty first page
check commit;
return <http:NotFound>{};
return {'\@self: string `${serverHost}/experiments/${experimentId}/timeline`, items: [], itemCount: stepCount};
} else {
steps = check database:getTimelineStepList(experimentId, plugin\-name, 'version, status, uncleared\-substep, result\-quality, 'limit = item\-count, offset = offset, sort = intSort);
check commit;
// else is requred for type checker to be happy...
if (offset >= stepCount) {
// page is out of range!
check commit;
return <http:NotFound>{};
} else {
steps = check database:getTimelineStepList(experimentId, plugin\-name, 'version, status, uncleared\-substep, result\-quality, 'limit = item\-count, offset = offset, sort = intSort);
check commit;
}
}

} on fail error err {
Expand Down
82 changes: 82 additions & 0 deletions modules/database/database.bal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -679,6 +679,88 @@ public isolated transactional function getData(int experimentId, string name, st
return error(string `Experiment data with experimentId: ${experimentId}, name: ${name} and version: ${'version == () ? "latest" : 'version} was not found!`);
}

public isolated transactional function getAllDataVersions(int experimentId, string name) returns ExperimentDataFull[]|error {
sql:ParameterizedQuery baseQuery = `SELECT dataId, experimentId, name, version, location, type, contentType
FROM ExperimentData WHERE experimentId=${experimentId} AND name=${name} ORDER BY version DESC;`;

stream<ExperimentDataFull, sql:Error?> experimentData = experimentDB->query(baseQuery);

ExperimentDataFull[]? experimentDataList = check from var data in experimentData
select data;

check experimentData.close();

if experimentDataList != () {
return experimentDataList;
}

return [];
}

public isolated transactional function getRelatedData(int experimentId, string name, string|int 'version, "pre"|"exact"|"post"|"any" relation, boolean includeSelf=false, string? dataType=(), string? contentType=()) returns ExperimentDataFull[]|error {
final var data = check getData(experimentId, name, 'version);

// get all steps where this data object was created/changed
sql:ParameterizedQuery[] subQuery = [
`SELECT stepId FROM StepData JOIN ExperimentData ON StepData.dataId = ExperimentData.dataId
WHERE ExperimentData.experimentId=${data.experimentId} AND ExperimentData.name=${data.name} `
];
if (relation == "pre") { // include steps before the specific version
subQuery.push(` AND ExperimentData.version <= ${data.version} `);
}
if (relation == "exact") { // only the step of the specific version
subQuery.push(` AND ExperimentData.version = ${data.version} `);
}
if (relation == "post") { // include steps after the specific verion
subQuery.push(` AND ExperimentData.version >= ${data.version} `);
}
// default case "any", include steps for all versions


sql:ParameterizedQuery[] baseQuery = [
`SELECT DISTINCT ExperimentData.dataId, experimentId, name, version, location, type, contentType
FROM ExperimentData JOIN StepData ON StepData.dataId = ExperimentData.dataId
WHERE experimentId=${experimentId} `
];
baseQuery.push(` AND stepId IN ( `);
baseQuery.push(sql:queryConcat(...subQuery));
baseQuery.push(` ) `);

// exclude different versions and current version from results
if !includeSelf {
baseQuery.push(` AND name != ${data.name} `);
}

// only include certain file types
if !(dataType is ()) {
var dataTypeFilter = mimetypeLikeToDbLikeString(dataType);
if dataTypeFilter != () {
baseQuery.push(` AND type LIKE ${dataTypeFilter} `);
}
}
if !(contentType is ()) {
var contentTypeFilter = mimetypeLikeToDbLikeString(contentType);
if contentTypeFilter != () {
baseQuery.push(` AND contentType LIKE ${contentTypeFilter} `);
}
}

baseQuery.push(`;`);

stream<ExperimentDataFull, sql:Error?> experimentData = experimentDB->query(sql:queryConcat(...baseQuery));

ExperimentDataFull[]? experimentDataList = check from var relatedData in experimentData
select relatedData;

check experimentData.close();

if experimentDataList != () {
return experimentDataList;
}

return [];
}

public isolated transactional function getProducingStepOfData(int|ExperimentDataFull data) returns int|error {
stream<record {int producingStep;}, sql:Error?> step;

Expand Down
Loading