Skip to content

Commit

Permalink
chaining promises
Browse files Browse the repository at this point in the history
  • Loading branch information
Daniel Pollithy committed Nov 21, 2017
1 parent a755a41 commit e03f239
Show file tree
Hide file tree
Showing 6 changed files with 106 additions and 61 deletions.
2 changes: 2 additions & 0 deletions ToDos.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
- empty xml files after restore (refactor the restore logic concerning files)
- Den Kanten fehlen automatische Attribute, wenn man sie nicht per Copy-n-connect in der ersten Hand erstellt hat
5 changes: 5 additions & 0 deletions docs/sphinx/sources/CHANGELOG.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,8 @@ v1.0.1

Improved the SQL export (restructured the FOREIGN KEY part of the queries).

v1.1.1
......

Chained batch-add and chained heatmap queries in order to make them deterministic.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "giant",
"productName": "GIAnT",
"version": "0.0.3",
"version": "1.1.1",
"description": "A graphical image annotation tool based on mxgraph, neo4j and electron",
"main": "src/atom.js",
"directories": {
Expand Down
21 changes: 12 additions & 9 deletions src/codec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ Codec.mxgraph_to_layered_object = function(filename) {
Codec.mxgraph_to_flattened_object = function(filename) {
return Codec.mxgraph_to_layered_object(filename).then(function (graph) {
if (!(graph && graph.mxGraphModel)) {
return Promise.reject('Graph is empty');
return Promise.reject('Graph is empty '+filename);
}
graph.mxGraphModel.data = graph.mxGraphModel.root[0].mxCell[0].children;

Expand Down Expand Up @@ -456,12 +456,12 @@ Codec.mxgraph_to_graphml = function(filename) {
* @return {Promise}
*/
Codec.add_all_completed_fragments_to_neo4j = function() {
var all_promises = [];
var num_changed = 0;
var num_not_changed = 0;
var p = new Promise(function(resolve, reject){
database.get_all_completed_fragments().then(
function(records) {
var promise_chain = [];
records.forEach(function(record){
var image_id = record.get('image_id');
var fragment_id = record.get('fragment_id');
Expand All @@ -471,15 +471,18 @@ Codec.add_all_completed_fragments_to_neo4j = function() {
num_not_changed++;
} else {
num_changed++;
database.remove_fragment(image_id, fragment_id, true).then(function (success) {
all_promises.push(Codec.mxgraph_to_neo4j(image_id, fragment_id));
}, function (err) {
log.error(err);
reject(err);
});
var promise_function = function() {
return database.remove_fragment(image_id, fragment_id, true).then(function (success) {
return Codec.mxgraph_to_neo4j(image_id, fragment_id);
}, function (err) {
log.error(err);
reject(err);
});
};
promise_chain.push(promise_function);
}
});
Promise.all(all_promises).then(function(all) {
utils.chain_promises(promise_chain).then(function(all) {
resolve({'num_changed':num_changed, 'num_not_changed':num_not_changed});
}).catch(function(err) {
reject(err);
Expand Down
113 changes: 69 additions & 44 deletions src/export.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ var readline = require('readline');
var exif_utils = require('./exif_utils');
var sizeOf = require('image-size');
var codec = require('./codec');
var utils = require('./utils');



Expand Down Expand Up @@ -737,14 +738,23 @@ Export._rebuild_image = function(image_path) {

Export._copy_xmls = function(fragment_mapping) {
// fragment_mapping is old_fragment_id -> new_fragment_id
var all_promises = [];
var backup_folder_name = 'backup_' + this._get_time();
var backup_folder = path.join(__dirname, '..', 'media', 'uploaded_xmls', backup_folder_name);
if (!fs.existsSync(backup_folder)) {
fs.mkdirSync(backup_folder, 0744);
log.info("Created folder: " + backup_folder)
}

Object.keys(fragment_mapping).forEach(function(old_fragment_id) {
var new_fragment_id = fragment_mapping[old_fragment_id];
var old_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.xml');
var old_path_backup = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.backup.xml');
var old_path_backup = path.join(backup_folder, old_fragment_id+'.xml');
var new_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', new_fragment_id+'.xml');

all_promises.push(new Promise(function(resolve, reject){
fs.copyFileSync(old_path, old_path_backup);
fs.copyFileSync(old_path, new_path);

/*all_promises.push(new Promise(function(resolve, reject){
var read = fs.createReadStream(old_path);
var write = fs.createWriteStream(old_path_backup);
write.on('error', reject);
Expand All @@ -758,10 +768,14 @@ Export._copy_xmls = function(fragment_mapping) {
write.on('error', reject);
write.on('close', resolve);
read.pipe(write);
}));
}));*/

});
return Promise.all(all_promises);

var mapping_backup_file = path.join(backup_folder, 'fragment_mapping.json');
fs.writeFileSync(mapping_backup_file, JSON.stringify(fragment_mapping));

return Promise.resolve();
};

Export.rebuild_database = function(relations_fn, node_props_fn) {
Expand All @@ -770,54 +784,65 @@ Export.rebuild_database = function(relations_fn, node_props_fn) {
// fragment_mapping: old_fragment_id -> new_fragment_id
var fragment_mapping = {};
var image_promises = [];
var promise_function;
Object.keys(images).forEach(function(image_id){
var image = images[image_id];
var file_path = image['file_path'];
if (file_path != undefined) {
var p = Export._rebuild_image(file_path)
.then(function(db_image_id){
// create all fragments
var fragment_promises = [];
Object.keys(image).forEach(function(fragment_id){
if (fragment_id != 'file_path') {
var fragment_name = images[image_id][fragment_id]['fragment_name'];
var comment = images[image_id][fragment_id]['comment'];
if (comment == 'UL') {comment = ''}
var completed = images[image_id][fragment_id]['completed'];
// Attention: is NULL stands in the cell it is shortened to a UL
if (completed == "UL") {completed = undefined}
if (fragment_name == undefined) {
log.error('REBUILD: Missing fragment_name of fragment_id='+fragment_id);
fragment_name = '';
}
var fp = database.add_fragment(db_image_id, fragment_name, comment, completed)
.then(function(record){
var db_fragment_id = record.get('ident').toString();
var overwrite_xml = '../media/uploaded_xmls/' + fragment_id + '.xml';
return codec.mxgraph_to_neo4j(db_image_id, db_fragment_id, overwrite_xml).then(function (data) {
fragment_mapping[fragment_id] = db_fragment_id;
return Promise.resolve();
}).catch(function(err){
log.error(err);
return Promise.reject(err);
promise_function = function() {
log.info('Now working at ' + file_path);
return Export._rebuild_image(file_path)
.then(function (db_image_id) {
// create all fragments
var fragment_promises = [];
Object.keys(image).forEach(function (fragment_id) {
if (fragment_id != 'file_path') {
var fragment_name = images[image_id][fragment_id]['fragment_name'];
var comment = images[image_id][fragment_id]['comment'];
if (comment == 'UL') {
comment = ''
}
var completed = images[image_id][fragment_id]['completed'];
// Attention: is NULL stands in the cell it is shortened to a UL
if (completed == "UL") {
completed = undefined
}
if (fragment_name == undefined) {
log.error('REBUILD: Missing fragment_name of fragment_id=' + fragment_id);
fragment_name = '';
}
var fp = database.add_fragment(db_image_id, fragment_name, comment, completed)
.then(function (record) {
var db_fragment_id = record.get('ident').toString();
var overwrite_xml = '../media/uploaded_xmls/' + fragment_id + '.xml';
return codec.mxgraph_to_neo4j(db_image_id, db_fragment_id, overwrite_xml).then(function (data) {
fragment_mapping[fragment_id] = db_fragment_id;
return Promise.resolve();
}).catch(function (err) {
log.error(err);
return Promise.reject(err);
});
}).catch(function (err) {
log.error('REBUILD: Error creating fragment fragment_id=' + fragment_id);
log.error('REBUILD: ' + err);
});
}).catch(function(err) {
log.error('REBUILD: Error creating fragment fragment_id='+fragment_id);
log.error('REBUILD: ' + err);
});
fragment_promises.push(fp);
}
fragment_promises.push(fp);
}
});
return Promise.all(fragment_promises);
}).catch(function (err) {
log.error('REBUILD: image ' + image_id + ' error creating: ' + err);
});
return Promise.all(fragment_promises);
}).catch(function(err){
log.error('REBUILD: image ' + image_id + ' error creating: ' + err);
});
image_promises.push(p);
} else {
};
image_promises.push(promise_function);
}
else {
log.error('REBUILD: image ' + image_id + ' has no file_path');
}
});
return Promise.all(image_promises)


return utils.chain_promises(image_promises)
.then(function(){
log.info('REBUILD: images were added successfully.');
return Export._copy_xmls(fragment_mapping);
Expand Down
24 changes: 17 additions & 7 deletions src/heatmap.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ var BOUNDING_BOX_CACHE = BOUNDING_BOX_CACHE || {
*/
function clean_caches() {
// cache all images with their tokens
IMAGE_FRAGMENT_CACHE = IMAGE_FRAGMENT_CACHE || {
IMAGE_FRAGMENT_CACHE = {
images: {
// image_id -> image
},
Expand Down Expand Up @@ -557,7 +557,6 @@ function normalization_on_bounding_box_centered(token, normalization, target_wid
'height': normalized.height
};


return new_normalized;
}, function (err) {
log.error(err)
Expand Down Expand Up @@ -694,9 +693,11 @@ function process_heatmap_query(query, normalization, width, height, pixel_size)
num_errors++;
} else {
// calcalute all point spanned by the rects x,y,width,height
for (var x_ = normalized.x; x_ < normalized.x + normalized.width; x_++) {
for (var y_ = normalized.y; y_ < normalized.y + normalized.height; y_++) {
// log.log(x_, y_);
for (var x_ = (normalized.x >= 0)? normalized.x:0; x_ < normalized.x + normalized.width && x_ < width; x_++) {
for (var y_ = (normalized.y >= 0)? normalized.y:0; y_ < normalized.y + normalized.height && y_ < height; y_++) {
if (x_ >= 160 || x_ < 0) {
a = 3;
}
heat_map[x_][y_]++;
}
}
Expand All @@ -719,20 +720,29 @@ function process_heatmap_query(query, normalization, width, height, pixel_size)
log.info("Normalize the output matrix...");

// calculate the maximum entry of the matrix
var max = 0;
var max = -Infinity;
var min = Infinity;
for (var x = 0; x < heat_map.length; x++) {
for (var y = 0; y < heat_map[0].length; y++) {
if (heat_map[x][y] > max) {
max = heat_map[x][y];
}
if (heat_map[x][y] < min) {
min = heat_map[x][y];
}
}
}

// maximum distance
var interval = (max - min) * 1.0;
log.log("interval")
log.log(interval)

// transform the values of the matrix into the interval [0;1]
// by dividing every entry by max
for (var x = 0; x < heat_map.length; x++) {
for (var y = 0; y < heat_map[0].length; y++) {
heat_map[x][y] = heat_map[x][y] / max;
heat_map[x][y] = (heat_map[x][y] - min) / interval;
}
}
var d3js_heat_map = format_heat_map_to_d3js(heat_map);
Expand Down

0 comments on commit e03f239

Please sign in to comment.