From e03f239fbbf3920b276b3a6739fc3c7bfe5181d4 Mon Sep 17 00:00:00 2001 From: Daniel Pollithy Date: Wed, 22 Nov 2017 00:40:43 +0100 Subject: [PATCH] chaining promises --- ToDos.txt | 2 + docs/sphinx/sources/CHANGELOG.rst | 5 ++ package.json | 2 +- src/codec.js | 21 +++--- src/export.js | 113 ++++++++++++++++++------------ src/heatmap.js | 24 +++++-- 6 files changed, 106 insertions(+), 61 deletions(-) create mode 100644 ToDos.txt diff --git a/ToDos.txt b/ToDos.txt new file mode 100644 index 0000000..380bc6d --- /dev/null +++ b/ToDos.txt @@ -0,0 +1,2 @@ +- empty xml files after restore (refactor the restore logic concerning files) +- Den Kanten fehlen automatische Attribute, wenn man sie nicht per Copy-n-connect in der ersten Hand erstellt hat diff --git a/docs/sphinx/sources/CHANGELOG.rst b/docs/sphinx/sources/CHANGELOG.rst index a358e0f..818645a 100644 --- a/docs/sphinx/sources/CHANGELOG.rst +++ b/docs/sphinx/sources/CHANGELOG.rst @@ -11,3 +11,8 @@ v1.0.1 Improved the SQL export (restructured the FOREIGN KEY part of the queries). +v1.1.1 +...... + +Chained batch-add and chained heatmap queries in order to make them deterministic. + diff --git a/package.json b/package.json index 367eca8..31618b1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "giant", "productName": "GIAnT", - "version": "0.0.3", + "version": "1.1.1", "description": "A graphical image annotation tool based on mxgraph, neo4j and electron", "main": "src/atom.js", "directories": { diff --git a/src/codec.js b/src/codec.js index 3044ded..127ea1e 100644 --- a/src/codec.js +++ b/src/codec.js @@ -268,7 +268,7 @@ Codec.mxgraph_to_layered_object = function(filename) { Codec.mxgraph_to_flattened_object = function(filename) { return Codec.mxgraph_to_layered_object(filename).then(function (graph) { if (!(graph && graph.mxGraphModel)) { - return Promise.reject('Graph is empty'); + return Promise.reject('Graph is empty '+filename); } graph.mxGraphModel.data = graph.mxGraphModel.root[0].mxCell[0].children; @@ -456,12 +456,12 @@ Codec.mxgraph_to_graphml = function(filename) { * @return {Promise} */ Codec.add_all_completed_fragments_to_neo4j = function() { - var all_promises = []; var num_changed = 0; var num_not_changed = 0; var p = new Promise(function(resolve, reject){ database.get_all_completed_fragments().then( function(records) { + var promise_chain = []; records.forEach(function(record){ var image_id = record.get('image_id'); var fragment_id = record.get('fragment_id'); @@ -471,15 +471,18 @@ Codec.add_all_completed_fragments_to_neo4j = function() { num_not_changed++; } else { num_changed++; - database.remove_fragment(image_id, fragment_id, true).then(function (success) { - all_promises.push(Codec.mxgraph_to_neo4j(image_id, fragment_id)); - }, function (err) { - log.error(err); - reject(err); - }); + var promise_function = function() { + return database.remove_fragment(image_id, fragment_id, true).then(function (success) { + return Codec.mxgraph_to_neo4j(image_id, fragment_id); + }, function (err) { + log.error(err); + reject(err); + }); + }; + promise_chain.push(promise_function); } }); - Promise.all(all_promises).then(function(all) { + utils.chain_promises(promise_chain).then(function(all) { resolve({'num_changed':num_changed, 'num_not_changed':num_not_changed}); }).catch(function(err) { reject(err); diff --git a/src/export.js b/src/export.js index 86f61c0..b2e686b 100644 --- a/src/export.js +++ b/src/export.js @@ -58,6 +58,7 @@ var readline = require('readline'); var exif_utils = require('./exif_utils'); var sizeOf = require('image-size'); var codec = require('./codec'); +var utils = require('./utils'); @@ -737,14 +738,23 @@ Export._rebuild_image = function(image_path) { Export._copy_xmls = function(fragment_mapping) { // fragment_mapping is old_fragment_id -> new_fragment_id - var all_promises = []; + var backup_folder_name = 'backup_' + this._get_time(); + var backup_folder = path.join(__dirname, '..', 'media', 'uploaded_xmls', backup_folder_name); + if (!fs.existsSync(backup_folder)) { + fs.mkdirSync(backup_folder, 0744); + log.info("Created folder: " + backup_folder) + } + Object.keys(fragment_mapping).forEach(function(old_fragment_id) { var new_fragment_id = fragment_mapping[old_fragment_id]; var old_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.xml'); - var old_path_backup = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.backup.xml'); + var old_path_backup = path.join(backup_folder, old_fragment_id+'.xml'); var new_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', new_fragment_id+'.xml'); - all_promises.push(new Promise(function(resolve, reject){ + fs.copyFileSync(old_path, old_path_backup); + fs.copyFileSync(old_path, new_path); + + /*all_promises.push(new Promise(function(resolve, reject){ var read = fs.createReadStream(old_path); var write = fs.createWriteStream(old_path_backup); write.on('error', reject); @@ -758,10 +768,14 @@ Export._copy_xmls = function(fragment_mapping) { write.on('error', reject); write.on('close', resolve); read.pipe(write); - })); + }));*/ }); - return Promise.all(all_promises); + + var mapping_backup_file = path.join(backup_folder, 'fragment_mapping.json'); + fs.writeFileSync(mapping_backup_file, JSON.stringify(fragment_mapping)); + + return Promise.resolve(); }; Export.rebuild_database = function(relations_fn, node_props_fn) { @@ -770,54 +784,65 @@ Export.rebuild_database = function(relations_fn, node_props_fn) { // fragment_mapping: old_fragment_id -> new_fragment_id var fragment_mapping = {}; var image_promises = []; + var promise_function; Object.keys(images).forEach(function(image_id){ var image = images[image_id]; var file_path = image['file_path']; if (file_path != undefined) { - var p = Export._rebuild_image(file_path) - .then(function(db_image_id){ - // create all fragments - var fragment_promises = []; - Object.keys(image).forEach(function(fragment_id){ - if (fragment_id != 'file_path') { - var fragment_name = images[image_id][fragment_id]['fragment_name']; - var comment = images[image_id][fragment_id]['comment']; - if (comment == 'UL') {comment = ''} - var completed = images[image_id][fragment_id]['completed']; - // Attention: is NULL stands in the cell it is shortened to a UL - if (completed == "UL") {completed = undefined} - if (fragment_name == undefined) { - log.error('REBUILD: Missing fragment_name of fragment_id='+fragment_id); - fragment_name = ''; - } - var fp = database.add_fragment(db_image_id, fragment_name, comment, completed) - .then(function(record){ - var db_fragment_id = record.get('ident').toString(); - var overwrite_xml = '../media/uploaded_xmls/' + fragment_id + '.xml'; - return codec.mxgraph_to_neo4j(db_image_id, db_fragment_id, overwrite_xml).then(function (data) { - fragment_mapping[fragment_id] = db_fragment_id; - return Promise.resolve(); - }).catch(function(err){ - log.error(err); - return Promise.reject(err); + promise_function = function() { + log.info('Now working at ' + file_path); + return Export._rebuild_image(file_path) + .then(function (db_image_id) { + // create all fragments + var fragment_promises = []; + Object.keys(image).forEach(function (fragment_id) { + if (fragment_id != 'file_path') { + var fragment_name = images[image_id][fragment_id]['fragment_name']; + var comment = images[image_id][fragment_id]['comment']; + if (comment == 'UL') { + comment = '' + } + var completed = images[image_id][fragment_id]['completed']; + // Attention: is NULL stands in the cell it is shortened to a UL + if (completed == "UL") { + completed = undefined + } + if (fragment_name == undefined) { + log.error('REBUILD: Missing fragment_name of fragment_id=' + fragment_id); + fragment_name = ''; + } + var fp = database.add_fragment(db_image_id, fragment_name, comment, completed) + .then(function (record) { + var db_fragment_id = record.get('ident').toString(); + var overwrite_xml = '../media/uploaded_xmls/' + fragment_id + '.xml'; + return codec.mxgraph_to_neo4j(db_image_id, db_fragment_id, overwrite_xml).then(function (data) { + fragment_mapping[fragment_id] = db_fragment_id; + return Promise.resolve(); + }).catch(function (err) { + log.error(err); + return Promise.reject(err); + }); + }).catch(function (err) { + log.error('REBUILD: Error creating fragment fragment_id=' + fragment_id); + log.error('REBUILD: ' + err); }); - }).catch(function(err) { - log.error('REBUILD: Error creating fragment fragment_id='+fragment_id); - log.error('REBUILD: ' + err); - }); - fragment_promises.push(fp); - } + fragment_promises.push(fp); + } + }); + return Promise.all(fragment_promises); + }).catch(function (err) { + log.error('REBUILD: image ' + image_id + ' error creating: ' + err); }); - return Promise.all(fragment_promises); - }).catch(function(err){ - log.error('REBUILD: image ' + image_id + ' error creating: ' + err); - }); - image_promises.push(p); - } else { + }; + image_promises.push(promise_function); + } + else { log.error('REBUILD: image ' + image_id + ' has no file_path'); } }); - return Promise.all(image_promises) + + + return utils.chain_promises(image_promises) .then(function(){ log.info('REBUILD: images were added successfully.'); return Export._copy_xmls(fragment_mapping); diff --git a/src/heatmap.js b/src/heatmap.js index d039001..55572e0 100644 --- a/src/heatmap.js +++ b/src/heatmap.js @@ -47,7 +47,7 @@ var BOUNDING_BOX_CACHE = BOUNDING_BOX_CACHE || { */ function clean_caches() { // cache all images with their tokens - IMAGE_FRAGMENT_CACHE = IMAGE_FRAGMENT_CACHE || { + IMAGE_FRAGMENT_CACHE = { images: { // image_id -> image }, @@ -557,7 +557,6 @@ function normalization_on_bounding_box_centered(token, normalization, target_wid 'height': normalized.height }; - return new_normalized; }, function (err) { log.error(err) @@ -694,9 +693,11 @@ function process_heatmap_query(query, normalization, width, height, pixel_size) num_errors++; } else { // calcalute all point spanned by the rects x,y,width,height - for (var x_ = normalized.x; x_ < normalized.x + normalized.width; x_++) { - for (var y_ = normalized.y; y_ < normalized.y + normalized.height; y_++) { - // log.log(x_, y_); + for (var x_ = (normalized.x >= 0)? normalized.x:0; x_ < normalized.x + normalized.width && x_ < width; x_++) { + for (var y_ = (normalized.y >= 0)? normalized.y:0; y_ < normalized.y + normalized.height && y_ < height; y_++) { + if (x_ >= 160 || x_ < 0) { + a = 3; + } heat_map[x_][y_]++; } } @@ -719,20 +720,29 @@ function process_heatmap_query(query, normalization, width, height, pixel_size) log.info("Normalize the output matrix..."); // calculate the maximum entry of the matrix - var max = 0; + var max = -Infinity; + var min = Infinity; for (var x = 0; x < heat_map.length; x++) { for (var y = 0; y < heat_map[0].length; y++) { if (heat_map[x][y] > max) { max = heat_map[x][y]; } + if (heat_map[x][y] < min) { + min = heat_map[x][y]; + } } } + // maximum distance + var interval = (max - min) * 1.0; + log.log("interval") + log.log(interval) + // transform the values of the matrix into the interval [0;1] // by dividing every entry by max for (var x = 0; x < heat_map.length; x++) { for (var y = 0; y < heat_map[0].length; y++) { - heat_map[x][y] = heat_map[x][y] / max; + heat_map[x][y] = (heat_map[x][y] - min) / interval; } } var d3js_heat_map = format_heat_map_to_d3js(heat_map);