Add progress indicator to featurization

Author: stefanv

cesium_app/handlers/feature.py

@@ -8,6 +8,8 @@
 from baselayer.app.access import auth_or_token
 from ..models import DBSession, Dataset, Featureset, Project
 
+from .progressbar import WebSocketProgressBar
+
 from os.path import join as pjoin
 import uuid
 import datetime
@@ -32,7 +34,12 @@ async def _await_featurization(self, future, fset):
         That said, we can push notifications through to the frontend
         using flow.
         """
+        def progressbar_update(payload):
+            payload.update({'fsetID': fset.id})
+            self.action('cesium/FEATURIZE_PROGRESS', payload=payload)
+
         try:
+            WebSocketProgressBar([future], progressbar_update, interval=2)
             result = await future
 
             fset = DBSession().merge(fset)

cesium_app/handlers/progressbar.py

@@ -0,0 +1,67 @@
+# Derived from `distributed.diagnostics.progressbar` which is
+
+# Copyright (c) 2015-2017, Anaconda, Inc. and contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# Neither the name of Anaconda nor the names of any contributors may
+# be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+
+from contextlib import contextmanager
+
+from distributed.diagnostics.progressbar import ProgressBar
+from distributed.utils import LoopRunner
+
+from tornado.ioloop import IOLoop
+
+import sys
+
+
+def format_time(t):
+    "Format seconds into a human readable form."
+    m, s = divmod(t, 60)
+    h, m = divmod(m, 60)
+    return f'{int(h):02d}:{int(m):02d}:{int(s):02d}'
+
+
+class WebSocketProgressBar(ProgressBar):
+    def __init__(self, keys, update_callback, scheduler=None, interval=1,
+                 loop=None, complete=True, start=True):
+        super(WebSocketProgressBar, self).__init__(keys, scheduler, interval,
+                                                   complete)
+        self.update_callback = update_callback
+        self.loop = loop or IOLoop.current()
+
+        if start:
+            self.loop.add_callback(self.listen)
+
+    def _draw_bar(self, remaining, all, **kwargs):
+        frac = (1 - remaining / all) if all else 1.0
+        percent = frac * 100
+        elapsed = format_time(self.elapsed)
+        self.update_callback({'percent': f'{percent:2.1f}', 'elapsed': elapsed})

static/js/CesiumMessageHandler.js

@@ -23,6 +23,10 @@ let CesiumMessageHandler = dispatch => {
       case Action.FETCH_PREDICTIONS:
         dispatch(Action.fetchPredictions());
         break;
+      case Action.FEATURIZE_PROGRESS:
+        let time_update = message.payload;
+        dispatch(Action.featurizeUpdateProgress(time_update));
+        break;
       default:
         console.log('Unknown message received through flow:',
                     message);

static/js/actions.js

@@ -49,6 +49,8 @@ export const CLICK_FEATURE_TAG_CHECKBOX = 'cesium/CLICK_FEATURE_TAG_CHECKBOX';
 export const FETCH_USER_PROFILE = 'cesium/FETCH_USER_PROFILE';
 export const RECEIVE_USER_PROFILE = 'cesium/FETCH_USER_PROFILE';
 
+export const FEATURIZE_PROGRESS = 'cesium/FEATURIZE_PROGRESS';
+
 import { showNotification, reduceNotifications } from 'baselayer/components/Notifications';
 import promiseAction from './action_tools';
 import { objectType } from './utils';
@@ -294,6 +296,13 @@ function receiveFeaturesets(featuresets) {
   };
 }
 
+// Receive updates on featurization
+export function featurizeUpdateProgress(time_update) {
+  return {
+    type: FEATURIZE_PROGRESS,
+    payload: time_update
+  };
+}
 
 export function createModel(form) {
   return dispatch =>

static/js/components/Features.jsx

@@ -224,7 +224,19 @@ export let FeatureTable = props => (
               </td>
             </tr>);
 
-          const status = done ? <td>Completed {reformatDatetime(featureset.finished)}</td> : <td>In progress</td>;
+          let elapsed = "", percent = "";
+          if (featureset.progress) {
+            ({ elapsed, percent } = { ...featureset.progress });
+          }
+
+          let status;
+          if (done) {
+            status = <td>Completed { reformatDatetime(featureset.finished) }</td>;
+          } else if (elapsed == "") {
+            status = <td>In progress...</td>;
+          } else {
+            status = <td>In progress: { percent }%, { elapsed }s</td>;
+          }
 
           return (
             <FoldableRow key={idx}>

static/js/reducers.js

@@ -40,6 +40,20 @@ function featuresets(state=[], action) {
   switch (action.type) {
     case Action.RECEIVE_FEATURESETS:
       return action.payload;
+    case Action.FEATURIZE_PROGRESS:
+      const newState = [ ...state ];
+
+      const { percent, elapsed } = { ...action.payload };
+      const featureIdx = newState.findIndex((element) => (
+        element.id == action.payload.fsetID
+      ));
+
+      if (featureIdx != -1) {
+        const feature = newState[featureIdx];
+        feature.progress = {percent, elapsed};
+      }
+
+      return newState;
     default:
       return state;
   }