tests, html and README improvements

README.md

@@ -89,29 +89,35 @@ they do not exist
 
 In the output directory, the following files will be saved:
 
-- **station-energy-magnitude.hdf** A tabular files where each row represents a
-  station/waveform and each column the station computed data and metadata,
+- **station-energy-magnitude.hdf** A tabular file where each row represents a
+  station(^) and each column the station computed data and metadata,
   including the station energy magnitude.
-  Note that the program assumes that a single channel (the vertical) is
-  downloaded per station, so that 1 waveform <=> 1 station
+
+  (^) Note: technically speaking, a single HDF row represents a waveform. However, 
+  there is no distinction because for each station a single channel (the vertical 
+  component `BHZ`) is downloaded (just consider this if you increase the station 
+  channels to download in `download.yaml`)
+
 
-
-- **energy-magnitude.csv** A tabular file (one row per event) aggregating the result
-  of the previous file into the final event energy magnitude. The event Me
-  is the mean of all station energy magnitudes within the 5-95 percentiles
+- **energy-magnitude.csv** A tabular file where each row represents a seismic event, 
+  aggregating the result of the previous file into the final event energy magnitude. 
+  The event Me is the mean of all station energy magnitudes within the 5-95 percentiles.
+  Empty or non-numeric Me values indicate that the energy magnitude could not be 
+  computed or resulted in invalid values (NaN, null, +-inf)
 
 
 - **energy-magnitude.html** A report that can be opened in the user browser to
   visualize the computed energy magnitudes on maps and HTML tables
 
 
 - **[eventid1].xml, ..., [eventid1].xml** All processed events saved in QuakeMl
-  format, updated with the information of their energy magnitude
+  format, updated with the information of their energy magnitude. Only events with
+  valid Me will be saved
 
 
 - **energy-magnitude.log** the log file where the info, errors and warnings
   of the routine are stored. The core energy magnitude computation at station
-  level (performed via stream2segment utilities) has a separated and more
+  level (performed via `stream2segment` utilities) has a separated and more
   detailed log file (see below)
 
 

mecompute/base-config/report.template.html

@@ -70,6 +70,11 @@
 			{% endfor %}
 		</table>
 	</div>
+	<div style="flex: 1 1 0"></div>
+	<div class="text-muted small">
+		Note: an empty Me means that the value could not be computed<br>or
+		resulted to be invalid (NaN, null or +-inf).
+	</div>
 </div>
 <div class="d-flex flex-column" style="flex: 1 0 auto">
 	<div id="map" class='d-flex' style='flex: 1 0 auto;'></div>
@@ -87,16 +92,14 @@
 				<div class="text-muted small">(click on symbol for details)</div>
 				<div>Station color:</div>
 				<div class="d-flex flex-row align-items-baseline">
-					<div>&Delta;Me:</div>
-					<table style="flex: 1 1 0">
+					<div class='me-2'>&Delta;Me:</div>
+					<table style='flex: 1 1 0'>
 						<tr>
-							<td class='text-end'></td>
-							<td style="background-color: blue"></td>
+							<td style="width:1rem;background-color: blue"></td>
 							<td style="background: linear-gradient(to right, blue, white 50%, red);padding:.5rem;"></td>
-							<td style="background-color: red"></td>
+							<td style="width:1rem;background-color: red"></td>
 						</tr>
 						<tr>
-							<td></td>
 							<td class='text-end'>-1</td>
 							<td class='text-center'>0</td>
 							<td class='text-start'>1</td>

mecompute/run.py

@@ -223,7 +223,7 @@ def process(dconfig, start, end, dest_dir,
                            evt['Me'], evt['Me_stddev'], evt['Me_waveforms_used'],
                            author_uri, force_overwrite)
         except (OSError, HTTPError, HTTPException, URLError) as exc:
-            logger.warning(f'Unable to create QuakeML for "ev_catalog_id": {exc}')
+            logger.warning(f'Unable to create QuakeML for {ev_catalog_id}: {exc}')
 
         html_evts[evt['db_id']] = [[evt[h] for h in ev_headers], stations]
         if sel_event_id is None:
@@ -324,6 +324,10 @@ def _compute_station_me(outfile, dburl, segments_selection, p_config=None):
 
 def _write_quekeml(dest_file, event_url, me, me_u=None, me_stations=None,
                    author="", force_overwrite=False):
+    with pd.option_context('mode.use_inf_as_na', True):
+        if pd.isna(me):
+            raise URLError('Me is N/A (nan, +-inf, None)')
+
     if isfile(dest_file) and not force_overwrite:
         return dest_file
 

mecompute/stats.py

@@ -91,10 +91,8 @@ def get_report_rows(hdf_path_or_df):
 
         # anomalyscores = np.asarray(df_['signal_amplitude_anomaly_score'].values)
         me, me_std, num_waveforms = Stats.Me_p.compute(values)
-        # print(values)
-        # print([me, me_std, num_waveforms])
-        # print('!')
-        invalid_me = pd.isna(me) or not np.isfinite(me)
+        with pd.option_context('mode.use_inf_as_na', True):
+            invalid_me = pd.isna(me)
 
         event['Me'] = None if invalid_me else float(np.round(me, 2))
         event['Me_stddev'] = None if invalid_me else float(np.round(me_std, 3))

test/test_workflow.py

@@ -1,13 +1,15 @@
 import os
-from datetime import timedelta, datetime
-from os.path import dirname, join, abspath, isdir, isfile, splitext, basename
+from datetime import datetime, timedelta
+
+from os.path import dirname, join, abspath, isfile
 
 import pandas as pd
 import pytest
 from click.testing import CliRunner
 
 from mecompute.run import cli
 from mecompute.stats import ParabolicScore2Weight, LinearScore2Weight
+from unittest.mock import patch
 
 TEST_DATA_DIR = abspath(join(dirname(__file__), 'data'))
 TEST_DOWNLOAD_CONFIG_PATH = join(TEST_DATA_DIR, 'download.yaml')
@@ -17,9 +19,23 @@
 TEST_TMP_ROOT_DIR = abspath(join(TEST_DATA_DIR, 'tmp'))
 
 
+@patch('mecompute.run.process')
+def test_proc_ess_params(mock_process, capsys):
+    runner = CliRunner()
+    now = datetime.utcnow().replace(microsecond=0,hour=0, minute=0, second=0)
+    days = 365
+    result = runner.invoke(cli, ['-f',
+                                 '-d', TEST_DOWNLOAD_CONFIG_PATH, '-t', str(days),
+                                 TEST_TMP_ROOT_DIR])
+    assert mock_process.called
+    start, end = mock_process.call_args[0][1], mock_process.call_args[0][2]
+    assert now == datetime.fromisoformat(end)
+    assert datetime.fromisoformat(end) - datetime.fromisoformat(start) == \
+           timedelta(days=days)
+
+
 @pytest.mark.parametrize('params', [
     ['-s', '2022-05-20T09:00:00', '-e', '2022-05-20T13:00:00'],  # process all db events
-    ['-t', 100*365]  # huge time span in the past in order to process all db events
 ])
 def test_process(params, capsys):
     """test the processing routine"""