Merge pull request #425 from GEMScienceTools/fix-tests

Fix some tests and improve installation instructions

.github/workflows/test_win64.yaml

@@ -33,7 +33,8 @@ jobs:
         $curDir = Get-Location
         Write-Host "Current Working Directory: $curDir"
         Get-ChildItem -Path .\windows –recurse
-        .\windows\install_oqmbtk.ps1
+        Write-Host "Install oqmbtk on branch ${{ github.ref_name }}"
+        .\windows\install_oqmbtk.ps1 -mbtk_branch ${{ github.ref_name }}
         $MYPATH = $HOME + "\mbtk"
         Write-Host "Print USERPROFILE folder: $env:USERPROFILE"
         Write-Host "Print MYPATH folder: $MYPATH"

docs/_sources/contents/installation.rst.txt

@@ -4,10 +4,29 @@ The *oq-mbt* is installed with the procedure described in the following.
 Note that this procedure implies the installation of the OpenQuake engine. 
 It was tested on Mac OS and Linux systems.
 
-* Open a terminal and move to the folder where to intend to install the tools;
-* Create a virtual environment with ``python3 -m venv venv``
-* Activate the virtual environment ``source venv/bin/activate``
-* Update pip ``pip install -U pip``
-* Enter the virtual environment ``cd venv`` and create a directory for storing source code ``mkdir src; cd src``
-* Clone the OpenQuake engine ``git clone [email protected]:gem/oq-engine.git``
-* Complete a development installation with ``cd ..`` then ``pip install -r ./src/oq-engine/requirements-py36-macos.txt`` and finally ``pip install -e ./src/oq-engine/``
+* Open a terminal and move to the folder where you intend to install the tools;
+* Upgrade pip:
+
+```bash
+python -m pip install --upgrade pip
+```
+
+* Install the OpenQuake engine and activate its virtual environment:
+
+```bash
+$ git clone --depth=1 https://github.com/gem/oq-engine.git
+$ cd oq-engine
+$ python3 install.py devel
+$ source ~/openquake/bin/activate
+```
+
+* Go to the folder where you cloned the oq-mbtk repository and complete the
+installation running the following commands,
+making sure to replace `requirements-py311-linux.txt` with the name of
+the file corresponding to the correct python version and operating system:
+
+```bash
+$ pip install -e .
+$ pip install -r requirements-py311-linux.txt
+```
+

openquake/mbt/tests/tools/fault_modeler/test_fault_source_modeler.py

@@ -27,12 +27,12 @@
 
 import os
 import json
-import filecmp
 import unittest
 import configparser
 import tempfile
 import pathlib
 
+from openquake.utils import are_equal_ignoring_line_endings
 import openquake.mbt.tools.fault_modeler.fault_source_modeler as fsm
 
 # -----------------------------------------------------------------------------
@@ -61,7 +61,6 @@ class TestDatabaseIO(unittest.TestCase):
     defaults = {'m_min': 6.0}
 
     def test_fault_database(self):
-        filecmp.clear_cache()
 
         # Target and reference files
         _, test_file = tempfile.mkstemp()
@@ -97,7 +96,6 @@ def test_fault_database(self):
 
 #    @unittest.skip('find better way to compare outputs!')
     def test_build_model_from_db(self):
-        filecmp.clear_cache()
 
         # Target and reference files
         _, test_file = tempfile.mkstemp()
@@ -130,11 +128,10 @@ def test_build_model_from_db(self):
                                 defaults=self.defaults)
 
         # Compare files
-        self.assertTrue(filecmp.cmp(base_file, test_file))
+        self.assertTrue(are_equal_ignoring_line_endings(base_file, test_file))
 
     @unittest.skip('RS to check the output file which has traces reverted')
     def test_build_source_model_single_args(self):
-        filecmp.clear_cache()
 
         # Target and reference files
         _, test_file = tempfile.mkstemp()
@@ -153,11 +150,10 @@ def test_build_source_model_single_args(self):
                               lower_seismogenic_depth=30.)
 
         # Compare files
-        self.assertTrue(filecmp.cmp(base_file, test_file))
+        self.assertTrue(are_equal_ignoring_line_endings(base_file, test_file))
 
     @unittest.skip('RS to check the output file which has traces reverted')
     def test_build_source_model_dictionary(self):
-        filecmp.clear_cache()
 
         # Target and reference files
         _, test_file = tempfile.mkstemp()
@@ -175,10 +171,9 @@ def test_build_source_model_dictionary(self):
                                         'm_min': 6.0})
 
         # Compare files
-        self.assertTrue(filecmp.cmp(base_file, test_file))
+        self.assertTrue(are_equal_ignoring_line_endings(base_file, test_file))
 
     def test_build_source_model_config_file(self):
-        filecmp.clear_cache()
 
         # Configuration, target and reference files
         conf_file = os.path.join(BASE_DATA_PATH, 'config.ini')
@@ -191,9 +186,7 @@ def test_build_source_model_config_file(self):
         new_config_fname = test_dir / 'config.ini'
         data_path = pathlib.Path(BASE_DATA_PATH)
         geojson_original_path = data_path / config['config']['geojson_file']
-        tmp = os.path.relpath(
-            str(geojson_original_path), str(new_config_fname))
-        config['config']['geojson_file'] = tmp
+        config['config']['geojson_file'] = str(geojson_original_path)
 
         with open(new_config_fname, 'w') as configfile:
             config.write(configfile)
@@ -205,4 +198,4 @@ def test_build_source_model_config_file(self):
         fsm.build_fault_model(cfg_file=new_config_fname)
 
         # Compare files
-        self.assertTrue(filecmp.cmp(base_file, test_file))
+        self.assertTrue(are_equal_ignoring_line_endings(base_file, test_file))

openquake/mbt/tools/fault_modeler/fault_source_modeler.py

@@ -24,11 +24,9 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 # coding: utf-8
 
-import sys
 import ast
 import json
 from copy import deepcopy
-import pathlib
 import configparser
 import warnings
 
@@ -264,12 +262,11 @@ def build_fault_model(*, cfg_file=None,
     # Import arguments from INI configuration file
     if cfg_file is not None:
         cfg_dict = read_config_file(cfg_file)
-        basedir = pathlib.Path(cfg_file).parent
 
         if 'config' in cfg_dict:
             config = cfg_dict['config']
             if 'geojson_file' in config:
-                geojson_file = basedir / config['geojson_file']
+                geojson_file = config['geojson_file']
             if 'xml_output' in config:
                 xml_output = config['xml_output']
             if 'black_list' in config:
@@ -315,8 +312,6 @@ def build_fault_model(*, cfg_file=None,
     if xml_output is None:
         return srcl
 
-
-
     msg = 'Parameter configuration file (.ini)'
     build_fault_model.cfg_file = msg
     msg = 'Fault database in geojson format'

openquake/sub/make_cs_coords.py

@@ -32,6 +32,7 @@
 from openquake.baselib import sap
 from openquake.hazardlib.geo.geodetic import azimuth
 
+
 def make_cs_coords(cs_dir, outfi, ini_fname, cs_length=300., cs_depth=300.):
     """
     Creates cs_coords file of format to be used by plotting script, in the
@@ -47,9 +48,10 @@ def make_cs_coords(cs_dir, outfi, ini_fname, cs_length=300., cs_depth=300.):
             continue
 
         df = pd.read_csv(fi, sep=' ', names=["lon", "lat", "depth"])
-        az = azimuth(df.lon[0], df.lat[0], df.lon.values[-1], df.lat.values[-1])
+        az = azimuth(
+            df.lon[0], df.lat[0], df.lon.values[-1], df.lat.values[-1])
 
-        csid = fi.split('/')[-1][3:].replace('.csv','')
+        csid = fi.split(os.path.sep)[-1][3:].replace('.csv', '')
         line = f'{df.lon[0]} {df.lat[0]} {cs_length} {cs_depth} '
         line += f'{az:.4} {csid} {ini_fname} \n'
         lines.append(line)
@@ -61,6 +63,7 @@ def make_cs_coords(cs_dir, outfi, ini_fname, cs_length=300., cs_depth=300.):
             f.write(line)
     print(f'Written to {outfi}')
 
+
 make_cs_coords.cs_dir = 'directory with cross section coordinates'
 make_cs_coords.outfi = 'output filename'
 make_cs_coords.ini_fname = 'name of ini file specifying data paths'

openquake/sub/slab/rupture.py

@@ -9,17 +9,18 @@
 import re
 import h5py
 import numpy as np
+# import pandas as pd
 import rtree
 import logging
 import configparser
 
 import matplotlib.pyplot as plt
-from mpl_toolkits.mplot3d import Axes3D
+# from mpl_toolkits.mplot3d import Axes3D
 
-#from mayavi import mlab
+# from mayavi import mlab
 from pyproj import Proj
-##from openquake.sub.plotting.tools import plot_mesh
-#from openquake.sub.plotting.tools import plot_mesh_mayavi
+# from openquake.sub.plotting.tools import plot_mesh
+# from openquake.sub.plotting.tools import plot_mesh_mayavi
 
 from openquake.sub.misc.edge import create_from_profiles
 from openquake.sub.quad.msh import create_lower_surface_mesh
@@ -42,10 +43,10 @@
 from openquake.wkf.utils import create_folder
 
 PLOTTING = True
-#PLOTTING = False
+# PLOTTING = False
 
 
-def get_catalogue(cat_pickle_fname, treg_filename=None, label='', 
+def get_catalogue(cat_pickle_fname, treg_filename=None, label='',
                   sort_cat=False):
     """
     :param cat_pickle_fname:
@@ -61,7 +62,7 @@ def get_catalogue(cat_pickle_fname, treg_filename=None, label='',
     #
     # loading the catalogue
     catalogue = load_catalogue(cat_pickle_fname)
-    if sort_cat == True:
+    if sort_cat is True:
         catalogue.sort_catalogue_chronologically()
     #
     # if a label and a TR are provided we filter the catalogue
@@ -371,7 +372,6 @@ def create_ruptures(mfd, dips, sampling, msr, asprs, float_strike, float_dip,
     # Assign probability of occurrence
     for mag, occr in mfd.get_annual_occurrence_rates():
 
-
         # Create the label
         lab = '{:.2f}'.format(mag)
 
@@ -541,7 +541,7 @@ def calculate_ruptures(ini_fname, only_plt=False, ref_fdr=None, agr=None,
     cat_pickle_fname = os.path.abspath(os.path.join(ref_fdr, cat_pickle_fname))
     try:
         sort_cat = bool(config.get('main', 'sort_catalogue'))
-    except:
+    except Exception:
         sort_cat = False
 
     # Output
@@ -553,11 +553,11 @@ def calculate_ruptures(ini_fname, only_plt=False, ref_fdr=None, agr=None,
     tmps = os.path.join(ref_fdr, out_hdf5_smoothing_fname)
     out_hdf5_smoothing_fname = os.path.abspath(tmps)
     # create the smoothing directory if it doesn't exist
-    smoothing_dir = '/'.join(out_hdf5_smoothing_fname.split('/')[:-1])
+    smoothing_dir = os.path.sep.join(
+        out_hdf5_smoothing_fname.split(os.path.sep)[:-1])
     if not os.path.exists(smoothing_dir):
         os.makedirs(smoothing_dir)
 
-
     # Tectonic regionalisation
     treg_filename = config.get('main', 'treg_fname')
     if not re.search('[a-z]', treg_filename):
@@ -592,14 +592,12 @@ def calculate_ruptures(ini_fname, only_plt=False, ref_fdr=None, agr=None,
     logging.info('Creating ruptures on virtual faults')
     print('Creating ruptures on virtual faults')
     ohs = create_inslab_meshes(msh, dips, slab_thickness, sampling)
-    #breakpoint()
 
-    #if only_plt:
+    # if only_plt:
     #    pass
     if False:
+        # TODO consider replacing wiith pyvista
 
-    # TODO consider replacing wiith pyvista
-
         azim = 10.
         elev = 20.
         dist = 20.
@@ -633,7 +631,6 @@ def calculate_ruptures(ini_fname, only_plt=False, ref_fdr=None, agr=None,
         mlab.show()
 
         exit(0)
-
 
     if PLOTTING:
         vsc = 0.01
@@ -680,8 +677,8 @@ def calculate_ruptures(ini_fname, only_plt=False, ref_fdr=None, agr=None,
     # mlo, mla, mde = msh3d.select_nodes_within_two_meshesa(omsh, olmsh)
     mlo, mla, mde = msh3d.get_coordinates_vectors()
     if False:
-        df = pd.DataFrame({'mlo': mlo, 'mla': mla, 'mde': mde}) 
-        df.to_csv('mesh_coords.csv') 
+        df = pd.DataFrame({'mlo': mlo, 'mla': mla, 'mde': mde})
+        df.to_csv('mesh_coords.csv')
 
     # save data on hdf5 file
     if os.path.exists(hdf5_filename):
@@ -698,7 +695,7 @@ def calculate_ruptures(ini_fname, only_plt=False, ref_fdr=None, agr=None,
     fh5.close()
 
     # Get catalogue
-    catalogue = get_catalogue(cat_pickle_fname, treg_filename, label, 
+    catalogue = get_catalogue(cat_pickle_fname, treg_filename, label,
                               sort_cat)
 
     # smoothing

openquake/sub/tests/cross_section_test.py

@@ -1,7 +1,6 @@
 import os
 import re
 import numpy
-import filecmp
 import unittest
 import tempfile
 
@@ -13,6 +12,7 @@
 from openquake.hazardlib.geo.line import Line
 from openquake.sub.tests import __file__ as tests__init__
 from openquake.sub.make_cs_coords import make_cs_coords
+from openquake.utils import are_equal_ignoring_line_endings
 
 BASE_PATH = os.path.dirname(__file__)
 tmp = 'data/crust/crust_except.xyz'
@@ -91,7 +91,7 @@ def test_simple_cs(self):
         """
         cs = CrossSection(10.0, 45.0, [100], [45])
         computed = cs.get_mm()
-        #print(cs.plo[0], cs.plo[1], cs.pla[0], cs.pla[1])
+        # print(cs.plo[0], cs.plo[1], cs.pla[0], cs.pla[1])
         expected = [cs.plo[0], cs.plo[1], cs.pla[0], cs.pla[1], 0]
         numpy.testing.assert_almost_equal(computed, expected)
 
@@ -101,19 +101,21 @@ def test_cs_across_idl(self):
         """
         cs = CrossSection(-179.0, -50.0, [500], [-90])
         computed = cs.get_mm()
-        #print(cs.plo[0], cs.plo[1], cs.pla[0], cs.pla[1])
+        # print(cs.plo[0], cs.plo[1], cs.pla[0], cs.pla[1])
         expected = [cs.plo[0], cs.plo[1], cs.pla[0], cs.pla[1], 1]
         numpy.testing.assert_almost_equal(computed, expected)
 
     def test_cs_across_idl_with_delta(self):
         """
         Test cross section across idl + delta
         """
-        # If this dips 90 degrees it doesn't cross idl (so dip -90 like previous test)
+        # If this dips 90 degrees it doesn't cross idl (so dip -90 like
+        # previous test)
         cs = CrossSection(-179.5, -50.0, [500], [-90])
         computed = cs.get_mm(1.0)
         print(cs.plo[0], cs.plo[1], cs.pla[0], cs.pla[1])
-        #expected = [179.5, -175.70311203864779, -51.0, -48.966369263787726, 1]
+        # expected = [
+        #     179.5, -175.70311203864779, -51.0, -48.966369263787726, 1]
         expected = [-178.5, 172.5247868, -51.0, -48.790282, 1]
         numpy.testing.assert_almost_equal(computed, expected)
 
@@ -150,4 +152,4 @@ def test_cs_generator(self):
         cs_dir = os.path.join(BASE_PATH, 'data', 'cs')
         outfi = os.path.join('/tmp/cs_file.cs')
         make_cs_coords(cs_dir, outfi, 'cs.ini')
-        assert filecmp.cmp(outfi, reference_file) == True
+        self.assertTrue(are_equal_ignoring_line_endings(outfi, reference_file))