Merge pull request #139 from Kenneth-T-Moore/mpi1

Fixed a couple of MPI issues including a hang.

aviary/interface/methods_for_level2.py

@@ -13,6 +13,7 @@
 
 import openmdao.api as om
 from openmdao.core.component import Component
+from openmdao.utils.mpi import MPI
 from openmdao.utils.units import convert_units
 
 from aviary.constants import GRAV_ENGLISH_LBM, RHO_SEA_LEVEL_ENGLISH
@@ -142,14 +143,18 @@ def configure(self):
             all_prom_inputs.extend(var_prom)
 
         # Component promotes aren't handled until this group resolves.
-        # Here, we address anything promoted in AviaryProblem.
+        # Here, we address anything promoted with an alias in AviaryProblem.
         for system in self.system_iter(recurse=False, typ=Component):
             input_meta = system._var_promotes['input']
             var_prom = [v[0][1] for v in input_meta if isinstance(v[0], tuple)]
             all_prom_inputs.extend(var_prom)
             var_prom = [v[0] for v in input_meta if not isinstance(v[0], tuple)]
             all_prom_inputs.extend(var_prom)
 
+        if MPI and self.comm.size > 1:
+            # Under MPI, promotion info only lives on rank 0, so broadcast.
+            all_prom_inputs = self.comm.bcast(all_prom_inputs, root=0)
+
         for key in aviary_metadata:
 
             if ':' not in key or key.startswith('dynamic:'):
@@ -174,6 +179,19 @@ def configure(self):
 
             self.set_input_defaults(key, val=val, units=units)
 
+        # Set a more appropriate solver for dymos when the phases are linked.
+        if MPI and isinstance(self.traj.phases.linear_solver, om.PETScKrylov):
+
+            # When any phase is connected with input_initial = True, dymos puts
+            # a jacobi solver in the phases group. This is necessary in case
+            # you the phases are cyclic. However, this causes some problems
+            # with the newton solvers in Aviary, exacerbating issues with
+            # solver tolerances at multiple levels. Since Aviary's phases
+            # are basically in series, the jacobi solver is a much better
+            # choice and should be able to handle it in a couple of
+            # iterations.
+            self.traj.phases.linear_solver = om.LinearBlockJac(maxiter=5)
+
 
 class AviaryProblem(om.Problem):
     """

aviary/validation_cases/benchmark_tests/test_bench_FwFm.py

@@ -1,15 +1,21 @@
 import unittest
 
 import numpy as np
+
+from openmdao.utils.mpi import MPI
 from openmdao.utils.testing_utils import use_tempdirs
 from openmdao.utils.testing_utils import require_pyoptsparse
 
 from aviary.interface.methods_for_level1 import run_aviary
 from aviary.validation_cases.benchmark_utils import \
     compare_against_expected_values
 
+try:
+    from openmdao.vectors.petsc_vector import PETScVector
+except ImportError:
+    PETScVector = None
+
 
-@use_tempdirs
 class ProblemPhaseTestCase(unittest.TestCase):
     def setUp(self):
         expected_dict = {}
@@ -349,8 +355,12 @@ def setUp(self):
 
         self.phase_info = phase_info
 
+
+@use_tempdirs
+class TestBenchFwFmSerial(ProblemPhaseTestCase):
+
     @require_pyoptsparse(optimizer="IPOPT")
-    def test_bench_FwFm(self):
+    def test_bench_FwFm_IPOPT(self):
         prob = run_aviary(
             'models/test_aircraft/aircraft_for_bench_FwFm.csv', self.phase_info, max_iter=50, optimizer='IPOPT')
 
@@ -364,6 +374,20 @@ def test_bench_FwFm_SNOPT(self):
         compare_against_expected_values(prob, self.expected_dict)
 
 
+@use_tempdirs
+@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.")
+class TestBenchFwFmParallel(ProblemPhaseTestCase):
+
+    N_PROCS = 3
+
+    @require_pyoptsparse(optimizer="SNOPT")
+    def test_bench_FwFm_SNOPT_MPI(self):
+        prob = run_aviary(
+            'models/test_aircraft/aircraft_for_bench_FwFm.csv', self.phase_info, max_iter=50, optimizer='SNOPT')
+
+        compare_against_expected_values(prob, self.expected_dict)
+
+
 if __name__ == '__main__':
     test = ProblemPhaseTestCase()
     test.setUp()

aviary/validation_cases/benchmark_utils.py

@@ -21,19 +21,20 @@ def compare_against_expected_values(prob, expected_dict):
 
     for idx, phase in enumerate(['climb', 'cruise', 'descent']):
 
-        times.extend(prob.get_val(f'traj.{phase}.timeseries.time', units='s'))
+        times.extend(prob.get_val(f'traj.{phase}.timeseries.time', units='s',
+                                  get_remote=True))
         try:
             altitudes.extend(prob.get_val(
-                f'traj.{phase}.timeseries.altitude', units='m'))
+                f'traj.{phase}.timeseries.altitude', units='m', get_remote=True))
         except KeyError:
             altitudes.extend(prob.get_val(
-                f'traj.{phase}.timeseries.altitude', units='m'))
+                f'traj.{phase}.timeseries.altitude', units='m', get_remote=True))
         velocities.extend(prob.get_val(
-            f'traj.{phase}.timeseries.velocity', units='m/s'))
+            f'traj.{phase}.timeseries.velocity', units='m/s', get_remote=True))
         masses.extend(
-            prob.get_val(f'traj.{phase}.timeseries.mass', units='kg'))
+            prob.get_val(f'traj.{phase}.timeseries.mass', units='kg', get_remote=True))
         ranges.extend(
-            prob.get_val(f'traj.{phase}.timeseries.distance', units='m'))
+            prob.get_val(f'traj.{phase}.timeseries.distance', units='m', get_remote=True))
 
     times = np.array(times)
     altitudes = np.array(altitudes)