romio: implement binding generation for ROMIO

.gitignore

@@ -384,6 +384,7 @@ Makefile.am-stamp
 
 # /src/binding/
 /src/binding/c/c_binding.c
+/src/binding/c/io.c
 
 # /src/binding/cxx/
 /src/binding/cxx/Makefile.sm

Makefile.am

@@ -51,9 +51,22 @@ errnames_txt_files =
 external_subdirs = @mplsrcdir@ @hwlocsrcdir@ @jsonsrcdir@ @yaksasrcdir@ @pmisrcdir@
 external_ldflags = @mpllibdir@ @yaksalibdir@
 external_libs = @WRAPPER_LIBS@
-mpi_convenience_libs =
+
+# convenience libs that do not depend on MPI
 pmpi_convenience_libs = @mpllib@ @hwloclib@ @jsonlib@ @yaksalib@ @pmilib@
 
+# convenience libs that depend on MPI (either MPI functions or MPI constants)
+mpi_convenience_libs =
+abi_convenience_libs =
+
+if BUILD_ROMIO
+    external_subdirs += src/mpi/romio
+    mpi_convenience_libs += src/mpi/romio/libromio.la
+if BUILD_ABI_LIB
+    abi_convenience_libs += src/mpi/romio/libromio_abi.la
+endif
+endif
+
 # NOTE on our semi-unconventional usage of DIST_SUBDIRS:
 # The automake manual recommends thinking of DIST_SUBDIRS as the list of all
 # *configured* subdirectories.  The normal autotools model involves
@@ -124,8 +137,8 @@ if BUILD_PROFILING_LIB
 lib_LTLIBRARIES += lib/lib@[email protected]
 lib_lib@PMPILIBNAME@_la_SOURCES = $(mpi_sources) $(mpi_core_sources)
 lib_lib@PMPILIBNAME@_la_LDFLAGS = $(external_ldflags) $(ABIVERSIONFLAGS)
-lib_lib@PMPILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs)
-EXTRA_lib_lib@PMPILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs)
+lib_lib@PMPILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs) $(mpi_convenience_libs)
+EXTRA_lib_lib@PMPILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs) $(mpi_convenience_libs)
 
 # lib@[email protected] might depend on lib@[email protected].  We add them
 # in that order to lib_LTLIBRARIES so libtool doesn't get
@@ -134,14 +147,8 @@ lib_LTLIBRARIES += lib/lib@[email protected]
 lib_lib@MPILIBNAME@_la_SOURCES = $(mpi_sources)
 lib_lib@MPILIBNAME@_la_LDFLAGS = $(ABIVERSIONFLAGS)
 lib_lib@MPILIBNAME@_la_CPPFLAGS = $(AM_CPPFLAGS) -DMPICH_MPI_FROM_PMPI
-lib_lib@MPILIBNAME@_la_LIBADD = lib/lib@[email protected] $(mpi_convenience_libs)
+lib_lib@MPILIBNAME@_la_LIBADD = lib/lib@[email protected]
 
-if BUILD_ROMIO
-    # libpromio contains the PMPI symbols (unlike libpmpi, which contains MPI
-    # symbols) and should be added to libpmpi
-    lib_lib@PMPILIBNAME@_la_LIBADD += src/mpi/romio/libpromio.la
-    lib_lib@MPILIBNAME@_la_LIBADD += src/mpi/romio/libromio.la
-endif
 else !BUILD_PROFILING_LIB
 
 lib_LTLIBRARIES += lib/lib@[email protected]
@@ -151,9 +158,6 @@ lib_lib@MPILIBNAME@_la_CPPFLAGS = $(AM_CPPFLAGS)
 lib_lib@MPILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs) $(mpi_convenience_libs)
 EXTRA_lib_lib@MPILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs) $(mpi_convenience_libs)
 
-if BUILD_ROMIO
-    lib_lib@MPILIBNAME@_la_LIBADD += src/mpi/romio/libromio.la
-endif
 endif !BUILD_PROFILING_LIB
 
 if BUILD_ABI_LIB
@@ -165,8 +169,8 @@ lib_LTLIBRARIES += lib/lib@[email protected]
 lib_lib@PMPIABILIBNAME@_la_SOURCES = $(mpi_abi_sources) $(mpi_f77_sources) $(mpi_core_sources)
 lib_lib@PMPIABILIBNAME@_la_LDFLAGS = $(external_ldflags) $(ABIVERSIONFLAGS)
 lib_lib@PMPIABILIBNAME@_la_CPPFLAGS = $(AM_CPPFLAGS) -DF77_USE_PMPI $(abi_cppflags)
-lib_lib@PMPIABILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs)
-EXTRA_lib_lib@PMPIABILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs)
+lib_lib@PMPIABILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs) $(abi_convenience_libs)
+EXTRA_lib_lib@PMPIABILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs) $(abi_convenience_libs)
 
 # lib@[email protected] might depend on lib@[email protected].  We add them
 # in that order to lib_LTLIBRARIES so libtool doesn't get
@@ -175,24 +179,17 @@ lib_LTLIBRARIES += lib/lib@[email protected]
 lib_lib@MPIABILIBNAME@_la_SOURCES = $(mpi_abi_sources)
 lib_lib@MPIABILIBNAME@_la_LDFLAGS = $(ABIVERSIONFLAGS)
 lib_lib@MPIABILIBNAME@_la_CPPFLAGS = $(AM_CPPFLAGS) -DMPICH_MPI_FROM_PMPI $(abi_cppflags)
-lib_lib@MPIABILIBNAME@_la_LIBADD = lib/lib@[email protected] $(mpi_convenience_libs)
+lib_lib@MPIABILIBNAME@_la_LIBADD = lib/lib@[email protected]
 
-if BUILD_ROMIO
-    lib_lib@PMPIABILIBNAME@_la_LIBADD += src/mpi/romio/libpromio_abi.la
-    lib_lib@MPIABILIBNAME@_la_LIBADD += src/mpi/romio/libromio_abi.la
-endif
 else !BUILD_PROFILING_LIB
 
 lib_LTLIBRARIES += lib/lib@[email protected]
 lib_lib@MPIABILIBNAME@_la_SOURCES = $(mpi_abi_sources) $(mpi_core_sources)
 lib_lib@MPIABILIBNAME@_la_LDFLAGS = $(external_ldflags) $(ABIVERSIONFLAGS)
 lib_lib@MPIABILIBNAME@_la_CPPFLAGS = $(AM_CPPFLAGS) $(abi_cppflags)
-lib_lib@MPIABILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs) $(mpi_convenience_libs)
-EXTRA_lib_lib@MPIABILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs) $(mpi_convenience_libs)
+lib_lib@MPIABILIBNAME@_la_LIBADD = $(external_libs) $(pmpi_convenience_libs) $(abi_convenience_libs)
+EXTRA_lib_lib@MPIABILIBNAME@_la_DEPENDENCIES = $(pmpi_convenience_libs) $(abi_convenience_libs)
 
-if BUILD_ROMIO
-    lib_lib@MPIABILIBNAME@_la_LIBADD += src/mpi/romio/libromio_abi.la
-endif
 endif !BUILD_PROFILING_LIB
 endif BUILD_ABI_LIB
 

dummy

@@ -0,0 +1 @@
+1

maint/gen_abi.py

@@ -25,6 +25,7 @@ def search(pat, str, flags=0):
 def main():
     load_mpi_abi_h("src/binding/abi/mpi_abi.h")
     dump_mpi_abi_internal_h("src/binding/abi/mpi_abi_internal.h")
+    dump_io_abi_internal_h("src/binding/abi/io_abi_internal.h")
     dump_romio_abi_internal_h("src/mpi/romio/include/romio_abi_internal.h")
     dump_mpi_abi_util_c("src/binding/abi/mpi_abi_util.c")
 
@@ -101,7 +102,7 @@ def gen_mpi_abi_internal_h(out):
         for line in output_lines:
             print(line, file=Out)
         print("", file=Out)
-        
+
         print("#endif /* MPI_ABI_INTERNAL_H_INCLUDED */", file=Out)
 
 def dump_romio_abi_internal_h(romio_abi_internal_h):
@@ -181,6 +182,52 @@ def add_other(out):
         print("#define ROMIO_ABI_INTERNAL_H_INCLUDED", file=Out)
         print("", file=Out)
 
+        for line in output_lines:
+            print(line, file=Out)
+        print("", file=Out)
+
+        print("#endif /* ROMIO_ABI_INTERNAL_H_INCLUDED */", file=Out)
+
+# similar to romio_abi_internal.h but for use in the mpich io binding
+def dump_io_abi_internal_h(io_abi_internal_h):
+    def gen_io_abi_internal_h(out):
+        for line in G.abi_h_lines:
+            if RE.search(r'MPI_ABI_H_INCLUDED', line):
+                # skip the include guard, harmless
+                pass
+            elif RE.match(r'typedef struct.*\bMPI_File;\s*$', line):
+                out.append("typedef struct ADIOI_FileD *MPI_File;")
+            elif RE.match(r'(int|double|MPI_\w+) (P?MPI\w+)\((.*)\);', line):
+                # prototypes, rename param prefix, add MPICH_API_PUBLIC
+                (T, name, param) = RE.m.group(1,2,3)
+                if RE.match(r'P?MPI_(File_\w+|Register_datarep\w*)', name):
+                    out.append("%s %s(%s) MPICH_API_PUBLIC;" % (T, name, param))
+                else:
+                    out.append("%s %s(%s);" % (T, name, param))
+            else:
+                # replace param prefix
+                out.append(line.rstrip())
+
+    def add_mpich_visibility(out):
+        out.append("#if defined(HAVE_VISIBILITY)")
+        out.append("#define MPICH_API_PUBLIC __attribute__((visibility (\"default\")))")
+        out.append("#else")
+        out.append("#define MPICH_API_PUBLIC")
+        out.append("#endif")
+        out.append("")
+
+    # ----
+    output_lines = []
+    add_mpich_visibility(output_lines)
+    gen_io_abi_internal_h(output_lines)
+
+    print(" --> [%s]" % io_abi_internal_h)
+    with open(io_abi_internal_h, "w") as Out:
+        dump_copyright(Out)
+        print("#ifndef ROMIO_ABI_INTERNAL_H_INCLUDED", file=Out)
+        print("#define ROMIO_ABI_INTERNAL_H_INCLUDED", file=Out)
+        print("", file=Out)
+
         for line in output_lines:
             print(line, file=Out)
         print("", file=Out)

maint/gen_binding_c.py

@@ -50,10 +50,11 @@ def main():
             repl_func['_replaces'].append(func)
 
 
+    # We generate io functions separately for now
+    io_func_list = [f for f in func_list if f['dir'] == 'io']
+    func_list = [f for f in func_list if f['dir'] != 'io']
+
     # -- Generating code --
-    G.out = []
-    G.out.append("#include \"mpiimpl.h\"")
-    G.out.append("")
     G.doc3_src_txt = []
     G.poly_aliases = [] # large-count mansrc aliases
     G.need_dump_romio_reference = True
@@ -64,9 +65,6 @@ def dump_out(file_path):
         dump_c_file(file_path, G.out)
         # add to mpi_sources for dump_Makefile_mk()
         G.mpi_sources.append(file_path)
-        G.out = []
-        G.out.append("#include \"mpiimpl.h\"")
-        G.out.append("")
         G.need_dump_romio_reference = True
 
     def dump_func(func, manpage_out):
@@ -95,52 +93,91 @@ def dump_func_abi(func):
             dump_mpi_c(func, True)
         del func['_is_abi']
 
-    # ----
-    for func in func_list:
-        G.err_codes = {}
-        manpage_out = []
-
-        if 'replace' in func and 'body' not in func:
-            continue
+    def dump_c_binding():
+        G.out = []
+        G.out.append("#include \"mpiimpl.h\"")
+        G.out.append("")
+        for func in func_list:
+            G.err_codes = {}
+            manpage_out = []
 
-        dump_func(func, manpage_out)
-        if '_replaces' in func:
-            for t_func in func['_replaces']:
-                dump_func(t_func, manpage_out)
+            if 'replace' in func and 'body' not in func:
+                continue
 
-        if 'single-source' not in G.opts:
-            # dump individual functions in separate source files
-            dump_out(get_func_file_path(func, c_dir))
+            dump_func(func, manpage_out)
+            if '_replaces' in func:
+                for t_func in func['_replaces']:
+                    dump_func(t_func, manpage_out)
 
-    if 'single-source' in G.opts:
-        # otherwise, dump all functions in binding.c
-        dump_out(c_dir + "/c_binding.c")
+            if 'single-source' not in G.opts:
+                # dump individual functions in separate source files
+                dump_out(get_func_file_path(func, c_dir))
+                G.out = []
+                G.out.append("#include \"mpiimpl.h\"")
+                G.out.append("")
 
-    # ---- dump c_binding_abi.c
-    G.out = []
-    G.out.append("#include \"mpiimpl.h\"")
-    G.out.append("#include \"mpi_abi_util.h\"")
-    G.out.append("")
+        if 'single-source' in G.opts:
+            # otherwise, dump all functions in binding.c
+            dump_out(c_dir + "/c_binding.c")
 
-    for func in func_list:
-        if 'replace' in func and 'body' not in func:
-            continue
+    def dump_c_binding_abi():
+        G.out = []
+        G.out.append("#include \"mpiimpl.h\"")
+        G.out.append("#include \"mpi_abi_util.h\"")
+        G.out.append("")
 
-        if re.match(r'MPIX_', func['name']):
-            if re.match(r'MPIX_(Grequest_|Type_iov)', func['name']):
-                # needed by ROMIO
-                pass
-            else:
+        for func in func_list:
+            if 'replace' in func and 'body' not in func:
                 continue
-        dump_func_abi(func)
-        if '_replaces' in func:
-            for t_func in func['_replaces']:
-                dump_func_abi(t_func)
 
-    abi_file_path = abi_dir + "/c_binding_abi.c"
-    G.check_write_path(abi_file_path)
-    dump_c_file(abi_file_path, G.out)
-    G.out = []
+            if re.match(r'MPIX_', func['name']):
+                if re.match(r'MPIX_(Grequest_|Type_iov)', func['name']):
+                    # needed by ROMIO
+                    pass
+                else:
+                    continue
+            dump_func_abi(func)
+            if '_replaces' in func:
+                for t_func in func['_replaces']:
+                    dump_func_abi(t_func)
+
+        abi_file_path = abi_dir + "/c_binding_abi.c"
+        G.check_write_path(abi_file_path)
+        dump_c_file(abi_file_path, G.out)
+
+    def dump_io_funcs():
+        G.out = []
+        G.out.append("#include \"mpiimpl.h\"")
+        G.out.append("#include \"mpir_io_impl.h\"")
+        G.out.append("")
+
+        for func in io_func_list:
+            G.err_codes = {}
+            manpage_out = []
+            dump_func(func, manpage_out)
+
+        dump_out(c_dir + "/io.c")
+
+    def dump_io_funcs_abi():
+        G.out = []
+        G.out.append("#include \"mpichconf.h\"")
+        G.out.append("#include \"io_abi_internal.h\"")
+        G.out.append("#include \"mpir_io_impl.h\"")
+        G.out.append("#include <limits.h>")
+        G.out.append("")
+
+        for func in io_func_list:
+            dump_func_abi(func)
+
+        abi_file_path = abi_dir + "/io_abi.c"
+        G.check_write_path(abi_file_path)
+        dump_c_file(abi_file_path, G.out)
+
+    # ----
+    dump_c_binding()
+    dump_c_binding_abi()
+    dump_io_funcs()
+    dump_io_funcs_abi()
 
     if 'output-mansrc' in G.opts:
         f = c_dir + '/mansrc/' + 'poly_aliases.lst'
@@ -154,6 +191,7 @@ def dump_func_abi(func):
     G.check_write_path("src/include/mpi_proto.h")
     dump_Makefile_mk("%s/Makefile.mk" % c_dir)
     dump_mpir_impl_h("src/include/mpir_impl.h")
+    dump_mpir_io_impl_h("src/include/mpir_io_impl.h")
     dump_errnames_txt("%s/errnames.txt" % c_dir)
     dump_qmpi_register_h("src/mpi_t/qmpi_register.h")
     dump_mpi_proto_h("src/include/mpi_proto.h")

maint/gen_binding_f08.py

@@ -19,13 +19,6 @@ def main():
     G.check_write_path("%s/wrappers_f/" % f08_dir)
     G.check_write_path("%s/wrappers_c/" % f08_dir)
     func_list = load_C_func_list(binding_dir, True) # suppress noise
-    if "no-mpiio" in G.opts:
-        # a few MPI_File_xxx functions are already in (MPI_File_xxx_errhandler)
-        func_list = [f for f in func_list if not f['name'].startswith('MPI_File_')]
-    else:
-        # FIXME: until romio interface is generated
-        func_list.extend(get_mpiio_func_list())
-    func_list.append(G.FUNCS['mpi_f_sync_reg'])
 
     # preprocess
     get_real_POLY_kinds()

maint/gen_binding_f77.py

@@ -19,14 +19,7 @@ def main():
 
     func_list = load_C_func_list(binding_dir, True) # suppress noise
 
-    if "no-mpiio" in G.opts:
-        # a few MPI_File_xxx functions are already in (MPI_File_xxx_errhandler)
-        func_list = [f for f in func_list if not f['name'].startswith('MPI_File_')]
-    else:
-        # FIXME: until romio interface is generated
-        func_list.extend(get_mpiio_func_list())
     func_list.extend(get_f77_dummy_func_list())
-    func_list.append(G.FUNCS['mpi_f_sync_reg'])
 
     # preprocess
     for func in func_list:

maint/gen_binding_f90.py

@@ -17,10 +17,6 @@ def main():
     f90_dir = "src/binding/fortran/use_mpi"
     G.check_write_path("%s/" % f90_dir)
     func_list = load_C_func_list(binding_dir, True) # suppress noise
-    if "no-mpiio" in G.opts:
-        func_list = [f for f in func_list if not f['name'].startswith('MPI_File_')]
-    else:
-        func_list.extend(get_mpiio_func_list())
     func_list.extend(get_f77_dummy_func_list())
 
     def has_cptr(func):

maint/local_python/__init__.py

@@ -62,6 +62,7 @@ def check_write_path(path):
     mpi_sources = []
     mpi_declares = []
     impl_declares = []
+    io_impl_declares = []
     mpi_errnames = []
     mpix_symbols = {}