From bc357d6dffc6d02730df6b46f7a3fd9e6f92ac23 Mon Sep 17 00:00:00 2001 From: James Corbett Date: Tue, 31 Jan 2023 23:25:43 +0000 Subject: [PATCH 1/3] build: add DWS python files to build Problem: the build system doesn't know anything about the Python scripts and packages used for interacting with DWS. This means they are not included when running 'make dist' and are not installed. Add the Python modules to the build. --- configure.ac | 12 ++++++++---- src/Makefile.am | 5 ++++- src/cmd/Makefile.am | 2 ++ src/modules/Makefile.am | 2 ++ src/python/Makefile.am | 5 +++++ src/python/flux_k8s/Makefile.am | 24 ++++++++++++++++++++++++ 6 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 src/cmd/Makefile.am create mode 100644 src/modules/Makefile.am create mode 100644 src/python/Makefile.am create mode 100644 src/python/flux_k8s/Makefile.am diff --git a/configure.ac b/configure.ac index 0740f5ce..0febad84 100644 --- a/configure.ac +++ b/configure.ac @@ -119,15 +119,15 @@ AC_SUBST(fluxrc1dir) AS_VAR_SET(fluxrc3dir, $sysconfdir/flux/rc3.d) AC_SUBST(fluxrc3dir) -AS_VAR_SET(fluxcoral2pydir, $pyexecdir/fluxcoral2) -AC_SUBST(fluxcoral2pydir) +AS_VAR_SET(fluxk8spydir, $pyexecdir/flux_k8s) +AC_SUBST(fluxk8spydir) # Target of PYTHONPATH set by flux(1) cmddriver, so flux(1) # doesn't inadvertently insert system python paths (or any # other python path for that matter) first in PYTHONPATH. # -AS_VAR_SET(fluxcoral2pylinkdir, $fluxlibdir/python$PYTHON_VERSION) -AC_SUBST(fluxcoral2pylinkdir) +AS_VAR_SET(fluxk8spylinkdir, $fluxlibdir/python$PYTHON_VERSION) +AC_SUBST(fluxk8spylinkdir) ## # Macros to avoid repetition in Makefiles.am's @@ -147,6 +147,10 @@ AC_CONFIG_FILES([Makefile src/job-manager/plugins/Makefile src/shell/Makefile src/shell/plugins/Makefile + src/cmd/Makefile + src/modules/Makefile + src/python/Makefile + src/python/flux_k8s/Makefile t/Makefile]) AC_OUTPUT diff --git a/src/Makefile.am b/src/Makefile.am index a74da119..539d8466 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,6 +1,9 @@ .NOTPARALLEL: SUBDIRS = job-manager \ - shell + shell \ + cmd \ + modules \ + python check-local: all diff --git a/src/cmd/Makefile.am b/src/cmd/Makefile.am new file mode 100644 index 00000000..019c00a6 --- /dev/null +++ b/src/cmd/Makefile.am @@ -0,0 +1,2 @@ +dist_fluxcmd_SCRIPTS = \ + flux-dws2jgf.py diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am new file mode 100644 index 00000000..da2f5226 --- /dev/null +++ b/src/modules/Makefile.am @@ -0,0 +1,2 @@ +dist_bin_SCRIPTS = \ + dws.py diff --git a/src/python/Makefile.am b/src/python/Makefile.am new file mode 100644 index 00000000..0ec8da05 --- /dev/null +++ b/src/python/Makefile.am @@ -0,0 +1,5 @@ +.NOTPARALLEL: + +SUBDIRS = flux_k8s + +check-local: all diff --git a/src/python/flux_k8s/Makefile.am b/src/python/flux_k8s/Makefile.am new file mode 100644 index 00000000..438b49ff --- /dev/null +++ b/src/python/flux_k8s/Makefile.am @@ -0,0 +1,24 @@ +nobase_fluxk8spy_PYTHON = \ + __init__.py \ + crd.py \ + directivebreakdown.py \ + watch.py + + +clean-local: + -rm -f *.pyc */*.pyc *.pyo */*.pyo + -rm -rf __pycache__ */__pycache__ + +install-data-hook: + $(AM_V_at)echo Linking flux_k8s py in non-standard location... && \ + $(INSTALL) -d -m 0755 "$(DESTDIR)$(fluxk8spylinkdir)" && \ + target=$(fluxk8spydir) && \ + f=$${target##*/} && \ + cd "$(DESTDIR)$(fluxk8spylinkdir)" && \ + rm -f $$f && \ + $(LN_S) $$target . + +uninstall-local: + $(AM_V_at)target=$(fluxk8spydir) && f=$${target##*/} && \ + echo "Removing $(fluxk8spylinkdir)/$$f" && \ + rm -rf $(fluxk8spylinkdir)/$$f From b416543a26f4d4c19ea09e45f1d96635a79b5345 Mon Sep 17 00:00:00 2001 From: James Corbett Date: Tue, 31 Jan 2023 16:43:48 -0800 Subject: [PATCH 2/3] dws: rename dws.py to coral2_dws.py Problem: dws.py will be installed in a common location, but its name is very generic. Rename it. --- src/job-manager/plugins/dws-jobtap.c | 2 +- src/modules/Makefile.am | 2 +- src/modules/{dws.py => coral2_dws.py} | 0 t/t1002-dws-workflow-obj.t | 2 +- t/t1003-dws-nnf-watch.t | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/modules/{dws.py => coral2_dws.py} (100%) diff --git a/src/job-manager/plugins/dws-jobtap.c b/src/job-manager/plugins/dws-jobtap.c index 65bcdc2d..9b124e61 100644 --- a/src/job-manager/plugins/dws-jobtap.c +++ b/src/job-manager/plugins/dws-jobtap.c @@ -309,7 +309,7 @@ static int run_cb (flux_plugin_t *p, flux_future_destroy (fetch_R_future); flux_log_error (h, "dws-jobtap: " - "Error creating future to send R to dws.py"); + "Error creating future to send R to coral2_dws.py"); dws_prolog_finish (h, p, id, 0, "", prolog_active); return -1; } diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am index da2f5226..1c0631b8 100644 --- a/src/modules/Makefile.am +++ b/src/modules/Makefile.am @@ -1,2 +1,2 @@ dist_bin_SCRIPTS = \ - dws.py + coral2_dws.py diff --git a/src/modules/dws.py b/src/modules/coral2_dws.py similarity index 100% rename from src/modules/dws.py rename to src/modules/coral2_dws.py diff --git a/t/t1002-dws-workflow-obj.t b/t/t1002-dws-workflow-obj.t index 2bbb8bf0..f3cd6695 100755 --- a/t/t1002-dws-workflow-obj.t +++ b/t/t1002-dws-workflow-obj.t @@ -16,7 +16,7 @@ test_under_flux ${FLUX_SIZE} job flux setattr log-stderr-level 1 PLUGINPATH=${FLUX_BUILD_DIR}/src/job-manager/plugins/.libs -DWS_MODULE_PATH=${FLUX_SOURCE_DIR}/src/modules/dws.py +DWS_MODULE_PATH=${FLUX_SOURCE_DIR}/src/modules/coral2_dws.py RPC=${FLUX_BUILD_DIR}/t/util/rpc CREATE_DEP_NAME="dws-create" diff --git a/t/t1003-dws-nnf-watch.t b/t/t1003-dws-nnf-watch.t index 523d7868..155669c5 100755 --- a/t/t1003-dws-nnf-watch.t +++ b/t/t1003-dws-nnf-watch.t @@ -11,7 +11,7 @@ test_under_flux ${FLUX_SIZE} job flux setattr log-stderr-level 1 DATA_DIR=${SHARNESS_TEST_SRCDIR}/data/nnf-watch/ -DWS_MODULE_PATH=${FLUX_SOURCE_DIR}/src/modules/dws.py +DWS_MODULE_PATH=${FLUX_SOURCE_DIR}/src/modules/coral2_dws.py RPC=${FLUX_BUILD_DIR}/t/util/rpc check_dmesg_for_pattern() { From b52eeb746c53b74a3fccccf12fb0b7084a85c6bf Mon Sep 17 00:00:00 2001 From: James Corbett Date: Tue, 31 Jan 2023 16:50:03 -0800 Subject: [PATCH 3/3] dws: improve error message Problem: when coral2_dws.py is not running as a Flux job, the error message is unclear. Clarify the error. --- src/modules/coral2_dws.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/modules/coral2_dws.py b/src/modules/coral2_dws.py index a6558e87..577d63dc 100755 --- a/src/modules/coral2_dws.py +++ b/src/modules/coral2_dws.py @@ -370,6 +370,10 @@ def main(): parser.add_argument("--watch-interval", type=int, default=5) parser.add_argument("--verbose", "-v", action="count", default=0) args = parser.parse_args() + try: + jobid = id_parse(os.environ["FLUX_JOB_ID"]) + except KeyError as keyerr: + raise RuntimeError("this script is meant to be run as a Flux job") from keyerr log_level = logging.WARNING if args.verbose > 1: log_level = logging.INFO @@ -428,7 +432,6 @@ def main(): # process starting and the `dws` service being registered. Once # https://github.com/flux-framework/flux-core/issues/3821 is # implemented/closed, this can be replaced with that solution. - jobid = id_parse(os.environ["FLUX_JOB_ID"]) Future(fh.job_raise(jobid, "exception", 7, "dws watchers setup")).get() fh.reactor_run()