From 5acb1d0c70a1c5606c483960b665b93711a5894f Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Tue, 24 Oct 2023 16:40:19 -0700 Subject: [PATCH 1/3] Fix incorrect splitting of system cmd having string type Signed-off-by: Partho Sarthi --- .../src/spark_rapids_pytools/cloud_api/sp_types.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py index b8f4f6a29..766ccc3f6 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py @@ -20,7 +20,7 @@ from dataclasses import dataclass, field from enum import Enum from logging import Logger -from typing import Type, Any, List, Callable +from typing import Type, Any, List, Callable, Union from spark_rapids_tools import EnumeratedType, CspEnv from spark_rapids_pytools.common.prop_manager import AbstractPropertiesContainer, JSONPropertiesContainer, \ @@ -369,7 +369,7 @@ def validate_env(self): self._handle_inconsistent_configurations(incorrect_envs) def run_sys_cmd(self, - cmd, + cmd: Union[str, list[str]], cmd_input: str = None, fail_ok: bool = False, env_vars: dict = None) -> str: @@ -393,7 +393,11 @@ def process_streams(std_out, std_err): if len(stdout_splits) > 0: std_out_lines = Utils.gen_multiline_str([f'\t| {line}' for line in stdout_splits]) stdout_str = f'\n\t\n{std_out_lines}' - cmd_log_str = Utils.gen_joined_str(' ', process_credentials_option(cmd)) + if isinstance(cmd, list): + cmd_list = cmd + else: + cmd_list = cmd.split(' ') + cmd_log_str = Utils.gen_joined_str(' ', process_credentials_option(cmd_list)) if len(stderr_splits) > 0: std_err_lines = Utils.gen_multiline_str([f'\t| {line}' for line in stderr_splits]) stderr_str = f'\n\t\n{std_err_lines}' From f06f3d3f46b525a8330bfe7060792d300f3afe82 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Tue, 24 Oct 2023 17:00:28 -0700 Subject: [PATCH 2/3] Fix subscriptable type Signed-off-by: Partho Sarthi --- user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py index 766ccc3f6..a884582da 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py @@ -369,7 +369,7 @@ def validate_env(self): self._handle_inconsistent_configurations(incorrect_envs) def run_sys_cmd(self, - cmd: Union[str, list[str]], + cmd: Union[str, list], cmd_input: str = None, fail_ok: bool = False, env_vars: dict = None) -> str: From 585d326603a3e2843d88b3e2a3c5a944b7d457cd Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 27 Oct 2023 08:40:47 -0700 Subject: [PATCH 3/3] Concise if-else condition Signed-off-by: Partho Sarthi --- user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py index a884582da..2ea9e4864 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/sp_types.py @@ -393,10 +393,8 @@ def process_streams(std_out, std_err): if len(stdout_splits) > 0: std_out_lines = Utils.gen_multiline_str([f'\t| {line}' for line in stdout_splits]) stdout_str = f'\n\t\n{std_out_lines}' - if isinstance(cmd, list): - cmd_list = cmd - else: - cmd_list = cmd.split(' ') + # if the command is already a list, use it as-is. Otherwise, split the string into a list. + cmd_list = cmd if isinstance(cmd, list) else cmd.split(' ') cmd_log_str = Utils.gen_joined_str(' ', process_credentials_option(cmd_list)) if len(stderr_splits) > 0: std_err_lines = Utils.gen_multiline_str([f'\t| {line}' for line in stderr_splits])