Add monitoring configuration to emr create-cluster (#9891)

Author: justinmae

awscli/customizations/emr/argumentschema.py

@@ -868,3 +868,47 @@
         }
     },
 }
+
+MONITORING_CONFIGURATION_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "CloudWatchLogConfiguration": {
+            "type": "object",
+            "description": "CloudWatch log configuration settings and metadata that specify settings like log files to monitor and where to send them.",
+            "properties": {
+                "Enabled": {
+                    "type": "boolean",
+                    "description": "Specifies if CloudWatch logging is enabled.",
+                    "required": True
+                },
+                "LogGroupName": {
+                    "type": "string",
+                    "description": "The name of the CloudWatch log group where logs are published."
+                },
+                "LogStreamNamePrefix": {
+                    "type": "string",
+                    "description": "The prefix of the log stream name."
+                },
+                "EncryptionKeyArn": {
+                    "type": "string",
+                    "description": "The ARN of the encryption key used to encrypt the logs."
+                },
+                "LogTypes": {
+                    "type": "map",
+                    "key": {
+                        "type": "string",
+                        "description": "Log type category"
+                    },
+                    "value": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        },
+                        "description": "File names (STDOUT or STDERR) for the log type"
+                    },
+                    "description": "A map of log types to file names for publishing logs to the standard output or standard error streams for CloudWatch. Valid log types include STEP_LOGS, SPARK_DRIVER, and SPARK_EXECUTOR. Valid file names for each type include STDOUT and STDERR."
+                }
+            }
+        }
+    }
+}

awscli/customizations/emr/createcluster.py

@@ -212,6 +212,11 @@ class CreateCluster(Command):
             'schema': argumentschema.AUTO_TERMINATION_POLICY_SCHEMA,
             'help_text': helptext.AUTO_TERMINATION_POLICY,
         },
+        {
+            'name': 'monitoring-configuration',
+            'schema': argumentschema.MONITORING_CONFIGURATION_SCHEMA,
+            'help_text': helptext.MONITORING_CONFIGURATION,
+        },
         {
             'name': 'extended-support',
             'action': 'store_true',
@@ -553,6 +558,13 @@ def _run_main_command(self, parsed_args, parsed_globals):
                 parsed_args.auto_termination_policy,
             )
 
+        if parsed_args.monitoring_configuration is not None:
+            emrutils.apply_dict(
+                params,
+                'MonitoringConfiguration',
+                parsed_args.monitoring_configuration,
+            )
+
         self._validate_required_applications(parsed_args)
 
         run_job_flow_response = emrutils.call(

awscli/customizations/emr/helptext.py

@@ -570,3 +570,13 @@
 )
 
 EXTENDED_SUPPORT = '<p>Reserved.</p> '
+
+MONITORING_CONFIGURATION = (
+    '<p>Monitoring configuration for an Amazon EMR cluster. '
+    'The configuration specifies CloudWatch logging settings for the cluster. '
+    'You can configure the CloudWatchLogConfiguration which includes '
+    'the Enabled flag (required), LogGroupName, LogStreamNamePrefix, '
+    'EncryptionKeyArn, and LogTypes. The LogTypes parameter is a map '
+    'of log type categories (e.g., "STEP_LOGS", "SPARK_DRIVER", '
+    '"SPARK_EXECUTOR") to a list of file names (e.g., "STDOUT", "STDERR").</p>'
+)

tests/unit/customizations/emr/test_create_cluster_release_label.py

@@ -1828,6 +1828,39 @@ def test_create_cluster_with_auto_termination_policy(self):
         }
         self.assert_params_for_cmd(cmd, result)
 
+    def test_create_cluster_with_monitoring_configuration(self):
+        cmd = (
+            self.prefix
+            + '--release-label emr-5.34.0 '
+            + '--monitoring-configuration '
+            + 'CloudWatchLogConfiguration={Enabled=true,LogGroupName=MyLogGroup,'
+            + 'LogStreamNamePrefix=MyPrefix,EncryptionKeyArn=arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012,'
+            + 'LogTypes={STEP_LOGS=[STDOUT,STDERR],SPARK_DRIVER=[STDOUT],SPARK_EXECUTOR=[STDERR]}} '
+            + '--instance-groups '
+            + DEFAULT_INSTANCE_GROUPS_ARG
+        )
+        result = {
+            'Name': DEFAULT_CLUSTER_NAME,
+            'Instances': DEFAULT_INSTANCES,
+            'ReleaseLabel': 'emr-5.34.0',
+            'VisibleToAllUsers': True,
+            'Tags': [],
+            'MonitoringConfiguration': {
+                'CloudWatchLogConfiguration': {
+                    'Enabled': True,
+                    'LogGroupName': 'MyLogGroup',
+                    'LogStreamNamePrefix': 'MyPrefix',
+                    'EncryptionKeyArn': 'arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012',
+                    'LogTypes': {
+                        'STEP_LOGS': ['STDOUT', 'STDERR'],
+                        'SPARK_DRIVER': ['STDOUT'],
+                        'SPARK_EXECUTOR': ['STDERR'],
+                    },
+                },
+            },
+        }
+        self.assert_params_for_cmd(cmd, result)
+
     def test_create_cluster_with_log_encryption_kms_key_id(self):
         test_log_uri = 's3://test/logs'
         test_log_encryption_kms_key_id = 'valid_kms_key'