[Gh 884] IAM policy splitting for requestor IAM policies

backend/dataall/base/utils/iam_cdk_utils.py

@@ -1,34 +1,67 @@
 from typing import Dict, Any, List
 from aws_cdk import aws_iam as iam
 
-from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks
+from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks, \
+    split_policy_with_resources_in_statements, split_policy_with_mutiple_value_condition_in_statements
 
 
 def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[Any, Any]):
     return iam.PolicyStatement(
         sid=iam_policy.get('Sid'),
-        effect=iam_policy.get('Effect'),
-        actions=iam_policy.get('Action'),
-        resources=iam_policy.get('Resource'),
+        effect=iam.Effect.ALLOW if iam_policy.get('Effect').casefold() == 'Allow'.casefold() else iam.Effect.DENY,
+        actions=_convert_to_array(str, iam_policy.get('Action')),
+        resources=_convert_to_array(str, iam_policy.get('Resource')),
         conditions=iam_policy.get('Condition'),
     )
 
 
 def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]):
     return iam.PolicyStatement(
         sid=iam_policy.get('Sid'),
-        effect=iam_policy.get('Effect'),
-        actions=iam_policy.get('Action'),
-        resources=iam_policy.get('Resource'),
+        effect=iam.Effect.ALLOW if iam_policy.get('Effect').casefold() == 'Allow'.casefold() else iam.Effect.DENY,
+        actions=_convert_to_array(str, iam_policy.get('Action')),
+        resources=_convert_to_array(str, iam_policy.get('Resource')),
     )
 
 
 def process_and_split_statements_in_chunks(statements: List[Dict]):
-    statement_chunks_json = split_policy_statements_in_chunks(statements)
-    statements_chunks = []
-    for statement_js in statement_chunks_json:
-        if not statement_js.get('Condition', None):
-            statements_chunks.append(convert_from_json_to_iam_policy_statement_with_conditions(statement_js))
-        else:
-            statements_chunks.append(convert_from_json_to_iam_policy_statement(statement_js))
+    statement_chunks_json: List[List[Dict]] = split_policy_statements_in_chunks(statements)
+    statements_chunks: List[List[iam.PolicyStatement]] = []
+    for statement_js_chunk in statement_chunks_json:
+        statements: List[iam.PolicyStatement] = []
+        for statement in statement_js_chunk:
+            if statement.get('Condition', None):
+                statements.append(convert_from_json_to_iam_policy_statement_with_conditions(statement))
+            else:
+                statements.append(convert_from_json_to_iam_policy_statement(statement))
+        statements_chunks.append(statements)
     return statements_chunks
+
+
+def process_and_split_policy_with_resources_in_statements(base_sid: str, effect: str, actions: List[str],
+                                                          resources: List[str], condition_dict: Dict = None):
+    if condition_dict is not None:
+        print(f"Condition dictionary is: {condition_dict}")
+        json_statements = split_policy_with_mutiple_value_condition_in_statements(base_sid=base_sid, effect=effect,
+                                                                                  actions=actions,
+                                                                                  resources=resources,
+                                                                                  condition_dict=condition_dict)
+    else:
+        json_statements = split_policy_with_resources_in_statements(base_sid=base_sid, effect=effect, actions=actions,
+                                                                    resources=resources)
+    iam_statements: [iam.PolicyStatement] = []
+    for json_statement in json_statements:
+        if json_statement.get('Condition', None):
+            iam_policy_statement = convert_from_json_to_iam_policy_statement_with_conditions(json_statement)
+        else:
+            iam_policy_statement = convert_from_json_to_iam_policy_statement(json_statement)
+        iam_statements.append(iam_policy_statement)
+    return iam_statements
+
+
+# If item is of item type i.e. single instance if present, then wrap in an array.
+# This is helpful at places where array is required even if one element is present
+def _convert_to_array(item_type, item):
+    if isinstance(item, item_type):
+        return [item]
+    return item

backend/dataall/core/environment/cdk/pivot_role_stack.py

@@ -41,7 +41,7 @@ def generate_policies(self) -> List[iam.ManagedPolicy]:
             logger.info(f'statements: {str(service.get_statements(self))}')
 
         statements_json = [statement.to_json() for statement in statements]
-        statements_chunks = process_and_split_statements_in_chunks(statements_json)
+        statements_chunks: List[List[iam.PolicyStatement]] = process_and_split_statements_in_chunks(statements_json)
 
         for index, chunk in enumerate(statements_chunks):
             policies.append(

backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py

@@ -3,8 +3,7 @@
 
 from dataall.base import db
 from dataall.base.aws.sts import SessionHelper
-from dataall.base.utils.iam_cdk_utils import convert_from_json_to_iam_policy_statement
-from dataall.base.utils.iam_policy_utils import split_policy_with_resources_in_statements
+from dataall.base.utils.iam_cdk_utils import process_and_split_policy_with_resources_in_statements
 from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet
 from dataall.modules.redshift_datasets.db.redshift_connection_repositories import RedshiftConnectionRepository
 from dataall.modules.redshift_datasets.aws.redshift_serverless import redshift_serverless_client
@@ -82,7 +81,7 @@ def get_statements(self):
                 workgroup_arns = [
                     rs_client.get_workgroup_arn(workgroup_name=conn.workgroup) for conn in connections if conn.workgroup
                 ]
-                redshift_data_statement_json = split_policy_with_resources_in_statements(
+                redshift_data_statements = process_and_split_policy_with_resources_in_statements(
                     base_sid='RedshiftData',
                     effect=iam.Effect.ALLOW.value,
                     actions=[
@@ -93,7 +92,6 @@ def get_statements(self):
                     ],
                     resources=cluster_arns + workgroup_arns,
                 )
-                redshift_data_statement = convert_from_json_to_iam_policy_statement(redshift_data_statement_json)
-                additional_statements.extend(redshift_data_statement)
+                additional_statements.extend(redshift_data_statements)
 
             return base_statements + additional_statements

backend/dataall/modules/redshift_datasets_shares/cdk/pivot_role_redshift_data_sharing_policy.py

@@ -1,6 +1,6 @@
 import os
 from dataall.base import db
-from dataall.base.utils.iam_policy_utils import split_policy_with_resources_in_statements
+from dataall.base.utils.iam_cdk_utils import process_and_split_policy_with_resources_in_statements
 from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet
 from dataall.modules.redshift_datasets.db.redshift_connection_repositories import RedshiftConnectionRepository
 
@@ -39,9 +39,9 @@ def get_statements(self):
                     for conn in connections
                 ]
                 additional_statements.extend(
-                    split_policy_with_resources_in_statements(
+                    process_and_split_policy_with_resources_in_statements(
                         base_sid='RedshiftDataShare',
-                        effect=iam.Effect.ALLOW,
+                        effect=iam.Effect.ALLOW.value,
                         actions=['redshift:AuthorizeDataShare'],
                         resources=source_datashare_arns,
                     )

backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py

@@ -1,11 +1,9 @@
 import os
 from dataall.base import db
 from dataall.base.utils.iam_cdk_utils import (
-    convert_from_json_to_iam_policy_statement,
-    convert_from_json_to_iam_policy_statement_with_conditions,
+    convert_from_json_to_iam_policy_statement_with_conditions, process_and_split_policy_with_resources_in_statements,
 )
 from dataall.base.utils.iam_policy_utils import (
-    split_policy_with_resources_in_statements,
     split_policy_with_mutiple_value_condition_in_statements,
 )
 from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet
@@ -157,7 +155,7 @@ def get_statements(self):
                         imported_kms_alias.append(f'alias/{dataset.KmsAlias}')
 
         if imported_buckets:
-            dataset_statement_json = split_policy_with_resources_in_statements(
+            dataset_statements = process_and_split_policy_with_resources_in_statements(
                 base_sid='ImportedDatasetBuckets',
                 effect=iam.Effect.ALLOW.value,
                 actions=[
@@ -172,10 +170,9 @@ def get_statements(self):
                 ],
                 resources=imported_buckets,
             )
-            dataset_statement = convert_from_json_to_iam_policy_statement(dataset_statement_json)
-            statements.extend(dataset_statement)
+            statements.extend(dataset_statements)
         if imported_kms_alias:
-            kms_statement_json = split_policy_with_mutiple_value_condition_in_statements(
+            kms_statements = process_and_split_policy_with_resources_in_statements(
                 base_sid='KMSImportedDataset',
                 effect=iam.Effect.ALLOW.value,
                 actions=[
@@ -195,7 +192,6 @@ def get_statements(self):
                     'values': imported_kms_alias,
                 },
             )
-            kms_statement = convert_from_json_to_iam_policy_statement_with_conditions(kms_statement_json)
-            statements.extend(kms_statement)
+            statements.extend(kms_statements)
 
         return statements

backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py

@@ -245,8 +245,8 @@ def merge_statements_and_update_policies(
         self, target_sid: str, target_s3_statements: List[Any], target_s3_kms_statements: List[Any]
     ):
         """
-        This method is responsible for merging policy statement, re-generating chunks comprizing of statements.
-        Creates policies if needed and then updates policies with statement chunks.
+        This method is responsible for merging policy statements, re-generating chunks consisting of statements.
+        Creates new policies (if needed) and then updates existing policies with statement chunks.
         Based on target_sid:
         1. This method merges all the S3 statments
         2. Splits the policy into policy chunks, where each chunk is <= size of the policy ( this is approximately true )
@@ -493,6 +493,10 @@ def add_resources_and_generate_split_statements(self, statements, target_resourc
         for target_resource in target_resources:
             if target_resource not in s3_statements_resources:
                 s3_statements_resources.append(target_resource)
+
+        if len(s3_statements_resources) == 0:
+            return []
+
         statement_chunks = split_policy_with_resources_in_statements(
             base_sid=sid,
             effect='Allow',

backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py

@@ -140,9 +140,10 @@ def _validate_iam_role_and_policy(
             # In this case it could also happen that the role is the Admin of the environment
             old_managed_policy_name = policy_manager.generate_old_policy_name()
             old_managed_policy_exists = policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name)
+            share_managed_policies_exist = True if policy_manager.get_managed_policies() else False
             # If old managed policy doesn't exist and also new managed policies do not exist.
             # Then there might be inline policies, convert them to managed indexed policies
-            if not old_managed_policy_exists and not policy_manager.check_if_managed_policies_exists():
+            if not old_managed_policy_exists and not share_managed_policies_exist:
                 policy_manager.create_managed_policy_from_inline_and_delete_inline()
             # End of backwards compatibility
 
@@ -154,11 +155,11 @@ def _validate_iam_role_and_policy(
                 policy_manager.create_managed_indexed_policy_from_managed_policy_delete_old_policy()
             # End of backwards compatibility
 
-            attached = policy_manager.check_if_policies_attached()
-            if not attached and not managed and not attachMissingPolicies:
+            unattached = policy_manager.get_policies_unattached_to_role()
+            if unattached and not managed and not attachMissingPolicies:
                 raise Exception(
                     f'Required customer managed policies {policy_manager.get_policies_unattached_to_role()} are not attached to role {principal_role_name}'
                 )
-            elif not attached:
+            elif unattached:
                 managed_policy_list = policy_manager.get_policies_unattached_to_role()
                 policy_manager.attach_policies(managed_policy_list)

backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py

@@ -351,21 +351,6 @@ def grant_target_role_access_policy(self):
         if kms_key_id:
             kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}']
 
-        managed_policy_exists = True if share_policy_service.get_managed_policies() else False
-
-        if not managed_policy_exists:
-            logger.info('Managed policies do not exist. Creating one')
-            # Create a managed policy with naming convention and index
-            share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0)
-            empty_policy = share_policy_service.generate_empty_policy()
-            IAM.create_managed_policy(
-                self.target_account_id,
-                self.target_environment.region,
-                share_resource_policy_name,
-                json.dumps(empty_policy),
-            )
-
-        s3_kms_statement_chunks = []
         s3_statements = share_policy_service.total_s3_access_point_stmts
         s3_statement_chunks = share_policy_service.add_resources_and_generate_split_statements(
             statements=s3_statements,
@@ -376,16 +361,16 @@ def grant_target_role_access_policy(self):
         logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}')
         logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}')
 
-        if kms_target_resources:
-            s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts
-            s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements(
-                statements=s3_kms_statements,
-                target_resources=kms_target_resources,
-                sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS',
-                resource_type='kms',
-            )
-            logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}')
-            logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}')
+        s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts
+        s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements(
+            statements=s3_kms_statements,
+            target_resources=kms_target_resources,
+            sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS',
+            resource_type='kms',
+        )
+        logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}')
+        logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}')
+
         try:
             share_policy_service.merge_statements_and_update_policies(
                 target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID,
@@ -737,9 +722,7 @@ def revoke_target_role_access_policy(self):
             logger.info(f'Managed policies for share with uri: {self.share.shareUri} are not found')
             return
 
-        s3_kms_statement_chunks = []
         s3_statements = share_policy_service.total_s3_access_point_stmts
-
         s3_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements(
             statements=s3_statements,
             target_resources=s3_target_resources,
@@ -749,17 +732,15 @@ def revoke_target_role_access_policy(self):
         logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}')
         logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}')
 
-        if kms_target_resources:
-            s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts
-
-            s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements(
-                statements=s3_kms_statements,
-                target_resources=kms_target_resources,
-                sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS',
-                resource_type='kms',
-            )
-            logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}')
-            logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}')
+        s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts
+        s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements(
+            statements=s3_kms_statements,
+            target_resources=kms_target_resources,
+            sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS',
+            resource_type='kms',
+        )
+        logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}')
+        logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}')
 
         share_policy_service.merge_statements_and_update_policies(
             target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID,