From 34ac3e2c91d80f03c2122db86924712f11d1381b Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 7 Oct 2024 17:29:12 -0500 Subject: [PATCH 01/30] Changes for splitting IAM role changes --- backend/dataall/base/aws/iam.py | 36 ++ .../dataall/base/utils/iam_policy_utils.py | 2 +- .../dataall/base/utils/naming_convention.py | 11 + .../services/managed_iam_policies.py | 168 ++++-- .../s3_share_managed_policy_service.py | 482 ++++++++++++++++-- .../services/s3_share_validator.py | 27 +- .../services/share_managers/__init__.py | 2 +- .../s3_access_point_share_manager.py | 311 ++++++----- .../share_managers/s3_bucket_share_manager.py | 299 ++++++----- backend/docker/dev/Dockerfile | 10 +- backend/requirements.txt | 3 +- docker-compose.yaml | 5 +- .../Catalog/components/RequestAccessModal.js | 31 +- .../test_s3_access_point_share_manager.py | 6 +- 14 files changed, 1006 insertions(+), 387 deletions(-) diff --git a/backend/dataall/base/aws/iam.py b/backend/dataall/base/aws/iam.py index bedfd4620..624dd6ecc 100644 --- a/backend/dataall/base/aws/iam.py +++ b/backend/dataall/base/aws/iam.py @@ -66,6 +66,25 @@ def get_role_policy( log.error(f'Failed to get policy {policy_name} of role {role_name} : {e}') return None + @staticmethod + def list_policy_names_by_policy_pattern(account_id: str, region: str, policy_name: str): + try: + client = IAM.client(account_id, region) + # Setting Scope to 'Local' to fetch all the policies created in this account + paginator = client.get_paginator('list_policies') + policies = [] + for page in paginator.paginate(Scope='Local'): + policies.extend(page['Policies']) + policy_names = [policy.get('PolicyName') for policy in policies] + return [policy_nm for policy_nm in policy_names if policy_name in policy_nm] + except ClientError as e: + if e.response['Error']['Code'] == 'AccessDenied': + raise Exception( + f'Data.all Environment Pivot Role does not have permissions to get policies with pattern {policy_name} due to: {e}' + ) + log.error(f'Failed to get policies for policy pattern due to: {e}') + return [] + @staticmethod def delete_role_policy( account_id: str, @@ -101,6 +120,23 @@ def get_managed_policy_by_name(account_id: str, region: str, policy_name: str): log.error(f'Failed to get policy {policy_name}: {e}') return None + @staticmethod + def get_managed_policy_document_by_name(account_id: str, region: str, policy_name: str): + try: + arn = f'arn:aws:iam::{account_id}:policy/{policy_name}' + client = IAM.client(account_id, region) + policy = IAM.get_managed_policy_by_name(account_id, region, policy_name) + policyVersionId = policy['DefaultVersionId'] + response = client.get_policy_version(PolicyArn=arn, VersionId=policyVersionId) + return response['PolicyVersion']['Document'] + except ClientError as e: + if e.response['Error']['Code'] == 'AccessDenied': + raise Exception( + f'Data.all Environment Pivot Role does not have permissions to to get policy {policy_name}: {e}' + ) + log.error(f'Failed to get policy {policy_name}: {e}') + return None + @staticmethod def create_managed_policy(account_id: str, region: str, policy_name: str, policy: str): try: diff --git a/backend/dataall/base/utils/iam_policy_utils.py b/backend/dataall/base/utils/iam_policy_utils.py index 2d0479f6a..0d9d8b792 100644 --- a/backend/dataall/base/utils/iam_policy_utils.py +++ b/backend/dataall/base/utils/iam_policy_utils.py @@ -9,7 +9,7 @@ MAXIMUM_NUMBER_MANAGED_POLICIES = 20 # Soft limit 10, hard limit 20 -def split_policy_statements_in_chunks(statements: List): +def split_policy_statements_in_chunks(statements: List[iam.PolicyStatement]): """ Splitter used for IAM policies with an undefined number of statements - Ensures that the size of the IAM policy remains below the POLICY LIMIT diff --git a/backend/dataall/base/utils/naming_convention.py b/backend/dataall/base/utils/naming_convention.py index b60cf05aa..bb5dfb359 100644 --- a/backend/dataall/base/utils/naming_convention.py +++ b/backend/dataall/base/utils/naming_convention.py @@ -56,6 +56,17 @@ def build_compliant_name(self) -> str: suffix = f'-{self.target_uri}' if len(self.target_uri) else '' return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}" + def build_compliant_name_with_index(self, index: int = None) -> str: + """ + Builds a compliant AWS resource name + """ + regex = NamingConventionPattern[self.service].value['regex'] + separator = NamingConventionPattern[self.service].value['separator'] + max_length = NamingConventionPattern[self.service].value['max_length'] + suffix = f'-{self.target_uri}' if len(self.target_uri) else '' + suffix = suffix + f'-{index}' if index is not None else suffix + return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}" + def validate_name(self): regex = NamingConventionPattern[self.service].value['regex'] max_length = NamingConventionPattern[self.service].value['max_length'] diff --git a/backend/dataall/core/environment/services/managed_iam_policies.py b/backend/dataall/core/environment/services/managed_iam_policies.py index 5962f58fd..acd309936 100644 --- a/backend/dataall/core/environment/services/managed_iam_policies.py +++ b/backend/dataall/core/environment/services/managed_iam_policies.py @@ -29,12 +29,26 @@ def policy_type(self): ... @abstractmethod - def generate_policy_name(self) -> str: + def generate_old_policy_name(self) -> str: + """ + Returns string and needs to be implemented in the ManagedPolicies inherited classes. + Used for backwards compatibility. It should be deprecated in the future releases. + """ + + @abstractmethod + def generate_base_policy_name(self) -> str: """ Returns string and needs to be implemented in the ManagedPolicies inherited classes """ ... + @abstractmethod + def generate_indexed_policy_name(self, index) -> str: + """ + Returns string of policy name with index at the end .Needs to be implemented in the ManagedPolicies inherited classes + """ + ... + @abstractmethod def generate_empty_policy(self) -> dict: """ @@ -50,21 +64,57 @@ def create_managed_policy_from_inline_and_delete_inline(self) -> str: """ ... - def check_if_policy_exists(self) -> bool: - policy_name = self.generate_policy_name() + @abstractmethod + def create_managed_indexed_policy_from_managed_policy(self) -> str: + """ + Returns policy ARNs and needs to be implemented in the ManagedPolicies inherited classes + It is used for backwards compatibility. It should be deprecated and removed in future releases. + """ + ... + + def check_if_policy_exists(self, policy_name) -> bool: share_policy = IAM.get_managed_policy_by_name(self.account, self.region, policy_name) return share_policy is not None - def check_if_policy_attached(self): - policy_name = self.generate_policy_name() - return IAM.is_policy_attached(self.account, self.region, policy_name, self.role_name) + def check_if_managed_policies_exists(self) -> bool: + # Fetch the policy name which was created without indexes and filter through all policies + policy_pattern = self.generate_base_policy_name() + share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + return True if share_policies else False + + def get_managed_policies(self) -> List[str]: + policy_pattern = self.generate_base_policy_name() + share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + return share_policies + + def check_if_policy_attached(self, policy_name): + is_policy_attached = IAM.is_policy_attached(self.account, self.region, policy_name, self.role_name) + return is_policy_attached + + def check_if_policies_attached(self): + policy_pattern = self.generate_base_policy_name() + share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + return all( + IAM.is_policy_attached(self.account, self.region, share_policy_name, self.role_name) + for share_policy_name in share_policies + ) + + def get_policies_unattached_to_role(self): + policy_pattern = self.generate_base_policy_name() + share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + unattached_policies = [] + for share_policy_name in share_policies: + if not IAM.is_policy_attached(self.account, self.region, share_policy_name, self.role_name): + unattached_policies.append(share_policy_name) + return unattached_policies - def attach_policy(self): - policy_arn = f'arn:aws:iam::{self.account}:policy/{self.generate_policy_name()}' - try: - IAM.attach_role_policy(self.account, self.region, self.role_name, policy_arn) - except Exception as e: - raise Exception(f"Required customer managed policy {policy_arn} can't be attached: {e}") + def attach_policies(self, managed_policies_list: List[str]): + for policy_name in managed_policies_list: + policy_arn = f'arn:aws:iam::{self.account}:policy/{policy_name}' + try: + IAM.attach_role_policy(self.account, self.region, self.role_name, policy_arn) + except Exception as e: + raise Exception(f"Required customer managed policy {policy_arn} can't be attached: {e}") class PolicyManager(object): @@ -99,12 +149,11 @@ def create_all_policies(self, managed) -> bool: Manager that registers and calls all policies created by data.all modules and that need to be created for consumption roles and team roles """ - try: - for Policy in self.initializedPolicies: - empty_policy = Policy.generate_empty_policy() - policy_name = Policy.generate_policy_name() - logger.info(f'Creating policy {policy_name}') - + for policy_manager in self.initializedPolicies: + empty_policy = policy_manager.generate_empty_policy() + policy_name = policy_manager.generate_indexed_policy_name(index=0) + logger.info(f'Creating policy: {policy_name}') + try: IAM.create_managed_policy( account_id=self.account, region=self.region, @@ -119,8 +168,9 @@ def create_all_policies(self, managed) -> bool: role_name=self.role_name, policy_arn=f'arn:aws:iam::{self.account}:policy/{policy_name}', ) - except Exception as e: - raise e + except Exception as e: + logger.error(f'Error while creating and attaching policies due to: {e}') + raise e return True def delete_all_policies(self) -> bool: @@ -128,23 +178,38 @@ def delete_all_policies(self) -> bool: Manager that registers and calls all policies created by data.all modules and that need to be deleted for consumption roles and team roles """ - try: - for Policy in self.initializedPolicies: - policy_name = Policy.generate_policy_name() + for policy_manager in self.initializedPolicies: + policy_name_list = policy_manager.get_managed_policies() + + # Check if policy with old naming format exists + if not policy_name_list: + old_managed_policy_name = policy_manager.generate_old_policy_name() + policy_name_list = ( + [old_managed_policy_name] + if policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) + else [] + ) + + for policy_name in policy_name_list: logger.info(f'Deleting policy {policy_name}') - if Policy.check_if_policy_attached(): - IAM.detach_policy_from_role( - account_id=self.account, region=self.region, role_name=self.role_name, policy_name=policy_name - ) - if Policy.check_if_policy_exists(): - IAM.delete_managed_policy_non_default_versions( - account_id=self.account, region=self.region, policy_name=policy_name - ) - IAM.delete_managed_policy_by_name( - account_id=self.account, region=self.region, policy_name=policy_name - ) - except Exception as e: - raise e + try: + if policy_manager.check_if_policy_attached(policy_name=policy_name): + IAM.detach_policy_from_role( + account_id=self.account, + region=self.region, + role_name=self.role_name, + policy_name=policy_name, + ) + if policy_manager.check_if_policy_exists(policy_name=policy_name): + IAM.delete_managed_policy_non_default_versions( + account_id=self.account, region=self.region, policy_name=policy_name + ) + IAM.delete_managed_policy_by_name( + account_id=self.account, region=self.region, policy_name=policy_name + ) + except Exception as e: + logger.error(f'Error while deleting managed policies due to: {e}') + raise e return True def get_all_policies(self) -> List[dict]: @@ -152,14 +217,27 @@ def get_all_policies(self) -> List[dict]: Manager that registers and calls all policies created by data.all modules and that need to be listed for consumption roles and team roles """ + logger.info('I am here in the managed policy') all_policies = [] - for Policy in self.initializedPolicies: - policy_dict = { - 'policy_name': Policy.generate_policy_name(), - 'policy_type': Policy.policy_type, - 'exists': Policy.check_if_policy_exists(), - 'attached': Policy.check_if_policy_attached(), - } - all_policies.append(policy_dict) - logger.info(f'All policies currently added to role {str(all_policies)}') + for policy_manager in self.initializedPolicies: + policy_name_list = policy_manager.get_managed_policies() + + # Check if policy with old naming format exists + if not policy_name_list: + old_managed_policy_name = policy_manager.generate_old_policy_name() + policy_name_list = ( + [old_managed_policy_name] + if policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) + else [] + ) + + for policy_name in policy_name_list: + policy_dict = { + 'policy_name': policy_name, + 'policy_type': policy_manager.policy_type, + 'exists': policy_manager.check_if_policy_exists(policy_name=policy_name), + 'attached': policy_manager.check_if_policy_attached(policy_name=policy_name), + } + all_policies.append(policy_dict) + logger.info(f'All policies currently added to role: {self.role_name} are: {str(all_policies)}') return all_policies diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 11d0e68c3..1395384aa 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -1,8 +1,15 @@ import json +from typing import Any, List, Set + from dataall.base.aws.iam import IAM +from dataall.base.utils.iam_policy_utils import ( + split_policy_with_resources_in_statements, + split_policy_statements_in_chunks, +) from dataall.base.utils.naming_convention import NamingConventionService, NamingConventionPattern from dataall.core.environment.services.managed_iam_policies import ManagedPolicy import logging +from aws_cdk import aws_iam as iam log = logging.getLogger(__name__) @@ -27,12 +34,47 @@ def __init__(self, role_name, account, region, environmentUri, resource_prefix): self.region = region self.environmentUri = environmentUri self.resource_prefix = resource_prefix + self.policy_version_map = {} + self.total_s3_stmts: List[Any] = [] + self.total_s3_kms_stmts: List[Any] = [] + self.total_s3_access_point_stmts: List[Any] = [] + self.total_s3_access_point_kms_stmts: List[Any] = [] + + def initialize_statements(self): + log.info('Extracting policy statement from all managed policies') + share_managed_policies_name_list = self.get_managed_policies() + + for share_managed_policy in share_managed_policies_name_list: + version_id, policy_document = IAM.get_managed_policy_default_version( + account_id=self.account, region=self.region, policy_name=share_managed_policy + ) + self.policy_version_map[share_managed_policy] = version_id + s3_statements, s3_kms_statements, s3_access_point_statements, s3_kms_access_point_statements = ( + S3SharePolicyService._get_segregated_policy_statements_from_policy(policy_document) + ) + self.total_s3_stmts.extend(s3_statements) + self.total_s3_kms_stmts.extend(s3_kms_statements) + self.total_s3_access_point_stmts.extend(s3_access_point_statements) + self.total_s3_access_point_kms_stmts.extend(s3_kms_access_point_statements) + + log.debug(f'Total S3 Bucket sharing statements: {self.total_s3_stmts}') + log.debug(f'Total KMS Bucket sharing statements : {self.total_s3_kms_stmts}') + log.debug(f'Total S3 Access-point sharing statements : {self.total_s3_access_point_stmts}') + log.debug(f'Total KMS Access-point sharing statements : {self.total_s3_access_point_kms_stmts}') @property def policy_type(self) -> str: return 'SharePolicy' - def generate_policy_name(self) -> str: + def generate_old_policy_name(self) -> str: + return NamingConventionService( + target_label=f'env-{self.environmentUri}-share-policy', + target_uri=self.role_name, + pattern=NamingConventionPattern.IAM_POLICY, + resource_prefix=self.resource_prefix, + ).build_compliant_name() + + def generate_base_policy_name(self) -> str: # In this case it is not possible to build a too long policy because the IAM role can be max 64 chars # However it is good practice to use the standard utility to build the name return NamingConventionService( @@ -42,6 +84,14 @@ def generate_policy_name(self) -> str: resource_prefix=self.resource_prefix, ).build_compliant_name() + def generate_indexed_policy_name(self, index: int = 0) -> str: + return NamingConventionService( + target_label=f'env-{self.environmentUri}-share-policy', + target_uri=self.role_name, + pattern=NamingConventionPattern.IAM_POLICY, + resource_prefix=self.resource_prefix, + ).build_compliant_name_with_index(index) + def generate_empty_policy(self) -> dict: return { 'Version': '2012-10-17', @@ -64,7 +114,7 @@ def add_missing_resources_to_policy_statement( :param existing_policy_statement: dict :return """ - policy_name = self.generate_policy_name() + policy_name = self.generate_base_policy_name() policy_actions = S3_ALLOWED_ACTIONS if resource_type == 's3' else [f'{resource_type}:*'] index = self._get_statement_by_sid(policy_document, statement_sid) if index is None: @@ -93,7 +143,7 @@ def add_missing_resources_to_policy_statement( ) def remove_resource_from_statement(self, target_resources: list, statement_sid: str, policy_document: dict): - policy_name = self.generate_policy_name() + policy_name = self.generate_base_policy_name() index = self._get_statement_by_sid(policy_document, statement_sid) log.info(f'Removing {target_resources} from Statement[{index}] in Managed policy {policy_name} ...') if index is None: @@ -116,34 +166,47 @@ def remove_resource_from_statement(self, target_resources: list, statement_sid: policy_document['Statement'] = empty_policy_document['Statement'] @staticmethod - def check_resource_in_policy_statement(target_resources: list, existing_policy_statement: dict) -> bool: + def check_resource_in_policy_statements(target_resources: list, existing_policy_statements: List[Any]) -> bool: """ - Checks if the resources are in the existing policy + Checks if the resources are in the existing policy statements :param target_resources: list - :param existing_policy_statement: dict + :param existing_policy_statements: dict :return True if all target_resources in the existing policy else False """ + policy_resources = [ + resource for statement in existing_policy_statements for resource in statement.get('Resource') + ] for target_resource in target_resources: - if target_resource not in existing_policy_statement['Resource']: + if target_resource not in policy_resources: return False return True @staticmethod - def check_s3_actions_in_policy_statement(existing_policy_statement: dict) -> (bool, str, str): + def check_s3_actions_in_policy_statement(existing_policy_statements: List[Any]) -> (bool, str, str): """ Checks if all required s3 actions are allowed in the existing policy and there is no unallowed actions - :param existing_policy_statement: + :param existing_policy_statements: :return: bool, allowed missing actions string, not allowed actions string """ - statement_actions = set(existing_policy_statement['Action']) - allowed_actions = set(S3_ALLOWED_ACTIONS) - missing_actions = allowed_actions - statement_actions - extra_actions = statement_actions - allowed_actions - return ( - not (missing_actions or extra_actions), - ','.join(missing_actions), - ','.join(extra_actions), - ) + s3_actions_checker_dict = {} + for statement in existing_policy_statements: + statement_actions = set(statement.get('Action')) + allowed_actions = set(S3_ALLOWED_ACTIONS) + missing_actions = allowed_actions - statement_actions + extra_actions = statement_actions - allowed_actions + s3_actions_checker_dict[statement.get('Sid')] = { + 'policy_check': (missing_actions or extra_actions), + 'missing_permissions': ','.join(missing_actions), + 'extra_permissions': ','.join(extra_actions), + } + return s3_actions_checker_dict + + @staticmethod + def check_if_sid_exists(sid: str, statements): + for statement in statements: + if sid in statement.get('Sid', ''): + return True + return False @staticmethod def _get_statement_by_sid(policy, sid): @@ -161,20 +224,304 @@ def create_managed_policy_from_inline_and_delete_inline(self): Finally, delete the old obsolete inline policies from the role """ try: - policy_document = self._generate_managed_policy_from_inline_policies() - log.info(f'Creating policy from inline backwards compatibility. Policy = {str(policy_document)}') - policy_arn = IAM.create_managed_policy( - self.account, self.region, self.generate_policy_name(), json.dumps(policy_document) + policy_statements = self._generate_managed_policy_statements_from_inline_policies() + log.info( + f'Creating policy from inline backwards compatibility. Policy Statements = {str(policy_statements)}' ) - + policy_arns = self._create_indexed_managed_policies(policy_statements) # Delete obsolete inline policies log.info(f'Deleting {OLD_IAM_ACCESS_POINT_ROLE_POLICY} and {OLD_IAM_S3BUCKET_ROLE_POLICY}') self._delete_old_inline_policies() except Exception as e: raise Exception(f'Error creating policy from inline policies: {e}') - return policy_arn + return policy_arns + + def create_managed_indexed_policy_from_managed_policy(self): + """ + Previously, only one managed policy was created for a role on which share policy statement were attached. + Convert these old managed policies into indexed policies by splitting statements into chunks + """ + policy_name = self.generate_old_policy_name() + log.info(f'Converting old managed policy with name: {policy_name} to indexed managed policy with index: 0') + policy_document = IAM.get_managed_policy_document_by_name( + account_id=self.account, region=self.region, policy_name=policy_name + ) + + if not policy_document: + raise Exception('Failed to fetch policy document while converting to indexed managed policy') + + s3_statements, s3_kms_statements, s3_access_point_statements, s3_kms_access_point_statements = ( + S3SharePolicyService._get_segregated_policy_statements_from_policy(policy_document) + ) + + log.debug(f'Total S3 Bucket sharing statements: {s3_statements}') + log.debug(f'Total KMS Bucket sharing statements : {s3_kms_statements}') + log.debug(f'Total S3 Access-point sharing statements : {s3_access_point_statements}') + log.debug(f'Total KMS Access-point sharing statements : {s3_kms_access_point_statements}') + + policy_statements = [] + if len(s3_statements + s3_access_point_statements) > 0: + existing_bucket_s3_statements = self._split_and_generate_statement_chunks( + statements_s3=s3_statements, statements_kms=s3_kms_statements, sid=IAM_S3_BUCKETS_STATEMENT_SID + ) + existing_bucket_s3_access_point_statement = self._split_and_generate_statement_chunks( + statements_s3=s3_access_point_statements, + statements_kms=s3_kms_access_point_statements, + sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, + ) + policy_statements = existing_bucket_s3_statements + existing_bucket_s3_access_point_statement + + log.info( + f'Found policy statements for existing managed policy. Number of policy statements after splitting: {len(policy_statements)}' + ) + self._create_indexed_managed_policies(policy_statements) + + if self.check_if_policy_attached(policy_name=policy_name): + IAM.detach_policy_from_role( + account_id=self.account, region=self.region, role_name=self.role_name, policy_name=policy_name + ) + + IAM.delete_managed_policy_non_default_versions( + account_id=self.account, region=self.region, policy_name=policy_name + ) + IAM.delete_managed_policy_by_name(account_id=self.account, region=self.region, policy_name=policy_name) + + def merge_statements_and_update_policies( + self, target_sid: str, target_s3_statements: List[Any], target_s3_kms_statements: List[Any] + ): + share_managed_policies_name_list = self.get_managed_policies() + total_s3_iam_policy_stmts: List[iam.PolicyStatement] = [] + total_s3_iam_policy_kms_stmts: List[iam.PolicyStatement] = [] + total_s3_iam_policy_access_point_stmts: List[iam.PolicyStatement] = [] + total_s3_iam_policy_access_point_kms_stmts: List[iam.PolicyStatement] = [] + + if target_sid == IAM_S3_BUCKETS_STATEMENT_SID: + total_s3_iam_policy_stmts = target_s3_statements + total_s3_iam_policy_kms_stmts = target_s3_kms_statements + total_s3_iam_policy_access_point_stmts.extend( + self._convert_to_iam_policy_statement(self.total_s3_access_point_stmts) + ) + total_s3_iam_policy_access_point_kms_stmts.extend( + self._convert_to_iam_policy_statement(self.total_s3_access_point_kms_stmts) + ) + else: + total_s3_iam_policy_access_point_stmts = target_s3_statements + total_s3_iam_policy_access_point_kms_stmts = target_s3_kms_statements + total_s3_iam_policy_stmts.extend(self._convert_to_iam_policy_statement(self.total_s3_stmts)) + total_s3_iam_policy_kms_stmts.extend(self._convert_to_iam_policy_statement(self.total_s3_kms_stmts)) + + aggregated_iam_policy_statements = ( + total_s3_iam_policy_stmts + + total_s3_iam_policy_kms_stmts + + total_s3_iam_policy_access_point_stmts + + total_s3_iam_policy_access_point_kms_stmts + ) + log.info(f'Total number of policy statements after merging: {len(aggregated_iam_policy_statements)}') + + if len(aggregated_iam_policy_statements) == 0: + aggregated_iam_policy_statements.append( + iam.PolicyStatement( + actions=['none:null'], resources=['*'], sid=EMPTY_STATEMENT_SID, effect=iam.Effect.ALLOW + ) + ) - def _generate_managed_policy_from_inline_policies(self): + policy_statement_chunks = split_policy_statements_in_chunks(aggregated_iam_policy_statements) + log.info(f'Number of policy chunks created: {len(policy_statement_chunks)}') + + log.info('Checking if there are any missing policies.') + # Check if there are policies which do not exist but should have existed + current_policy_indexes = [int(policy[-1]) for policy in share_managed_policies_name_list] + integer_indexes = list(range(0, len(share_managed_policies_name_list))) + missing_policies_indexes = [index for index in integer_indexes if index not in current_policy_indexes] + if len(missing_policies_indexes) > 0: + log.info(f'Creating missing policies for indexes: {missing_policies_indexes}') + self._create_empty_policies_with_indexes(indexes=missing_policies_indexes) + + # Check if the number of policies required are greater than currently present + if len(policy_statement_chunks) > len(share_managed_policies_name_list): + additional_policy_indexes = list(range(len(share_managed_policies_name_list), len(policy_statement_chunks))) + log.info( + f'Number of policies needed are more than existing number of policies. Creating policies with indexes: {additional_policy_indexes}' + ) + self._create_empty_policies_with_indexes(indexes=additional_policy_indexes) + + updated_share_managed_policies_name_list = self.get_managed_policies() + + # Update the dict tracking the policy version for new policies which were created + for managed_policy_name in updated_share_managed_policies_name_list: + if managed_policy_name not in self.policy_version_map: + self.policy_version_map[managed_policy_name] = 'v1' + + for index, statement_chunk in enumerate(policy_statement_chunks): + policy_document = self._generate_policy_document_from_statements(statement_chunk) + # If statement length is greater than 1 then check if has empty statements sid and remove it + if len(policy_document.get('Statement')) > 1: + policy_document = S3SharePolicyService.remove_empty_statement( + policy_doc=policy_document, statement_sid=EMPTY_STATEMENT_SID + ) + policy_name = self.generate_indexed_policy_name(index=index) + IAM.update_managed_policy_default_version( + account_id=self.account, + region=self.region, + policy_name=policy_name, + old_version_id=self.policy_version_map.get(policy_name), + policy_document=json.dumps(policy_document), + ) + + # Deleting excess policies + if len(policy_statement_chunks) < len(updated_share_managed_policies_name_list): + excess_policies_indexes = list( + range(len(policy_statement_chunks), len(updated_share_managed_policies_name_list)) + ) + log.info(f'Found more policies than needed. Deleting policies with indexes: {excess_policies_indexes}') + self._delete_policies_with_indexes(indexes=excess_policies_indexes) + + def _convert_to_iam_policy_statement(self, statements): + iam_policy_statements: List[iam.PolicyStatement] = [] + for statement in statements: + iam_policy_statements.append( + iam.PolicyStatement( + sid=statement.get('Sid'), + actions=S3SharePolicyService._convert_to_array(str, statement.get('Action')), + resources=S3SharePolicyService._convert_to_array(str, statement.get('Resource')), + effect=iam.Effect.ALLOW, + ) + ) + return iam_policy_statements + + def _delete_policies_with_indexes(self, indexes): + for index in indexes: + policy_name = self.generate_indexed_policy_name(index=index) + log.info(f'Deleting policy {policy_name}') + # Checking if policy exist or not first before deleting + if self.check_if_policy_exists(policy_name=policy_name): + if self.check_if_policy_attached(policy_name=policy_name): + IAM.detach_policy_from_role( + account_id=self.account, region=self.region, role_name=self.role_name, policy_name=policy_name + ) + IAM.delete_managed_policy_non_default_versions( + account_id=self.account, region=self.region, policy_name=policy_name + ) + IAM.delete_managed_policy_by_name( + account_id=self.account, region=self.region, policy_name=policy_name + ) + else: + log.info(f'Policy with name {policy_name} does not exist') + + def _create_empty_policies_with_indexes(self, indexes): + for index in indexes: + policy_name = self.generate_indexed_policy_name(index=index) + policy_document = self.generate_empty_policy() + IAM.create_managed_policy(self.account, self.region, policy_name, json.dumps(policy_document)) + + def _create_indexed_managed_policies(self, policy_statements: List[iam.PolicyStatement]): + # Todo : Call the split in chunks with a maximum number of policy thing + # Todo : If the number of policies exceed that send an email to the dataset requester that the IAM role policy limit is going to exceed + + if not policy_statements: + policy_statements.append( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, sid=EMPTY_STATEMENT_SID, actions=['none:null'], resources=['*'] + ) + ) + + policy_statement_chunks = split_policy_statements_in_chunks(policy_statements) + + log.info(f'Number of Policy chunks made: {len(policy_statement_chunks)}') + + policy_arns = [] + for index, statement_chunk in enumerate(policy_statement_chunks): + policy_document = self._generate_policy_document_from_statements(statement_chunk) + indexed_policy_name = self.generate_indexed_policy_name(index=index) + policy_arns.append( + IAM.create_managed_policy(self.account, self.region, indexed_policy_name, json.dumps(policy_document)) + ) + + return policy_arns + + @staticmethod + def _get_segregated_policy_statements_from_policy(policy_document): + policy_statements = policy_document.get('Statement', []) + s3_statements = [ + policy_stmt + for policy_stmt in policy_statements + if f'{IAM_S3_BUCKETS_STATEMENT_SID}S3' in policy_stmt.get('Sid', '') + ] + s3_kms_statements = [ + policy_stmt + for policy_stmt in policy_statements + if f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS' in policy_stmt.get('Sid', '') + ] + s3_access_point_statements = [ + policy_stmt + for policy_stmt in policy_statements + if f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3' in policy_stmt.get('Sid', '') + ] + s3_kms_access_point_statements = [ + policy_stmt + for policy_stmt in policy_statements + if f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS' in policy_stmt.get('Sid', '') + ] + + return s3_statements, s3_kms_statements, s3_access_point_statements, s3_kms_access_point_statements + + def add_resources_and_generate_split_statements(self, statements, target_resources, sid, resource_type): + # Using _convert_to_array to convert to array if single resource is present and its not in array + # This is required while creating iam.PolicyStatement in the split_policy_with_resources_in_statements function. + # iam.PolicyStatement throws error if the resource & actions is not an array type + s3_statements_resources: List[str] = [ + resource + for statement in statements + for resource in S3SharePolicyService._convert_to_array(str, statement.get('Resource')) + ] + for target_resource in target_resources: + if target_resource not in s3_statements_resources: + s3_statements_resources.append(target_resource) + statement_chunks: List[iam.PolicyStatement] = split_policy_with_resources_in_statements( + base_sid=sid, + effect=iam.Effect.ALLOW, + actions=S3_ALLOWED_ACTIONS if resource_type == 's3' else [f'{resource_type}:*'], + resources=s3_statements_resources, + ) + return statement_chunks + + def remove_resources_and_generate_split_statements(self, statements, target_resources, sid, resource_type): + # Using _convert_to_array to convert to array if single resource is present and its not in array + # This is required while creating iam.PolicyStatement in the split_policy_with_resources_in_statements function. + # iam.PolicyStatement throws error if the resource & actions is not an array type + s3_statements_resources = [ + resource + for statement in statements + for resource in S3SharePolicyService._convert_to_array(str, statement.get('Resource')) + ] + s3_statements_resources = [resource for resource in s3_statements_resources if resource not in target_resources] + + if len(s3_statements_resources) == 0: + return [] + + statement_chunks = split_policy_with_resources_in_statements( + base_sid=sid, + effect=iam.Effect.ALLOW, + actions=S3_ALLOWED_ACTIONS if resource_type == 's3' else [f'{resource_type}:*'], + resources=s3_statements_resources, + ) + return statement_chunks + + # If item is of item type i.e. single instance if present, then wrap in an array. + # This is helpful at places where array is required even if one element is present + @staticmethod + def _convert_to_array(item_type, item): + if isinstance(item, item_type): + return [item] + return item + + def _generate_policy_document_from_statements(self, statements: List[iam.PolicyStatement]): + if statements is None: + statements = [] + policy_doc = iam.PolicyDocument(statements=statements) + return policy_doc.to_json() + + def _generate_managed_policy_statements_from_inline_policies(self): """ Get resources shared in previous inline policies If there are already shared resources, add them to the empty policy and remove the fake statement @@ -190,23 +537,50 @@ def _generate_managed_policy_from_inline_policies(self): log.info( f'Back-filling S3ACCESS POINTS sharing resources: S3={existing_access_points_s3}, KMS={existing_access_points_kms}' ) + policy_statements = [] if len(existing_bucket_s3 + existing_access_points_s3) > 0: - new_policy = {'Version': '2012-10-17', 'Statement': []} - updated_policy = self._update_policy_resources_from_inline_policy( - policy=new_policy, - statement_sid=IAM_S3_BUCKETS_STATEMENT_SID, - existing_s3=existing_bucket_s3, - existing_kms=existing_bucket_kms, + # Split the statements in chunks + existing_bucket_s3_statements = self._split_and_generate_statement_chunks( + statements_s3=existing_bucket_s3, statements_kms=existing_bucket_kms, sid=IAM_S3_BUCKETS_STATEMENT_SID ) - updated_policy = self._update_policy_resources_from_inline_policy( - policy=updated_policy, - statement_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, - existing_s3=existing_access_points_s3, - existing_kms=existing_access_points_kms, + existing_bucket_s3_access_point_statement = self._split_and_generate_statement_chunks( + statements_s3=existing_access_points_s3, + statements_kms=existing_access_points_kms, + sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, ) - else: - updated_policy = self.generate_empty_policy() - return updated_policy + policy_statements = existing_bucket_s3_statements + existing_bucket_s3_access_point_statement + + log.debug(f'Created policy statements with length: {len(policy_statements)}') + return policy_statements + + def _split_and_generate_statement_chunks(self, statements_s3, statements_kms, sid): + aggregate_statements = [] + if len(statements_s3) > 0: + statement_resources = [ + resource + for statement in statements_s3 + for resource in S3SharePolicyService._convert_to_array(str, statement.get('Resource')) + ] + aggregate_statements.extend( + split_policy_with_resources_in_statements( + base_sid=f'{sid}S3', + effect=iam.Effect.ALLOW, + actions=['s3:List*', 's3:Describe*', 's3:GetObject'], + resources=statement_resources, + ) + ) + if len(statements_kms) > 0: + statement_resources = [ + resource + for statement in statements_kms + for resource in S3SharePolicyService._convert_to_array(str, statement.get('Resource')) + ] + aggregate_statements.extend( + split_policy_with_resources_in_statements( + base_sid=f'{sid}KMS', effect=iam.Effect.ALLOW, actions=['kms:*'], resources=statement_resources + ) + ) + return aggregate_statements def _get_policy_resources_from_inline_policy(self, policy_name): # This function can only be used for backwards compatibility where policies had statement[0] for s3 @@ -250,10 +624,38 @@ def _delete_old_inline_policies(self): try: existing_policy = IAM.get_role_policy(self.account, self.region, self.role_name, policy_name) if existing_policy is not None: - log.info(f'Deleting inline policy {policy_name}...') + log.info(f'Deleting inline policy: {policy_name}') IAM.delete_role_policy(self.account, self.region, self.role_name, policy_name) else: pass except Exception as e: log.error(f'Failed to retrieve the existing policy {policy_name}: {e} ') return True + + def process_backwards_compatibility_for_target_iam_roles(self): + """ + Backwards compatibility + we check if a managed share policy exists. If False, the role was introduced to data.all before this update + we create the policy from the inline statements and attach it to the role + """ + log.info('Checking if inline policies are present') + old_managed_policy_name = self.generate_old_policy_name() + old_managed_policy_exists = self.check_if_policy_exists(policy_name=old_managed_policy_name) + # If old managed policy doesn't exist and also new managed policies do not exist. + # Then there might be inline policies, convert them to managed indexed policies + if not old_managed_policy_exists and not self.check_if_managed_policies_exists(): + self.create_managed_policy_from_inline_and_delete_inline() + managed_policies_list = self.get_managed_policies() + self.attach_policies(managed_policies_list) + # End of backwards compatibility + + """ + Backwards compatibility + After 2.6, the IAM managed policies are created with indexes on them. This was made to solve this issue decribed here - https://github.com/data-dot-all/dataall/issues/884 + If an old managed policy exists then + """ + log.info(f'Old managed policy: {old_managed_policy_exists}') + if old_managed_policy_exists: + self.create_managed_indexed_policy_from_managed_policy() + managed_policies_list = self.get_managed_policies() + self.attach_policies(managed_policies_list) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index 657afb4ec..e7dc38c1f 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -131,21 +131,34 @@ def _validate_iam_role_and_policy( region=environment.region, resource_prefix=environment.resourcePrefix, ) - for Policy in [ + for policy_manager in [ Policy for Policy in share_policy_manager.initializedPolicies if Policy.policy_type == 'SharePolicy' ]: - # Backwards compatibility + # Backwards compatibility - 1 # we check if a managed share policy exists. If False, the role was introduced to data.all before this update # We create the policy from the inline statements # In this case it could also happen that the role is the Admin of the environment - if not Policy.check_if_policy_exists(): - Policy.create_managed_policy_from_inline_and_delete_inline() + old_managed_policy_name = policy_manager.generate_old_policy_name() + old_managed_policy_exists = policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) + # If old managed policy doesn't exist and also new managed policies do not exist. + # Then there might be inline policies, convert them to managed indexed policies + if not old_managed_policy_exists and not policy_manager.check_if_managed_policies_exists(): + policy_manager.create_managed_policy_from_inline_and_delete_inline() # End of backwards compatibility - attached = Policy.check_if_policy_attached() + # Backwards compatibility - 2 + # Check if an already existing managed policy is present in old format + # If yes, convert it to the indexed format + old_managed_policy_name = policy_manager.generate_old_policy_name() + if policy_manager.check_if_policy_exists(old_managed_policy_name): + policy_manager.create_managed_indexed_policy_from_managed_policy() + # End of backwards compatibility + + attached = policy_manager.check_if_policies_attached() if not attached and not managed and not attachMissingPolicies: raise Exception( - f'Required customer managed policy {Policy.generate_policy_name()} is not attached to role {principal_role_name}' + f'Required customer managed policy {policy_manager.get_policies_unattached_to_role()} is not attached to role {principal_role_name}' ) elif not attached: - Policy.attach_policy() + managed_policy_list = policy_manager.get_policies_unattached_to_role() + policy_manager.attach_policies(managed_policy_list) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/__init__.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/__init__.py index df0af76bf..99b109480 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/__init__.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/__init__.py @@ -1,3 +1,3 @@ -from .s3_access_point_share_manager import S3AccessPointShareManager from .lf_share_manager import LFShareManager +from .s3_access_point_share_manager import S3AccessPointShareManager from .s3_bucket_share_manager import S3BucketShareManager diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index e5d70e5ef..851a07e6f 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -2,7 +2,8 @@ import json import time from itertools import count - +from typing import List +from warnings import warn from dataall.core.environment.services.environment_service import EnvironmentService from dataall.base.db import utils from dataall.base.aws.sts import SessionHelper @@ -32,7 +33,6 @@ from dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service import ( S3SharePolicyService, IAM_S3_ACCESS_POINTS_STATEMENT_SID, - EMPTY_STATEMENT_SID, ) from dataall.modules.shares_base.services.shares_enums import PrincipalType from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, S3Dataset @@ -174,20 +174,43 @@ def check_target_role_access_policy(self) -> None: role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, ) - share_resource_policy_name = share_policy_service.generate_policy_name() + share_policy_service.initialize_statements() - if not share_policy_service.check_if_policy_exists(): - logger.info(f'IAM Policy {share_resource_policy_name} does not exist') + share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + + # Checking if managed policies without indexes are present. This is used for backward compatibility + # Check this with AWS team + warn( + "Convert all your share's requestor policies to managed policies with indexes. Deprecation >= ?? ", + DeprecationWarning, + stacklevel=2, + ) + old_managed_policy_name = share_policy_service.generate_old_policy_name() + if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + logger.info( + f'No managed policy exists for the role: {self.target_requester_IAMRoleName}, Reapply share create managed policies.' + ) self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - if not share_policy_service.check_if_policy_attached(): + if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): logger.info( - f'IAM Policy {share_resource_policy_name} exists but is not attached to role {self.share.principalRoleName}' + f'Older version of managed policy present which is without index. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' ) - self.folder_errors.append( - ShareErrorFormatter.dne_error_msg('IAM Policy attached', share_resource_policy_name) + self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return + + if not share_policy_service.check_if_managed_policies_exists(): + logger.info(f'IAM Policy {share_resource_policy_name} does not exist') + self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return + + unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() + if len(unattached_policies) > 0: + logger.info( + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' ) + self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) return s3_target_resources = [ @@ -197,101 +220,97 @@ def check_target_role_access_policy(self) -> None: f'arn:aws:s3:{self.dataset_region}:{self.dataset_account_id}:accesspoint/{self.access_point_name}/*', ] - version_id, policy_document = IAM.get_managed_policy_default_version( - self.target_environment.AwsAccountId, self.target_environment.region, share_resource_policy_name - ) - logger.info(f'Policy... {policy_document}') - - s3_statement_index = S3SharePolicyService._get_statement_by_sid( - policy_document, f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3' - ) - - if s3_statement_index is None: - logger.info(f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 does not exist') + if not S3SharePolicyService.check_if_sid_exists( + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', share_policy_service.total_s3_stmts + ): + logger.info(f'IAM Policy Statement with base Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 does not exist') self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, - 'IAM Policy Statement', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + 'IAM Policy Statement Sid', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3-', 'S3 Bucket', f'{self.bucket_name}', ) ) - - elif not share_policy_service.check_resource_in_policy_statement( + elif not share_policy_service.check_resource_in_policy_statements( target_resources=s3_target_resources, - existing_policy_statement=policy_document['Statement'][s3_statement_index], + existing_policy_statements=share_policy_service.total_s3_stmts, ): logger.info( - f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 does not contain resources {s3_target_resources}' + f'IAM Policy Statement with Sid {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3- does not contain resources {s3_target_resources}' ) self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, - 'IAM Policy Resource', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + 'IAM Policy Resource(s)', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3-', 'S3 Bucket', f'{self.bucket_name}', ) ) else: - policy_check, missing_permissions, extra_permissions = ( - share_policy_service.check_s3_actions_in_policy_statement( - existing_policy_statement=policy_document['Statement'][s3_statement_index] - ) + policy_sid_actions_map = share_policy_service.check_s3_actions_in_policy_statement( + existing_policy_statements=share_policy_service.total_s3_stmts ) - if not policy_check: - logger.info(f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 has invalid actions') - if missing_permissions: - self.folder_errors.append( - ShareErrorFormatter.missing_permission_error_msg( - self.target_requester_IAMRoleName, - 'IAM Policy Action', - missing_permissions, - 'S3 Bucket', - f'{self.bucket_name}', + for sid in policy_sid_actions_map: + policy_check = policy_sid_actions_map[sid].get('policy_check') + missing_permissions = policy_sid_actions_map[sid].get('missing_permissions') + extra_permissions = policy_sid_actions_map[sid].get('extra_permissions') + # Check if policy violations are present + if policy_check: + logger.info(f'IAM Policy Statement {sid} has invalid actions') + if missing_permissions: + self.folder_errors.append( + ShareErrorFormatter.missing_permission_error_msg( + self.target_requester_IAMRoleName, + 'IAM Policy Action', + missing_permissions, + 'S3 Bucket', + f'{self.bucket_name}', + ) ) - ) - if extra_permissions: - self.folder_errors.append( - ShareErrorFormatter.not_allowed_permission_error_msg( - self.target_requester_IAMRoleName, - 'IAM Policy Action', - extra_permissions, - 'S3 Bucket', - f'{self.bucket_name}', + if extra_permissions: + self.folder_errors.append( + ShareErrorFormatter.not_allowed_permission_error_msg( + self.target_requester_IAMRoleName, + 'IAM Policy Action', + extra_permissions, + 'S3 Bucket', + f'{self.bucket_name}', + ) ) - ) if kms_key_id: - kms_statement_index = S3SharePolicyService._get_statement_by_sid( - policy_document, f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS' - ) kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] - if not kms_statement_index: - logger.info(f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS does not exist') + + if not S3SharePolicyService.check_if_sid_exists( + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', share_policy_service.total_s3_kms_stmts + ): + logger.info( + f'IAM Policy Statement with base Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS- does not exist' + ) self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Statement', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS-', 'KMS Key', f'{kms_key_id}', ) ) - - elif not share_policy_service.check_resource_in_policy_statement( + elif not share_policy_service.check_resource_in_policy_statements( target_resources=kms_target_resources, - existing_policy_statement=policy_document['Statement'][kms_statement_index], + existing_policy_statements=share_policy_service.total_s3_kms_stmts, ): logger.info( - f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS does not contain resources {kms_target_resources}' + f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS- does not contain resources {kms_target_resources}' ) self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Resource', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS-', 'KMS Key', f'{kms_key_id}', ) @@ -311,29 +330,8 @@ def grant_target_role_access_policy(self): role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, ) - - # Backwards compatibility - # we check if a managed share policy exists. If False, the role was introduced to data.all before this update - # We create the policy from the inline statements and attach it to the role - if not share_policy_service.check_if_policy_exists(): - share_policy_service.create_managed_policy_from_inline_and_delete_inline() - share_policy_service.attach_policy() - # End of backwards compatibility - - if not share_policy_service.check_if_policy_attached(): - if self.share.principalType == PrincipalType.Group.value: - share_policy_service.attach_policy() - else: - consumption_role = EnvironmentService.get_consumption_role( - session=self.session, uri=self.share.principalId - ) - if consumption_role.dataallManaged: - share_policy_service.attach_policy() - - share_resource_policy_name = share_policy_service.generate_policy_name() - version_id, policy_document = IAM.get_managed_policy_default_version( - self.target_account_id, self.target_environment.region, share_resource_policy_name - ) + share_policy_service.process_backwards_compatibility_for_target_iam_roles() + share_policy_service.initialize_statements() key_alias = f'alias/{self.dataset.KmsAlias}' kms_client = KmsClient(self.dataset_account_id, self.source_environment.region) @@ -345,33 +343,66 @@ def grant_target_role_access_policy(self): f'arn:aws:s3:{self.dataset_region}:{self.dataset_account_id}:accesspoint/{self.access_point_name}', f'arn:aws:s3:{self.dataset_region}:{self.dataset_account_id}:accesspoint/{self.access_point_name}/*', ] + kms_target_resources = [] + if kms_key_id: + kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] - share_policy_service.add_missing_resources_to_policy_statement( - resource_type='s3', + managed_policy_exists = share_policy_service.check_if_managed_policies_exists() + + if not managed_policy_exists: + logger.info('Managed policies do not exist. Creating one') + # Create a managed policy with naming convention and index + share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + IAM.create_managed_policy( + self.target_account_id, + self.target_environment.region, + share_resource_policy_name, + json.dumps(share_policy_service.generate_empty_policy()), + ) + + s3_kms_statement_chunks = [] + s3_statements = share_policy_service.total_s3_access_point_stmts + s3_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( + statements=s3_statements, target_resources=s3_target_resources, - statement_sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', - policy_document=policy_document, + sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + resource_type='s3', ) + logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') + logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') - share_policy_service.remove_empty_statement(policy_doc=policy_document, statement_sid=EMPTY_STATEMENT_SID) - - if kms_key_id: - kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] - share_policy_service.add_missing_resources_to_policy_statement( - resource_type='kms', + if kms_target_resources: + s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts + s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( + statements=s3_kms_statements, target_resources=kms_target_resources, - statement_sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', - policy_document=policy_document, + sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + resource_type='kms', ) + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') - IAM.update_managed_policy_default_version( - self.target_account_id, - self.target_environment.region, - share_resource_policy_name, - version_id, - json.dumps(policy_document), + share_policy_service.merge_statements_and_update_policies( + target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, + target_s3_statements=s3_statement_chunks, + target_s3_kms_statements=s3_kms_statement_chunks, ) + if not share_policy_service.check_if_policies_attached(): + logger.info( + f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' + ) + if self.share.principalType == PrincipalType.Group.value: + share_managed_policies = share_policy_service.get_managed_policies() + share_policy_service.attach_policies(share_managed_policies) + else: + consumption_role = EnvironmentService.get_consumption_role( + session=self.session, uri=self.share.principalId + ) + if consumption_role.dataallManaged: + share_managed_policies = share_policy_service.get_managed_policies() + share_policy_service.attach_policies(share_managed_policies) + def check_access_point_and_policy(self) -> None: """ Checks if access point created with correct permissions @@ -661,28 +692,8 @@ def revoke_target_role_access_policy(self): role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, ) - - role_arn = IAM.get_role_arn_by_name( - self.target_account_id, self.target_environment.region, self.target_requester_IAMRoleName - ) - - # Backwards compatibility - # we check if a managed share policy exists. If False, the role was introduced to data.all before this update - # We create the policy from the inline statements and attach it to the role - if not share_policy_service.check_if_policy_exists() and role_arn: - share_policy_service.create_managed_policy_from_inline_and_delete_inline() - share_policy_service.attach_policy() - # End of backwards compatibility - - share_resource_policy_name = share_policy_service.generate_policy_name() - - version_id, policy_document = IAM.get_managed_policy_default_version( - self.target_account_id, self.target_environment.region, share_resource_policy_name - ) - - if not policy_document: - logger.info(f'Policy {share_resource_policy_name} is not found') - return + share_policy_service.process_backwards_compatibility_for_target_iam_roles() + share_policy_service.initialize_statements() key_alias = f'alias/{self.dataset.KmsAlias}' kms_client = KmsClient(self.dataset_account_id, self.source_environment.region) @@ -695,24 +706,44 @@ def revoke_target_role_access_policy(self): f'arn:aws:s3:{self.dataset_region}:{self.dataset_account_id}:accesspoint/{self.access_point_name}/*', ] - share_policy_service.remove_resource_from_statement( - target_resources=s3_target_resources, - statement_sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', - policy_document=policy_document, - ) + kms_target_resources = [] if kms_key_id: kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] - share_policy_service.remove_resource_from_statement( + + managed_policy_exists = share_policy_service.check_if_managed_policies_exists() + + if not managed_policy_exists: + logger.info(f'Managed policies for share with uri: {self.share.shareUri} are not found') + return + + s3_kms_statement_chunks = [] + s3_statements = share_policy_service.total_s3_access_point_stmts + + s3_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( + statements=s3_statements, + target_resources=s3_target_resources, + sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + resource_type='s3', + ) + logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') + logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') + + if kms_target_resources: + s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts + + s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( + statements=s3_kms_statements, target_resources=kms_target_resources, - statement_sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', - policy_document=policy_document, + sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + resource_type='kms', ) - IAM.update_managed_policy_default_version( - self.target_account_id, - self.target_environment.region, - share_resource_policy_name, - version_id, - json.dumps(policy_document), + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') + + share_policy_service.merge_statements_and_update_policies( + target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, + target_s3_statements=s3_statement_chunks, + target_s3_kms_statements=s3_kms_statement_chunks, ) def delete_dataset_bucket_key_policy( diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index ca9de13d3..1db189ac9 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -1,7 +1,8 @@ import json import logging from itertools import count - +from typing import List +from warnings import warn from dataall.base.aws.iam import IAM from dataall.base.aws.sts import SessionHelper from dataall.core.environment.db.environment_models import Environment @@ -16,7 +17,6 @@ from dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service import ( S3SharePolicyService, IAM_S3_BUCKETS_STATEMENT_SID, - EMPTY_STATEMENT_SID, ) from dataall.modules.s3_datasets_shares.services.share_managers.s3_utils import ( generate_policy_statement, @@ -83,116 +83,137 @@ def check_s3_iam_access(self) -> None: role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, ) - share_resource_policy_name = share_policy_service.generate_policy_name() + share_policy_service.initialize_statements() - if not share_policy_service.check_if_policy_exists(): - logger.info(f'IAM Policy {share_resource_policy_name} does not exist') + share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + + warn( + "Convert all your share's requestor policies to managed policies with indexes. Deprecation >= ?? ", + DeprecationWarning, + stacklevel=2, + ) + old_managed_policy_name = share_policy_service.generate_old_policy_name() + if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + logger.info( + f'No managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' + ) self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - if not share_policy_service.check_if_policy_attached(): + if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): logger.info( - f'IAM Policy {share_resource_policy_name} exists but is not attached to role {self.share.principalRoleName}' + f'Older version of managed policy present which is without index. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' ) - self.bucket_errors.append( - ShareErrorFormatter.dne_error_msg('IAM Policy attached', share_resource_policy_name) + self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return + + if not share_policy_service.check_if_managed_policies_exists(): + logger.info(f'IAM Policy {share_resource_policy_name} does not exist') + self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return + + unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() + if len(unattached_policies) > 0: + logger.info( + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' ) + self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) return s3_target_resources = [f'arn:aws:s3:::{self.bucket_name}', f'arn:aws:s3:::{self.bucket_name}/*'] - version_id, policy_document = IAM.get_managed_policy_default_version( - self.target_environment.AwsAccountId, self.target_environment.region, share_resource_policy_name - ) - s3_statement_index = S3SharePolicyService._get_statement_by_sid( - policy_document, f'{IAM_S3_BUCKETS_STATEMENT_SID}S3' - ) - - if s3_statement_index is None: - logger.info(f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}S3 does not exist') + if not S3SharePolicyService.check_if_sid_exists( + f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', share_policy_service.total_s3_stmts + ): + logger.info(f'IAM Policy Statement with base Sid: {IAM_S3_BUCKETS_STATEMENT_SID}S3 does not exist') self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, - 'IAM Policy Statement', - f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'IAM Policy Statement Sid', + f'{IAM_S3_BUCKETS_STATEMENT_SID}S3-', 'S3 Bucket', f'{self.bucket_name}', ) ) - elif not share_policy_service.check_resource_in_policy_statement( + elif not share_policy_service.check_resource_in_policy_statements( target_resources=s3_target_resources, - existing_policy_statement=policy_document['Statement'][s3_statement_index], + existing_policy_statements=share_policy_service.total_s3_stmts, ): logger.info( - f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}S3 does not contain resources {s3_target_resources}' + f'IAM Policy Statement with Sid {IAM_S3_BUCKETS_STATEMENT_SID}S3- does not contain resources {s3_target_resources}' ) self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, - 'IAM Policy Resource', - f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'IAM Policy Resource(s)', + f'{IAM_S3_BUCKETS_STATEMENT_SID}S3-', 'S3 Bucket', f'{self.bucket_name}', ) ) else: - policy_check, missing_permissions, extra_permissions = ( - share_policy_service.check_s3_actions_in_policy_statement( - existing_policy_statement=policy_document['Statement'][s3_statement_index] - ) + policy_sid_actions_map = share_policy_service.check_s3_actions_in_policy_statement( + existing_policy_statements=share_policy_service.total_s3_stmts ) - if not policy_check: - logger.info(f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}S3 has invalid actions') - if missing_permissions: - self.bucket_errors.append( - ShareErrorFormatter.missing_permission_error_msg( - self.target_requester_IAMRoleName, - 'IAM Policy Action', - missing_permissions, - 'S3 Bucket', - f'{self.bucket_name}', + + for sid in policy_sid_actions_map: + policy_check = policy_sid_actions_map[sid].get('policy_check') + missing_permissions = policy_sid_actions_map[sid].get('missing_permissions') + extra_permissions = policy_sid_actions_map[sid].get('extra_permissions') + # Check if policy violations are present + if policy_check: + logger.info(f'IAM Policy Statement {sid} has invalid actions') + if missing_permissions: + self.bucket_errors.append( + ShareErrorFormatter.missing_permission_error_msg( + self.target_requester_IAMRoleName, + 'IAM Policy Action', + missing_permissions, + 'S3 Bucket', + f'{self.bucket_name}', + ) ) - ) - if extra_permissions: - self.bucket_errors.append( - ShareErrorFormatter.not_allowed_permission_error_msg( - self.target_requester_IAMRoleName, - 'IAM Policy Action', - extra_permissions, - 'S3 Bucket', - f'{self.bucket_name}', + if extra_permissions: + self.bucket_errors.append( + ShareErrorFormatter.not_allowed_permission_error_msg( + self.target_requester_IAMRoleName, + 'IAM Policy Action', + extra_permissions, + 'S3 Bucket', + f'{self.bucket_name}', + ) ) - ) if kms_key_id: - kms_statement_index = S3SharePolicyService._get_statement_by_sid( - policy_document, f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS' - ) kms_target_resources = [f'arn:aws:kms:{self.bucket_region}:{self.source_account_id}:key/{kms_key_id}'] - if kms_statement_index is None: - logger.info(f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}KMS does not exist') + + if not S3SharePolicyService.check_if_sid_exists( + f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', share_policy_service.total_s3_kms_stmts + ): + logger.info( + f'IAM Policy Statement with base Sid: {IAM_S3_BUCKETS_STATEMENT_SID}KMS- does not exist' + ) self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Statement', - f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS-', 'KMS Key', f'{kms_key_id}', ) ) - - elif not share_policy_service.check_resource_in_policy_statement( + elif not share_policy_service.check_resource_in_policy_statements( target_resources=kms_target_resources, - existing_policy_statement=policy_document['Statement'][kms_statement_index], + existing_policy_statements=share_policy_service.total_s3_kms_stmts, ): logger.info( - f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}KMS does not contain resources {kms_target_resources}' + f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}KMS- does not contain resources {kms_target_resources}' ) self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Resource', - f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS-', 'KMS Key', f'{kms_key_id}', ) @@ -213,31 +234,8 @@ def grant_s3_iam_access(self): role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, ) - - # Backwards compatibility - # we check if a managed share policy exists. If False, the role was introduced to data.all before this update - # We create the policy from the inline statements and attach it to the role - if not share_policy_service.check_if_policy_exists(): - share_policy_service.create_managed_policy_from_inline_and_delete_inline() - share_policy_service.attach_policy() - # End of backwards compatibility - - if not share_policy_service.check_if_policy_attached(): - if self.share.principalType == PrincipalType.Group.value: - share_policy_service.attach_policy() - else: - consumption_role = EnvironmentService.get_consumption_role( - session=self.session, uri=self.share.principalId - ) - if consumption_role.dataallManaged: - share_policy_service.attach_policy() - - share_resource_policy_name = share_policy_service.generate_policy_name() - - logger.info(f'Share policy name is {share_resource_policy_name}') - version_id, policy_document = IAM.get_managed_policy_default_version( - self.target_account_id, self.target_environment.region, share_resource_policy_name - ) + share_policy_service.process_backwards_compatibility_for_target_iam_roles() + share_policy_service.initialize_statements() key_alias = f'alias/{self.target_bucket.KmsAlias}' kms_client = KmsClient(self.source_account_id, self.source_environment.region) @@ -245,32 +243,66 @@ def grant_s3_iam_access(self): s3_target_resources = [f'arn:aws:s3:::{self.bucket_name}', f'arn:aws:s3:::{self.bucket_name}/*'] - share_policy_service.add_missing_resources_to_policy_statement( - resource_type='s3', + kms_target_resources = [] + if kms_key_id: + kms_target_resources = [f'arn:aws:kms:{self.bucket_region}:{self.source_account_id}:key/{kms_key_id}'] + + managed_policy_exists = share_policy_service.check_if_managed_policies_exists() + + if not managed_policy_exists: + logger.info('Managed policies do not exist. Creating one') + # Create a managed policy with naming convention and index + share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + IAM.create_managed_policy( + self.target_account_id, + self.target_environment.region, + share_resource_policy_name, + json.dumps(share_policy_service.generate_empty_policy()), + ) + + s3_kms_statement_chunks = [] + s3_statements = share_policy_service.total_s3_stmts + s3_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( + statements=s3_statements, target_resources=s3_target_resources, - statement_sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', - policy_document=policy_document, + sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + resource_type='s3', ) + logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') + logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') - share_policy_service.remove_empty_statement(policy_doc=policy_document, statement_sid=EMPTY_STATEMENT_SID) - - if kms_key_id: - kms_target_resources = [f'arn:aws:kms:{self.bucket_region}:{self.source_account_id}:key/{kms_key_id}'] - share_policy_service.add_missing_resources_to_policy_statement( - resource_type='kms', + if kms_target_resources: + s3_kms_statements = share_policy_service.total_s3_kms_stmts + s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( + statements=s3_kms_statements, target_resources=kms_target_resources, - statement_sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', - policy_document=policy_document, + sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + resource_type='kms', ) + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') - IAM.update_managed_policy_default_version( - self.target_account_id, - self.target_environment.region, - share_resource_policy_name, - version_id, - json.dumps(policy_document), + share_policy_service.merge_statements_and_update_policies( + target_sid=IAM_S3_BUCKETS_STATEMENT_SID, + target_s3_statements=s3_statement_chunks, + target_s3_kms_statements=s3_kms_statement_chunks, ) + if not share_policy_service.check_if_policies_attached(): + logger.info( + f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' + ) + if self.share.principalType == PrincipalType.Group.value: + share_managed_policies = share_policy_service.get_managed_policies() + share_policy_service.attach_policies(share_managed_policies) + else: + consumption_role = EnvironmentService.get_consumption_role( + session=self.session, uri=self.share.principalId + ) + if consumption_role.dataallManaged: + share_managed_policies = share_policy_service.get_managed_policies() + share_policy_service.attach_policies(share_managed_policies) + def get_bucket_policy_or_default(self): """ Fetches the existing bucket policy for the S3 bucket if one exists otherwise returns the default bucket policy @@ -511,18 +543,8 @@ def delete_target_role_access_policy( environmentUri=target_environment.environmentUri, resource_prefix=target_environment.resourcePrefix, ) - # Backwards compatibility - # we check if a managed share policy exists. If False, the role was introduced to data.all before this update - # We create the policy from the inline statements and attach it to the role - if not share_policy_service.check_if_policy_exists(): - share_policy_service.create_managed_policy_from_inline_and_delete_inline() - share_policy_service.attach_policy() - # End of backwards compatibility - - share_resource_policy_name = share_policy_service.generate_policy_name() - version_id, policy_document = IAM.get_managed_policy_default_version( - self.target_account_id, self.target_environment.region, share_resource_policy_name - ) + share_policy_service.process_backwards_compatibility_for_target_iam_roles() + share_policy_service.initialize_statements() key_alias = f'alias/{target_bucket.KmsAlias}' kms_client = KmsClient(target_bucket.AwsAccountId, target_bucket.region) @@ -532,25 +554,44 @@ def delete_target_role_access_policy( f'arn:aws:s3:::{target_bucket.S3BucketName}', f'arn:aws:s3:::{target_bucket.S3BucketName}/*', ] - share_policy_service.remove_resource_from_statement( - target_resources=s3_target_resources, - statement_sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', - policy_document=policy_document, - ) + + kms_target_resources = [] if kms_key_id: kms_target_resources = [f'arn:aws:kms:{target_bucket.region}:{target_bucket.AwsAccountId}:key/{kms_key_id}'] - share_policy_service.remove_resource_from_statement( + + managed_policy_exists = share_policy_service.check_if_managed_policies_exists() + + if not managed_policy_exists: + logger.info(f'Managed policies for share with uri: {share.shareUri} are not found') + return + + s3_kms_statement_chunks = [] + s3_statements = share_policy_service.total_s3_stmts + + s3_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( + statements=s3_statements, + target_resources=s3_target_resources, + sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + resource_type='s3', + ) + logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') + logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') + + if kms_target_resources: + s3_kms_statements = share_policy_service.total_s3_kms_stmts + s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( + statements=s3_kms_statements, target_resources=kms_target_resources, - statement_sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', - policy_document=policy_document, + sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + resource_type='kms', ) + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') - IAM.update_managed_policy_default_version( - self.target_account_id, - self.target_environment.region, - share_resource_policy_name, - version_id, - json.dumps(policy_document), + share_policy_service.merge_statements_and_update_policies( + target_sid=IAM_S3_BUCKETS_STATEMENT_SID, + target_s3_statements=s3_statement_chunks, + target_s3_kms_statements=s3_kms_statement_chunks, ) def delete_target_role_bucket_key_policy( diff --git a/backend/docker/dev/Dockerfile b/backend/docker/dev/Dockerfile index 6c7d3719a..0b6a4c46b 100644 --- a/backend/docker/dev/Dockerfile +++ b/backend/docker/dev/Dockerfile @@ -55,12 +55,10 @@ RUN /bin/bash -c ". ~/.nvm/nvm.sh && \ nvm alias default node && nvm cache clear" RUN echo export PATH="\ -/root/.nvm/versions/node/${NODE_VERSION}/bin:\ -$(${PYTHON_VERSION} -m site --user-base)/bin:\ -$(python3 -m site --user-base)/bin:\ -$(python -m site --user-base)/bin:\ -/home/${CONTAINER_USER}/.local/bin:\ -$PATH" >> ~/.bashrc && \ + /root/.nvm/versions/node/${NODE_VERSION}/bin:\ + $(${PYTHON_VERSION} -m site --user-base)/bin:\ + $(python3 -m site --user-base)/bin:\ + $PATH" >> ~/.bashrc && \ echo "nvm use ${NODE_VERSION} 1> /dev/null" >> ~/.bashrc RUN /bin/bash -c ". ~/.nvm/nvm.sh && cdk --version" diff --git a/backend/requirements.txt b/backend/requirements.txt index 94f7927ca..847ec5127 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -15,4 +15,5 @@ requests==2.32.2 requests_aws4auth==1.1.1 sqlalchemy==1.3.24 starlette==0.36.3 -alembic==1.13.1 \ No newline at end of file +alembic==1.13.1 +aws-cdk-lib==2.99.0 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 6b10cfeac..f53157047 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: dockerfile: docker/dev/Dockerfile args: CONTAINER_UID: ${UID} - entrypoint: /bin/bash -c 'aws configure set region "eu-west-1" &&. ~/.nvm/nvm.sh && uvicorn cdkproxymain:app --host 0.0.0.0 --port 2805 --reload' + entrypoint: /bin/bash -c 'aws configure set region "eu-west-1" && . ~/.nvm/nvm.sh && uvicorn cdkproxymain:app --host 0.0.0.0 --port 2805 --reload' expose: - 2805 ports: @@ -37,7 +37,7 @@ services: dockerfile: docker/dev/Dockerfile args: CONTAINER_UID: ${UID} - entrypoint: /bin/bash -c "../build/wait-for-it.sh elasticsearch:9200 -t 30 && python3.9 local_graphql_server.py" + entrypoint: /bin/bash -c "../build/wait-for-it.sh elasticsearch:9200 -t 30 && . ~/.nvm/nvm.sh && python3.9 local_graphql_server.py" expose: - 5000 ports: @@ -51,6 +51,7 @@ services: volumes: - ./backend/dataall:/dataall - $HOME/.aws/credentials:/home/cuser/.aws/credentials:ro + - $HOME/.aws/config:/home/cuser/.aws/config - ./config.json:/config.json depends_on: - db diff --git a/frontend/src/modules/Catalog/components/RequestAccessModal.js b/frontend/src/modules/Catalog/components/RequestAccessModal.js index 1d3184a8a..316df29a2 100644 --- a/frontend/src/modules/Catalog/components/RequestAccessModal.js +++ b/frontend/src/modules/Catalog/components/RequestAccessModal.js @@ -49,7 +49,7 @@ export const RequestAccessModal = (props) => { const [loadingPolicies, setLoadingPolicies] = useState(false); const [roleOptions, setRoleOptions] = useState([]); const [isSharePolicyAttached, setIsSharePolicyAttached] = useState(true); - const [policyName, setPolicyName] = useState(''); + const [unAttachedPolicyNames, setUnAttachedPolicyNames] = useState(''); const [step, setStep] = useState(0); const [share, setShare] = useState(false); @@ -175,15 +175,21 @@ export const RequestAccessModal = (props) => { }) ); if (!response.errors) { - var isSharePolicyAttached = - response.data.getConsumptionRolePolicies.find( - (policy) => policy.policy_type === 'SharePolicy' - ).attached; - setIsSharePolicyAttached(isSharePolicyAttached); - var policyName = response.data.getConsumptionRolePolicies.find( - (policy) => policy.policy_type === 'SharePolicy' - ).policy_name; - setPolicyName(policyName); + let isSharePoliciesAttached = response.data.getConsumptionRolePolicies + .filter((policy) => policy.policy_type === 'SharePolicy') + .map((policy) => policy.attached); + const isAllPoliciesAttached = isSharePoliciesAttached.every( + (value) => value === true + ); + setIsSharePolicyAttached(isAllPoliciesAttached); + let policyNameList = response.data.getConsumptionRolePolicies + .filter((policy) => { + return ( + policy.policy_type === 'SharePolicy' && policy.attached === false + ); + }) + .map((policy) => policy.policy_name); + setUnAttachedPolicyNames(policyNameList); } else { dispatch({ type: SET_ERROR, error: response.errors[0].message }); } @@ -614,7 +620,7 @@ export const RequestAccessModal = (props) => { ); } else { setFieldValue('consumptionRole', ''); - setPolicyName(''); + setUnAttachedPolicyNames(''); } }} renderInput={(params) => ( @@ -746,7 +752,8 @@ export const RequestAccessModal = (props) => { Selected consumption role is managed by customer, but the share policy{' '} - {policyName} is not attached. + {unAttachedPolicyNames} is + not attached.
Please attach it or let Data.all attach it for you. diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py index d93e2f6f2..5b9ce07c8 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py @@ -385,7 +385,7 @@ def test_grant_target_role_access_policy_test_empty_policy( account=target_environment.AwsAccountId, region=target_environment.region, resource_prefix=target_environment.resourcePrefix, - ).generate_policy_name() + ).generate_base_policy_name() # Then iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -939,7 +939,7 @@ def test_delete_target_role_access_policy_no_remaining_statement( account=target_environment.AwsAccountId, region=target_environment.region, resource_prefix=target_environment.resourcePrefix, - ).generate_policy_name() + ).generate_base_policy_name() iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -1043,7 +1043,7 @@ def test_delete_target_role_access_policy_with_remaining_statement( account=target_environment.AwsAccountId, region=target_environment.region, resource_prefix=target_environment.resourcePrefix, - ).generate_policy_name() + ).generate_base_policy_name() iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, From 44d120526d54f564f52b5ebb63b27e08cc168e63 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 15 Oct 2024 13:29:16 -0500 Subject: [PATCH 02/30] Sycing latest chanegs from aws dev for iam splitting --- backend/dataall/base/aws/iam.py | 13 + backend/dataall/base/aws/service_quota.py | 56 +++ .../dataall/base/utils/iam_policy_utils.py | 62 +--- .../base/utils/iam_policy_utils_cdk.py | 160 +++++++++ .../dataall/base/utils/naming_convention.py | 5 +- .../core/environment/cdk/pivot_role_stack.py | 2 +- .../services/managed_iam_policies.py | 16 +- .../cdk/pivot_role_datasets_policy.py | 2 +- .../s3_share_managed_policy_service.py | 319 +++++++++--------- .../services/s3_share_validator.py | 2 +- .../s3_access_point_share_manager.py | 75 ++-- .../share_managers/s3_bucket_share_manager.py | 54 +-- .../services/share_notification_service.py | 33 ++ backend/docker/dev/Dockerfile | 1 - .../components/EnvironmentTeams.js | 134 +++++--- .../listAllEnvironmentConsumptionRoles.js | 5 - 16 files changed, 636 insertions(+), 303 deletions(-) create mode 100644 backend/dataall/base/aws/service_quota.py create mode 100644 backend/dataall/base/utils/iam_policy_utils_cdk.py diff --git a/backend/dataall/base/aws/iam.py b/backend/dataall/base/aws/iam.py index 624dd6ecc..d34f64ec2 100644 --- a/backend/dataall/base/aws/iam.py +++ b/backend/dataall/base/aws/iam.py @@ -311,3 +311,16 @@ def remove_invalid_role_ids(account_id: str, region: str, principal_list): if 'AROA' in p_id: if p_id not in all_role_ids: principal_list.remove(p_id) + + @staticmethod + def get_attached_managed_policies_to_role(account_id: str, region: str, role_name: str): + try: + client = IAM.client(account_id, region) + response = client.list_attached_role_policies(RoleName=role_name) + return [policy.get('PolicyName') for policy in response['AttachedPolicies']] + except ClientError as e: + if e.response['Error']['Code'] == 'AccessDenied': + raise Exception( + f'Data.all Environment Pivot Role does not have permissions to get attached managed policies for {role_name}: {e}' + ) + raise Exception(f'Failed to get attached managed policies for {role_name}: {e}') diff --git a/backend/dataall/base/aws/service_quota.py b/backend/dataall/base/aws/service_quota.py new file mode 100644 index 000000000..f9ee1b33c --- /dev/null +++ b/backend/dataall/base/aws/service_quota.py @@ -0,0 +1,56 @@ +import logging +from botocore.exceptions import ClientError + +from .sts import SessionHelper + +log = logging.getLogger(__name__) + + +class ServiceQuota: + def __init__(self, account_id, region): + session = SessionHelper.remote_session(accountid=account_id, region=region) + self.client = session.client('service-quotas') + + def list_services(self): + try: + log.info('Fetching services list with service codes in aws account') + services_list = [] + paginator = self.client.get_paginator('list_services') + for page in paginator.paginate(): + services_list.extend(page.get('Services')) + return services_list + except ClientError as e: + if e.response['Error']['Code'] == 'AccessDenied': + raise Exception( + f'Data.all Environment Pivot Role does not have permissions to list services for getting service code : {e}' + ) + log.error(f'Failed list services and service codes due to: {e}') + return [] + + def list_service_quota(self, service_code): + try: + log.info('Fetching services quota code in aws account') + service_quota_code_list = [] + paginator = self.client.get_paginator('list_service_quotas') + for page in paginator.paginate(ServiceCode=service_code): + service_quota_code_list.extend(page.get('Quotas')) + log.debug(f'Services quota list: {service_quota_code_list}') + return service_quota_code_list + except ClientError as e: + if e.response['Error']['Code'] == 'AccessDenied': + raise Exception(f'Data.all Environment Pivot Role does not have permissions to list quota codes: {e}') + log.error(f'Failed list quota codes to: {e}') + return [] + + def get_service_quota_value(self, service_code, service_quota_code): + try: + log.info( + f'Getting service quota for service code: {service_code} and service quota code: {service_quota_code}' + ) + response = self.client.get_service_quota(ServiceCode=service_code, QuotaCode=service_quota_code) + return response['Quota']['Value'] + except ClientError as e: + if e.response['Error']['Code'] == 'AccessDenied': + raise Exception(f'Data.all Environment Pivot Role does not have permissions to list quota codes: {e}') + log.error(f'Failed list quota codes to: {e}') + return None diff --git a/backend/dataall/base/utils/iam_policy_utils.py b/backend/dataall/base/utils/iam_policy_utils.py index 0d9d8b792..ea4b70f98 100644 --- a/backend/dataall/base/utils/iam_policy_utils.py +++ b/backend/dataall/base/utils/iam_policy_utils.py @@ -1,6 +1,5 @@ -from typing import List, Callable +from typing import List, Callable, Any, Dict import logging -from aws_cdk import aws_iam as iam logger = logging.getLogger(__name__) @@ -9,7 +8,7 @@ MAXIMUM_NUMBER_MANAGED_POLICIES = 20 # Soft limit 10, hard limit 20 -def split_policy_statements_in_chunks(statements: List[iam.PolicyStatement]): +def split_policy_statements_in_chunks(statements: List[Any]): """ Splitter used for IAM policies with an undefined number of statements - Ensures that the size of the IAM policy remains below the POLICY LIMIT @@ -18,7 +17,7 @@ def split_policy_statements_in_chunks(statements: List[iam.PolicyStatement]): """ chunks = [] index = 0 - statements_list_of_strings = [str(s.to_json()) for s in statements] + statements_list_of_strings = [str(s) for s in statements] total_length = len(', '.join(statements_list_of_strings)) logger.info(f'Number of statements = {len(statements)}') logger.info(f'Total length of statements = {total_length}') @@ -31,13 +30,12 @@ def split_policy_statements_in_chunks(statements: List[iam.PolicyStatement]): chunk = [] chunk_size = 0 while ( - index < len(statements) - and chunk_size + len(str(statements[index].to_json())) < POLICY_LIMIT - POLICY_HEADERS_BUFFER + index < len(statements) and chunk_size + len(str(statements[index])) < POLICY_LIMIT - POLICY_HEADERS_BUFFER ): # Appends a statement to the chunk until we reach its maximum size. # It compares, current size of the statements < allowed size for the statements section of a policy chunk.append(statements[index]) - chunk_size += len(str(statements[index].to_json())) + chunk_size += len(str(statements[index])) index += 1 chunks.append(chunk) logger.info(f'Total number of managed policies = {len(chunks)}') @@ -46,15 +44,13 @@ def split_policy_statements_in_chunks(statements: List[iam.PolicyStatement]): return chunks -def split_policy_with_resources_in_statements( - base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str] -): +def split_policy_with_resources_in_statements(base_sid: str, effect: str, actions: List[str], resources: List[str]): """ The variable part of the policy is in the resources parameter of the PolicyStatement """ def _build_statement(split, subset): - return iam.PolicyStatement(sid=base_sid + str(split), effect=effect, actions=actions, resources=subset) + return {'Sid': base_sid + str(split), 'Effect': effect, 'Action': actions, 'Resource': subset} total_length, base_length = _policy_analyzer(resources, _build_statement) extra_chars = len('" ," ') @@ -71,51 +67,13 @@ def _build_statement(split, subset): return resulting_statements -def split_policy_with_mutiple_value_condition_in_statements( - base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str], condition_dict: dict -): - """ - The variable part of the policy is in the conditions parameter of the PolicyStatement - conditions_dict passes the different components of the condition mapping - """ - - def _build_statement(split, subset): - return iam.PolicyStatement( - sid=base_sid + str(split), - effect=effect, - actions=actions, - resources=resources, - conditions={condition_dict.get('key'): {condition_dict.get('resource'): subset}}, - ) - - total_length, base_length = _policy_analyzer(condition_dict.get('values'), _build_statement) - extra_chars = len( - str(f'"Condition": {{ "{condition_dict.get("key")}": {{"{condition_dict.get("resource")}": }} }}') - ) - - if total_length < POLICY_LIMIT - POLICY_HEADERS_BUFFER: - logger.info('Not exceeding policy limit, returning statement ...') - resulting_statement = _build_statement(1, condition_dict.get('values')) - return [resulting_statement] - else: - logger.info('Exceeding policy limit, splitting values ...') - resulting_statements = _policy_splitter( - base_length=base_length, - resources=condition_dict.get('values'), - extra_chars=extra_chars, - statement_builder=_build_statement, - ) - - return resulting_statements - - -def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], iam.PolicyStatement]): +def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], Dict]): """ Calculates the policy size with the resources (total_length) and without resources (base_length) """ statement_without_resources = statement_builder(1, ['*']) resources_str = '" ," '.join(r for r in resources) - base_length = len(str(statement_without_resources.to_json())) + base_length = len(str(statement_without_resources)) total_length = base_length + len(resources_str) logger.info(f'Policy base length = {base_length}') logger.info(f'Resources as string length = {len(resources_str)}') @@ -128,7 +86,7 @@ def _policy_splitter( base_length: int, resources: List[str], extra_chars: int, - statement_builder: Callable[[int, List[str]], iam.PolicyStatement], + statement_builder: Callable[[int, List[str]], Dict], ): """ Splitter used for IAM policy statements with an undefined number of resources one of the parameters of the policy. diff --git a/backend/dataall/base/utils/iam_policy_utils_cdk.py b/backend/dataall/base/utils/iam_policy_utils_cdk.py new file mode 100644 index 000000000..0d9d8b792 --- /dev/null +++ b/backend/dataall/base/utils/iam_policy_utils_cdk.py @@ -0,0 +1,160 @@ +from typing import List, Callable +import logging +from aws_cdk import aws_iam as iam + +logger = logging.getLogger(__name__) + +POLICY_LIMIT = 6144 +POLICY_HEADERS_BUFFER = 144 # The policy headers take around 60 chars. An extra buffer of 84 chars is added for any additional spacing or char that is unaccounted. +MAXIMUM_NUMBER_MANAGED_POLICIES = 20 # Soft limit 10, hard limit 20 + + +def split_policy_statements_in_chunks(statements: List[iam.PolicyStatement]): + """ + Splitter used for IAM policies with an undefined number of statements + - Ensures that the size of the IAM policy remains below the POLICY LIMIT + - If it exceeds the POLICY LIMIT, it breaks the policy into multiple policies (chunks) + - Note the POLICY_HEADERS_BUFFER to account for the headers of the policy which usually take around ~60chars + """ + chunks = [] + index = 0 + statements_list_of_strings = [str(s.to_json()) for s in statements] + total_length = len(', '.join(statements_list_of_strings)) + logger.info(f'Number of statements = {len(statements)}') + logger.info(f'Total length of statements = {total_length}') + max_length = max(statements_list_of_strings, key=len) + if len(max_length) > POLICY_LIMIT - POLICY_HEADERS_BUFFER: + raise Exception(f'Policy statement {max_length} exceeds maximum policy size') + while index < len(statements): + # Iterating until all statements are assigned to a chunk. + # "index" represents the statement position in the statements list + chunk = [] + chunk_size = 0 + while ( + index < len(statements) + and chunk_size + len(str(statements[index].to_json())) < POLICY_LIMIT - POLICY_HEADERS_BUFFER + ): + # Appends a statement to the chunk until we reach its maximum size. + # It compares, current size of the statements < allowed size for the statements section of a policy + chunk.append(statements[index]) + chunk_size += len(str(statements[index].to_json())) + index += 1 + chunks.append(chunk) + logger.info(f'Total number of managed policies = {len(chunks)}') + if len(chunks) > MAXIMUM_NUMBER_MANAGED_POLICIES: + raise Exception('The number of policies calculated exceeds the allowed maximum number of managed policies') + return chunks + + +def split_policy_with_resources_in_statements( + base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str] +): + """ + The variable part of the policy is in the resources parameter of the PolicyStatement + """ + + def _build_statement(split, subset): + return iam.PolicyStatement(sid=base_sid + str(split), effect=effect, actions=actions, resources=subset) + + total_length, base_length = _policy_analyzer(resources, _build_statement) + extra_chars = len('" ," ') + + if total_length < POLICY_LIMIT - POLICY_HEADERS_BUFFER: + logger.info('Not exceeding policy limit, returning statement ...') + resulting_statement = _build_statement(1, resources) + return [resulting_statement] + else: + logger.info('Exceeding policy limit, splitting statement ...') + resulting_statements = _policy_splitter( + base_length=base_length, resources=resources, extra_chars=extra_chars, statement_builder=_build_statement + ) + return resulting_statements + + +def split_policy_with_mutiple_value_condition_in_statements( + base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str], condition_dict: dict +): + """ + The variable part of the policy is in the conditions parameter of the PolicyStatement + conditions_dict passes the different components of the condition mapping + """ + + def _build_statement(split, subset): + return iam.PolicyStatement( + sid=base_sid + str(split), + effect=effect, + actions=actions, + resources=resources, + conditions={condition_dict.get('key'): {condition_dict.get('resource'): subset}}, + ) + + total_length, base_length = _policy_analyzer(condition_dict.get('values'), _build_statement) + extra_chars = len( + str(f'"Condition": {{ "{condition_dict.get("key")}": {{"{condition_dict.get("resource")}": }} }}') + ) + + if total_length < POLICY_LIMIT - POLICY_HEADERS_BUFFER: + logger.info('Not exceeding policy limit, returning statement ...') + resulting_statement = _build_statement(1, condition_dict.get('values')) + return [resulting_statement] + else: + logger.info('Exceeding policy limit, splitting values ...') + resulting_statements = _policy_splitter( + base_length=base_length, + resources=condition_dict.get('values'), + extra_chars=extra_chars, + statement_builder=_build_statement, + ) + + return resulting_statements + + +def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], iam.PolicyStatement]): + """ + Calculates the policy size with the resources (total_length) and without resources (base_length) + """ + statement_without_resources = statement_builder(1, ['*']) + resources_str = '" ," '.join(r for r in resources) + base_length = len(str(statement_without_resources.to_json())) + total_length = base_length + len(resources_str) + logger.info(f'Policy base length = {base_length}') + logger.info(f'Resources as string length = {len(resources_str)}') + logger.info(f'Total length approximated as base length + resources string length = {total_length}') + + return total_length, base_length + + +def _policy_splitter( + base_length: int, + resources: List[str], + extra_chars: int, + statement_builder: Callable[[int, List[str]], iam.PolicyStatement], +): + """ + Splitter used for IAM policy statements with an undefined number of resources one of the parameters of the policy. + - Ensures that the size of the IAM statement is below the POLICY LIMIT + - If it exceeds the POLICY LIMIT, it breaks the statement in multiple statements with a subset of resources + - Note the POLICY_HEADERS_BUFFER to account for the headers of the policy which usually take around ~60chars + """ + index = 0 + split = 0 + resulting_statements = [] + while index < len(resources): + # Iterating until all values are defined in a policy statement. + # "index" represents the position of the value in the values list + size = 0 + subset = [] + while ( + index < len(resources) + and (size + len(resources[index]) + extra_chars) < POLICY_LIMIT - POLICY_HEADERS_BUFFER - base_length + ): + # Appending a resource to the subset list until we reach the maximum size for the condition section + # It compares: current size of subset versus the allowed size of the condition section in a statement + subset.append(resources[index]) + size += len(resources[index]) + extra_chars + index += 1 + resulting_statement = statement_builder(split=split, subset=subset) + split += 1 + resulting_statements.append(resulting_statement) + logger.info(f'Statement divided into {split+1} smaller statements') + return resulting_statements diff --git a/backend/dataall/base/utils/naming_convention.py b/backend/dataall/base/utils/naming_convention.py index bb5dfb359..2f5ef752d 100644 --- a/backend/dataall/base/utils/naming_convention.py +++ b/backend/dataall/base/utils/naming_convention.py @@ -63,9 +63,10 @@ def build_compliant_name_with_index(self, index: int = None) -> str: regex = NamingConventionPattern[self.service].value['regex'] separator = NamingConventionPattern[self.service].value['separator'] max_length = NamingConventionPattern[self.service].value['max_length'] + index_string = f'-{index}' if index is not None else '- ' suffix = f'-{self.target_uri}' if len(self.target_uri) else '' - suffix = suffix + f'-{index}' if index is not None else suffix - return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}" + suffix = suffix + index_string if index is not None else suffix + return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri + index_string))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}" def validate_name(self): regex = NamingConventionPattern[self.service].value['regex'] diff --git a/backend/dataall/core/environment/cdk/pivot_role_stack.py b/backend/dataall/core/environment/cdk/pivot_role_stack.py index d6368ea55..f32d9a46e 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_stack.py +++ b/backend/dataall/core/environment/cdk/pivot_role_stack.py @@ -3,7 +3,7 @@ from typing import List from constructs import Construct from aws_cdk import Duration, aws_iam as iam, NestedStack -from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks +from dataall.base.utils.iam_policy_utils_cdk import split_policy_statements_in_chunks logger = logging.getLogger(__name__) diff --git a/backend/dataall/core/environment/services/managed_iam_policies.py b/backend/dataall/core/environment/services/managed_iam_policies.py index acd309936..c0450c0c8 100644 --- a/backend/dataall/core/environment/services/managed_iam_policies.py +++ b/backend/dataall/core/environment/services/managed_iam_policies.py @@ -79,12 +79,12 @@ def check_if_policy_exists(self, policy_name) -> bool: def check_if_managed_policies_exists(self) -> bool: # Fetch the policy name which was created without indexes and filter through all policies policy_pattern = self.generate_base_policy_name() - share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + share_policies = self._get_share_policy_names(policy_pattern) return True if share_policies else False def get_managed_policies(self) -> List[str]: policy_pattern = self.generate_base_policy_name() - share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + share_policies = self._get_share_policy_names(policy_pattern) return share_policies def check_if_policy_attached(self, policy_name): @@ -93,7 +93,7 @@ def check_if_policy_attached(self, policy_name): def check_if_policies_attached(self): policy_pattern = self.generate_base_policy_name() - share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + share_policies = self._get_share_policy_names(policy_pattern) return all( IAM.is_policy_attached(self.account, self.region, share_policy_name, self.role_name) for share_policy_name in share_policies @@ -101,7 +101,7 @@ def check_if_policies_attached(self): def get_policies_unattached_to_role(self): policy_pattern = self.generate_base_policy_name() - share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + share_policies = self._get_share_policy_names(policy_pattern) unattached_policies = [] for share_policy_name in share_policies: if not IAM.is_policy_attached(self.account, self.region, share_policy_name, self.role_name): @@ -116,6 +116,13 @@ def attach_policies(self, managed_policies_list: List[str]): except Exception as e: raise Exception(f"Required customer managed policy {policy_arn} can't be attached: {e}") + def _get_share_policy_names(self, policy_pattern): + share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) + # Filter through all policies which have the old policy name + # This is to check that old policies are not included + old_policy_name = self.generate_old_policy_name() + return [policy for policy in share_policies if policy != old_policy_name] + class PolicyManager(object): def __init__( @@ -217,7 +224,6 @@ def get_all_policies(self) -> List[dict]: Manager that registers and calls all policies created by data.all modules and that need to be listed for consumption roles and team roles """ - logger.info('I am here in the managed policy') all_policies = [] for policy_manager in self.initializedPolicies: policy_name_list = policy_manager.get_managed_policies() diff --git a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py index e09360fc6..f0b957569 100644 --- a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py @@ -1,6 +1,6 @@ import os from dataall.base import db -from dataall.base.utils.iam_policy_utils import ( +from dataall.base.utils.iam_policy_utils_cdk import ( split_policy_with_resources_in_statements, split_policy_with_mutiple_value_condition_in_statements, ) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 1395384aa..27c7fb515 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -1,15 +1,17 @@ import json -from typing import Any, List, Set +from typing import Any, List, Dict from dataall.base.aws.iam import IAM +from dataall.base.aws.service_quota import ServiceQuota from dataall.base.utils.iam_policy_utils import ( - split_policy_with_resources_in_statements, split_policy_statements_in_chunks, + split_policy_with_resources_in_statements, ) from dataall.base.utils.naming_convention import NamingConventionService, NamingConventionPattern from dataall.core.environment.services.managed_iam_policies import ManagedPolicy import logging -from aws_cdk import aws_iam as iam + +from dataall.modules.shares_base.services.share_notification_service import ShareNotificationService log = logging.getLogger(__name__) @@ -25,15 +27,20 @@ EMPTY_STATEMENT_SID = 'EmptyStatement' S3_ALLOWED_ACTIONS = ['s3:List*', 's3:Describe*', 's3:GetObject'] +IAM_SERVICE_NAME = 'AWS Identity and Access Management (IAM)' +IAM_SERVICE_QUOTA_NAME = 'Managed policies per role' +DEFAULT_MAX_ATTACHABLE_MANAGED_POLICIES_ACCOUNT = 10 class S3SharePolicyService(ManagedPolicy): - def __init__(self, role_name, account, region, environmentUri, resource_prefix): + def __init__(self, role_name, account, region, environmentUri, resource_prefix, share=None, dataset=None): self.role_name = role_name self.account = account self.region = region self.environmentUri = environmentUri self.resource_prefix = resource_prefix + self.share = share + self.dataset = dataset self.policy_version_map = {} self.total_s3_stmts: List[Any] = [] self.total_s3_kms_stmts: List[Any] = [] @@ -75,14 +82,12 @@ def generate_old_policy_name(self) -> str: ).build_compliant_name() def generate_base_policy_name(self) -> str: - # In this case it is not possible to build a too long policy because the IAM role can be max 64 chars - # However it is good practice to use the standard utility to build the name return NamingConventionService( target_label=f'env-{self.environmentUri}-share-policy', target_uri=self.role_name, pattern=NamingConventionPattern.IAM_POLICY, resource_prefix=self.resource_prefix, - ).build_compliant_name() + ).build_compliant_name_with_index() def generate_indexed_policy_name(self, index: int = 0) -> str: return NamingConventionService( @@ -95,7 +100,7 @@ def generate_indexed_policy_name(self, index: int = 0) -> str: def generate_empty_policy(self) -> dict: return { 'Version': '2012-10-17', - 'Statement': [{'Sid': EMPTY_STATEMENT_SID, 'Effect': 'Allow', 'Action': 'none:null', 'Resource': '*'}], + 'Statement': [{'Sid': EMPTY_STATEMENT_SID, 'Effect': 'Allow', 'Action': ['none:null'], 'Resource': ['*']}], } @staticmethod @@ -105,66 +110,6 @@ def remove_empty_statement(policy_doc: dict, statement_sid: str) -> dict: policy_doc['Statement'].pop(statement_index) return policy_doc - def add_missing_resources_to_policy_statement( - self, resource_type: str, target_resources: list, statement_sid: str, policy_document: dict - ): - """ - Checks if the resources are in the existing statement. Otherwise, it will add it. - :param target_resources: list - :param existing_policy_statement: dict - :return - """ - policy_name = self.generate_base_policy_name() - policy_actions = S3_ALLOWED_ACTIONS if resource_type == 's3' else [f'{resource_type}:*'] - index = self._get_statement_by_sid(policy_document, statement_sid) - if index is None: - log.info(f'{statement_sid} does NOT exists for Managed policy {policy_name} ' f'creating statement...') - additional_policy = { - 'Sid': statement_sid, - 'Effect': 'Allow', - 'Action': policy_actions, - 'Resource': target_resources, - } - policy_document['Statement'].append(additional_policy) - else: - # Enforce, that actions are valid - policy_document['Statement'][index]['Action'] = policy_actions - for target_resource in target_resources: - if target_resource not in policy_document['Statement'][index]['Resource']: - log.info( - f'{statement_sid} exists for Managed policy {policy_name} ' - f'but {target_resource} is not included, updating...' - ) - policy_document['Statement'][index]['Resource'].extend([target_resource]) - else: - log.info( - f'{statement_sid} exists for Managed policy {policy_name} ' - f'and {target_resource} is included, skipping...' - ) - - def remove_resource_from_statement(self, target_resources: list, statement_sid: str, policy_document: dict): - policy_name = self.generate_base_policy_name() - index = self._get_statement_by_sid(policy_document, statement_sid) - log.info(f'Removing {target_resources} from Statement[{index}] in Managed policy {policy_name} ...') - if index is None: - log.info(f'{statement_sid} does NOT exists for Managed policy {policy_name} ' f'skipping...') - else: - policy_statement = policy_document['Statement'][index] - for target_resource in target_resources: - if target_resource in policy_statement['Resource']: - log.info( - f'{statement_sid} exists for Managed policy {policy_name} ' - f'and {target_resource} is included, removing...' - ) - policy_statement['Resource'].remove(target_resource) - if len(policy_statement['Resource']) == 0: - log.info(f'No more resources in {statement_sid}, removing statement...') - policy_document['Statement'].pop(index) - if len(policy_document['Statement']) == 0: - log.info(f'No more statements in {policy_document}, adding empty statement...') - empty_policy_document = self.generate_empty_policy() - policy_document['Statement'] = empty_policy_document['Statement'] - @staticmethod def check_resource_in_policy_statements(target_resources: list, existing_policy_statements: List[Any]) -> bool: """ @@ -216,7 +161,6 @@ def _get_statement_by_sid(policy, sid): return None # Backwards compatibility - def create_managed_policy_from_inline_and_delete_inline(self): """ For existing consumption and team roles, the IAM managed policy won't be created. @@ -236,6 +180,7 @@ def create_managed_policy_from_inline_and_delete_inline(self): raise Exception(f'Error creating policy from inline policies: {e}') return policy_arns + # Backwards compatibility def create_managed_indexed_policy_from_managed_policy(self): """ Previously, only one managed policy was created for a role on which share policy statement were attached. @@ -289,26 +234,31 @@ def create_managed_indexed_policy_from_managed_policy(self): def merge_statements_and_update_policies( self, target_sid: str, target_s3_statements: List[Any], target_s3_kms_statements: List[Any] ): + """ + Based on target_sid: {} + 1. This method merges all the S3 statments + 2. Splits the policy into policy chunks, where each chunk is <= size of the policy ( this is approximately true ) + 3. Check if there are any missing policies and creates them + 4. Check if extra policies are required and also checks if those policies can be attached to the role (At the time of writing, IAM role has limit of 10 managed policies but can be increased to 20 ) + 5. Once policies are creates, fill the policies with the policy chunks + 6. Delete ( if any ) extra policies which are remaining + """ share_managed_policies_name_list = self.get_managed_policies() - total_s3_iam_policy_stmts: List[iam.PolicyStatement] = [] - total_s3_iam_policy_kms_stmts: List[iam.PolicyStatement] = [] - total_s3_iam_policy_access_point_stmts: List[iam.PolicyStatement] = [] - total_s3_iam_policy_access_point_kms_stmts: List[iam.PolicyStatement] = [] + total_s3_iam_policy_stmts: List[Dict] = [] + total_s3_iam_policy_kms_stmts: List[Dict] = [] + total_s3_iam_policy_access_point_stmts: List[Dict] = [] + total_s3_iam_policy_access_point_kms_stmts: List[Dict] = [] if target_sid == IAM_S3_BUCKETS_STATEMENT_SID: total_s3_iam_policy_stmts = target_s3_statements total_s3_iam_policy_kms_stmts = target_s3_kms_statements - total_s3_iam_policy_access_point_stmts.extend( - self._convert_to_iam_policy_statement(self.total_s3_access_point_stmts) - ) - total_s3_iam_policy_access_point_kms_stmts.extend( - self._convert_to_iam_policy_statement(self.total_s3_access_point_kms_stmts) - ) + total_s3_iam_policy_access_point_stmts.extend(self.total_s3_access_point_stmts) + total_s3_iam_policy_access_point_kms_stmts.extend(self.total_s3_access_point_kms_stmts) else: total_s3_iam_policy_access_point_stmts = target_s3_statements total_s3_iam_policy_access_point_kms_stmts = target_s3_kms_statements - total_s3_iam_policy_stmts.extend(self._convert_to_iam_policy_statement(self.total_s3_stmts)) - total_s3_iam_policy_kms_stmts.extend(self._convert_to_iam_policy_statement(self.total_s3_kms_stmts)) + total_s3_iam_policy_stmts.extend(self.total_s3_stmts) + total_s3_iam_policy_kms_stmts.extend(self.total_s3_kms_stmts) aggregated_iam_policy_statements = ( total_s3_iam_policy_stmts @@ -319,14 +269,14 @@ def merge_statements_and_update_policies( log.info(f'Total number of policy statements after merging: {len(aggregated_iam_policy_statements)}') if len(aggregated_iam_policy_statements) == 0: - aggregated_iam_policy_statements.append( - iam.PolicyStatement( - actions=['none:null'], resources=['*'], sid=EMPTY_STATEMENT_SID, effect=iam.Effect.ALLOW - ) - ) + log.info('Attaching empty policy statement') + empty_policy = self.generate_empty_policy() + log.info(empty_policy['Statement']) + aggregated_iam_policy_statements = empty_policy['Statement'] policy_statement_chunks = split_policy_statements_in_chunks(aggregated_iam_policy_statements) log.info(f'Number of policy chunks created: {len(policy_statement_chunks)}') + log.debug(policy_statement_chunks) log.info('Checking if there are any missing policies.') # Check if there are policies which do not exist but should have existed @@ -337,6 +287,10 @@ def merge_statements_and_update_policies( log.info(f'Creating missing policies for indexes: {missing_policies_indexes}') self._create_empty_policies_with_indexes(indexes=missing_policies_indexes) + log.info('Checking service quota limit for number of managed policies which can be attached to role') + # Check if managed policies can be attached to target requester role and new service policies do not exceed service quota limit + self._check_iam_managed_policy_attachment_limit(policy_statement_chunks) + # Check if the number of policies required are greater than currently present if len(policy_statement_chunks) > len(share_managed_policies_name_list): additional_policy_indexes = list(range(len(share_managed_policies_name_list), len(policy_statement_chunks))) @@ -356,10 +310,12 @@ def merge_statements_and_update_policies( policy_document = self._generate_policy_document_from_statements(statement_chunk) # If statement length is greater than 1 then check if has empty statements sid and remove it if len(policy_document.get('Statement')) > 1: + log.info('Removing empty policy statements') policy_document = S3SharePolicyService.remove_empty_statement( policy_doc=policy_document, statement_sid=EMPTY_STATEMENT_SID ) policy_name = self.generate_indexed_policy_name(index=index) + log.info(f'Policy document before putting is: {policy_document}') IAM.update_managed_policy_default_version( account_id=self.account, region=self.region, @@ -376,19 +332,6 @@ def merge_statements_and_update_policies( log.info(f'Found more policies than needed. Deleting policies with indexes: {excess_policies_indexes}') self._delete_policies_with_indexes(indexes=excess_policies_indexes) - def _convert_to_iam_policy_statement(self, statements): - iam_policy_statements: List[iam.PolicyStatement] = [] - for statement in statements: - iam_policy_statements.append( - iam.PolicyStatement( - sid=statement.get('Sid'), - actions=S3SharePolicyService._convert_to_array(str, statement.get('Action')), - resources=S3SharePolicyService._convert_to_array(str, statement.get('Resource')), - effect=iam.Effect.ALLOW, - ) - ) - return iam_policy_statements - def _delete_policies_with_indexes(self, indexes): for index in indexes: policy_name = self.generate_indexed_policy_name(index=index) @@ -414,21 +357,19 @@ def _create_empty_policies_with_indexes(self, indexes): policy_document = self.generate_empty_policy() IAM.create_managed_policy(self.account, self.region, policy_name, json.dumps(policy_document)) - def _create_indexed_managed_policies(self, policy_statements: List[iam.PolicyStatement]): - # Todo : Call the split in chunks with a maximum number of policy thing - # Todo : If the number of policies exceed that send an email to the dataset requester that the IAM role policy limit is going to exceed - + def _create_indexed_managed_policies(self, policy_statements: List[Dict]): if not policy_statements: - policy_statements.append( - iam.PolicyStatement( - effect=iam.Effect.ALLOW, sid=EMPTY_STATEMENT_SID, actions=['none:null'], resources=['*'] - ) - ) + empty_policy = self.generate_empty_policy() + policy_statements = empty_policy['Statement'] policy_statement_chunks = split_policy_statements_in_chunks(policy_statements) - log.info(f'Number of Policy chunks made: {len(policy_statement_chunks)}') + log.info( + 'Checking service quota limit for number of managed policies which can be attached to role before converting' + ) + self._check_iam_managed_policy_attachment_limit(policy_statement_chunks) + policy_arns = [] for index, statement_chunk in enumerate(policy_statement_chunks): policy_document = self._generate_policy_document_from_statements(statement_chunk) @@ -439,6 +380,72 @@ def _create_indexed_managed_policies(self, policy_statements: List[iam.PolicySta return policy_arns + def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): + number_of_policies_needed = len(policy_document_chunks) + log.info(f'number_of_policies_needed: {number_of_policies_needed}') + policies_present = self.get_managed_policies() + log.info(f'policies_present: {policies_present}') + managed_policies_attached_to_role = IAM.get_attached_managed_policies_to_role( + account_id=self.account, region=self.region, role_name=self.role_name + ) + number_of_non_share_managed_policies_attached_to_role = len( + [policy for policy in managed_policies_attached_to_role if policy not in policies_present] + ) + log.info(f'number_of_non_share_managed_policies_attached_to_role: {number_of_non_share_managed_policies_attached_to_role}') + + managed_iam_policy_quota = self._get_managed_policy_quota() + if number_of_policies_needed + number_of_non_share_managed_policies_attached_to_role > managed_iam_policy_quota: + # Send an email notification to the requestors to increase the quota and then try again + log.error( + f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}' + ) + try: + ShareNotificationService( + session=None, dataset=self.dataset, share=self.share + ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) + except Exception as e: + log.error(f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}') + raise Exception( + f'Failed to process share as number of needed attached policies to the role is greater than the service quota limit: {managed_iam_policy_quota}' + ) + + log.info( + f'Role: {self.role_name} has capacity to attach managed policies for share with URI: {self.share.shareUri}' + ) + + def _get_managed_policy_quota(self): + # Get the number of managed policies which can be attached to the IAM role + service_quota_client = ServiceQuota(account_id=self.account, region=self.region) + service_code_list = service_quota_client.list_services() + log.info(f'service_code_list: {service_code_list}') + service_code = None + for service in service_code_list: + if service.get('ServiceName') == IAM_SERVICE_NAME: + service_code = service.get('ServiceCode') + break + + log.info(f'Found service code : {service_code}') + service_quota_code = None + if service_code: + service_quota_codes = service_quota_client.list_service_quota(service_code=service_code) + for service_quota_cd in service_quota_codes: + if service_quota_cd.get('QuotaName') == IAM_SERVICE_QUOTA_NAME: + service_quota_code = service_quota_cd.get('QuotaCode') + break + + log.info(f'service_quota_code: {service_quota_code}') + managed_iam_policy_quota = None + if service_quota_code: + managed_iam_policy_quota = service_quota_client.get_service_quota_value( + service_code=service_code, service_quota_code=service_quota_code + ) + + if managed_iam_policy_quota is None: + log.info('Defaulting to default max values') + managed_iam_policy_quota = DEFAULT_MAX_ATTACHABLE_MANAGED_POLICIES_ACCOUNT + + return managed_iam_policy_quota + @staticmethod def _get_segregated_policy_statements_from_policy(policy_document): policy_statements = policy_document.get('Statement', []) @@ -467,8 +474,6 @@ def _get_segregated_policy_statements_from_policy(policy_document): def add_resources_and_generate_split_statements(self, statements, target_resources, sid, resource_type): # Using _convert_to_array to convert to array if single resource is present and its not in array - # This is required while creating iam.PolicyStatement in the split_policy_with_resources_in_statements function. - # iam.PolicyStatement throws error if the resource & actions is not an array type s3_statements_resources: List[str] = [ resource for statement in statements @@ -477,9 +482,9 @@ def add_resources_and_generate_split_statements(self, statements, target_resourc for target_resource in target_resources: if target_resource not in s3_statements_resources: s3_statements_resources.append(target_resource) - statement_chunks: List[iam.PolicyStatement] = split_policy_with_resources_in_statements( + statement_chunks = split_policy_with_resources_in_statements( base_sid=sid, - effect=iam.Effect.ALLOW, + effect='Allow', actions=S3_ALLOWED_ACTIONS if resource_type == 's3' else [f'{resource_type}:*'], resources=s3_statements_resources, ) @@ -487,8 +492,6 @@ def add_resources_and_generate_split_statements(self, statements, target_resourc def remove_resources_and_generate_split_statements(self, statements, target_resources, sid, resource_type): # Using _convert_to_array to convert to array if single resource is present and its not in array - # This is required while creating iam.PolicyStatement in the split_policy_with_resources_in_statements function. - # iam.PolicyStatement throws error if the resource & actions is not an array type s3_statements_resources = [ resource for statement in statements @@ -501,7 +504,7 @@ def remove_resources_and_generate_split_statements(self, statements, target_reso statement_chunks = split_policy_with_resources_in_statements( base_sid=sid, - effect=iam.Effect.ALLOW, + effect='Allow', actions=S3_ALLOWED_ACTIONS if resource_type == 's3' else [f'{resource_type}:*'], resources=s3_statements_resources, ) @@ -515,11 +518,10 @@ def _convert_to_array(item_type, item): return [item] return item - def _generate_policy_document_from_statements(self, statements: List[iam.PolicyStatement]): + def _generate_policy_document_from_statements(self, statements: List[Dict]): if statements is None: - statements = [] - policy_doc = iam.PolicyDocument(statements=statements) - return policy_doc.to_json() + raise Exception('Provide valid statements while generating policy document from statement') + return {'Version': '2012-10-17', 'Statement': statements} def _generate_managed_policy_statements_from_inline_policies(self): """ @@ -527,28 +529,39 @@ def _generate_managed_policy_statements_from_inline_policies(self): If there are already shared resources, add them to the empty policy and remove the fake statement return: IAM policy document """ - existing_bucket_s3, existing_bucket_kms = self._get_policy_resources_from_inline_policy( + existing_bucket_s3_resources, existing_bucket_kms_resources = self._get_policy_resources_from_inline_policy( OLD_IAM_S3BUCKET_ROLE_POLICY ) - existing_access_points_s3, existing_access_points_kms = self._get_policy_resources_from_inline_policy( - OLD_IAM_ACCESS_POINT_ROLE_POLICY + existing_access_points_s3_resources, existing_access_points_kms_resources = ( + self._get_policy_resources_from_inline_policy(OLD_IAM_ACCESS_POINT_ROLE_POLICY) ) - log.info(f'Back-filling S3BUCKET sharing resources: S3={existing_bucket_s3}, KMS={existing_bucket_kms}') log.info( - f'Back-filling S3ACCESS POINTS sharing resources: S3={existing_access_points_s3}, KMS={existing_access_points_kms}' + f'Back-filling S3BUCKET sharing resources: S3={existing_bucket_s3_resources}, KMS={existing_bucket_kms_resources}' + ) + log.info( + f'Back-filling S3ACCESS POINTS sharing resources: S3={existing_access_points_s3_resources}, KMS={existing_access_points_kms_resources}' + ) + bucket_s3_statement, bucket_kms_statement = self._generate_statement_from_inline_resources( + existing_bucket_s3_resources, existing_bucket_kms_resources, IAM_S3_BUCKETS_STATEMENT_SID ) + access_points_s3_statement, access_points_kms_statement = self._generate_statement_from_inline_resources( + existing_access_points_s3_resources, + existing_access_points_kms_resources, + IAM_S3_ACCESS_POINTS_STATEMENT_SID, + ) + policy_statements = [] - if len(existing_bucket_s3 + existing_access_points_s3) > 0: + if len(existing_bucket_s3_resources + existing_access_points_s3_resources) > 0: # Split the statements in chunks existing_bucket_s3_statements = self._split_and_generate_statement_chunks( - statements_s3=existing_bucket_s3, statements_kms=existing_bucket_kms, sid=IAM_S3_BUCKETS_STATEMENT_SID + statements_s3=bucket_s3_statement, statements_kms=bucket_kms_statement, sid=IAM_S3_BUCKETS_STATEMENT_SID ) - existing_bucket_s3_access_point_statement = self._split_and_generate_statement_chunks( - statements_s3=existing_access_points_s3, - statements_kms=existing_access_points_kms, + existing_bucket_s3_access_point_statements = self._split_and_generate_statement_chunks( + statements_s3=access_points_s3_statement, + statements_kms=access_points_kms_statement, sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, ) - policy_statements = existing_bucket_s3_statements + existing_bucket_s3_access_point_statement + policy_statements = existing_bucket_s3_statements + existing_bucket_s3_access_point_statements log.debug(f'Created policy statements with length: {len(policy_statements)}') return policy_statements @@ -564,7 +577,7 @@ def _split_and_generate_statement_chunks(self, statements_s3, statements_kms, si aggregate_statements.extend( split_policy_with_resources_in_statements( base_sid=f'{sid}S3', - effect=iam.Effect.ALLOW, + effect='Allow', actions=['s3:List*', 's3:Describe*', 's3:GetObject'], resources=statement_resources, ) @@ -577,11 +590,30 @@ def _split_and_generate_statement_chunks(self, statements_s3, statements_kms, si ] aggregate_statements.extend( split_policy_with_resources_in_statements( - base_sid=f'{sid}KMS', effect=iam.Effect.ALLOW, actions=['kms:*'], resources=statement_resources + base_sid=f'{sid}KMS', effect='Allow', actions=['kms:*'], resources=statement_resources ) ) return aggregate_statements + def _generate_statement_from_inline_resources(self, bucket_s3_resources, bucket_kms_resources, base_sid): + bucket_s3_statement = [] + bucket_kms_statement = [] + if len(bucket_s3_resources) > 0: + bucket_s3_statement.append( + { + 'Sid': f'{base_sid}S3', + 'Effect': 'Allow', + 'Action': S3_ALLOWED_ACTIONS, + 'Resource': bucket_s3_resources, + } + ) + if len(bucket_kms_resources) > 0: + bucket_kms_statement.append( + {'Sid': f'{base_sid}KMS', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': bucket_kms_resources} + ) + log.info(f'Generated statement from resources: S3: {bucket_s3_statement}, KMS: {bucket_kms_statement}') + return bucket_s3_statement, bucket_kms_statement + def _get_policy_resources_from_inline_policy(self, policy_name): # This function can only be used for backwards compatibility where policies had statement[0] for s3 # and statement[1] for KMS permissions @@ -598,27 +630,6 @@ def _get_policy_resources_from_inline_policy(self, policy_name): log.error(f'Failed to retrieve the existing policy {policy_name}: {e} ') return [], [] - def _update_policy_resources_from_inline_policy(self, policy, statement_sid, existing_s3, existing_kms): - # This function can only be used for backwards compatibility where policies had statement[0] for s3 - # and statement[1] for KMS permissions - if len(existing_s3) > 0: - additional_policy = { - 'Sid': f'{statement_sid}S3', - 'Effect': 'Allow', - 'Action': ['s3:List*', 's3:Describe*', 's3:GetObject'], - 'Resource': existing_s3, - } - policy['Statement'].append(additional_policy) - if len(existing_kms) > 0: - additional_policy = { - 'Sid': f'{statement_sid}KMS', - 'Effect': 'Allow', - 'Action': ['kms:*'], - 'Resource': existing_kms, - } - policy['Statement'].append(additional_policy) - return policy - def _delete_old_inline_policies(self): for policy_name in [OLD_IAM_S3BUCKET_ROLE_POLICY, OLD_IAM_ACCESS_POINT_ROLE_POLICY]: try: diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index e7dc38c1f..26bbdd423 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -157,7 +157,7 @@ def _validate_iam_role_and_policy( attached = policy_manager.check_if_policies_attached() if not attached and not managed and not attachMissingPolicies: raise Exception( - f'Required customer managed policy {policy_manager.get_policies_unattached_to_role()} is not attached to role {principal_role_name}' + f'Required customer managed policies {policy_manager.get_policies_unattached_to_role()} are not attached to role {principal_role_name}' ) elif not attached: managed_policy_list = policy_manager.get_policies_unattached_to_role() diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 2b00b1163..d692b025d 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -1,5 +1,6 @@ import logging import json +import time from itertools import count from typing import List from warnings import warn @@ -38,6 +39,9 @@ from dataall.modules.shares_base.services.sharing_service import ShareData logger = logging.getLogger(__name__) +ACCESS_POINT_CREATION_TIME = 30 +ACCESS_POINT_CREATION_RETRIES = 10 +ACCESS_POINT_BACKOFF_COEFFICIENT = 1.1 # every time increase retry delay by 10% class S3AccessPointShareManager: @@ -169,34 +173,45 @@ def check_target_role_access_policy(self) -> None: region=self.target_environment.region, role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, + share=self.share, + dataset=self.dataset, ) share_policy_service.initialize_statements() share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + is_managed_policies_exists = share_policy_service.check_if_managed_policies_exists() # Checking if managed policies without indexes are present. This is used for backward compatibility # Check this with AWS team - warn( - "Convert all your share's requestor policies to managed policies with indexes. Deprecation >= ?? ", - DeprecationWarning, - stacklevel=2, - ) - old_managed_policy_name = share_policy_service.generate_old_policy_name() - if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): - logger.info( - f'No managed policy exists for the role: {self.target_requester_IAMRoleName}, Reapply share create managed policies.' + if not is_managed_policies_exists: + warn( + "Convert all your share's requestor policies to managed policies with indexes. Deprecation >= ?? ", + DeprecationWarning, + stacklevel=2, ) - self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return + old_managed_policy_name = share_policy_service.generate_old_policy_name() + if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + logger.info( + f'No managed policy exists for the role: {self.target_requester_IAMRoleName}, Reapply share create managed policies.' + ) + self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return - if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): - logger.info( - f'Older version of managed policy present which is without index. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' - ) - self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return + if share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + logger.info( + f'Old managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' + ) + self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return - if not share_policy_service.check_if_managed_policies_exists(): + if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): + logger.info( + f'Older version of managed policy present which is without index. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' + ) + self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return + + if not is_managed_policies_exists: logger.info(f'IAM Policy {share_resource_policy_name} does not exist') self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return @@ -325,6 +340,8 @@ def grant_target_role_access_policy(self): region=self.target_environment.region, role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, + share=self.share, + dataset=self.dataset, ) share_policy_service.process_backwards_compatibility_for_target_iam_roles() share_policy_service.initialize_statements() @@ -349,11 +366,12 @@ def grant_target_role_access_policy(self): logger.info('Managed policies do not exist. Creating one') # Create a managed policy with naming convention and index share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + empty_policy = share_policy_service.generate_empty_policy() IAM.create_managed_policy( self.target_account_id, self.target_environment.region, share_resource_policy_name, - json.dumps(share_policy_service.generate_empty_policy()), + json.dumps(empty_policy), ) s3_kms_statement_chunks = [] @@ -469,9 +487,21 @@ def manage_access_point_and_policy(self): """ s3_client = S3ControlClient(self.source_account_id, self.source_environment.region) - access_point_arn = s3_client.create_bucket_access_point(self.bucket_name, self.access_point_name) + access_point_arn = s3_client.get_bucket_access_point_arn(self.access_point_name) if not access_point_arn: - raise Exception('Failed to create access point') + logger.info(f'Access point {self.access_point_name} does not exists, creating...') + access_point_arn = s3_client.create_bucket_access_point(self.bucket_name, self.access_point_name) + # Access point creation is slow + retries = 1 + sleep_coeff = 1 + while ( + not s3_client.get_bucket_access_point_arn(self.access_point_name) + and retries < ACCESS_POINT_CREATION_RETRIES + ): + logger.info('Waiting 30s for access point creation to complete..') + time.sleep(ACCESS_POINT_CREATION_TIME * sleep_coeff) + sleep_coeff = sleep_coeff * ACCESS_POINT_BACKOFF_COEFFICIENT + retries += 1 existing_policy = s3_client.get_access_point_policy(self.access_point_name) # requester will use this role to access resources target_requester_id = SessionHelper.get_role_id( @@ -516,7 +546,6 @@ def manage_access_point_and_policy(self): self.s3_prefix, perms_to_actions(self.share.permissions, SidType.BucketPolicy), ) - s3_client.attach_access_point_policy( access_point_name=self.access_point_name, policy=json.dumps(access_point_policy) ) @@ -676,6 +705,8 @@ def revoke_target_role_access_policy(self): region=self.target_environment.region, role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, + share=self.share, + dataset=self.dataset, ) share_policy_service.process_backwards_compatibility_for_target_iam_roles() share_policy_service.initialize_statements() diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index 1db189ac9..5a5e1ea8c 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -82,32 +82,43 @@ def check_s3_iam_access(self) -> None: region=self.target_environment.region, role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, + share=self.share, + dataset=self.dataset, ) share_policy_service.initialize_statements() share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + is_managed_policies_exists = share_policy_service.check_if_managed_policies_exists() - warn( - "Convert all your share's requestor policies to managed policies with indexes. Deprecation >= ?? ", - DeprecationWarning, - stacklevel=2, - ) - old_managed_policy_name = share_policy_service.generate_old_policy_name() - if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): - logger.info( - f'No managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' + if not is_managed_policies_exists: + warn( + "Convert all your share's requestor policies to managed policies with indexes. Deprecation >= ?? ", + DeprecationWarning, + stacklevel=2, ) - self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return + old_managed_policy_name = share_policy_service.generate_old_policy_name() + if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + logger.info( + f'No managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' + ) + self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return - if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): - logger.info( - f'Older version of managed policy present which is without index. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' - ) - self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return + if share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + logger.info( + f'Old managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' + ) + self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return + + if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): + logger.info( + f'Older version of managed policy without index present. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' + ) + self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) + return - if not share_policy_service.check_if_managed_policies_exists(): + if not is_managed_policies_exists: logger.info(f'IAM Policy {share_resource_policy_name} does not exist') self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return @@ -233,6 +244,8 @@ def grant_s3_iam_access(self): region=self.target_environment.region, role_name=self.target_requester_IAMRoleName, resource_prefix=self.target_environment.resourcePrefix, + share=self.share, + dataset=self.dataset, ) share_policy_service.process_backwards_compatibility_for_target_iam_roles() share_policy_service.initialize_statements() @@ -253,11 +266,12 @@ def grant_s3_iam_access(self): logger.info('Managed policies do not exist. Creating one') # Create a managed policy with naming convention and index share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) + empty_policy = share_policy_service.generate_empty_policy() IAM.create_managed_policy( self.target_account_id, self.target_environment.region, share_resource_policy_name, - json.dumps(share_policy_service.generate_empty_policy()), + json.dumps(empty_policy), ) s3_kms_statement_chunks = [] @@ -542,6 +556,8 @@ def delete_target_role_access_policy( region=self.target_environment.region, environmentUri=target_environment.environmentUri, resource_prefix=target_environment.resourcePrefix, + share=self.share, + dataset=self.dataset, ) share_policy_service.process_backwards_compatibility_for_target_iam_roles() share_policy_service.initialize_statements() diff --git a/backend/dataall/modules/shares_base/services/share_notification_service.py b/backend/dataall/modules/shares_base/services/share_notification_service.py index f8e664883..ee4e9c734 100644 --- a/backend/dataall/modules/shares_base/services/share_notification_service.py +++ b/backend/dataall/modules/shares_base/services/share_notification_service.py @@ -24,6 +24,7 @@ class DataSharingNotificationType(enum.Enum): SHARE_OBJECT_EXTENSION_REJECTED = 'SHARE_OBJECT_EXTENSION_REJECTED' SHARE_OBJECT_REJECTED = 'SHARE_OBJECT_REJECTED' SHARE_OBJECT_PENDING_APPROVAL = 'SHARE_OBJECT_PENDING_APPROVAL' + SHARE_OBJECT_FAILED = 'SHARE_OBJECT_FAILED' DATASET_VERSION = 'DATASET_VERSION' @@ -109,6 +110,38 @@ def notify_persistent_email_reminder(self, email_id: str): ) return notifications + def notify_managed_policy_limit_exceeded_action(self, email_id: str): + share_link_text = '' + if os.environ.get('frontend_domain_url'): + share_link_text = ( + f'

Please visit data.all share link ' + f'to view more details.' + ) + + msg_intro = f"""Dear User, + + We are contacting you because for a share requested by {email_id} failed because no new managed policy can be attached to your IAM role {self.share.principalRoleName}. + Please check the service quota for the managed policies that can be attached to a role in your aws account and increase the limit. + For reference please take a look at this link - https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-quotas-entities + + Note - Previously made shares are not affected but all new shares will be failed till the time you increase the IAM quota limit. + """ + + msg_end = """Your prompt attention in this matter is greatly appreciated. +

Best regards, +
The Data.all Team + """ + + subject = 'URGENT: Data.all | Action Required due to failing share' + email_notification_msg = msg_intro + share_link_text + msg_end + + self._create_and_send_email_notifications( + subject=subject, + msg=email_notification_msg, + recipient_groups_list=[self.share.groupUri], + ) + def notify_share_object_approval(self, email_id: str): share_link_text = '' if os.environ.get('frontend_domain_url'): diff --git a/backend/docker/dev/Dockerfile b/backend/docker/dev/Dockerfile index 0b6a4c46b..0e33ba5fd 100644 --- a/backend/docker/dev/Dockerfile +++ b/backend/docker/dev/Dockerfile @@ -27,7 +27,6 @@ RUN chown -R ${CONTAINER_USER}:root /tmp USER ${CONTAINER_USER} - ## Add source WORKDIR /build diff --git a/frontend/src/modules/Environments/components/EnvironmentTeams.js b/frontend/src/modules/Environments/components/EnvironmentTeams.js index eb89902b8..2ce2a3cc5 100644 --- a/frontend/src/modules/Environments/components/EnvironmentTeams.js +++ b/frontend/src/modules/Environments/components/EnvironmentTeams.js @@ -43,7 +43,11 @@ import { } from 'design'; import { SET_ERROR, useDispatch } from 'globalErrors'; import { isFeatureEnabled } from 'utils'; -import { useClient, useFetchGroups } from 'services'; +import { + getConsumptionRolePolicies, + useClient, + useFetchGroups +} from 'services'; import { generateEnvironmentAccessToken, getEnvironmentAssumeRoleUrl, @@ -262,6 +266,91 @@ TeamRow.propTypes = { isTeamEditModalOpenId: PropTypes.string }; +export const IAMRolePolicyDataGridCell = ({ environmentUri, IAMRoleName }) => { + const [isLoading, setLoading] = useState(true); + const [managedPolicyDetails, setManagedPolicyDetails] = useState(null); + const dispatch = useDispatch(); + const { enqueueSnackbar } = useSnackbar(); + const client = useClient(); + + useEffect(() => { + if (client) { + getRolePolicies().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); + } + }, [client, dispatch, enqueueSnackbar]); + + const getRolePolicies = async () => { + setLoading(true); + try { + const response = await client.query( + getConsumptionRolePolicies({ + environmentUri: environmentUri, + IAMRoleName: IAMRoleName + }) + ); + if (!response.errors) { + setManagedPolicyDetails(response.data.getConsumptionRolePolicies); + } else { + dispatch({ type: SET_ERROR, error: response.errors[0].message }); + } + } catch (e) { + dispatch({ type: SET_ERROR, error: e.message }); + } finally { + setLoading(false); + } + }; + + return ( + + {isLoading ? ( + + ) : ( + + + { + await navigator.clipboard.writeText( + managedPolicyDetails.map((policy) => policy.policy_name) + ); + enqueueSnackbar('Policy Name is copied to clipboard', { + anchorOrigin: { + horizontal: 'right', + vertical: 'top' + }, + variant: 'success' + }); + }} + > + + + + )} + + ); +}; + +IAMRolePolicyDataGridCell.propTypes = { + environmentUri: PropTypes.any, + IAMRoleName: PropTypes.any +}; + export const EnvironmentTeams = ({ environment }) => { const client = useClient(); const dispatch = useDispatch(); @@ -720,45 +809,10 @@ export const EnvironmentTeams = ({ environment }) => { headerName: 'IAM Policies', flex: 0.5, renderCell: (params: GridRenderCellParams) => ( - - - { - await navigator.clipboard.writeText( - params.row.managedPolicies.map( - (policy) => policy.policy_name - ) - ); - enqueueSnackbar( - 'Policy Name is copied to clipboard', - { - anchorOrigin: { - horizontal: 'right', - vertical: 'top' - }, - variant: 'success' - } - ); - }} - > - - - + ) }, { diff --git a/frontend/src/modules/Environments/services/listAllEnvironmentConsumptionRoles.js b/frontend/src/modules/Environments/services/listAllEnvironmentConsumptionRoles.js index 04cd522ea..010bebdfb 100644 --- a/frontend/src/modules/Environments/services/listAllEnvironmentConsumptionRoles.js +++ b/frontend/src/modules/Environments/services/listAllEnvironmentConsumptionRoles.js @@ -29,11 +29,6 @@ export const listAllEnvironmentConsumptionRoles = ({ groupUri IAMRoleArn dataallManaged - managedPolicies { - policy_type - policy_name - attached - } } } } From e131da7780029174e90e741b298483ae8ddcc309 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 16 Oct 2024 14:31:09 -0500 Subject: [PATCH 03/30] Corrections to unit tests --- .../s3_share_managed_policy_service.py | 24 +- .../s3_access_point_share_manager.py | 10 +- .../share_managers/s3_bucket_share_manager.py | 6 - .../test_s3_access_point_share_manager.py | 206 ++++++++- .../tasks/test_s3_bucket_share_manager.py | 430 +++++++++++++++--- .../modules/s3_datasets_shares/test_share.py | 80 +++- 6 files changed, 625 insertions(+), 131 deletions(-) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 27c7fb515..c04db57c6 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -317,11 +317,11 @@ def merge_statements_and_update_policies( policy_name = self.generate_indexed_policy_name(index=index) log.info(f'Policy document before putting is: {policy_document}') IAM.update_managed_policy_default_version( - account_id=self.account, - region=self.region, - policy_name=policy_name, - old_version_id=self.policy_version_map.get(policy_name), - policy_document=json.dumps(policy_document), + self.account, + self.region, + policy_name, + self.policy_version_map.get(policy_name, 'v1'), + json.dumps(policy_document), ) # Deleting excess policies @@ -342,12 +342,10 @@ def _delete_policies_with_indexes(self, indexes): IAM.detach_policy_from_role( account_id=self.account, region=self.region, role_name=self.role_name, policy_name=policy_name ) - IAM.delete_managed_policy_non_default_versions( - account_id=self.account, region=self.region, policy_name=policy_name - ) - IAM.delete_managed_policy_by_name( - account_id=self.account, region=self.region, policy_name=policy_name - ) + IAM.delete_managed_policy_non_default_versions( + account_id=self.account, region=self.region, policy_name=policy_name + ) + IAM.delete_managed_policy_by_name(account_id=self.account, region=self.region, policy_name=policy_name) else: log.info(f'Policy with name {policy_name} does not exist') @@ -391,7 +389,9 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): number_of_non_share_managed_policies_attached_to_role = len( [policy for policy in managed_policies_attached_to_role if policy not in policies_present] ) - log.info(f'number_of_non_share_managed_policies_attached_to_role: {number_of_non_share_managed_policies_attached_to_role}') + log.info( + f'number_of_non_share_managed_policies_attached_to_role: {number_of_non_share_managed_policies_attached_to_role}' + ) managed_iam_policy_quota = self._get_managed_policy_quota() if number_of_policies_needed + number_of_non_share_managed_policies_attached_to_role > managed_iam_policy_quota: diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index d692b025d..f06d35e48 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -232,7 +232,7 @@ def check_target_role_access_policy(self) -> None: ] if not S3SharePolicyService.check_if_sid_exists( - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', share_policy_service.total_s3_stmts + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', share_policy_service.total_s3_access_point_stmts ): logger.info(f'IAM Policy Statement with base Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 does not exist') self.folder_errors.append( @@ -246,7 +246,7 @@ def check_target_role_access_policy(self) -> None: ) elif not share_policy_service.check_resource_in_policy_statements( target_resources=s3_target_resources, - existing_policy_statements=share_policy_service.total_s3_stmts, + existing_policy_statements=share_policy_service.total_s3_access_point_stmts, ): logger.info( f'IAM Policy Statement with Sid {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3- does not contain resources {s3_target_resources}' @@ -262,7 +262,7 @@ def check_target_role_access_policy(self) -> None: ) else: policy_sid_actions_map = share_policy_service.check_s3_actions_in_policy_statement( - existing_policy_statements=share_policy_service.total_s3_stmts + existing_policy_statements=share_policy_service.total_s3_access_point_stmts ) for sid in policy_sid_actions_map: policy_check = policy_sid_actions_map[sid].get('policy_check') @@ -296,7 +296,7 @@ def check_target_role_access_policy(self) -> None: kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] if not S3SharePolicyService.check_if_sid_exists( - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', share_policy_service.total_s3_kms_stmts + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', share_policy_service.total_s3_access_point_kms_stmts ): logger.info( f'IAM Policy Statement with base Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS- does not exist' @@ -312,7 +312,7 @@ def check_target_role_access_policy(self) -> None: ) elif not share_policy_service.check_resource_in_policy_statements( target_resources=kms_target_resources, - existing_policy_statements=share_policy_service.total_s3_kms_stmts, + existing_policy_statements=share_policy_service.total_s3_access_point_kms_stmts, ): logger.info( f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS- does not contain resources {kms_target_resources}' diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index 5a5e1ea8c..5aec5262f 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -575,12 +575,6 @@ def delete_target_role_access_policy( if kms_key_id: kms_target_resources = [f'arn:aws:kms:{target_bucket.region}:{target_bucket.AwsAccountId}:key/{kms_key_id}'] - managed_policy_exists = share_policy_service.check_if_managed_policies_exists() - - if not managed_policy_exists: - logger.info(f'Managed policies for share with uri: {share.shareUri} are not found') - return - s3_kms_statement_chunks = [] s3_statements = share_policy_service.total_s3_stmts diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py index 65f7dcce7..d3e546757 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py @@ -241,7 +241,7 @@ def _create_target_dataset_access_control_policy(bucket_name, access_point_name) 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': S3_ALLOWED_ACTIONS, 'Resource': [ @@ -252,7 +252,7 @@ def _create_target_dataset_access_control_policy(bucket_name, access_point_name) ], }, { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:eu-west-1:{SOURCE_ENV_ACCOUNT}:key/some-key-2112'], @@ -335,16 +335,39 @@ def test_grant_target_role_access_policy_test_empty_policy( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', initial_policy_document) ) + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + iam_update_role_policy_mock = mocker.patch( 'dataall.base.aws.iam.IAM.update_managed_policy_default_version', return_value=None, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) @@ -354,7 +377,7 @@ def test_grant_target_role_access_policy_test_empty_policy( 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': S3_ALLOWED_ACTIONS, 'Resource': [ @@ -365,7 +388,7 @@ def test_grant_target_role_access_policy_test_empty_policy( ], }, { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:{dataset1.region}:{dataset1.AwsAccountId}:key/kms-key'], @@ -385,7 +408,7 @@ def test_grant_target_role_access_policy_test_empty_policy( account=target_environment.AwsAccountId, region=target_environment.region, resource_prefix=target_environment.resourcePrefix, - ).generate_base_policy_name() + ).generate_indexed_policy_name(index=0) # Then iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -405,12 +428,33 @@ def test_grant_target_role_access_policy_existing_policy_bucket_not_included( ) mocker.patch('dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', iam_policy)) + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) @@ -427,26 +471,27 @@ def test_grant_target_role_access_policy_existing_policy_bucket_not_included( # Then iam_update_role_policy_mock.assert_called() + policy_object = json.loads(iam_update_role_policy_mock.call_args.args[4]) - # Iam function is called with str from object so we transform back to object - policy_object = iam_policy s3_index = S3SharePolicyService._get_statement_by_sid( - policy=policy_object, sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3' + policy=policy_object, sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S31' ) kms_index = S3SharePolicyService._get_statement_by_sid( - policy=policy_object, sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS' + policy=policy_object, sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS1' ) # Assert that bucket_name is inside the resource array of policy object assert location1.S3BucketName in ','.join(policy_object['Statement'][s3_index]['Resource']) assert ( f'arn:aws:kms:{dataset1.region}:{dataset1.AwsAccountId}:key/kms-key' - in iam_policy['Statement'][kms_index]['Resource'] - and 'kms:*' in iam_policy['Statement'][kms_index]['Action'] + in policy_object['Statement'][kms_index]['Resource'] + and 'kms:*' in policy_object['Statement'][kms_index]['Action'] ) # Assert that statements for S3 bucket sharing are unaffected - s3_index = S3SharePolicyService._get_statement_by_sid(policy=policy_object, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3') + s3_index = S3SharePolicyService._get_statement_by_sid( + policy=policy_object, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S31' + ) def test_grant_target_role_access_policy_existing_policy_bucket_included(mocker, share_manager): @@ -461,9 +506,32 @@ def test_grant_target_role_access_policy_existing_policy_bucket_included(mocker, 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0', 'policy-2'] + ) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0', 'policy-2']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) @@ -883,7 +951,7 @@ def test_delete_target_role_access_policy_no_remaining_statement( 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [ @@ -894,7 +962,7 @@ def test_delete_target_role_access_policy_no_remaining_statement( ], }, { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:{dataset1.region}:{dataset1.AwsAccountId}:key/kms-key'], @@ -904,7 +972,7 @@ def test_delete_target_role_access_policy_no_remaining_statement( expected_remaining_target_role_policy = { 'Version': '2012-10-17', - 'Statement': [{'Sid': EMPTY_STATEMENT_SID, 'Effect': 'Allow', 'Action': 'none:null', 'Resource': '*'}], + 'Statement': [{'Sid': EMPTY_STATEMENT_SID, 'Effect': 'Allow', 'Action': ['none:null'], 'Resource': ['*']}], } # Given @@ -913,6 +981,18 @@ def test_delete_target_role_access_policy_no_remaining_statement( ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) @@ -921,6 +1001,18 @@ def test_delete_target_role_access_policy_no_remaining_statement( return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch( + 'dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0', 'policy-2'] + ) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0', 'policy-2']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + iam_update_role_policy_mock = mocker.patch( 'dataall.base.aws.iam.IAM.update_managed_policy_default_version', return_value=None, @@ -943,7 +1035,7 @@ def test_delete_target_role_access_policy_no_remaining_statement( account=target_environment.AwsAccountId, region=target_environment.region, resource_prefix=target_environment.resourcePrefix, - ).generate_base_policy_name() + ).generate_indexed_policy_name(index=0) iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -968,7 +1060,7 @@ def test_delete_target_role_access_policy_with_remaining_statement( 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [ @@ -980,7 +1072,7 @@ def test_delete_target_role_access_policy_with_remaining_statement( ], }, { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [ @@ -995,13 +1087,13 @@ def test_delete_target_role_access_policy_with_remaining_statement( 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S31', 'Effect': 'Allow', - 'Action': ['s3:*'], + 'Action': ['s3:List*', 's3:Describe*', 's3:GetObject'], 'Resource': ['arn:aws:s3:::UNRELATED_BUCKET_ARN'], }, { - 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:us-east-1:121231131212:key/some-key-2112'], @@ -1016,7 +1108,7 @@ def test_delete_target_role_access_policy_with_remaining_statement( mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( @@ -1024,6 +1116,29 @@ def test_delete_target_role_access_policy_with_remaining_statement( return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + iam_update_role_policy_mock = mocker.patch( 'dataall.base.aws.iam.IAM.update_managed_policy_default_version', return_value=None, @@ -1047,7 +1162,7 @@ def test_delete_target_role_access_policy_with_remaining_statement( account=target_environment.AwsAccountId, region=target_environment.region, resource_prefix=target_environment.resourcePrefix, - ).generate_base_policy_name() + ).generate_indexed_policy_name(index=0) iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -1224,6 +1339,19 @@ def test_check_target_role_access_policy(mocker, share_manager): _create_target_dataset_access_control_policy(share_manager.bucket_name, share_manager.access_point_name), ), ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) mocker.patch( 'dataall.base.aws.iam.IAM.get_role_arn_by_name', @@ -1253,6 +1381,20 @@ def test_check_target_role_access_policy_wrong_permissions(mocker, share_manager return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + bad_policy = _create_target_dataset_access_control_policy( share_manager.bucket_name, share_manager.access_point_name ) @@ -1300,6 +1442,20 @@ def test_check_target_role_access_policy_existing_policy_bucket_and_key_not_incl return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + # Gets policy with other S3 and KMS iam_get_policy_mock = mocker.patch( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py index a8d175e56..7522a5c54 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py @@ -420,6 +420,29 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): # Backwards compatibility: check that the create_managed_policy_from_inline_and_delete_inline is called # Check if the get and update_role_policy func are called and policy statements are added + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=False, + ) + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, @@ -436,12 +459,12 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): return_value='arn:iam::someArn', ) share_policy_service_mock_2 = mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policy', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) share_policy_service_mock_3 = mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=False, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, ) iam_update_role_policy_mock_1 = mocker.patch( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', empty_policy_document) @@ -457,11 +480,10 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): share_policy_service_mock_3.assert_called_once() iam_update_role_policy_mock_1.assert_called_once() iam_update_role_policy_mock_2.assert_called_once() + iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) - iam_policy = empty_policy_document - - s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3') - kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS') + s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S31') + kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1') # Assert if the IAM role policy with S3 and KMS permissions was created assert len(iam_policy['Statement']) == 2 @@ -482,18 +504,42 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): def test_grant_s3_iam_access_with_empty_policy(mocker, dataset2, share2_manager): # Given - # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns True) + # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns False but check_if_managed_policies_exists returns True, indicating that IAM indexed IAM policies exist) # And the IAM Policy is empty (get_managed_policy_default_version returns initial_policy_document) # Check if the get and update_role_policy func are called and policy statements are added + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) + kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' @@ -513,11 +559,10 @@ def test_grant_s3_iam_access_with_empty_policy(mocker, dataset2, share2_manager) # Assert IAM called iam_update_role_policy_mock_1.assert_called_once() iam_update_role_policy_mock_2.assert_called_once() + iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) - iam_policy = initial_policy_document - - s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3') - kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS') + s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S31') + kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1') # Assert if the IAM role policy with S3 and KMS permissions was created assert len(iam_policy['Statement']) == 2 @@ -538,7 +583,7 @@ def test_grant_s3_iam_access_with_empty_policy(mocker, dataset2, share2_manager) def test_grant_s3_iam_access_with_policy_and_target_resources_not_present(mocker, dataset2, share2_manager): # Given - # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns True) + # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns False but check_if_managed_policies_exists returns True, indicating that IAM indexed IAM policies exist) # And the IAM Policy is NOT empty (get_managed_policy_default_version returns policy) # Check if the get and update_role_policy func are called and policy statements are added to the existing ones @@ -546,13 +591,13 @@ def test_grant_s3_iam_access_with_policy_and_target_resources_not_present(mocker 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [f'arn:aws:s3:::S3Bucket', f'arn:aws:s3:::S3Bucket/*'], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:us-east-1:12121121121:key/some-kms-key'], @@ -560,16 +605,44 @@ def test_grant_s3_iam_access_with_policy_and_target_resources_not_present(mocker ], } + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) - s3_index = S3SharePolicyService._get_statement_by_sid(policy=policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3') - kms_index = S3SharePolicyService._get_statement_by_sid(policy=policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS') + + s3_index = S3SharePolicyService._get_statement_by_sid(policy=policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S31') + kms_index = S3SharePolicyService._get_statement_by_sid(policy=policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1') assert len(policy['Statement']) == 2 assert len(policy['Statement'][s3_index]['Resource']) == 2 @@ -590,7 +663,7 @@ def test_grant_s3_iam_access_with_policy_and_target_resources_not_present(mocker iam_update_role_policy_mock_1.assert_called_once() iam_update_role_policy_mock_2.assert_called_once() - iam_policy = policy + iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) # Assert that new resources were appended assert len(policy['Statement']) == 2 @@ -605,7 +678,7 @@ def test_grant_s3_iam_access_with_policy_and_target_resources_not_present(mocker def test_grant_s3_iam_access_with_complete_policy_present(mocker, dataset2, share2_manager): # Given - # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns True) + # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns False but check_if_managed_policies_exists returns True, indicating that indexed IAM policies exist) # And the IAM Policy is NOT empty and already contains all target resources (get_managed_policy_default_version returns policy) # Check if policy created after calling function and the existing Policy are the same @@ -613,9 +686,9 @@ def test_grant_s3_iam_access_with_complete_policy_present(mocker, dataset2, shar 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', - 'Action': ['s3:*'], + 'Action': S3_ALLOWED_ACTIONS, 'Resource': [ f'arn:aws:s3:::{dataset2.S3BucketName}', f'arn:aws:s3:::{dataset2.S3BucketName}/*', @@ -624,7 +697,7 @@ def test_grant_s3_iam_access_with_complete_policy_present(mocker, dataset2, shar ], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:{dataset2.region}:{dataset2.AwsAccountId}:key/kms-key'], @@ -632,10 +705,37 @@ def test_grant_s3_iam_access_with_complete_policy_present(mocker, dataset2, shar ], } + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, @@ -655,10 +755,14 @@ def test_grant_s3_iam_access_with_complete_policy_present(mocker, dataset2, shar iam_update_role_policy_mock_1.assert_called_once() iam_update_role_policy_mock_2.assert_called_once() - created_iam_policy = policy + created_iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) - s3_index = S3SharePolicyService._get_statement_by_sid(policy=policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3') - kms_index = S3SharePolicyService._get_statement_by_sid(policy=policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS') + s3_index = S3SharePolicyService._get_statement_by_sid( + policy=created_iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S31' + ) + kms_index = S3SharePolicyService._get_statement_by_sid( + policy=created_iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1' + ) assert len(created_iam_policy['Statement']) == 2 assert ( @@ -922,6 +1026,33 @@ def test_delete_target_role_access_no_policy_no_other_resources_shared( 'Statement': [{'Sid': EMPTY_STATEMENT_SID, 'Effect': 'Allow', 'Action': 'none:null', 'Resource': '*'}], } + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, @@ -934,8 +1065,9 @@ def test_delete_target_role_access_no_policy_no_other_resources_shared( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.create_managed_policy_from_inline_and_delete_inline', return_value='arn:iam::someArn', ) + share_policy_service_mock_2 = mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policy', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) @@ -957,16 +1089,10 @@ def test_delete_target_role_access_no_policy_no_other_resources_shared( iam_update_role_policy_mock_2.assert_called_once() # Get the updated IAM policy and compare it with the existing one - updated_iam_policy = policy_document - s3_index = S3SharePolicyService._get_statement_by_sid( - policy=updated_iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3' - ) - kms_index = S3SharePolicyService._get_statement_by_sid( - policy=updated_iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS' - ) + updated_iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) assert len(updated_iam_policy['Statement']) == 1 - assert '*' == updated_iam_policy['Statement'][0]['Resource'] + assert ['*'] == updated_iam_policy['Statement'][0]['Resource'] def test_delete_target_role_access_policy_no_resource_of_datasets_s3_bucket( @@ -980,13 +1106,13 @@ def test_delete_target_role_access_policy_no_resource_of_datasets_s3_bucket( 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [f'arn:aws:s3:::someOtherBucket', f'arn:aws:s3:::someOtherBucket/*'], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:us-east-1:121231131212:key/some-key-2112'], @@ -994,10 +1120,37 @@ def test_delete_target_role_access_policy_no_resource_of_datasets_s3_bucket( ], } + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, @@ -1021,9 +1174,10 @@ def test_delete_target_role_access_policy_no_resource_of_datasets_s3_bucket( iam_update_role_policy_mock_2.assert_called_once() # Get the updated IAM policy and compare it with the existing one - updated_iam_policy = iam_policy - s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3') - kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS') + updated_iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) + + s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S31') + kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1') assert len(updated_iam_policy['Statement']) == 2 assert 'arn:aws:s3:::someOtherBucket,arn:aws:s3:::someOtherBucket/*' == ','.join( @@ -1045,7 +1199,7 @@ def test_delete_target_role_access_policy_with_multiple_s3_buckets_in_policy( 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [ @@ -1056,7 +1210,7 @@ def test_delete_target_role_access_policy_with_multiple_s3_buckets_in_policy( ], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [ @@ -1067,10 +1221,37 @@ def test_delete_target_role_access_policy_with_multiple_s3_buckets_in_policy( ], } + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, @@ -1093,10 +1274,10 @@ def test_delete_target_role_access_policy_with_multiple_s3_buckets_in_policy( iam_update_role_policy_mock_1.assert_called_once() iam_update_role_policy_mock_2.assert_called_once() - updated_iam_policy = iam_policy + updated_iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) - s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S3') - kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS') + s3_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}S31') + kms_index = S3SharePolicyService._get_statement_by_sid(policy=iam_policy, sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1') assert f'arn:aws:s3:::{dataset2.S3BucketName}' not in updated_iam_policy['Statement'][s3_index]['Resource'] assert f'arn:aws:s3:::{dataset2.S3BucketName}/*' not in updated_iam_policy['Statement'][s3_index]['Resource'] @@ -1124,7 +1305,7 @@ def test_delete_target_role_access_policy_with_one_s3_bucket_and_one_kms_resourc 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [ @@ -1133,7 +1314,7 @@ def test_delete_target_role_access_policy_with_one_s3_bucket_and_one_kms_resourc ], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:{dataset2.region}:{dataset2.AwsAccountId}:key/kms-key'], @@ -1141,10 +1322,38 @@ def test_delete_target_role_access_policy_with_one_s3_bucket_and_one_kms_resourc ], } + mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) + + mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', + return_value=10, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, @@ -1384,19 +1593,33 @@ def test_check_s3_iam_access(mocker, dataset2, share2_manager): 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:List*', 's3:Describe*', 's3:GetObject'], 'Resource': [f'arn:aws:s3:::{dataset2.S3BucketName}', f'arn:aws:s3:::{dataset2.S3BucketName}/*'], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:{dataset2.region}:{dataset2.AwsAccountId}:key/kms-key'], }, ], } + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, @@ -1427,19 +1650,33 @@ def test_check_s3_iam_access_wrong_actions(mocker, dataset2, share2_manager): 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [f'arn:aws:s3:::{dataset2.S3BucketName}', f'arn:aws:s3:::{dataset2.S3BucketName}/*'], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:{dataset2.region}:{dataset2.AwsAccountId}:key/kms-key'], }, ], } + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, @@ -1474,7 +1711,7 @@ def test_check_s3_iam_access_no_policy(mocker, dataset2, share2_manager): # Check if the update_role_policy func is called and policy statements are added # When policy does not exist - iam_update_role_policy_mock_1 = mocker.patch( + share_managerpolicy_mock_1 = mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) @@ -1484,7 +1721,7 @@ def test_check_s3_iam_access_no_policy(mocker, dataset2, share2_manager): # When share2_manager.check_s3_iam_access() # Then - iam_update_role_policy_mock_1.assert_called_once() + share_managerpolicy_mock_1.assert_called_once() assert (len(share2_manager.bucket_errors)) == 1 assert 'IAM Policy Target Resource' in share2_manager.bucket_errors[0] @@ -1494,8 +1731,28 @@ def test_check_s3_iam_access_policy_not_attached(mocker, dataset2, share2_manage # There is not existing IAM policy in the requesters account for the dataset's S3bucket # Check if the update_role_policy func is called and policy statements are added + policy = { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', + 'Effect': 'Allow', + 'Action': ['s3:*'], + 'Resource': [f'arn:aws:s3:::{dataset2.S3BucketName}', f'arn:aws:s3:::{dataset2.S3BucketName}/*'], + }, + { + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', + 'Effect': 'Allow', + 'Action': ['kms:*'], + 'Resource': [f'arn:aws:kms:{dataset2.region}:{dataset2.AwsAccountId}:key/kms-key'], + }, + ], + } + + mocker.patch('dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', policy)) + # When policy does not exist - iam_update_role_policy_mock_1 = mocker.patch( + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) @@ -1504,6 +1761,20 @@ def test_check_s3_iam_access_policy_not_attached(mocker, dataset2, share2_manage return_value=False, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + iam_update_role_policy_mock_1 = mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' # When @@ -1523,7 +1794,7 @@ def test_check_s3_iam_access_missing_policy_statement(mocker, dataset2, share2_m 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:us-east-1:12121121121:key/some-kms-key'], @@ -1538,6 +1809,19 @@ def test_check_s3_iam_access_missing_policy_statement(mocker, dataset2, share2_m 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) # Gets policy with other S3 and KMS iam_update_role_policy_mock_1 = mocker.patch( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', policy) @@ -1550,7 +1834,8 @@ def test_check_s3_iam_access_missing_policy_statement(mocker, dataset2, share2_m # Then iam_update_role_policy_mock_1.assert_called_once() assert ( - f'missing IAM Policy Statement permissions: {IAM_S3_BUCKETS_STATEMENT_SID}S3' in share2_manager.bucket_errors[0] + f'missing IAM Policy Statement Sid permissions: {IAM_S3_BUCKETS_STATEMENT_SID}S3' + in share2_manager.bucket_errors[0] ) assert ( f'missing IAM Policy Resource permissions: {IAM_S3_BUCKETS_STATEMENT_SID}KMS' in share2_manager.bucket_errors[1] @@ -1565,19 +1850,33 @@ def test_check_s3_iam_access_missing_target_resource(mocker, dataset2, share2_ma 'Version': '2012-10-17', 'Statement': [ { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}S31', 'Effect': 'Allow', 'Action': ['s3:*'], 'Resource': [f'arn:aws:s3:::S3Bucket', f'arn:aws:s3:::S3Bucket/*'], }, { - 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + 'Sid': f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS1', 'Effect': 'Allow', 'Action': ['kms:*'], 'Resource': [f'arn:aws:kms:us-east-1:12121121121:key/some-kms-key'], }, ], } + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, @@ -1599,7 +1898,8 @@ def test_check_s3_iam_access_missing_target_resource(mocker, dataset2, share2_ma iam_update_role_policy_mock_1.assert_called_once() assert (len(share2_manager.bucket_errors)) == 2 assert ( - f'missing IAM Policy Resource permissions: {IAM_S3_BUCKETS_STATEMENT_SID}S3' in share2_manager.bucket_errors[0] + f'missing IAM Policy Resource(s) permissions: {IAM_S3_BUCKETS_STATEMENT_SID}S3' + in share2_manager.bucket_errors[0] ) assert ( f'missing IAM Policy Resource permissions: {IAM_S3_BUCKETS_STATEMENT_SID}KMS' in share2_manager.bucket_errors[1] diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 91bf6b18d..7f6475565 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -1275,12 +1275,16 @@ def test_create_share_object_as_requester(mocker, client, user2, group2, env2gro # When a user that belongs to environment and group creates request mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1308,12 +1312,16 @@ def test_create_share_object_as_approver_and_requester(mocker, client, user, gro ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1341,12 +1349,16 @@ def test_create_share_object_invalid_account(mocker, client, user, group2, env2g ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1375,12 +1387,16 @@ def test_create_share_object_with_item_authorized( ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1424,14 +1440,22 @@ def test_create_share_object_share_policy_not_attached_attachMissingPolicies_ena ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=False, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value='some-policy', + ) attach_mocker = mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policy', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) create_share_object_response = create_share_object( @@ -1465,14 +1489,22 @@ def test_create_share_object_share_policy_not_attached_attachMissingPolicies_dis ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=False, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=True, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value='some-policy', + ) attach_mocker = mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policy', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) create_share_object_response = create_share_object( @@ -1506,12 +1538,20 @@ def test_create_share_object_share_policy_not_attached_attachMissingPolicies_dis ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=False, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value='some-policy', + ) consumption_role = MagicMock(spec_set=ConsumptionRole) consumption_role.IAMRoleName = 'randomName' consumption_role.IAMRoleArn = 'randomArn' @@ -1533,8 +1573,8 @@ def test_create_share_object_share_policy_not_attached_attachMissingPolicies_dis principalType=PrincipalType.ConsumptionRole.value, ) # Then share object is not created and an error appears - assert 'Required customer managed policy' in create_share_object_response.errors[0].message - assert 'is not attached to role randomName' in create_share_object_response.errors[0].message + assert 'Required customer managed policies' in create_share_object_response.errors[0].message + assert 'are not attached to role randomName' in create_share_object_response.errors[0].message def test_create_share_object_with_share_expiration_added( @@ -1546,10 +1586,14 @@ def test_create_share_object_with_share_expiration_added( ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) create_share_object_response = create_share_object( @@ -1586,10 +1630,10 @@ def test_create_share_object_with_non_expiring_share( ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) create_share_object_response = create_share_object( @@ -1620,10 +1664,10 @@ def test_create_share_object_with_share_expiration_incorrect_share_expiration( ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=True, + return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) create_share_object_response = create_share_object( From 3087bc6fb96c6ef09d8e200cf56a1652952b7a6a Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 16 Oct 2024 14:37:36 -0500 Subject: [PATCH 04/30] Service Quota file --- .../cdk/pivot_role_core_policies/__init__.py | 3 ++- .../cdk/pivot_role_core_policies/iam.py | 2 +- .../pivot_role_core_policies/service_quota.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py diff --git a/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py b/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py index 609d5ad3b..f4160c851 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py +++ b/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py @@ -10,6 +10,7 @@ sqs, ssm, sts, + service_quota ) -__all__ = ['cloudformation', 'iam', 'kms', 'logging', 's3', 'sns', 'sqs', 'ssm', 'sts'] +__all__ = ['cloudformation', 'iam', 'kms', 'logging', 's3', 'sns', 'sqs', 'ssm', 'sts', 'service_quota'] diff --git a/backend/dataall/core/environment/cdk/pivot_role_core_policies/iam.py b/backend/dataall/core/environment/cdk/pivot_role_core_policies/iam.py index b84083bbf..d2342c761 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_core_policies/iam.py +++ b/backend/dataall/core/environment/cdk/pivot_role_core_policies/iam.py @@ -13,7 +13,7 @@ def get_statements(self): statements = [ # IAM - needed for consumption roles and for S3 sharing iam.PolicyStatement( - sid='IAMListGet', effect=iam.Effect.ALLOW, actions=['iam:ListRoles', 'iam:Get*'], resources=['*'] + sid='IAMListGet', effect=iam.Effect.ALLOW, actions=['iam:List*', 'iam:Get*'], resources=['*'] ), iam.PolicyStatement( sid='PassRole', diff --git a/backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py b/backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py new file mode 100644 index 000000000..b59339c3d --- /dev/null +++ b/backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py @@ -0,0 +1,19 @@ +from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet +from aws_cdk import aws_iam as iam + + +class ServiceQuotaPivotRole(PivotRoleStatementSet): + """ + Class including all permissions needed by the pivot role to work with AWS Service Quota. + It allows pivot role to: + - List and Get Service Quota details + """ + + def get_statements(self): + statements = [ + # Service Quota - Needed to determine the number of service quotas for managed policies which can be attached + iam.PolicyStatement( + sid='ServiceQuotaListGet', effect=iam.Effect.ALLOW, actions=['servicequotas:List*', 'servicequotas:Get*'], resources=['*'] + ) + ] + return statements From db36fb0d8c58ae1d59539100ed06f8f20bc0ab5e Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 17 Oct 2024 14:06:34 -0500 Subject: [PATCH 05/30] Adding comments and other changes --- .../cdk/pivot_role_core_policies/__init__.py | 2 +- .../pivot_role_core_policies/service_quota.py | 5 +- .../services/managed_iam_policies.py | 16 +-- .../s3_share_managed_policy_service.py | 114 +++++++++++------- .../services/s3_share_validator.py | 2 +- .../s3_access_point_share_manager.py | 25 ++-- .../share_managers/s3_bucket_share_manager.py | 29 ++--- 7 files changed, 101 insertions(+), 92 deletions(-) diff --git a/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py b/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py index f4160c851..e7c2e7c09 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py +++ b/backend/dataall/core/environment/cdk/pivot_role_core_policies/__init__.py @@ -10,7 +10,7 @@ sqs, ssm, sts, - service_quota + service_quota, ) __all__ = ['cloudformation', 'iam', 'kms', 'logging', 's3', 'sns', 'sqs', 'ssm', 'sts', 'service_quota'] diff --git a/backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py b/backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py index b59339c3d..9895064d3 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py +++ b/backend/dataall/core/environment/cdk/pivot_role_core_policies/service_quota.py @@ -13,7 +13,10 @@ def get_statements(self): statements = [ # Service Quota - Needed to determine the number of service quotas for managed policies which can be attached iam.PolicyStatement( - sid='ServiceQuotaListGet', effect=iam.Effect.ALLOW, actions=['servicequotas:List*', 'servicequotas:Get*'], resources=['*'] + sid='ServiceQuotaListGet', + effect=iam.Effect.ALLOW, + actions=['servicequotas:List*', 'servicequotas:Get*'], + resources=['*'], ) ] return statements diff --git a/backend/dataall/core/environment/services/managed_iam_policies.py b/backend/dataall/core/environment/services/managed_iam_policies.py index c0450c0c8..685b760ec 100644 --- a/backend/dataall/core/environment/services/managed_iam_policies.py +++ b/backend/dataall/core/environment/services/managed_iam_policies.py @@ -65,7 +65,7 @@ def create_managed_policy_from_inline_and_delete_inline(self) -> str: ... @abstractmethod - def create_managed_indexed_policy_from_managed_policy(self) -> str: + def create_managed_indexed_policy_from_managed_policy_delete_old_policy(self) -> str: """ Returns policy ARNs and needs to be implemented in the ManagedPolicies inherited classes It is used for backwards compatibility. It should be deprecated and removed in future releases. @@ -191,11 +191,8 @@ def delete_all_policies(self) -> bool: # Check if policy with old naming format exists if not policy_name_list: old_managed_policy_name = policy_manager.generate_old_policy_name() - policy_name_list = ( - [old_managed_policy_name] - if policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) - else [] - ) + if policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name): + policy_name_list.append(old_managed_policy_name) for policy_name in policy_name_list: logger.info(f'Deleting policy {policy_name}') @@ -231,11 +228,8 @@ def get_all_policies(self) -> List[dict]: # Check if policy with old naming format exists if not policy_name_list: old_managed_policy_name = policy_manager.generate_old_policy_name() - policy_name_list = ( - [old_managed_policy_name] - if policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) - else [] - ) + if policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name): + policy_name_list.append(old_managed_policy_name) for policy_name in policy_name_list: policy_dict = { diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index c04db57c6..97226abee 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -41,7 +41,7 @@ def __init__(self, role_name, account, region, environmentUri, resource_prefix, self.resource_prefix = resource_prefix self.share = share self.dataset = dataset - self.policy_version_map = {} + self.policy_version_map = {} # Policy version map helps while updating policies self.total_s3_stmts: List[Any] = [] self.total_s3_kms_stmts: List[Any] = [] self.total_s3_access_point_stmts: List[Any] = [] @@ -74,6 +74,7 @@ def policy_type(self) -> str: return 'SharePolicy' def generate_old_policy_name(self) -> str: + # This function should be deprecated and removed in the future return NamingConventionService( target_label=f'env-{self.environmentUri}-share-policy', target_uri=self.role_name, @@ -82,6 +83,10 @@ def generate_old_policy_name(self) -> str: ).build_compliant_name() def generate_base_policy_name(self) -> str: + """ + Returns the base name of managed policies. This base name is without the index. + build_compliant_name_with_index() function generated the name of the policy considering the length needed for index. + """ return NamingConventionService( target_label=f'env-{self.environmentUri}-share-policy', target_uri=self.role_name, @@ -129,9 +134,9 @@ def check_resource_in_policy_statements(target_resources: list, existing_policy_ @staticmethod def check_s3_actions_in_policy_statement(existing_policy_statements: List[Any]) -> (bool, str, str): """ - Checks if all required s3 actions are allowed in the existing policy and there is no unallowed actions + Checks if all required s3 actions are allowed in the existing policy and there is no disallowed actions :param existing_policy_statements: - :return: bool, allowed missing actions string, not allowed actions string + :return: List[{ bool, allowed missing actions string, not allowed actions string }] """ s3_actions_checker_dict = {} for statement in existing_policy_statements: @@ -181,15 +186,18 @@ def create_managed_policy_from_inline_and_delete_inline(self): return policy_arns # Backwards compatibility - def create_managed_indexed_policy_from_managed_policy(self): + def create_managed_indexed_policy_from_managed_policy_delete_old_policy(self): """ - Previously, only one managed policy was created for a role on which share policy statement were attached. - Convert these old managed policies into indexed policies by splitting statements into chunks + Previously, only one managed policy was created for a role. + Convert this old managed policy into indexed policies by splitting statements into chunks + After converting and splitting, delete the old managed policy """ - policy_name = self.generate_old_policy_name() - log.info(f'Converting old managed policy with name: {policy_name} to indexed managed policy with index: 0') + old_managed_policy_name = self.generate_old_policy_name() + log.info( + f'Converting old managed policy with name: {old_managed_policy_name} to indexed managed policy with index: 0' + ) policy_document = IAM.get_managed_policy_document_by_name( - account_id=self.account, region=self.region, policy_name=policy_name + account_id=self.account, region=self.region, policy_name=old_managed_policy_name ) if not policy_document: @@ -221,26 +229,33 @@ def create_managed_indexed_policy_from_managed_policy(self): ) self._create_indexed_managed_policies(policy_statements) - if self.check_if_policy_attached(policy_name=policy_name): + if self.check_if_policy_attached(policy_name=old_managed_policy_name): IAM.detach_policy_from_role( - account_id=self.account, region=self.region, role_name=self.role_name, policy_name=policy_name + account_id=self.account, + region=self.region, + role_name=self.role_name, + policy_name=old_managed_policy_name, ) IAM.delete_managed_policy_non_default_versions( - account_id=self.account, region=self.region, policy_name=policy_name + account_id=self.account, region=self.region, policy_name=old_managed_policy_name + ) + IAM.delete_managed_policy_by_name( + account_id=self.account, region=self.region, policy_name=old_managed_policy_name ) - IAM.delete_managed_policy_by_name(account_id=self.account, region=self.region, policy_name=policy_name) def merge_statements_and_update_policies( self, target_sid: str, target_s3_statements: List[Any], target_s3_kms_statements: List[Any] ): """ - Based on target_sid: {} + This method is responsible for merging policy statement, re-generating chunks comprizing of statements. + Creates policies if needed and then updates policies with statement chunks. + Based on target_sid: 1. This method merges all the S3 statments 2. Splits the policy into policy chunks, where each chunk is <= size of the policy ( this is approximately true ) - 3. Check if there are any missing policies and creates them - 4. Check if extra policies are required and also checks if those policies can be attached to the role (At the time of writing, IAM role has limit of 10 managed policies but can be increased to 20 ) - 5. Once policies are creates, fill the policies with the policy chunks + 3. Check if there are any missing policies and create them + 4. Check if extra policies are required and also checks if those policies can be attached to the role (At the time of writing, IAM role has limit of 10 managed policies and can be increased to 20 ) + 5. Once policies are created, fill/update the policies with the policy chunks 6. Delete ( if any ) extra policies which are remaining """ share_managed_policies_name_list = self.get_managed_policies() @@ -274,9 +289,9 @@ def merge_statements_and_update_policies( log.info(empty_policy['Statement']) aggregated_iam_policy_statements = empty_policy['Statement'] - policy_statement_chunks = split_policy_statements_in_chunks(aggregated_iam_policy_statements) - log.info(f'Number of policy chunks created: {len(policy_statement_chunks)}') - log.debug(policy_statement_chunks) + policy_document_chunks = split_policy_statements_in_chunks(aggregated_iam_policy_statements) + log.info(f'Number of policy chunks created: {len(policy_document_chunks)}') + log.debug(policy_document_chunks) log.info('Checking if there are any missing policies.') # Check if there are policies which do not exist but should have existed @@ -287,13 +302,13 @@ def merge_statements_and_update_policies( log.info(f'Creating missing policies for indexes: {missing_policies_indexes}') self._create_empty_policies_with_indexes(indexes=missing_policies_indexes) - log.info('Checking service quota limit for number of managed policies which can be attached to role') # Check if managed policies can be attached to target requester role and new service policies do not exceed service quota limit - self._check_iam_managed_policy_attachment_limit(policy_statement_chunks) + log.info('Checking service quota limit for number of managed policies which can be attached to role') + self._check_iam_managed_policy_attachment_limit(policy_document_chunks) # Check if the number of policies required are greater than currently present - if len(policy_statement_chunks) > len(share_managed_policies_name_list): - additional_policy_indexes = list(range(len(share_managed_policies_name_list), len(policy_statement_chunks))) + if len(policy_document_chunks) > len(share_managed_policies_name_list): + additional_policy_indexes = list(range(len(share_managed_policies_name_list), len(policy_document_chunks))) log.info( f'Number of policies needed are more than existing number of policies. Creating policies with indexes: {additional_policy_indexes}' ) @@ -301,12 +316,13 @@ def merge_statements_and_update_policies( updated_share_managed_policies_name_list = self.get_managed_policies() + log.info('Updating policy_version_map for any newly created policies') # Update the dict tracking the policy version for new policies which were created for managed_policy_name in updated_share_managed_policies_name_list: if managed_policy_name not in self.policy_version_map: self.policy_version_map[managed_policy_name] = 'v1' - for index, statement_chunk in enumerate(policy_statement_chunks): + for index, statement_chunk in enumerate(policy_document_chunks): policy_document = self._generate_policy_document_from_statements(statement_chunk) # If statement length is greater than 1 then check if has empty statements sid and remove it if len(policy_document.get('Statement')) > 1: @@ -315,7 +331,7 @@ def merge_statements_and_update_policies( policy_doc=policy_document, statement_sid=EMPTY_STATEMENT_SID ) policy_name = self.generate_indexed_policy_name(index=index) - log.info(f'Policy document before putting is: {policy_document}') + log.debug(f'Policy document for policy {policy_name}: {policy_document}') IAM.update_managed_policy_default_version( self.account, self.region, @@ -325,9 +341,9 @@ def merge_statements_and_update_policies( ) # Deleting excess policies - if len(policy_statement_chunks) < len(updated_share_managed_policies_name_list): + if len(policy_document_chunks) < len(updated_share_managed_policies_name_list): excess_policies_indexes = list( - range(len(policy_statement_chunks), len(updated_share_managed_policies_name_list)) + range(len(policy_document_chunks), len(updated_share_managed_policies_name_list)) ) log.info(f'Found more policies than needed. Deleting policies with indexes: {excess_policies_indexes}') self._delete_policies_with_indexes(indexes=excess_policies_indexes) @@ -357,19 +373,22 @@ def _create_empty_policies_with_indexes(self, indexes): def _create_indexed_managed_policies(self, policy_statements: List[Dict]): if not policy_statements: + log.info( + 'No policy statements supplied while creating indexed managed policies. Creating an empty policy statement.' + ) empty_policy = self.generate_empty_policy() policy_statements = empty_policy['Statement'] - policy_statement_chunks = split_policy_statements_in_chunks(policy_statements) - log.info(f'Number of Policy chunks made: {len(policy_statement_chunks)}') + policy_document_chunks = split_policy_statements_in_chunks(policy_statements) + log.info(f'Number of Policy chunks made: {len(policy_document_chunks)}') log.info( 'Checking service quota limit for number of managed policies which can be attached to role before converting' ) - self._check_iam_managed_policy_attachment_limit(policy_statement_chunks) + self._check_iam_managed_policy_attachment_limit(policy_document_chunks) policy_arns = [] - for index, statement_chunk in enumerate(policy_statement_chunks): + for index, statement_chunk in enumerate(policy_document_chunks): policy_document = self._generate_policy_document_from_statements(statement_chunk) indexed_policy_name = self.generate_indexed_policy_name(index=index) policy_arns.append( @@ -380,9 +399,7 @@ def _create_indexed_managed_policies(self, policy_statements: List[Dict]): def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): number_of_policies_needed = len(policy_document_chunks) - log.info(f'number_of_policies_needed: {number_of_policies_needed}') policies_present = self.get_managed_policies() - log.info(f'policies_present: {policies_present}') managed_policies_attached_to_role = IAM.get_attached_managed_policies_to_role( account_id=self.account, region=self.region, role_name=self.role_name ) @@ -395,7 +412,7 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): managed_iam_policy_quota = self._get_managed_policy_quota() if number_of_policies_needed + number_of_non_share_managed_policies_attached_to_role > managed_iam_policy_quota: - # Send an email notification to the requestors to increase the quota and then try again + # Send an email notification to the requestors to increase the quota log.error( f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}' ) @@ -417,14 +434,12 @@ def _get_managed_policy_quota(self): # Get the number of managed policies which can be attached to the IAM role service_quota_client = ServiceQuota(account_id=self.account, region=self.region) service_code_list = service_quota_client.list_services() - log.info(f'service_code_list: {service_code_list}') service_code = None for service in service_code_list: if service.get('ServiceName') == IAM_SERVICE_NAME: service_code = service.get('ServiceCode') break - log.info(f'Found service code : {service_code}') service_quota_code = None if service_code: service_quota_codes = service_quota_client.list_service_quota(service_code=service_code) @@ -433,7 +448,6 @@ def _get_managed_policy_quota(self): service_quota_code = service_quota_cd.get('QuotaCode') break - log.info(f'service_quota_code: {service_quota_code}') managed_iam_policy_quota = None if service_quota_code: managed_iam_policy_quota = service_quota_client.get_service_quota_value( @@ -441,13 +455,16 @@ def _get_managed_policy_quota(self): ) if managed_iam_policy_quota is None: - log.info('Defaulting to default max values') managed_iam_policy_quota = DEFAULT_MAX_ATTACHABLE_MANAGED_POLICIES_ACCOUNT return managed_iam_policy_quota @staticmethod def _get_segregated_policy_statements_from_policy(policy_document): + """Function to split the policy document and collect policy statements relating to S3 & KMS for bucket and access point shares + policy_document: IAM policy document + returns: s3_statements, s3_kms_statements, s3_access_point_statements, s3_kms_access_point_statements + """ policy_statements = policy_document.get('Statement', []) s3_statements = [ policy_stmt @@ -473,6 +490,10 @@ def _get_segregated_policy_statements_from_policy(policy_document): return s3_statements, s3_kms_statements, s3_access_point_statements, s3_kms_access_point_statements def add_resources_and_generate_split_statements(self, statements, target_resources, sid, resource_type): + """ + Method which adds target resources to the statements & splits the statements in chunks + returns : policy statements chunks + """ # Using _convert_to_array to convert to array if single resource is present and its not in array s3_statements_resources: List[str] = [ resource @@ -491,7 +512,10 @@ def add_resources_and_generate_split_statements(self, statements, target_resourc return statement_chunks def remove_resources_and_generate_split_statements(self, statements, target_resources, sid, resource_type): - # Using _convert_to_array to convert to array if single resource is present and its not in array + """ + Method which removes target resources from the statements & splits the statements in chunks + returns : policy statements chunks + """ s3_statements_resources = [ resource for statement in statements @@ -519,6 +543,9 @@ def _convert_to_array(item_type, item): return item def _generate_policy_document_from_statements(self, statements: List[Dict]): + """ + Helper method to generate a policy from statements + """ if statements is None: raise Exception('Provide valid statements while generating policy document from statement') return {'Version': '2012-10-17', 'Statement': statements} @@ -567,6 +594,9 @@ def _generate_managed_policy_statements_from_inline_policies(self): return policy_statements def _split_and_generate_statement_chunks(self, statements_s3, statements_kms, sid): + """ + Helper method to aggregate S3 and KMS statements for Bucket and Accesspoint shares and split into chunks + """ aggregate_statements = [] if len(statements_s3) > 0: statement_resources = [ @@ -665,8 +695,8 @@ def process_backwards_compatibility_for_target_iam_roles(self): After 2.6, the IAM managed policies are created with indexes on them. This was made to solve this issue decribed here - https://github.com/data-dot-all/dataall/issues/884 If an old managed policy exists then """ - log.info(f'Old managed policy: {old_managed_policy_exists}') + log.info(f'Old managed policy with name {old_managed_policy_name} exists: {old_managed_policy_exists}') if old_managed_policy_exists: - self.create_managed_indexed_policy_from_managed_policy() + self.create_managed_indexed_policy_from_managed_policy_delete_old_policy() managed_policies_list = self.get_managed_policies() self.attach_policies(managed_policies_list) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index 26bbdd423..6ffccaf96 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -151,7 +151,7 @@ def _validate_iam_role_and_policy( # If yes, convert it to the indexed format old_managed_policy_name = policy_manager.generate_old_policy_name() if policy_manager.check_if_policy_exists(old_managed_policy_name): - policy_manager.create_managed_indexed_policy_from_managed_policy() + policy_manager.create_managed_indexed_policy_from_managed_policy_delete_old_policy() # End of backwards compatibility attached = policy_manager.check_if_policies_attached() diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index f06d35e48..294304811 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -182,10 +182,9 @@ def check_target_role_access_policy(self) -> None: is_managed_policies_exists = share_policy_service.check_if_managed_policies_exists() # Checking if managed policies without indexes are present. This is used for backward compatibility - # Check this with AWS team if not is_managed_policies_exists: warn( - "Convert all your share's requestor policies to managed policies with indexes. Deprecation >= ?? ", + "Convert all your share's requestor policies to managed policies with indexes.", DeprecationWarning, stacklevel=2, ) @@ -204,22 +203,10 @@ def check_target_role_access_policy(self) -> None: self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): - logger.info( - f'Older version of managed policy present which is without index. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' - ) - self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return - - if not is_managed_policies_exists: - logger.info(f'IAM Policy {share_resource_policy_name} does not exist') - self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return - unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() if len(unattached_policies) > 0: logger.info( - f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalIAMRoleName}' ) self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) return @@ -330,7 +317,7 @@ def check_target_role_access_policy(self) -> None: def grant_target_role_access_policy(self): """ Updates requester IAM role policy to include requested S3 bucket and access point - :return: + :returns: None or raises exception if something fails """ logger.info(f'Grant target role {self.target_requester_IAMRoleName} access policy') @@ -343,7 +330,10 @@ def grant_target_role_access_policy(self): share=self.share, dataset=self.dataset, ) + # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() + + # Parses all policy documents and extracts s3 and kms statements share_policy_service.initialize_statements() key_alias = f'alias/{self.dataset.KmsAlias}' @@ -708,7 +698,10 @@ def revoke_target_role_access_policy(self): share=self.share, dataset=self.dataset, ) + # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() + + # Parses all policy documents and extracts s3 and kms statements share_policy_service.initialize_statements() key_alias = f'alias/{self.dataset.KmsAlias}' diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index 5aec5262f..fe0533043 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -85,6 +85,8 @@ def check_s3_iam_access(self) -> None: share=self.share, dataset=self.dataset, ) + + # Parses all policy documents and extracts s3 and kms statements share_policy_service.initialize_statements() share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) @@ -118,15 +120,10 @@ def check_s3_iam_access(self) -> None: self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - if not is_managed_policies_exists: - logger.info(f'IAM Policy {share_resource_policy_name} does not exist') - self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return - unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() if len(unattached_policies) > 0: logger.info( - f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalIAMRoleName}' ) self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) return @@ -247,7 +244,10 @@ def grant_s3_iam_access(self): share=self.share, dataset=self.dataset, ) + # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() + + # Parses all policy documents and extracts s3 and kms statements share_policy_service.initialize_statements() key_alias = f'alias/{self.target_bucket.KmsAlias}' @@ -260,20 +260,6 @@ def grant_s3_iam_access(self): if kms_key_id: kms_target_resources = [f'arn:aws:kms:{self.bucket_region}:{self.source_account_id}:key/{kms_key_id}'] - managed_policy_exists = share_policy_service.check_if_managed_policies_exists() - - if not managed_policy_exists: - logger.info('Managed policies do not exist. Creating one') - # Create a managed policy with naming convention and index - share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) - empty_policy = share_policy_service.generate_empty_policy() - IAM.create_managed_policy( - self.target_account_id, - self.target_environment.region, - share_resource_policy_name, - json.dumps(empty_policy), - ) - s3_kms_statement_chunks = [] s3_statements = share_policy_service.total_s3_stmts s3_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( @@ -559,7 +545,10 @@ def delete_target_role_access_policy( share=self.share, dataset=self.dataset, ) + # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() + + # Parses all policy documents and extracts s3 and kms statements share_policy_service.initialize_statements() key_alias = f'alias/{target_bucket.KmsAlias}' From 6af3c2e79e16d0d5d8f797dd504253e9d6c24bd7 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 17 Oct 2024 15:44:50 -0500 Subject: [PATCH 06/30] Correcting unit tests and making env chanegs for SES emails to work --- .../s3_access_point_share_manager.py | 2 +- .../share_managers/s3_bucket_share_manager.py | 2 +- deploy/stacks/container.py | 6 +-- .../test_s3_access_point_share_manager.py | 40 ++++++++++++++++++- .../tasks/test_s3_bucket_share_manager.py | 19 +++++++-- .../modules/s3_datasets_shares/test_share.py | 29 ++++++++++++++ 6 files changed, 89 insertions(+), 9 deletions(-) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 294304811..4b4ebcc15 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -206,7 +206,7 @@ def check_target_role_access_policy(self) -> None: unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() if len(unattached_policies) > 0: logger.info( - f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalIAMRoleName}' + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' ) self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) return diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index fe0533043..cb6cdca98 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -123,7 +123,7 @@ def check_s3_iam_access(self) -> None: unattached_policies: List[str] = share_policy_service.get_policies_unattached_to_role() if len(unattached_policies) > 0: logger.info( - f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalIAMRoleName}' + f'IAM Policies {unattached_policies} exists but are not attached to role {self.share.principalRoleName}' ) self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy attached', unattached_policies)) return diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index 0f875cabf..5811acd0b 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -260,7 +260,7 @@ def add_share_management_task(self): f'ShareManagementTaskContainer{self._envname}', container_name='container', image=ecs.ContainerImage.from_ecr_repository(repository=self._ecr_repository, tag=self._cdkproxy_image_tag), - environment=self._create_env('DEBUG'), + environment=self.env_vars, command=['python3.9', '-m', 'dataall.modules.shares_base.tasks.share_manager_task'], logging=ecs.LogDriver.aws_logs( stream_prefix='task', @@ -291,7 +291,7 @@ def add_share_verifier_task(self): command=['python3.9', '-m', 'dataall.modules.shares_base.tasks.share_verifier_task'], container_id='container', ecr_repository=self._ecr_repository, - environment=self._create_env('INFO'), + environment=self.env_vars, image_tag=self._cdkproxy_image_tag, log_group=self.create_log_group(self._envname, self._resource_prefix, log_group_name='share-verifier'), schedule_expression=Schedule.expression('rate(7 days)'), @@ -320,7 +320,7 @@ def add_share_reapplier_task(self): f'ShareReapplierTaskContainer{self._envname}', container_name='container', image=ecs.ContainerImage.from_ecr_repository(repository=self._ecr_repository, tag=self._cdkproxy_image_tag), - environment=self._create_env('INFO'), + environment=self.env_vars, command=['python3.9', '-m', 'dataall.modules.shares_base.tasks.share_reapplier_task'], logging=ecs.LogDriver.aws_logs( stream_prefix='task', diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py index d3e546757..ee004bd3b 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py @@ -1488,6 +1488,20 @@ def test_check_target_role_access_policy_test_no_policy(mocker, share_manager): return_value=False, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=False, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=[]) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=[]) + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, @@ -1504,10 +1518,34 @@ def test_check_target_role_access_policy_test_no_policy(mocker, share_manager): def test_check_target_role_access_policy_test_policy_not_attached(mocker, share_manager): # Given + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + # Gets policy with other S3 and KMS + iam_get_policy_mock = mocker.patch( + 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', + return_value=( + 'v1', + _create_target_dataset_access_control_policy(share_manager.bucket_name, share_manager.access_point_name), + ), + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, + ) # Policy is not attached mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py index 7522a5c54..1b6f604f0 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py @@ -1710,8 +1710,21 @@ def test_check_s3_iam_access_no_policy(mocker, dataset2, share2_manager): # There is not existing IAM policy in the requesters account for the dataset's S3bucket # Check if the update_role_policy func is called and policy statements are added - # When policy does not exist share_managerpolicy_mock_1 = mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + return_value=False, + ) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], + ) + + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=[]) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=[]) + + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) @@ -1771,9 +1784,9 @@ def test_check_s3_iam_access_policy_not_attached(mocker, dataset2, share2_manage return_value=['policy-0'], ) - mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) + mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=[]) - mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=[]) kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 7f6475565..455d801af 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -1285,6 +1285,13 @@ def test_create_share_object_as_requester(mocker, client, user2, group2, env2gro 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1322,6 +1329,13 @@ def test_create_share_object_as_approver_and_requester(mocker, client, user, gro 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1359,6 +1373,12 @@ def test_create_share_object_invalid_account(mocker, client, user, group2, env2g 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1397,6 +1417,13 @@ def test_create_share_object_with_item_authorized( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) + create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1670,6 +1697,8 @@ def test_create_share_object_with_share_expiration_incorrect_share_expiration( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + create_share_object_response = create_share_object( mocker=mocker, client=client, From e072f25781876c071c885adedba2e0f030143b07 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 17 Oct 2024 16:36:58 -0500 Subject: [PATCH 07/30] Changes observed during testing --- .../services/s3_share_managed_policy_service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 97226abee..12c71dc69 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -417,9 +417,10 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}' ) try: - ShareNotificationService( - session=None, dataset=self.dataset, share=self.share - ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) + if self.share is not None and self.dataset is not None: + ShareNotificationService( + session=None, dataset=self.dataset, share=self.share + ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) except Exception as e: log.error(f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}') raise Exception( @@ -427,7 +428,7 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): ) log.info( - f'Role: {self.role_name} has capacity to attach managed policies for share with URI: {self.share.shareUri}' + f'Role: {self.role_name} has capacity to attach managed policies' ) def _get_managed_policy_quota(self): From d83c805e59c17b31cdd2551ccd4746b8baab81ab Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 21 Oct 2024 19:14:32 -0500 Subject: [PATCH 08/30] Adding new file and changes for IAM policy utils --- .../dataall/base/utils/iam_policy_utils.py | 38 +++++ .../base/utils/iam_policy_utils_cdk.py | 160 ------------------ .../core/environment/cdk/pivot_role_stack.py | 17 +- .../cdk/pivot_role_redshift_policy.py | 25 +-- .../cdk/pivot_role_datasets_policy.py | 16 +- .../s3_share_managed_policy_service.py | 4 +- 6 files changed, 78 insertions(+), 182 deletions(-) delete mode 100644 backend/dataall/base/utils/iam_policy_utils_cdk.py diff --git a/backend/dataall/base/utils/iam_policy_utils.py b/backend/dataall/base/utils/iam_policy_utils.py index ea4b70f98..22560ecc9 100644 --- a/backend/dataall/base/utils/iam_policy_utils.py +++ b/backend/dataall/base/utils/iam_policy_utils.py @@ -67,6 +67,44 @@ def _build_statement(split, subset): return resulting_statements +def split_policy_with_mutiple_value_condition_in_statements( + base_sid: str, effect: str, actions: List[str], resources: List[str], condition_dict: dict +): + """ + The variable part of the policy is in the conditions parameter of the PolicyStatement + conditions_dict passes the different components of the condition mapping + """ + + def _build_statement(split, subset): + return { + 'Sid': base_sid + str(split), + 'Effect': effect, + 'Action': actions, + 'Resource': resources, + 'Condition': {condition_dict.get('key'): {condition_dict.get('resource'): subset}}, + } + + total_length, base_length = _policy_analyzer(condition_dict.get('values'), _build_statement) + extra_chars = len( + str(f'"Condition": {{ "{condition_dict.get("key")}": {{"{condition_dict.get("resource")}": }} }}') + ) + + if total_length < POLICY_LIMIT - POLICY_HEADERS_BUFFER: + logger.info('Not exceeding policy limit, returning statement ...') + resulting_statement = _build_statement(1, condition_dict.get('values')) + return [resulting_statement] + else: + logger.info('Exceeding policy limit, splitting values ...') + resulting_statements = _policy_splitter( + base_length=base_length, + resources=condition_dict.get('values'), + extra_chars=extra_chars, + statement_builder=_build_statement, + ) + + return resulting_statements + + def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], Dict]): """ Calculates the policy size with the resources (total_length) and without resources (base_length) diff --git a/backend/dataall/base/utils/iam_policy_utils_cdk.py b/backend/dataall/base/utils/iam_policy_utils_cdk.py deleted file mode 100644 index 0d9d8b792..000000000 --- a/backend/dataall/base/utils/iam_policy_utils_cdk.py +++ /dev/null @@ -1,160 +0,0 @@ -from typing import List, Callable -import logging -from aws_cdk import aws_iam as iam - -logger = logging.getLogger(__name__) - -POLICY_LIMIT = 6144 -POLICY_HEADERS_BUFFER = 144 # The policy headers take around 60 chars. An extra buffer of 84 chars is added for any additional spacing or char that is unaccounted. -MAXIMUM_NUMBER_MANAGED_POLICIES = 20 # Soft limit 10, hard limit 20 - - -def split_policy_statements_in_chunks(statements: List[iam.PolicyStatement]): - """ - Splitter used for IAM policies with an undefined number of statements - - Ensures that the size of the IAM policy remains below the POLICY LIMIT - - If it exceeds the POLICY LIMIT, it breaks the policy into multiple policies (chunks) - - Note the POLICY_HEADERS_BUFFER to account for the headers of the policy which usually take around ~60chars - """ - chunks = [] - index = 0 - statements_list_of_strings = [str(s.to_json()) for s in statements] - total_length = len(', '.join(statements_list_of_strings)) - logger.info(f'Number of statements = {len(statements)}') - logger.info(f'Total length of statements = {total_length}') - max_length = max(statements_list_of_strings, key=len) - if len(max_length) > POLICY_LIMIT - POLICY_HEADERS_BUFFER: - raise Exception(f'Policy statement {max_length} exceeds maximum policy size') - while index < len(statements): - # Iterating until all statements are assigned to a chunk. - # "index" represents the statement position in the statements list - chunk = [] - chunk_size = 0 - while ( - index < len(statements) - and chunk_size + len(str(statements[index].to_json())) < POLICY_LIMIT - POLICY_HEADERS_BUFFER - ): - # Appends a statement to the chunk until we reach its maximum size. - # It compares, current size of the statements < allowed size for the statements section of a policy - chunk.append(statements[index]) - chunk_size += len(str(statements[index].to_json())) - index += 1 - chunks.append(chunk) - logger.info(f'Total number of managed policies = {len(chunks)}') - if len(chunks) > MAXIMUM_NUMBER_MANAGED_POLICIES: - raise Exception('The number of policies calculated exceeds the allowed maximum number of managed policies') - return chunks - - -def split_policy_with_resources_in_statements( - base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str] -): - """ - The variable part of the policy is in the resources parameter of the PolicyStatement - """ - - def _build_statement(split, subset): - return iam.PolicyStatement(sid=base_sid + str(split), effect=effect, actions=actions, resources=subset) - - total_length, base_length = _policy_analyzer(resources, _build_statement) - extra_chars = len('" ," ') - - if total_length < POLICY_LIMIT - POLICY_HEADERS_BUFFER: - logger.info('Not exceeding policy limit, returning statement ...') - resulting_statement = _build_statement(1, resources) - return [resulting_statement] - else: - logger.info('Exceeding policy limit, splitting statement ...') - resulting_statements = _policy_splitter( - base_length=base_length, resources=resources, extra_chars=extra_chars, statement_builder=_build_statement - ) - return resulting_statements - - -def split_policy_with_mutiple_value_condition_in_statements( - base_sid: str, effect: iam.Effect, actions: List[str], resources: List[str], condition_dict: dict -): - """ - The variable part of the policy is in the conditions parameter of the PolicyStatement - conditions_dict passes the different components of the condition mapping - """ - - def _build_statement(split, subset): - return iam.PolicyStatement( - sid=base_sid + str(split), - effect=effect, - actions=actions, - resources=resources, - conditions={condition_dict.get('key'): {condition_dict.get('resource'): subset}}, - ) - - total_length, base_length = _policy_analyzer(condition_dict.get('values'), _build_statement) - extra_chars = len( - str(f'"Condition": {{ "{condition_dict.get("key")}": {{"{condition_dict.get("resource")}": }} }}') - ) - - if total_length < POLICY_LIMIT - POLICY_HEADERS_BUFFER: - logger.info('Not exceeding policy limit, returning statement ...') - resulting_statement = _build_statement(1, condition_dict.get('values')) - return [resulting_statement] - else: - logger.info('Exceeding policy limit, splitting values ...') - resulting_statements = _policy_splitter( - base_length=base_length, - resources=condition_dict.get('values'), - extra_chars=extra_chars, - statement_builder=_build_statement, - ) - - return resulting_statements - - -def _policy_analyzer(resources: List[str], statement_builder: Callable[[int, List[str]], iam.PolicyStatement]): - """ - Calculates the policy size with the resources (total_length) and without resources (base_length) - """ - statement_without_resources = statement_builder(1, ['*']) - resources_str = '" ," '.join(r for r in resources) - base_length = len(str(statement_without_resources.to_json())) - total_length = base_length + len(resources_str) - logger.info(f'Policy base length = {base_length}') - logger.info(f'Resources as string length = {len(resources_str)}') - logger.info(f'Total length approximated as base length + resources string length = {total_length}') - - return total_length, base_length - - -def _policy_splitter( - base_length: int, - resources: List[str], - extra_chars: int, - statement_builder: Callable[[int, List[str]], iam.PolicyStatement], -): - """ - Splitter used for IAM policy statements with an undefined number of resources one of the parameters of the policy. - - Ensures that the size of the IAM statement is below the POLICY LIMIT - - If it exceeds the POLICY LIMIT, it breaks the statement in multiple statements with a subset of resources - - Note the POLICY_HEADERS_BUFFER to account for the headers of the policy which usually take around ~60chars - """ - index = 0 - split = 0 - resulting_statements = [] - while index < len(resources): - # Iterating until all values are defined in a policy statement. - # "index" represents the position of the value in the values list - size = 0 - subset = [] - while ( - index < len(resources) - and (size + len(resources[index]) + extra_chars) < POLICY_LIMIT - POLICY_HEADERS_BUFFER - base_length - ): - # Appending a resource to the subset list until we reach the maximum size for the condition section - # It compares: current size of subset versus the allowed size of the condition section in a statement - subset.append(resources[index]) - size += len(resources[index]) + extra_chars - index += 1 - resulting_statement = statement_builder(split=split, subset=subset) - split += 1 - resulting_statements.append(resulting_statement) - logger.info(f'Statement divided into {split+1} smaller statements') - return resulting_statements diff --git a/backend/dataall/core/environment/cdk/pivot_role_stack.py b/backend/dataall/core/environment/cdk/pivot_role_stack.py index f32d9a46e..f5a4b101d 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_stack.py +++ b/backend/dataall/core/environment/cdk/pivot_role_stack.py @@ -3,7 +3,13 @@ from typing import List from constructs import Construct from aws_cdk import Duration, aws_iam as iam, NestedStack -from dataall.base.utils.iam_policy_utils_cdk import split_policy_statements_in_chunks + +from dataall.base.utils.iam_cdk_utils import ( + process_and, + convert_from_json_to_iam_policy_statement_with_conditions, + convert_from_json_to_iam_policy_statement, +) +from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks logger = logging.getLogger(__name__) @@ -36,7 +42,14 @@ def generate_policies(self) -> List[iam.ManagedPolicy]: statements.extend(service.get_statements(self)) logger.info(f'statements: {str(service.get_statements(self))}') - statements_chunks = split_policy_statements_in_chunks(statements) + statements_json = [statement.to_json() for statement in statements] + statement_chunks_json = split_policy_statements_in_chunks(statements_json) + statements_chunks = [] + for statement_js in statement_chunks_json: + if not statement_js.get('Condition', None): + statements_chunks.append(convert_from_json_to_iam_policy_statement_with_conditions(statement_js)) + else: + statements_chunks.append(convert_from_json_to_iam_policy_statement(statement_js)) for index, chunk in enumerate(statements_chunks): policies.append( diff --git a/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py b/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py index 66c52b89c..5c9f7f10e 100644 --- a/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py +++ b/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py @@ -3,6 +3,7 @@ from dataall.base import db from dataall.base.aws.sts import SessionHelper +from dataall.base.utils.iam_cdk_utils import convert_from_json_to_iam_policy_statement from dataall.base.utils.iam_policy_utils import split_policy_with_resources_in_statements from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet from dataall.modules.redshift_datasets.db.redshift_connection_repositories import RedshiftConnectionRepository @@ -81,18 +82,18 @@ def get_statements(self): workgroup_arns = [ rs_client.get_workgroup_arn(workgroup_name=conn.workgroup) for conn in connections if conn.workgroup ] - additional_statements.extend( - split_policy_with_resources_in_statements( - base_sid='RedshiftData', - effect=iam.Effect.ALLOW, - actions=[ - 'redshift-data:ListSchemas', - 'redshift-data:ListTables', - 'redshift-data:ExecuteStatement', - 'redshift-data:DescribeTable', - ], - resources=cluster_arns + workgroup_arns, - ) + redshift_data_statement_json = split_policy_with_resources_in_statements( + base_sid='RedshiftData', + effect=iam.Effect.ALLOW.value, + actions=[ + 'redshift-data:ListSchemas', + 'redshift-data:ListTables', + 'redshift-data:ExecuteStatement', + 'redshift-data:DescribeTable', + ], + resources=cluster_arns + workgroup_arns, ) + redshift_data_statement = convert_from_json_to_iam_policy_statement(redshift_data_statement_json) + additional_statements.extend(redshift_data_statement) return base_statements + additional_statements diff --git a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py index f0b957569..dfc963bc5 100644 --- a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py @@ -1,6 +1,10 @@ import os from dataall.base import db -from dataall.base.utils.iam_policy_utils_cdk import ( +from dataall.base.utils.iam_cdk_utils import ( + convert_from_json_to_iam_policy_statement, + convert_from_json_to_iam_policy_statement_with_conditions, +) +from dataall.base.utils.iam_policy_utils import ( split_policy_with_resources_in_statements, split_policy_with_mutiple_value_condition_in_statements, ) @@ -153,9 +157,9 @@ def get_statements(self): imported_kms_alias.append(f'alias/{dataset.KmsAlias}') if imported_buckets: - dataset_statement = split_policy_with_resources_in_statements( + dataset_statement_json = split_policy_with_resources_in_statements( base_sid='ImportedDatasetBuckets', - effect=iam.Effect.ALLOW, + effect=iam.Effect.ALLOW.value, actions=[ 's3:List*', 's3:GetBucket*', @@ -168,11 +172,12 @@ def get_statements(self): ], resources=imported_buckets, ) + dataset_statement = convert_from_json_to_iam_policy_statement(dataset_statement_json) statements.extend(dataset_statement) if imported_kms_alias: - kms_statement = split_policy_with_mutiple_value_condition_in_statements( + kms_statement_json = split_policy_with_mutiple_value_condition_in_statements( base_sid='KMSImportedDataset', - effect=iam.Effect.ALLOW, + effect=iam.Effect.ALLOW.value, actions=[ 'kms:Decrypt', 'kms:Encrypt', @@ -190,6 +195,7 @@ def get_statements(self): 'values': imported_kms_alias, }, ) + kms_statement = convert_from_json_to_iam_policy_statement_with_conditions(kms_statement_json) statements.extend(kms_statement) return statements diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 12c71dc69..9cb39944d 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -427,9 +427,7 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): f'Failed to process share as number of needed attached policies to the role is greater than the service quota limit: {managed_iam_policy_quota}' ) - log.info( - f'Role: {self.role_name} has capacity to attach managed policies' - ) + log.info(f'Role: {self.role_name} has capacity to attach managed policies') def _get_managed_policy_quota(self): # Get the number of managed policies which can be attached to the IAM role From b51a428e2396e28a7b3785feec4bf916105abf0f Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 21 Oct 2024 19:27:11 -0500 Subject: [PATCH 09/30] Corrections --- backend/dataall/core/environment/cdk/pivot_role_stack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/dataall/core/environment/cdk/pivot_role_stack.py b/backend/dataall/core/environment/cdk/pivot_role_stack.py index f5a4b101d..365c59af9 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_stack.py +++ b/backend/dataall/core/environment/cdk/pivot_role_stack.py @@ -5,7 +5,6 @@ from aws_cdk import Duration, aws_iam as iam, NestedStack from dataall.base.utils.iam_cdk_utils import ( - process_and, convert_from_json_to_iam_policy_statement_with_conditions, convert_from_json_to_iam_policy_statement, ) From fe6c1fea9c9e700762c9342a72ac43f56e453396 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 21 Oct 2024 19:28:47 -0500 Subject: [PATCH 10/30] Adding converter file --- backend/dataall/base/utils/iam_cdk_utils.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 backend/dataall/base/utils/iam_cdk_utils.py diff --git a/backend/dataall/base/utils/iam_cdk_utils.py b/backend/dataall/base/utils/iam_cdk_utils.py new file mode 100644 index 000000000..da15635ad --- /dev/null +++ b/backend/dataall/base/utils/iam_cdk_utils.py @@ -0,0 +1,21 @@ +from typing import Dict, Any +from aws_cdk import aws_iam as iam + + +def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[Any, Any]): + return iam.PolicyStatement( + sid=iam_policy.get('Sid'), + effect=iam_policy.get('Effect'), + actions=iam_policy.get('Action'), + resources=iam_policy.get('Resource'), + conditions=iam_policy.get('Condition') + ) + + +def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]): + return iam.PolicyStatement( + sid=iam_policy.get('Sid'), + effect=iam_policy.get('Effect'), + actions=iam_policy.get('Action'), + resources=iam_policy.get('Resource'), + ) From 207032876ad7fc0ad38ae95d3893568da48a2942 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 21 Oct 2024 19:32:12 -0500 Subject: [PATCH 11/30] backend linting changes --- backend/dataall/base/utils/iam_cdk_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/dataall/base/utils/iam_cdk_utils.py b/backend/dataall/base/utils/iam_cdk_utils.py index da15635ad..4217b09e3 100644 --- a/backend/dataall/base/utils/iam_cdk_utils.py +++ b/backend/dataall/base/utils/iam_cdk_utils.py @@ -8,7 +8,7 @@ def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[A effect=iam_policy.get('Effect'), actions=iam_policy.get('Action'), resources=iam_policy.get('Resource'), - conditions=iam_policy.get('Condition') + conditions=iam_policy.get('Condition'), ) From 508f520f18c7799e104501ca75f95b2b4bcc13e9 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 21 Oct 2024 19:36:21 -0500 Subject: [PATCH 12/30] Unit test corrections --- tests/modules/s3_datasets_shares/test_share.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 455d801af..12801a834 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -1663,6 +1663,10 @@ def test_create_share_object_with_non_expiring_share( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + return_value=True, + ) create_share_object_response = create_share_object( mocker=mocker, client=client, From 9d8583d968c29dde2762d900dc8f5d07dfc6d7b0 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 21 Oct 2024 20:47:30 -0500 Subject: [PATCH 13/30] Correction in share --- tests/modules/s3_datasets_shares/test_share.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 12801a834..230b3e3e2 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -1667,6 +1667,8 @@ def test_create_share_object_with_non_expiring_share( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + create_share_object_response = create_share_object( mocker=mocker, client=client, From d40faab40e49a2fe4f1cc94393ba0bc8ce3b3caf Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 22 Oct 2024 11:07:08 -0500 Subject: [PATCH 14/30] Adding comments --- backend/dataall/base/utils/naming_convention.py | 3 ++- .../dataall/core/environment/services/managed_iam_policies.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/dataall/base/utils/naming_convention.py b/backend/dataall/base/utils/naming_convention.py index 2f5ef752d..0fc0c80ab 100644 --- a/backend/dataall/base/utils/naming_convention.py +++ b/backend/dataall/base/utils/naming_convention.py @@ -60,10 +60,11 @@ def build_compliant_name_with_index(self, index: int = None) -> str: """ Builds a compliant AWS resource name """ + # Todo : Add comments regex = NamingConventionPattern[self.service].value['regex'] separator = NamingConventionPattern[self.service].value['separator'] max_length = NamingConventionPattern[self.service].value['max_length'] - index_string = f'-{index}' if index is not None else '- ' + index_string = f'-{index}' if index is not None else '- ' # Add a buffer suffix = f'-{self.target_uri}' if len(self.target_uri) else '' suffix = suffix + index_string if index is not None else suffix return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri + index_string))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}" diff --git a/backend/dataall/core/environment/services/managed_iam_policies.py b/backend/dataall/core/environment/services/managed_iam_policies.py index 685b760ec..42a42e16c 100644 --- a/backend/dataall/core/environment/services/managed_iam_policies.py +++ b/backend/dataall/core/environment/services/managed_iam_policies.py @@ -79,7 +79,7 @@ def check_if_policy_exists(self, policy_name) -> bool: def check_if_managed_policies_exists(self) -> bool: # Fetch the policy name which was created without indexes and filter through all policies policy_pattern = self.generate_base_policy_name() - share_policies = self._get_share_policy_names(policy_pattern) + share_policies = self._get_share_policy_names(policy_pattern) # Todo : remove share name return True if share_policies else False def get_managed_policies(self) -> List[str]: From f78429ca153026850e7ce1a22324bd83b7c8f2d2 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 23 Oct 2024 16:24:48 -0500 Subject: [PATCH 15/30] Simplifying interface --- backend/dataall/base/db/exceptions.py | 11 +++ backend/dataall/base/utils/iam_cdk_utils.py | 14 +++- .../dataall/base/utils/iam_policy_utils.py | 4 +- .../dataall/base/utils/naming_convention.py | 11 +-- .../core/environment/cdk/pivot_role_stack.py | 13 +-- .../services/managed_iam_policies.py | 40 ++-------- .../s3_share_managed_policy_service.py | 19 ++--- .../services/s3_share_validator.py | 40 ---------- .../s3_access_point_share_manager.py | 79 ++++++++++--------- .../share_managers/s3_bucket_share_manager.py | 48 +++++------ .../test_s3_access_point_share_manager.py | 36 ++++----- 11 files changed, 120 insertions(+), 195 deletions(-) diff --git a/backend/dataall/base/db/exceptions.py b/backend/dataall/base/db/exceptions.py index 95c2c2a73..24153e7fc 100644 --- a/backend/dataall/base/db/exceptions.py +++ b/backend/dataall/base/db/exceptions.py @@ -146,6 +146,17 @@ def __init__(self, action, message): def __str__(self): return f'{self.message}' +class AWSResourceQuotaExceeded(Exception): + def __init__(self, action, message): + self.action = action + self.message = f""" + An error occurred (AWSResourceQuotaExceeded) when calling {self.action} operation: + {message} + """ + + def __str__(self): + return f'{self.message}' + class EnvironmentResourcesFound(Exception): def __init__(self, action, message): diff --git a/backend/dataall/base/utils/iam_cdk_utils.py b/backend/dataall/base/utils/iam_cdk_utils.py index 4217b09e3..333fc1e2a 100644 --- a/backend/dataall/base/utils/iam_cdk_utils.py +++ b/backend/dataall/base/utils/iam_cdk_utils.py @@ -1,6 +1,8 @@ -from typing import Dict, Any +from typing import Dict, Any, List from aws_cdk import aws_iam as iam +from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks + def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[Any, Any]): return iam.PolicyStatement( @@ -19,3 +21,13 @@ def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]): actions=iam_policy.get('Action'), resources=iam_policy.get('Resource'), ) + +def process_and_split_statements_in_chunks(statements: List[Dict]): + statement_chunks_json = split_policy_statements_in_chunks(statements) + statements_chunks = [] + for statement_js in statement_chunks_json: + if not statement_js.get('Condition', None): + statements_chunks.append(convert_from_json_to_iam_policy_statement_with_conditions(statement_js)) + else: + statements_chunks.append(convert_from_json_to_iam_policy_statement(statement_js)) + return statements_chunks \ No newline at end of file diff --git a/backend/dataall/base/utils/iam_policy_utils.py b/backend/dataall/base/utils/iam_policy_utils.py index 22560ecc9..c055aee2c 100644 --- a/backend/dataall/base/utils/iam_policy_utils.py +++ b/backend/dataall/base/utils/iam_policy_utils.py @@ -1,4 +1,4 @@ -from typing import List, Callable, Any, Dict +from typing import List, Callable, Dict import logging logger = logging.getLogger(__name__) @@ -8,7 +8,7 @@ MAXIMUM_NUMBER_MANAGED_POLICIES = 20 # Soft limit 10, hard limit 20 -def split_policy_statements_in_chunks(statements: List[Any]): +def split_policy_statements_in_chunks(statements: List[Dict]): """ Splitter used for IAM policies with an undefined number of statements - Ensures that the size of the IAM policy remains below the POLICY LIMIT diff --git a/backend/dataall/base/utils/naming_convention.py b/backend/dataall/base/utils/naming_convention.py index 0fc0c80ab..08189d3ba 100644 --- a/backend/dataall/base/utils/naming_convention.py +++ b/backend/dataall/base/utils/naming_convention.py @@ -58,16 +58,17 @@ def build_compliant_name(self) -> str: def build_compliant_name_with_index(self, index: int = None) -> str: """ - Builds a compliant AWS resource name + Builds a compliant AWS resource name with an index at the end of the policy name + IMP - If no index is provided, then this method provides a base policy name without index. Base policy name is calculated by considering the length of string required for index + This is done so that the base policy name doesn't change when an index is added to the string. """ - # Todo : Add comments regex = NamingConventionPattern[self.service].value['regex'] separator = NamingConventionPattern[self.service].value['separator'] max_length = NamingConventionPattern[self.service].value['max_length'] - index_string = f'-{index}' if index is not None else '- ' # Add a buffer + index_string_length = 2 # This is added to adjust the target label string even if the index is set to None. This helps in getting the base policy name when index is None + index_string = f'-{index}' if index is not None else '' suffix = f'-{self.target_uri}' if len(self.target_uri) else '' - suffix = suffix + index_string if index is not None else suffix - return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri + index_string))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}" + return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri) - index_string_length)] + suffix + index_string, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}" def validate_name(self): regex = NamingConventionPattern[self.service].value['regex'] diff --git a/backend/dataall/core/environment/cdk/pivot_role_stack.py b/backend/dataall/core/environment/cdk/pivot_role_stack.py index 365c59af9..e09fba3c7 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_stack.py +++ b/backend/dataall/core/environment/cdk/pivot_role_stack.py @@ -5,10 +5,9 @@ from aws_cdk import Duration, aws_iam as iam, NestedStack from dataall.base.utils.iam_cdk_utils import ( - convert_from_json_to_iam_policy_statement_with_conditions, - convert_from_json_to_iam_policy_statement, + process_and_split_statements_in_chunks, ) -from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks + logger = logging.getLogger(__name__) @@ -42,13 +41,7 @@ def generate_policies(self) -> List[iam.ManagedPolicy]: logger.info(f'statements: {str(service.get_statements(self))}') statements_json = [statement.to_json() for statement in statements] - statement_chunks_json = split_policy_statements_in_chunks(statements_json) - statements_chunks = [] - for statement_js in statement_chunks_json: - if not statement_js.get('Condition', None): - statements_chunks.append(convert_from_json_to_iam_policy_statement_with_conditions(statement_js)) - else: - statements_chunks.append(convert_from_json_to_iam_policy_statement(statement_js)) + statements_chunks = process_and_split_statements_in_chunks(statements_json) for index, chunk in enumerate(statements_chunks): policies.append( diff --git a/backend/dataall/core/environment/services/managed_iam_policies.py b/backend/dataall/core/environment/services/managed_iam_policies.py index 42a42e16c..c70c6e3b5 100644 --- a/backend/dataall/core/environment/services/managed_iam_policies.py +++ b/backend/dataall/core/environment/services/managed_iam_policies.py @@ -56,55 +56,25 @@ def generate_empty_policy(self) -> dict: """ ... - @abstractmethod - def create_managed_policy_from_inline_and_delete_inline(self) -> str: - """ - Returns policy arn and needs to be implemented in the ManagedPolicies inherited classes - It is used for backwards compatibility. It should be deprecated and removed in future releases. - """ - ... - - @abstractmethod - def create_managed_indexed_policy_from_managed_policy_delete_old_policy(self) -> str: - """ - Returns policy ARNs and needs to be implemented in the ManagedPolicies inherited classes - It is used for backwards compatibility. It should be deprecated and removed in future releases. - """ - ... - def check_if_policy_exists(self, policy_name) -> bool: share_policy = IAM.get_managed_policy_by_name(self.account, self.region, policy_name) return share_policy is not None - def check_if_managed_policies_exists(self) -> bool: - # Fetch the policy name which was created without indexes and filter through all policies - policy_pattern = self.generate_base_policy_name() - share_policies = self._get_share_policy_names(policy_pattern) # Todo : remove share name - return True if share_policies else False - def get_managed_policies(self) -> List[str]: policy_pattern = self.generate_base_policy_name() - share_policies = self._get_share_policy_names(policy_pattern) + share_policies = self._get_policy_names(policy_pattern) return share_policies def check_if_policy_attached(self, policy_name): is_policy_attached = IAM.is_policy_attached(self.account, self.region, policy_name, self.role_name) return is_policy_attached - def check_if_policies_attached(self): - policy_pattern = self.generate_base_policy_name() - share_policies = self._get_share_policy_names(policy_pattern) - return all( - IAM.is_policy_attached(self.account, self.region, share_policy_name, self.role_name) - for share_policy_name in share_policies - ) - def get_policies_unattached_to_role(self): policy_pattern = self.generate_base_policy_name() - share_policies = self._get_share_policy_names(policy_pattern) + share_policies = self._get_policy_names(policy_pattern) unattached_policies = [] for share_policy_name in share_policies: - if not IAM.is_policy_attached(self.account, self.region, share_policy_name, self.role_name): + if not self.check_if_policy_attached(share_policy_name): unattached_policies.append(share_policy_name) return unattached_policies @@ -116,9 +86,9 @@ def attach_policies(self, managed_policies_list: List[str]): except Exception as e: raise Exception(f"Required customer managed policy {policy_arn} can't be attached: {e}") - def _get_share_policy_names(self, policy_pattern): + def _get_policy_names(self, policy_pattern): share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) - # Filter through all policies which have the old policy name + # Filter through all policies and remove which have the old policy name # This is to check that old policies are not included old_policy_name = self.generate_old_policy_name() return [policy for policy in share_policies if policy != old_policy_name] diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 9cb39944d..26a9ae8a7 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -3,6 +3,7 @@ from dataall.base.aws.iam import IAM from dataall.base.aws.service_quota import ServiceQuota +from dataall.base.db.exceptions import AWSResourceQuotaExceeded from dataall.base.utils.iam_policy_utils import ( split_policy_statements_in_chunks, split_policy_with_resources_in_statements, @@ -33,14 +34,12 @@ class S3SharePolicyService(ManagedPolicy): - def __init__(self, role_name, account, region, environmentUri, resource_prefix, share=None, dataset=None): + def __init__(self, role_name, account, region, environmentUri, resource_prefix): self.role_name = role_name self.account = account self.region = region self.environmentUri = environmentUri self.resource_prefix = resource_prefix - self.share = share - self.dataset = dataset self.policy_version_map = {} # Policy version map helps while updating policies self.total_s3_stmts: List[Any] = [] self.total_s3_kms_stmts: List[Any] = [] @@ -416,16 +415,7 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): log.error( f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}' ) - try: - if self.share is not None and self.dataset is not None: - ShareNotificationService( - session=None, dataset=self.dataset, share=self.share - ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) - except Exception as e: - log.error(f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}') - raise Exception( - f'Failed to process share as number of needed attached policies to the role is greater than the service quota limit: {managed_iam_policy_quota}' - ) + raise AWSResourceQuotaExceeded(action='_check_iam_managed_policy_attachment_limit', message=f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}') log.info(f'Role: {self.role_name} has capacity to attach managed policies') @@ -681,9 +671,10 @@ def process_backwards_compatibility_for_target_iam_roles(self): log.info('Checking if inline policies are present') old_managed_policy_name = self.generate_old_policy_name() old_managed_policy_exists = self.check_if_policy_exists(policy_name=old_managed_policy_name) + share_managed_policies_exist = True if self.get_managed_policies() else False # If old managed policy doesn't exist and also new managed policies do not exist. # Then there might be inline policies, convert them to managed indexed policies - if not old_managed_policy_exists and not self.check_if_managed_policies_exists(): + if not old_managed_policy_exists and not share_managed_policies_exist: self.create_managed_policy_from_inline_and_delete_inline() managed_policies_list = self.get_managed_policies() self.attach_policies(managed_policies_list) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index 6ffccaf96..41d826bfc 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -122,43 +122,3 @@ def _validate_iam_role_and_policy( action=principal_type, message=f'The principal role {principal_role_name} is not found.', ) - - log.info('Verifying data.all managed share IAM policy is attached to IAM role...') - share_policy_manager = PolicyManager( - role_name=principal_role_name, - environmentUri=environment.environmentUri, - account=environment.AwsAccountId, - region=environment.region, - resource_prefix=environment.resourcePrefix, - ) - for policy_manager in [ - Policy for Policy in share_policy_manager.initializedPolicies if Policy.policy_type == 'SharePolicy' - ]: - # Backwards compatibility - 1 - # we check if a managed share policy exists. If False, the role was introduced to data.all before this update - # We create the policy from the inline statements - # In this case it could also happen that the role is the Admin of the environment - old_managed_policy_name = policy_manager.generate_old_policy_name() - old_managed_policy_exists = policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) - # If old managed policy doesn't exist and also new managed policies do not exist. - # Then there might be inline policies, convert them to managed indexed policies - if not old_managed_policy_exists and not policy_manager.check_if_managed_policies_exists(): - policy_manager.create_managed_policy_from_inline_and_delete_inline() - # End of backwards compatibility - - # Backwards compatibility - 2 - # Check if an already existing managed policy is present in old format - # If yes, convert it to the indexed format - old_managed_policy_name = policy_manager.generate_old_policy_name() - if policy_manager.check_if_policy_exists(old_managed_policy_name): - policy_manager.create_managed_indexed_policy_from_managed_policy_delete_old_policy() - # End of backwards compatibility - - attached = policy_manager.check_if_policies_attached() - if not attached and not managed and not attachMissingPolicies: - raise Exception( - f'Required customer managed policies {policy_manager.get_policies_unattached_to_role()} are not attached to role {principal_role_name}' - ) - elif not attached: - managed_policy_list = policy_manager.get_policies_unattached_to_role() - policy_manager.attach_policies(managed_policy_list) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 4b4ebcc15..0eb2554fa 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -4,6 +4,8 @@ from itertools import count from typing import List from warnings import warn + +from dataall.base.db.exceptions import AWSResourceQuotaExceeded from dataall.core.environment.services.environment_service import EnvironmentService from dataall.base.db import utils from dataall.base.aws.sts import SessionHelper @@ -34,6 +36,7 @@ S3SharePolicyService, IAM_S3_ACCESS_POINTS_STATEMENT_SID, ) +from dataall.modules.shares_base.services.share_notification_service import ShareNotificationService from dataall.modules.shares_base.services.shares_enums import PrincipalType from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, S3Dataset from dataall.modules.shares_base.services.sharing_service import ShareData @@ -167,19 +170,15 @@ def check_target_role_access_policy(self) -> None: key_alias = f'alias/{self.dataset.KmsAlias}' kms_client = KmsClient(self.dataset_account_id, self.source_environment.region) kms_key_id = kms_client.get_key_id(key_alias) - share_policy_service = S3SharePolicyService( - environmentUri=self.target_environment.environmentUri, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - role_name=self.target_requester_IAMRoleName, - resource_prefix=self.target_environment.resourcePrefix, - share=self.share, - dataset=self.dataset, - ) + share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix) share_policy_service.initialize_statements() share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) - is_managed_policies_exists = share_policy_service.check_if_managed_policies_exists() + is_managed_policies_exists = True if share_policy_service.get_managed_policies() else False # Checking if managed policies without indexes are present. This is used for backward compatibility if not is_managed_policies_exists: @@ -321,15 +320,11 @@ def grant_target_role_access_policy(self): """ logger.info(f'Grant target role {self.target_requester_IAMRoleName} access policy') - share_policy_service = S3SharePolicyService( - environmentUri=self.target_environment.environmentUri, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - role_name=self.target_requester_IAMRoleName, - resource_prefix=self.target_environment.resourcePrefix, - share=self.share, - dataset=self.dataset, - ) + share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() @@ -350,7 +345,7 @@ def grant_target_role_access_policy(self): if kms_key_id: kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] - managed_policy_exists = share_policy_service.check_if_managed_policies_exists() + managed_policy_exists = True if share_policy_service.get_managed_policies() else False if not managed_policy_exists: logger.info('Managed policies do not exist. Creating one') @@ -385,14 +380,26 @@ def grant_target_role_access_policy(self): ) logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') - - share_policy_service.merge_statements_and_update_policies( - target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, - target_s3_statements=s3_statement_chunks, - target_s3_kms_statements=s3_kms_statement_chunks, - ) - - if not share_policy_service.check_if_policies_attached(): + try: + share_policy_service.merge_statements_and_update_policies( + target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, + target_s3_statements=s3_statement_chunks, + target_s3_kms_statements=s3_kms_statement_chunks, + ) + except AWSResourceQuotaExceeded as e: + error_message = e.message + try: + ShareNotificationService( + session=None, dataset=self.dataset, share=self.share + ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) + except Exception as e: + logger.error(f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}') + finally: + raise error_message + + share_managed_polices = share_policy_service.get_managed_policies() + all_managed_policies_attached = all( share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices) + if not all_managed_policies_attached: logger.info( f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' ) @@ -689,15 +696,11 @@ def delete_access_point(self): def revoke_target_role_access_policy(self): logger.info('Deleting target role IAM statements...') - share_policy_service = S3SharePolicyService( - environmentUri=self.target_environment.environmentUri, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - role_name=self.target_requester_IAMRoleName, - resource_prefix=self.target_environment.resourcePrefix, - share=self.share, - dataset=self.dataset, - ) + share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() @@ -719,7 +722,7 @@ def revoke_target_role_access_policy(self): if kms_key_id: kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] - managed_policy_exists = share_policy_service.check_if_managed_policies_exists() + managed_policy_exists = True if share_policy_service.get_managed_policies() else False if not managed_policy_exists: logger.info(f'Managed policies for share with uri: {self.share.shareUri} are not found') diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index cb6cdca98..ceaceb431 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -76,21 +76,17 @@ def check_s3_iam_access(self) -> None: kms_client = KmsClient(self.source_account_id, self.source_environment.region) kms_key_id = kms_client.get_key_id(key_alias) - share_policy_service = S3SharePolicyService( - environmentUri=self.target_environment.environmentUri, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - role_name=self.target_requester_IAMRoleName, - resource_prefix=self.target_environment.resourcePrefix, - share=self.share, - dataset=self.dataset, - ) + share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix) # Parses all policy documents and extracts s3 and kms statements share_policy_service.initialize_statements() share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) - is_managed_policies_exists = share_policy_service.check_if_managed_policies_exists() + is_managed_policies_exists = True if share_policy_service.get_managed_policies() else False if not is_managed_policies_exists: warn( @@ -235,15 +231,11 @@ def grant_s3_iam_access(self): """ logger.info(f'Grant target role {self.target_requester_IAMRoleName} access policy') - share_policy_service = S3SharePolicyService( - environmentUri=self.target_environment.environmentUri, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - role_name=self.target_requester_IAMRoleName, - resource_prefix=self.target_environment.resourcePrefix, - share=self.share, - dataset=self.dataset, - ) + share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() @@ -288,7 +280,9 @@ def grant_s3_iam_access(self): target_s3_kms_statements=s3_kms_statement_chunks, ) - if not share_policy_service.check_if_policies_attached(): + share_managed_polices = share_policy_service.get_managed_policies() + all_managed_policies_attached = all(share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices) + if not all_managed_policies_attached: logger.info( f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' ) @@ -536,15 +530,11 @@ def delete_target_role_access_policy( ): logger.info('Deleting target role IAM statements...') - share_policy_service = S3SharePolicyService( - role_name=share.principalRoleName, - account=target_environment.AwsAccountId, - region=self.target_environment.region, - environmentUri=target_environment.environmentUri, - resource_prefix=target_environment.resourcePrefix, - share=self.share, - dataset=self.dataset, - ) + share_policy_service = S3SharePolicyService(role_name=share.principalRoleName, + account=target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py index ee004bd3b..45fc5592c 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py @@ -402,13 +402,11 @@ def test_grant_target_role_access_policy_test_empty_policy( # When share_manager.grant_target_role_access_policy() - expected_policy_name = S3SharePolicyService( - environmentUri=target_environment.environmentUri, - role_name=share1.principalRoleName, - account=target_environment.AwsAccountId, - region=target_environment.region, - resource_prefix=target_environment.resourcePrefix, - ).generate_indexed_policy_name(index=0) + expected_policy_name = S3SharePolicyService(role_name=share1.principalRoleName, + account=target_environment.AwsAccountId, + region=target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix).generate_indexed_policy_name(index=0) # Then iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -1029,13 +1027,11 @@ def test_delete_target_role_access_policy_no_remaining_statement( # When we revoke IAM access to the target IAM role share_manager.revoke_target_role_access_policy() - expected_policy_name = S3SharePolicyService( - environmentUri=target_environment.environmentUri, - role_name=share1.principalRoleName, - account=target_environment.AwsAccountId, - region=target_environment.region, - resource_prefix=target_environment.resourcePrefix, - ).generate_indexed_policy_name(index=0) + expected_policy_name = S3SharePolicyService(role_name=share1.principalRoleName, + account=target_environment.AwsAccountId, + region=target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix).generate_indexed_policy_name(index=0) iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -1156,13 +1152,11 @@ def test_delete_target_role_access_policy_with_remaining_statement( share_manager.revoke_target_role_access_policy() # Then - expected_policy_name = S3SharePolicyService( - environmentUri=target_environment.environmentUri, - role_name=share1.principalRoleName, - account=target_environment.AwsAccountId, - region=target_environment.region, - resource_prefix=target_environment.resourcePrefix, - ).generate_indexed_policy_name(index=0) + expected_policy_name = S3SharePolicyService(role_name=share1.principalRoleName, + account=target_environment.AwsAccountId, + region=target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix).generate_indexed_policy_name(index=0) iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, From 7303bba342cc1e99256ea6259160754ac3bc0d12 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 23 Oct 2024 18:59:21 -0500 Subject: [PATCH 16/30] Fixing few tests --- .../modules/s3_datasets_shares/test_share.py | 39 ++----------------- 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 230b3e3e2..8bae58186 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -1275,22 +1275,14 @@ def test_create_share_object_as_requester(mocker, client, user2, group2, env2gro # When a user that belongs to environment and group creates request mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=False, + side_effect=lambda input: True if input=='policy-0' else False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) create_share_object_response = create_share_object( mocker=mocker, @@ -1319,22 +1311,14 @@ def test_create_share_object_as_approver_and_requester(mocker, client, user, gro ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=False, + side_effect=lambda input: True if input=='policy-0' else False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) create_share_object_response = create_share_object( mocker=mocker, @@ -1361,24 +1345,7 @@ def test_create_share_object_invalid_account(mocker, client, user, group2, env2g 'dataall.base.aws.iam.IAM.get_role_arn_by_name', return_value='role_arn', ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=False, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) create_share_object_response = create_share_object( mocker=mocker, client=client, From 200313ba112799bcb023fb4a7695e9f79b4428ca Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 23 Oct 2024 19:02:05 -0500 Subject: [PATCH 17/30] Linting --- backend/dataall/base/db/exceptions.py | 1 + backend/dataall/base/utils/iam_cdk_utils.py | 3 +- .../core/environment/cdk/pivot_role_stack.py | 2 +- .../s3_share_managed_policy_service.py | 5 +- .../s3_access_point_share_manager.py | 50 +++++++++++-------- .../share_managers/s3_bucket_share_manager.py | 40 +++++++++------ .../test_s3_access_point_share_manager.py | 36 +++++++------ .../modules/s3_datasets_shares/test_share.py | 5 +- 8 files changed, 85 insertions(+), 57 deletions(-) diff --git a/backend/dataall/base/db/exceptions.py b/backend/dataall/base/db/exceptions.py index 24153e7fc..8a06876df 100644 --- a/backend/dataall/base/db/exceptions.py +++ b/backend/dataall/base/db/exceptions.py @@ -146,6 +146,7 @@ def __init__(self, action, message): def __str__(self): return f'{self.message}' + class AWSResourceQuotaExceeded(Exception): def __init__(self, action, message): self.action = action diff --git a/backend/dataall/base/utils/iam_cdk_utils.py b/backend/dataall/base/utils/iam_cdk_utils.py index 333fc1e2a..3fc7894f3 100644 --- a/backend/dataall/base/utils/iam_cdk_utils.py +++ b/backend/dataall/base/utils/iam_cdk_utils.py @@ -22,6 +22,7 @@ def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]): resources=iam_policy.get('Resource'), ) + def process_and_split_statements_in_chunks(statements: List[Dict]): statement_chunks_json = split_policy_statements_in_chunks(statements) statements_chunks = [] @@ -30,4 +31,4 @@ def process_and_split_statements_in_chunks(statements: List[Dict]): statements_chunks.append(convert_from_json_to_iam_policy_statement_with_conditions(statement_js)) else: statements_chunks.append(convert_from_json_to_iam_policy_statement(statement_js)) - return statements_chunks \ No newline at end of file + return statements_chunks diff --git a/backend/dataall/core/environment/cdk/pivot_role_stack.py b/backend/dataall/core/environment/cdk/pivot_role_stack.py index e09fba3c7..b2d67d856 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_stack.py +++ b/backend/dataall/core/environment/cdk/pivot_role_stack.py @@ -5,7 +5,7 @@ from aws_cdk import Duration, aws_iam as iam, NestedStack from dataall.base.utils.iam_cdk_utils import ( - process_and_split_statements_in_chunks, + process_and_split_statements_in_chunks, ) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 26a9ae8a7..855a90574 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -415,7 +415,10 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): log.error( f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}' ) - raise AWSResourceQuotaExceeded(action='_check_iam_managed_policy_attachment_limit', message=f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}') + raise AWSResourceQuotaExceeded( + action='_check_iam_managed_policy_attachment_limit', + message=f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}', + ) log.info(f'Role: {self.role_name} has capacity to attach managed policies') diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 0eb2554fa..f9c88920e 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -170,11 +170,13 @@ def check_target_role_access_policy(self) -> None: key_alias = f'alias/{self.dataset.KmsAlias}' kms_client = KmsClient(self.dataset_account_id, self.source_environment.region) kms_key_id = kms_client.get_key_id(key_alias) - share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - environmentUri=self.target_environment.environmentUri, - resource_prefix=self.target_environment.resourcePrefix) + share_policy_service = S3SharePolicyService( + role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix, + ) share_policy_service.initialize_statements() share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) @@ -320,11 +322,13 @@ def grant_target_role_access_policy(self): """ logger.info(f'Grant target role {self.target_requester_IAMRoleName} access policy') - share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - environmentUri=self.target_environment.environmentUri, - resource_prefix=self.target_environment.resourcePrefix) + share_policy_service = S3SharePolicyService( + role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix, + ) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() @@ -389,16 +393,20 @@ def grant_target_role_access_policy(self): except AWSResourceQuotaExceeded as e: error_message = e.message try: - ShareNotificationService( - session=None, dataset=self.dataset, share=self.share - ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) + ShareNotificationService( + session=None, dataset=self.dataset, share=self.share + ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) except Exception as e: - logger.error(f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}') + logger.error( + f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}' + ) finally: raise error_message share_managed_polices = share_policy_service.get_managed_policies() - all_managed_policies_attached = all( share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices) + all_managed_policies_attached = all( + share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices + ) if not all_managed_policies_attached: logger.info( f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' @@ -696,11 +704,13 @@ def delete_access_point(self): def revoke_target_role_access_policy(self): logger.info('Deleting target role IAM statements...') - share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - environmentUri=self.target_environment.environmentUri, - resource_prefix=self.target_environment.resourcePrefix) + share_policy_service = S3SharePolicyService( + role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix, + ) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index ceaceb431..208cbbcd9 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -76,11 +76,13 @@ def check_s3_iam_access(self) -> None: kms_client = KmsClient(self.source_account_id, self.source_environment.region) kms_key_id = kms_client.get_key_id(key_alias) - share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - environmentUri=self.target_environment.environmentUri, - resource_prefix=self.target_environment.resourcePrefix) + share_policy_service = S3SharePolicyService( + role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix, + ) # Parses all policy documents and extracts s3 and kms statements share_policy_service.initialize_statements() @@ -231,11 +233,13 @@ def grant_s3_iam_access(self): """ logger.info(f'Grant target role {self.target_requester_IAMRoleName} access policy') - share_policy_service = S3SharePolicyService(role_name=self.target_requester_IAMRoleName, - account=self.target_environment.AwsAccountId, - region=self.target_environment.region, - environmentUri=self.target_environment.environmentUri, - resource_prefix=self.target_environment.resourcePrefix) + share_policy_service = S3SharePolicyService( + role_name=self.target_requester_IAMRoleName, + account=self.target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=self.target_environment.environmentUri, + resource_prefix=self.target_environment.resourcePrefix, + ) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() @@ -281,7 +285,9 @@ def grant_s3_iam_access(self): ) share_managed_polices = share_policy_service.get_managed_policies() - all_managed_policies_attached = all(share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices) + all_managed_policies_attached = all( + share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices + ) if not all_managed_policies_attached: logger.info( f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' @@ -530,11 +536,13 @@ def delete_target_role_access_policy( ): logger.info('Deleting target role IAM statements...') - share_policy_service = S3SharePolicyService(role_name=share.principalRoleName, - account=target_environment.AwsAccountId, - region=self.target_environment.region, - environmentUri=target_environment.environmentUri, - resource_prefix=target_environment.resourcePrefix) + share_policy_service = S3SharePolicyService( + role_name=share.principalRoleName, + account=target_environment.AwsAccountId, + region=self.target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix, + ) # Process all backwards compatibility tasks and convert to indexed policies share_policy_service.process_backwards_compatibility_for_target_iam_roles() diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py index 45fc5592c..ff1d3caa2 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py @@ -402,11 +402,13 @@ def test_grant_target_role_access_policy_test_empty_policy( # When share_manager.grant_target_role_access_policy() - expected_policy_name = S3SharePolicyService(role_name=share1.principalRoleName, - account=target_environment.AwsAccountId, - region=target_environment.region, - environmentUri=target_environment.environmentUri, - resource_prefix=target_environment.resourcePrefix).generate_indexed_policy_name(index=0) + expected_policy_name = S3SharePolicyService( + role_name=share1.principalRoleName, + account=target_environment.AwsAccountId, + region=target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix, + ).generate_indexed_policy_name(index=0) # Then iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -1027,11 +1029,13 @@ def test_delete_target_role_access_policy_no_remaining_statement( # When we revoke IAM access to the target IAM role share_manager.revoke_target_role_access_policy() - expected_policy_name = S3SharePolicyService(role_name=share1.principalRoleName, - account=target_environment.AwsAccountId, - region=target_environment.region, - environmentUri=target_environment.environmentUri, - resource_prefix=target_environment.resourcePrefix).generate_indexed_policy_name(index=0) + expected_policy_name = S3SharePolicyService( + role_name=share1.principalRoleName, + account=target_environment.AwsAccountId, + region=target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix, + ).generate_indexed_policy_name(index=0) iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, @@ -1152,11 +1156,13 @@ def test_delete_target_role_access_policy_with_remaining_statement( share_manager.revoke_target_role_access_policy() # Then - expected_policy_name = S3SharePolicyService(role_name=share1.principalRoleName, - account=target_environment.AwsAccountId, - region=target_environment.region, - environmentUri=target_environment.environmentUri, - resource_prefix=target_environment.resourcePrefix).generate_indexed_policy_name(index=0) + expected_policy_name = S3SharePolicyService( + role_name=share1.principalRoleName, + account=target_environment.AwsAccountId, + region=target_environment.region, + environmentUri=target_environment.environmentUri, + resource_prefix=target_environment.resourcePrefix, + ).generate_indexed_policy_name(index=0) iam_update_role_policy_mock.assert_called_with( target_environment.AwsAccountId, diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 8bae58186..c1ee6d6e8 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -1275,7 +1275,7 @@ def test_create_share_object_as_requester(mocker, client, user2, group2, env2gro # When a user that belongs to environment and group creates request mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - side_effect=lambda input: True if input=='policy-0' else False, + side_effect=lambda input: True if input == 'policy-0' else False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', @@ -1283,7 +1283,6 @@ def test_create_share_object_as_requester(mocker, client, user2, group2, env2gro ) mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) - create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1311,7 +1310,7 @@ def test_create_share_object_as_approver_and_requester(mocker, client, user, gro ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - side_effect=lambda input: True if input=='policy-0' else False, + side_effect=lambda input: True if input == 'policy-0' else False, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', From 21af99232beb1c8ff728f2d951d04c8c90fd78c8 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 30 Oct 2024 11:35:47 -0500 Subject: [PATCH 18/30] Change after PR review --- backend/dataall/base/aws/service_quota.py | 6 ++-- .../s3_share_managed_policy_service.py | 4 +-- .../s3_access_point_share_manager.py | 28 ++++++++--------- .../share_managers/s3_bucket_share_manager.py | 31 +++++++------------ .../services/share_notification_service.py | 13 ++++---- 5 files changed, 37 insertions(+), 45 deletions(-) diff --git a/backend/dataall/base/aws/service_quota.py b/backend/dataall/base/aws/service_quota.py index f9ee1b33c..75dbc75e0 100644 --- a/backend/dataall/base/aws/service_quota.py +++ b/backend/dataall/base/aws/service_quota.py @@ -22,7 +22,7 @@ def list_services(self): except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': raise Exception( - f'Data.all Environment Pivot Role does not have permissions to list services for getting service code : {e}' + f'Data.all Environment Pivot Role does not have permissions to do list_services : {e}' ) log.error(f'Failed list services and service codes due to: {e}') return [] @@ -38,7 +38,7 @@ def list_service_quota(self, service_code): return service_quota_code_list except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': - raise Exception(f'Data.all Environment Pivot Role does not have permissions to list quota codes: {e}') + raise Exception(f'Data.all Environment Pivot Role does not have permissions to do list_service_quota : {e}') log.error(f'Failed list quota codes to: {e}') return [] @@ -51,6 +51,6 @@ def get_service_quota_value(self, service_code, service_quota_code): return response['Quota']['Value'] except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': - raise Exception(f'Data.all Environment Pivot Role does not have permissions to list quota codes: {e}') + raise Exception(f'Data.all Environment Pivot Role does not have permissions to do get_service_quota: {e}') log.error(f'Failed list quota codes to: {e}') return None diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index 855a90574..b2a7e7394 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -12,8 +12,6 @@ from dataall.core.environment.services.managed_iam_policies import ManagedPolicy import logging -from dataall.modules.shares_base.services.share_notification_service import ShareNotificationService - log = logging.getLogger(__name__) logging.basicConfig( @@ -131,7 +129,7 @@ def check_resource_in_policy_statements(target_resources: list, existing_policy_ return True @staticmethod - def check_s3_actions_in_policy_statement(existing_policy_statements: List[Any]) -> (bool, str, str): + def check_s3_actions_in_policy_statements(existing_policy_statements: List[Any]) -> (bool, str, str): """ Checks if all required s3 actions are allowed in the existing policy and there is no disallowed actions :param existing_policy_statements: diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index f9c88920e..1675a6c3e 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -190,16 +190,16 @@ def check_target_role_access_policy(self) -> None: stacklevel=2, ) old_managed_policy_name = share_policy_service.generate_old_policy_name() - if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + old_policy_exist = share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name) + if not old_policy_exist: logger.info( - f'No managed policy exists for the role: {self.target_requester_IAMRoleName}, Reapply share create managed policies.' + f'No managed policy exists for the role: {self.target_requester_IAMRoleName}, Reapply share to create indexed managed policies.' ) self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - - if share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + else: logger.info( - f'Old managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' + f'Old managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share to create indexed managed policies.' ) self.folder_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return @@ -222,12 +222,12 @@ def check_target_role_access_policy(self) -> None: if not S3SharePolicyService.check_if_sid_exists( f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', share_policy_service.total_s3_access_point_stmts ): - logger.info(f'IAM Policy Statement with base Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 does not exist') + logger.info(f'IAM Policy Statement with Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not exist') self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Statement Sid', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3-', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', 'S3 Bucket', f'{self.bucket_name}', ) @@ -237,19 +237,19 @@ def check_target_role_access_policy(self) -> None: existing_policy_statements=share_policy_service.total_s3_access_point_stmts, ): logger.info( - f'IAM Policy Statement with Sid {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3- does not contain resources {s3_target_resources}' + f'IAM Policy Statement with Sid {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not contain resources {s3_target_resources}' ) self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Resource(s)', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3-', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', 'S3 Bucket', f'{self.bucket_name}', ) ) else: - policy_sid_actions_map = share_policy_service.check_s3_actions_in_policy_statement( + policy_sid_actions_map = share_policy_service.check_s3_actions_in_policy_statements( existing_policy_statements=share_policy_service.total_s3_access_point_stmts ) for sid in policy_sid_actions_map: @@ -287,13 +287,13 @@ def check_target_role_access_policy(self) -> None: f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', share_policy_service.total_s3_access_point_kms_stmts ): logger.info( - f'IAM Policy Statement with base Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS- does not exist' + f'IAM Policy Statement with Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS - where can be 0,1,2.. - does not exist' ) self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Statement', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS-', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', 'KMS Key', f'{kms_key_id}', ) @@ -303,13 +303,13 @@ def check_target_role_access_policy(self) -> None: existing_policy_statements=share_policy_service.total_s3_access_point_kms_stmts, ): logger.info( - f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS- does not contain resources {kms_target_resources}' + f'IAM Policy Statement {IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS - where can be 0,1,2.. - does not contain resources {kms_target_resources}' ) self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Resource', - f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS-', + f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', 'KMS Key', f'{kms_key_id}', ) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index 208cbbcd9..b8ab499b8 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -97,23 +97,16 @@ def check_s3_iam_access(self) -> None: stacklevel=2, ) old_managed_policy_name = share_policy_service.generate_old_policy_name() - if not share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): + old_policy_exist = share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name) + if not old_policy_exist: logger.info( - f'No managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' + f'No managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share to create indexed managed policies.' ) self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return - - if share_policy_service.check_if_policy_exists(policy_name=old_managed_policy_name): - logger.info( - f'Old managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share create managed policies.' - ) - self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) - return - - if share_policy_service.check_if_policy_attached(policy_name=old_managed_policy_name): + else: logger.info( - f'Older version of managed policy without index present. Correct managed policy: {share_resource_policy_name}. Reapply share to correct managed policy' + f'Old managed policy exists for the role: {self.target_requester_IAMRoleName}. Reapply share to create indexed managed policies.' ) self.bucket_errors.append(ShareErrorFormatter.dne_error_msg('IAM Policy', share_resource_policy_name)) return @@ -131,12 +124,12 @@ def check_s3_iam_access(self) -> None: if not S3SharePolicyService.check_if_sid_exists( f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', share_policy_service.total_s3_stmts ): - logger.info(f'IAM Policy Statement with base Sid: {IAM_S3_BUCKETS_STATEMENT_SID}S3 does not exist') + logger.info(f'IAM Policy Statement with Sid: {IAM_S3_BUCKETS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not exist') self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Statement Sid', - f'{IAM_S3_BUCKETS_STATEMENT_SID}S3-', + f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', 'S3 Bucket', f'{self.bucket_name}', ) @@ -146,19 +139,19 @@ def check_s3_iam_access(self) -> None: existing_policy_statements=share_policy_service.total_s3_stmts, ): logger.info( - f'IAM Policy Statement with Sid {IAM_S3_BUCKETS_STATEMENT_SID}S3- does not contain resources {s3_target_resources}' + f'IAM Policy Statement with Sid {IAM_S3_BUCKETS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not contain resources {s3_target_resources}' ) self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, 'IAM Policy Resource(s)', - f'{IAM_S3_BUCKETS_STATEMENT_SID}S3-', + f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', 'S3 Bucket', f'{self.bucket_name}', ) ) else: - policy_sid_actions_map = share_policy_service.check_s3_actions_in_policy_statement( + policy_sid_actions_map = share_policy_service.check_s3_actions_in_policy_statements( existing_policy_statements=share_policy_service.total_s3_stmts ) @@ -197,7 +190,7 @@ def check_s3_iam_access(self) -> None: f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', share_policy_service.total_s3_kms_stmts ): logger.info( - f'IAM Policy Statement with base Sid: {IAM_S3_BUCKETS_STATEMENT_SID}KMS- does not exist' + f'IAM Policy Statement with Sid: {IAM_S3_BUCKETS_STATEMENT_SID}KMS - where can be 0,1,2.. - does not exist' ) self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( @@ -213,7 +206,7 @@ def check_s3_iam_access(self) -> None: existing_policy_statements=share_policy_service.total_s3_kms_stmts, ): logger.info( - f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}KMS- does not contain resources {kms_target_resources}' + f'IAM Policy Statement {IAM_S3_BUCKETS_STATEMENT_SID}KMS - where can be 0,1,2.. - does not contain resources {kms_target_resources}' ) self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( diff --git a/backend/dataall/modules/shares_base/services/share_notification_service.py b/backend/dataall/modules/shares_base/services/share_notification_service.py index ee4e9c734..6b2897f67 100644 --- a/backend/dataall/modules/shares_base/services/share_notification_service.py +++ b/backend/dataall/modules/shares_base/services/share_notification_service.py @@ -24,7 +24,6 @@ class DataSharingNotificationType(enum.Enum): SHARE_OBJECT_EXTENSION_REJECTED = 'SHARE_OBJECT_EXTENSION_REJECTED' SHARE_OBJECT_REJECTED = 'SHARE_OBJECT_REJECTED' SHARE_OBJECT_PENDING_APPROVAL = 'SHARE_OBJECT_PENDING_APPROVAL' - SHARE_OBJECT_FAILED = 'SHARE_OBJECT_FAILED' DATASET_VERSION = 'DATASET_VERSION' @@ -119,13 +118,15 @@ def notify_managed_policy_limit_exceeded_action(self, email_id: str): f'to view more details.' ) - msg_intro = f"""Dear User, + msg_intro = f"""Dear User,
+ + We are contacting you because a share requested by {email_id} failed because no new managed policy can be attached to your IAM role {self.share.principalRoleName}. + Please check the service quota for the number of managed policies that can be attached to a role in your aws account and increase the limit. + For reference please take a look at this link - https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-quotas-entities.
+ Or please remove any unused managed policies from that role.
- We are contacting you because for a share requested by {email_id} failed because no new managed policy can be attached to your IAM role {self.share.principalRoleName}. - Please check the service quota for the managed policies that can be attached to a role in your aws account and increase the limit. - For reference please take a look at this link - https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-quotas-entities - Note - Previously made shares are not affected but all new shares will be failed till the time you increase the IAM quota limit. + Note - Previously made shares are not affected but any newly added share items or new shares on requestor role {self.share.principalRoleName} will be fail till the time you increase the IAM quota limit or detach any other managed policy from that role. """ msg_end = """Your prompt attention in this matter is greatly appreciated. From 73f98b1798c85d786aeeb40ddbf6cda5c38d5365 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 30 Oct 2024 11:38:10 -0500 Subject: [PATCH 19/30] Reverting changes made --- .../services/s3_share_validator.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index 41d826bfc..1ba57ad0f 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -122,3 +122,44 @@ def _validate_iam_role_and_policy( action=principal_type, message=f'The principal role {principal_role_name} is not found.', ) + + log.info('Verifying data.all managed share IAM policy is attached to IAM role...') + share_policy_manager = PolicyManager( + role_name=principal_role_name, + environmentUri=environment.environmentUri, + account=environment.AwsAccountId, + region=environment.region, + resource_prefix=environment.resourcePrefix, + ) + for policy_manager in [ + Policy for Policy in share_policy_manager.initializedPolicies if Policy.policy_type == 'SharePolicy' + ]: + # Backwards compatibility - 1 + # we check if a managed share policy exists. If False, the role was introduced to data.all before this update + # We create the policy from the inline statements + # In this case it could also happen that the role is the Admin of the environment + old_managed_policy_name = policy_manager.generate_old_policy_name() + old_managed_policy_exists = policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) + # If old managed policy doesn't exist and also new managed policies do not exist. + # Then there might be inline policies, convert them to managed indexed policies + if not old_managed_policy_exists and not policy_manager.check_if_managed_policies_exists(): + policy_manager.create_managed_policy_from_inline_and_delete_inline() + # End of backwards compatibility + + # Backwards compatibility - 2 + # Check if an already existing managed policy is present in old format + # If yes, convert it to the indexed format + old_managed_policy_name = policy_manager.generate_old_policy_name() + if policy_manager.check_if_policy_exists(old_managed_policy_name): + policy_manager.create_managed_indexed_policy_from_managed_policy_delete_old_policy() + # End of backwards compatibility + + attached = policy_manager.check_if_policies_attached() + if not attached and not managed and not attachMissingPolicies: + raise Exception( + f'Required customer managed policies {policy_manager.get_policies_unattached_to_role()} are not attached to role {principal_role_name}' + ) + elif not attached: + managed_policy_list = policy_manager.get_policies_unattached_to_role() + policy_manager.attach_policies(managed_policy_list) + From e27b0aa6864c1a18f356ecaadd92ce40091f95d4 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 30 Oct 2024 12:06:08 -0500 Subject: [PATCH 20/30] More changes --- backend/dataall/base/aws/iam.py | 7 ++--- backend/dataall/base/aws/service_quota.py | 12 +++++---- .../services/managed_iam_policies.py | 26 +++++++++---------- .../services/s3_share_validator.py | 1 - .../s3_access_point_share_manager.py | 13 +++++----- .../share_managers/s3_bucket_share_manager.py | 11 ++++---- backend/docker/dev/Dockerfile | 2 ++ 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/backend/dataall/base/aws/iam.py b/backend/dataall/base/aws/iam.py index d34f64ec2..581638e22 100644 --- a/backend/dataall/base/aws/iam.py +++ b/backend/dataall/base/aws/iam.py @@ -1,5 +1,6 @@ import logging from botocore.exceptions import ClientError +import re from .sts import SessionHelper @@ -67,7 +68,7 @@ def get_role_policy( return None @staticmethod - def list_policy_names_by_policy_pattern(account_id: str, region: str, policy_name: str): + def list_policy_names_by_policy_pattern(account_id: str, region: str, policy_filter_pattern: str): try: client = IAM.client(account_id, region) # Setting Scope to 'Local' to fetch all the policies created in this account @@ -76,11 +77,11 @@ def list_policy_names_by_policy_pattern(account_id: str, region: str, policy_nam for page in paginator.paginate(Scope='Local'): policies.extend(page['Policies']) policy_names = [policy.get('PolicyName') for policy in policies] - return [policy_nm for policy_nm in policy_names if policy_name in policy_nm] + return [policy_nm for policy_nm in policy_names if re.search(policy_filter_pattern, policy_nm)] except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': raise Exception( - f'Data.all Environment Pivot Role does not have permissions to get policies with pattern {policy_name} due to: {e}' + f'Data.all Environment Pivot Role does not have permissions to get policies with pattern {policy_filter_pattern} due to: {e}' ) log.error(f'Failed to get policies for policy pattern due to: {e}') return [] diff --git a/backend/dataall/base/aws/service_quota.py b/backend/dataall/base/aws/service_quota.py index 75dbc75e0..dda57474a 100644 --- a/backend/dataall/base/aws/service_quota.py +++ b/backend/dataall/base/aws/service_quota.py @@ -21,9 +21,7 @@ def list_services(self): return services_list except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': - raise Exception( - f'Data.all Environment Pivot Role does not have permissions to do list_services : {e}' - ) + raise Exception(f'Data.all Environment Pivot Role does not have permissions to do list_services : {e}') log.error(f'Failed list services and service codes due to: {e}') return [] @@ -38,7 +36,9 @@ def list_service_quota(self, service_code): return service_quota_code_list except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': - raise Exception(f'Data.all Environment Pivot Role does not have permissions to do list_service_quota : {e}') + raise Exception( + f'Data.all Environment Pivot Role does not have permissions to do list_service_quota : {e}' + ) log.error(f'Failed list quota codes to: {e}') return [] @@ -51,6 +51,8 @@ def get_service_quota_value(self, service_code, service_quota_code): return response['Quota']['Value'] except ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': - raise Exception(f'Data.all Environment Pivot Role does not have permissions to do get_service_quota: {e}') + raise Exception( + f'Data.all Environment Pivot Role does not have permissions to do get_service_quota: {e}' + ) log.error(f'Failed list quota codes to: {e}') return None diff --git a/backend/dataall/core/environment/services/managed_iam_policies.py b/backend/dataall/core/environment/services/managed_iam_policies.py index c70c6e3b5..ebf7a90ce 100644 --- a/backend/dataall/core/environment/services/managed_iam_policies.py +++ b/backend/dataall/core/environment/services/managed_iam_policies.py @@ -57,13 +57,13 @@ def generate_empty_policy(self) -> dict: ... def check_if_policy_exists(self, policy_name) -> bool: - share_policy = IAM.get_managed_policy_by_name(self.account, self.region, policy_name) - return share_policy is not None + policy = IAM.get_managed_policy_by_name(self.account, self.region, policy_name) + return policy is not None def get_managed_policies(self) -> List[str]: policy_pattern = self.generate_base_policy_name() - share_policies = self._get_policy_names(policy_pattern) - return share_policies + policies = self._get_policy_names(policy_pattern) + return policies def check_if_policy_attached(self, policy_name): is_policy_attached = IAM.is_policy_attached(self.account, self.region, policy_name, self.role_name) @@ -71,11 +71,11 @@ def check_if_policy_attached(self, policy_name): def get_policies_unattached_to_role(self): policy_pattern = self.generate_base_policy_name() - share_policies = self._get_policy_names(policy_pattern) + policies = self._get_policy_names(policy_pattern) unattached_policies = [] - for share_policy_name in share_policies: - if not self.check_if_policy_attached(share_policy_name): - unattached_policies.append(share_policy_name) + for policy_name in policies: + if not self.check_if_policy_attached(policy_name): + unattached_policies.append(policy_name) return unattached_policies def attach_policies(self, managed_policies_list: List[str]): @@ -86,12 +86,10 @@ def attach_policies(self, managed_policies_list: List[str]): except Exception as e: raise Exception(f"Required customer managed policy {policy_arn} can't be attached: {e}") - def _get_policy_names(self, policy_pattern): - share_policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, policy_pattern) - # Filter through all policies and remove which have the old policy name - # This is to check that old policies are not included - old_policy_name = self.generate_old_policy_name() - return [policy for policy in share_policies if policy != old_policy_name] + def _get_policy_names(self, base_policy_name): + filter_pattern = r'{base_policy_name}-\d'.format(base_policy_name=base_policy_name) + policies = IAM.list_policy_names_by_policy_pattern(self.account, self.region, filter_pattern) + return policies class PolicyManager(object): diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index 1ba57ad0f..6ffccaf96 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -162,4 +162,3 @@ def _validate_iam_role_and_policy( elif not attached: managed_policy_list = policy_manager.get_policies_unattached_to_role() policy_manager.attach_policies(managed_policy_list) - diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 1675a6c3e..42834f07b 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -222,7 +222,9 @@ def check_target_role_access_policy(self) -> None: if not S3SharePolicyService.check_if_sid_exists( f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3', share_policy_service.total_s3_access_point_stmts ): - logger.info(f'IAM Policy Statement with Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not exist') + logger.info( + f'IAM Policy Statement with Sid: {IAM_S3_ACCESS_POINTS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not exist' + ) self.folder_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, @@ -401,13 +403,10 @@ def grant_target_role_access_policy(self): f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}' ) finally: - raise error_message + raise Exception(error_message) - share_managed_polices = share_policy_service.get_managed_policies() - all_managed_policies_attached = all( - share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices - ) - if not all_managed_policies_attached: + is_unattached_policies = share_policy_service.get_policies_unattached_to_role() + if is_unattached_policies: logger.info( f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' ) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index b8ab499b8..5d7640ac9 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -124,7 +124,9 @@ def check_s3_iam_access(self) -> None: if not S3SharePolicyService.check_if_sid_exists( f'{IAM_S3_BUCKETS_STATEMENT_SID}S3', share_policy_service.total_s3_stmts ): - logger.info(f'IAM Policy Statement with Sid: {IAM_S3_BUCKETS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not exist') + logger.info( + f'IAM Policy Statement with Sid: {IAM_S3_BUCKETS_STATEMENT_SID}S3 - where can be 0,1,2.. - does not exist' + ) self.bucket_errors.append( ShareErrorFormatter.missing_permission_error_msg( self.target_requester_IAMRoleName, @@ -277,11 +279,8 @@ def grant_s3_iam_access(self): target_s3_kms_statements=s3_kms_statement_chunks, ) - share_managed_polices = share_policy_service.get_managed_policies() - all_managed_policies_attached = all( - share_policy_service.check_if_policy_attached(managed_policy) for managed_policy in share_managed_polices - ) - if not all_managed_policies_attached: + is_unattached_policies = share_policy_service.get_policies_unattached_to_role() + if is_unattached_policies: logger.info( f'Found some policies are not attached to the target IAM role: {self.target_requester_IAMRoleName}. Attaching policies now' ) diff --git a/backend/docker/dev/Dockerfile b/backend/docker/dev/Dockerfile index 0e33ba5fd..9d475ac3d 100644 --- a/backend/docker/dev/Dockerfile +++ b/backend/docker/dev/Dockerfile @@ -57,6 +57,8 @@ RUN echo export PATH="\ /root/.nvm/versions/node/${NODE_VERSION}/bin:\ $(${PYTHON_VERSION} -m site --user-base)/bin:\ $(python3 -m site --user-base)/bin:\ + $(python -m site --user-base)/bin:\ + /home/${CONTAINER_USER}/.local/bin:\ $PATH" >> ~/.bashrc && \ echo "nvm use ${NODE_VERSION} 1> /dev/null" >> ~/.bashrc From 8abc4d62b9dd190bf5f5f757488b4e791181b2f8 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 30 Oct 2024 12:08:00 -0500 Subject: [PATCH 21/30] Minor changes --- backend/docker/dev/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/docker/dev/Dockerfile b/backend/docker/dev/Dockerfile index 9d475ac3d..d243db74e 100644 --- a/backend/docker/dev/Dockerfile +++ b/backend/docker/dev/Dockerfile @@ -54,12 +54,12 @@ RUN /bin/bash -c ". ~/.nvm/nvm.sh && \ nvm alias default node && nvm cache clear" RUN echo export PATH="\ - /root/.nvm/versions/node/${NODE_VERSION}/bin:\ - $(${PYTHON_VERSION} -m site --user-base)/bin:\ - $(python3 -m site --user-base)/bin:\ - $(python -m site --user-base)/bin:\ - /home/${CONTAINER_USER}/.local/bin:\ - $PATH" >> ~/.bashrc && \ +/root/.nvm/versions/node/${NODE_VERSION}/bin:\ +$(${PYTHON_VERSION} -m site --user-base)/bin:\ +$(python3 -m site --user-base)/bin:\ +$(python -m site --user-base)/bin:\ +/home/${CONTAINER_USER}/.local/bin:\ +$PATH" >> ~/.bashrc && \ echo "nvm use ${NODE_VERSION} 1> /dev/null" >> ~/.bashrc RUN /bin/bash -c ". ~/.nvm/nvm.sh && cdk --version" From 3b4dce2a6b65a62655d82108422bb6804d397090 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 30 Oct 2024 13:52:29 -0500 Subject: [PATCH 22/30] Removing managed policy from unused gql calls --- backend/dataall/core/environment/api/resolvers.py | 8 -------- backend/dataall/core/environment/api/types.py | 3 --- 2 files changed, 11 deletions(-) diff --git a/backend/dataall/core/environment/api/resolvers.py b/backend/dataall/core/environment/api/resolvers.py index 537e6de93..4f0140ffa 100644 --- a/backend/dataall/core/environment/api/resolvers.py +++ b/backend/dataall/core/environment/api/resolvers.py @@ -177,14 +177,6 @@ def get_parent_organization(context: Context, source, **kwargs): org = get_organization_simplified(context, source, organizationUri=source.organizationUri) return org - -# used from ConsumptionRole type as field resolver -def resolve_consumption_role_policies(context: Context, source, **kwargs): - return EnvironmentService.resolve_consumption_role_policies( - uri=source.environmentUri, IAMRoleName=source.IAMRoleName - ) - - # used from getConsumptionRolePolicies query -- query resolver def get_consumption_role_policies(context: Context, source, environmentUri, IAMRoleName): return EnvironmentService.resolve_consumption_role_policies(uri=environmentUri, IAMRoleName=IAMRoleName) diff --git a/backend/dataall/core/environment/api/types.py b/backend/dataall/core/environment/api/types.py index 229593dd8..2bc2fc922 100644 --- a/backend/dataall/core/environment/api/types.py +++ b/backend/dataall/core/environment/api/types.py @@ -180,9 +180,6 @@ gql.Field(name='created', type=gql.String), gql.Field(name='updated', type=gql.String), gql.Field(name='deleted', type=gql.String), - gql.Field( - name='managedPolicies', type=gql.ArrayType(RoleManagedPolicy), resolver=resolve_consumption_role_policies - ), ], ) From 1f5ec6e971bd54a4e467252d0c76f5eacf0eac81 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 30 Oct 2024 13:53:19 -0500 Subject: [PATCH 23/30] python linting --- backend/dataall/core/environment/api/resolvers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/dataall/core/environment/api/resolvers.py b/backend/dataall/core/environment/api/resolvers.py index 4f0140ffa..978bfd6b3 100644 --- a/backend/dataall/core/environment/api/resolvers.py +++ b/backend/dataall/core/environment/api/resolvers.py @@ -177,6 +177,7 @@ def get_parent_organization(context: Context, source, **kwargs): org = get_organization_simplified(context, source, organizationUri=source.organizationUri) return org + # used from getConsumptionRolePolicies query -- query resolver def get_consumption_role_policies(context: Context, source, environmentUri, IAMRoleName): return EnvironmentService.resolve_consumption_role_policies(uri=environmentUri, IAMRoleName=IAMRoleName) From a048fe9f02e6d96432fc34b0d0c40e1027e757ce Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 30 Oct 2024 14:57:34 -0500 Subject: [PATCH 24/30] Corrections --- backend/dataall/core/environment/api/types.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/dataall/core/environment/api/types.py b/backend/dataall/core/environment/api/types.py index 2bc2fc922..ad9997ce8 100644 --- a/backend/dataall/core/environment/api/types.py +++ b/backend/dataall/core/environment/api/types.py @@ -3,7 +3,6 @@ from dataall.core.environment.api.resolvers import ( get_environment_stack, get_parent_organization, - resolve_consumption_role_policies, resolve_environment_networks, resolve_parameters, resolve_user_role, From f48e07fa65c1f6849e17ff72c78d73c7dbd4e93a Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 31 Oct 2024 12:37:09 -0500 Subject: [PATCH 25/30] Naming changes in exceptions --- backend/dataall/base/db/exceptions.py | 2 +- .../services/s3_share_managed_policy_service.py | 4 ++-- .../services/share_managers/s3_access_point_share_manager.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/dataall/base/db/exceptions.py b/backend/dataall/base/db/exceptions.py index 8a06876df..c3c5d91e9 100644 --- a/backend/dataall/base/db/exceptions.py +++ b/backend/dataall/base/db/exceptions.py @@ -147,7 +147,7 @@ def __str__(self): return f'{self.message}' -class AWSResourceQuotaExceeded(Exception): +class AWSServiceQuotaExceeded(Exception): def __init__(self, action, message): self.action = action self.message = f""" diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index b2a7e7394..c7cea2a57 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -3,7 +3,7 @@ from dataall.base.aws.iam import IAM from dataall.base.aws.service_quota import ServiceQuota -from dataall.base.db.exceptions import AWSResourceQuotaExceeded +from dataall.base.db.exceptions import AWSServiceQuotaExceeded from dataall.base.utils.iam_policy_utils import ( split_policy_statements_in_chunks, split_policy_with_resources_in_statements, @@ -413,7 +413,7 @@ def _check_iam_managed_policy_attachment_limit(self, policy_document_chunks): log.error( f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}' ) - raise AWSResourceQuotaExceeded( + raise AWSServiceQuotaExceeded( action='_check_iam_managed_policy_attachment_limit', message=f'Number of policies which can be attached to the role is more than the service quota limit: {managed_iam_policy_quota}', ) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 42834f07b..fa8e7f508 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -5,7 +5,7 @@ from typing import List from warnings import warn -from dataall.base.db.exceptions import AWSResourceQuotaExceeded +from dataall.base.db.exceptions import AWSServiceQuotaExceeded from dataall.core.environment.services.environment_service import EnvironmentService from dataall.base.db import utils from dataall.base.aws.sts import SessionHelper @@ -392,7 +392,7 @@ def grant_target_role_access_policy(self): target_s3_statements=s3_statement_chunks, target_s3_kms_statements=s3_kms_statement_chunks, ) - except AWSResourceQuotaExceeded as e: + except AWSServiceQuotaExceeded as e: error_message = e.message try: ShareNotificationService( From 99556b35bc707b4f1f7538ce65a5fddef281587b Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 4 Nov 2024 12:16:01 -0600 Subject: [PATCH 26/30] Refactoring and corrections --- backend/dataall/base/utils/iam_cdk_utils.py | 61 +++++++++++++---- .../core/environment/cdk/pivot_role_stack.py | 2 +- .../cdk/pivot_role_redshift_policy.py | 8 +-- ...pivot_role_redshift_data_sharing_policy.py | 6 +- .../cdk/pivot_role_datasets_policy.py | 14 ++-- .../s3_share_managed_policy_service.py | 8 ++- .../services/s3_share_validator.py | 9 +-- .../s3_access_point_share_manager.py | 57 ++++++---------- .../share_managers/s3_bucket_share_manager.py | 66 +++++++++++-------- .../shares_base/tasks/share_manager_task.py | 3 +- 10 files changed, 129 insertions(+), 105 deletions(-) diff --git a/backend/dataall/base/utils/iam_cdk_utils.py b/backend/dataall/base/utils/iam_cdk_utils.py index 3fc7894f3..5881033d7 100644 --- a/backend/dataall/base/utils/iam_cdk_utils.py +++ b/backend/dataall/base/utils/iam_cdk_utils.py @@ -1,15 +1,16 @@ from typing import Dict, Any, List from aws_cdk import aws_iam as iam -from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks +from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks, \ + split_policy_with_resources_in_statements, split_policy_with_mutiple_value_condition_in_statements def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[Any, Any]): return iam.PolicyStatement( sid=iam_policy.get('Sid'), - effect=iam_policy.get('Effect'), - actions=iam_policy.get('Action'), - resources=iam_policy.get('Resource'), + effect=iam.Effect.ALLOW if iam_policy.get('Effect').casefold() == 'Allow'.casefold() else iam.Effect.DENY, + actions=_convert_to_array(str, iam_policy.get('Action')), + resources=_convert_to_array(str, iam_policy.get('Resource')), conditions=iam_policy.get('Condition'), ) @@ -17,18 +18,50 @@ def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[A def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]): return iam.PolicyStatement( sid=iam_policy.get('Sid'), - effect=iam_policy.get('Effect'), - actions=iam_policy.get('Action'), - resources=iam_policy.get('Resource'), + effect=iam.Effect.ALLOW if iam_policy.get('Effect').casefold() == 'Allow'.casefold() else iam.Effect.DENY, + actions=_convert_to_array(str, iam_policy.get('Action')), + resources=_convert_to_array(str, iam_policy.get('Resource')), ) def process_and_split_statements_in_chunks(statements: List[Dict]): - statement_chunks_json = split_policy_statements_in_chunks(statements) - statements_chunks = [] - for statement_js in statement_chunks_json: - if not statement_js.get('Condition', None): - statements_chunks.append(convert_from_json_to_iam_policy_statement_with_conditions(statement_js)) - else: - statements_chunks.append(convert_from_json_to_iam_policy_statement(statement_js)) + statement_chunks_json: List[List[Dict]] = split_policy_statements_in_chunks(statements) + statements_chunks: List[List[iam.PolicyStatement]] = [] + for statement_js_chunk in statement_chunks_json: + statements: List[iam.PolicyStatement] = [] + for statement in statement_js_chunk: + if statement.get('Condition', None): + statements.append(convert_from_json_to_iam_policy_statement_with_conditions(statement)) + else: + statements.append(convert_from_json_to_iam_policy_statement(statement)) + statements_chunks.append(statements) return statements_chunks + + +def process_and_split_policy_with_resources_in_statements(base_sid: str, effect: str, actions: List[str], + resources: List[str], condition_dict: Dict = None): + if condition_dict is not None: + print(f"Condition dictionary is: {condition_dict}") + json_statements = split_policy_with_mutiple_value_condition_in_statements(base_sid=base_sid, effect=effect, + actions=actions, + resources=resources, + condition_dict=condition_dict) + else: + json_statements = split_policy_with_resources_in_statements(base_sid=base_sid, effect=effect, actions=actions, + resources=resources) + iam_statements: [iam.PolicyStatement] = [] + for json_statement in json_statements: + if json_statement.get('Condition', None): + iam_policy_statement = convert_from_json_to_iam_policy_statement_with_conditions(json_statement) + else: + iam_policy_statement = convert_from_json_to_iam_policy_statement(json_statement) + iam_statements.append(iam_policy_statement) + return iam_statements + + +# If item is of item type i.e. single instance if present, then wrap in an array. +# This is helpful at places where array is required even if one element is present +def _convert_to_array(item_type, item): + if isinstance(item, item_type): + return [item] + return item diff --git a/backend/dataall/core/environment/cdk/pivot_role_stack.py b/backend/dataall/core/environment/cdk/pivot_role_stack.py index b2d67d856..073712898 100644 --- a/backend/dataall/core/environment/cdk/pivot_role_stack.py +++ b/backend/dataall/core/environment/cdk/pivot_role_stack.py @@ -41,7 +41,7 @@ def generate_policies(self) -> List[iam.ManagedPolicy]: logger.info(f'statements: {str(service.get_statements(self))}') statements_json = [statement.to_json() for statement in statements] - statements_chunks = process_and_split_statements_in_chunks(statements_json) + statements_chunks: List[List[iam.PolicyStatement]] = process_and_split_statements_in_chunks(statements_json) for index, chunk in enumerate(statements_chunks): policies.append( diff --git a/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py b/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py index 5c9f7f10e..b2ad8d9ab 100644 --- a/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py +++ b/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py @@ -3,8 +3,7 @@ from dataall.base import db from dataall.base.aws.sts import SessionHelper -from dataall.base.utils.iam_cdk_utils import convert_from_json_to_iam_policy_statement -from dataall.base.utils.iam_policy_utils import split_policy_with_resources_in_statements +from dataall.base.utils.iam_cdk_utils import process_and_split_policy_with_resources_in_statements from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet from dataall.modules.redshift_datasets.db.redshift_connection_repositories import RedshiftConnectionRepository from dataall.modules.redshift_datasets.aws.redshift_serverless import redshift_serverless_client @@ -82,7 +81,7 @@ def get_statements(self): workgroup_arns = [ rs_client.get_workgroup_arn(workgroup_name=conn.workgroup) for conn in connections if conn.workgroup ] - redshift_data_statement_json = split_policy_with_resources_in_statements( + redshift_data_statements = process_and_split_policy_with_resources_in_statements( base_sid='RedshiftData', effect=iam.Effect.ALLOW.value, actions=[ @@ -93,7 +92,6 @@ def get_statements(self): ], resources=cluster_arns + workgroup_arns, ) - redshift_data_statement = convert_from_json_to_iam_policy_statement(redshift_data_statement_json) - additional_statements.extend(redshift_data_statement) + additional_statements.extend(redshift_data_statements) return base_statements + additional_statements diff --git a/backend/dataall/modules/redshift_datasets_shares/cdk/pivot_role_redshift_data_sharing_policy.py b/backend/dataall/modules/redshift_datasets_shares/cdk/pivot_role_redshift_data_sharing_policy.py index c60ce5412..cd0251692 100644 --- a/backend/dataall/modules/redshift_datasets_shares/cdk/pivot_role_redshift_data_sharing_policy.py +++ b/backend/dataall/modules/redshift_datasets_shares/cdk/pivot_role_redshift_data_sharing_policy.py @@ -1,6 +1,6 @@ import os from dataall.base import db -from dataall.base.utils.iam_policy_utils import split_policy_with_resources_in_statements +from dataall.base.utils.iam_cdk_utils import process_and_split_policy_with_resources_in_statements from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet from dataall.modules.redshift_datasets.db.redshift_connection_repositories import RedshiftConnectionRepository @@ -39,9 +39,9 @@ def get_statements(self): for conn in connections ] additional_statements.extend( - split_policy_with_resources_in_statements( + process_and_split_policy_with_resources_in_statements( base_sid='RedshiftDataShare', - effect=iam.Effect.ALLOW, + effect=iam.Effect.ALLOW.value, actions=['redshift:AuthorizeDataShare'], resources=source_datashare_arns, ) diff --git a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py index dfc963bc5..4fd325f4b 100644 --- a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py @@ -1,11 +1,9 @@ import os from dataall.base import db from dataall.base.utils.iam_cdk_utils import ( - convert_from_json_to_iam_policy_statement, - convert_from_json_to_iam_policy_statement_with_conditions, + convert_from_json_to_iam_policy_statement_with_conditions, process_and_split_policy_with_resources_in_statements, ) from dataall.base.utils.iam_policy_utils import ( - split_policy_with_resources_in_statements, split_policy_with_mutiple_value_condition_in_statements, ) from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet @@ -157,7 +155,7 @@ def get_statements(self): imported_kms_alias.append(f'alias/{dataset.KmsAlias}') if imported_buckets: - dataset_statement_json = split_policy_with_resources_in_statements( + dataset_statements = process_and_split_policy_with_resources_in_statements( base_sid='ImportedDatasetBuckets', effect=iam.Effect.ALLOW.value, actions=[ @@ -172,10 +170,9 @@ def get_statements(self): ], resources=imported_buckets, ) - dataset_statement = convert_from_json_to_iam_policy_statement(dataset_statement_json) - statements.extend(dataset_statement) + statements.extend(dataset_statements) if imported_kms_alias: - kms_statement_json = split_policy_with_mutiple_value_condition_in_statements( + kms_statements = process_and_split_policy_with_resources_in_statements( base_sid='KMSImportedDataset', effect=iam.Effect.ALLOW.value, actions=[ @@ -195,7 +192,6 @@ def get_statements(self): 'values': imported_kms_alias, }, ) - kms_statement = convert_from_json_to_iam_policy_statement_with_conditions(kms_statement_json) - statements.extend(kms_statement) + statements.extend(kms_statements) return statements diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py index c7cea2a57..645f540d1 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_managed_policy_service.py @@ -245,8 +245,8 @@ def merge_statements_and_update_policies( self, target_sid: str, target_s3_statements: List[Any], target_s3_kms_statements: List[Any] ): """ - This method is responsible for merging policy statement, re-generating chunks comprizing of statements. - Creates policies if needed and then updates policies with statement chunks. + This method is responsible for merging policy statements, re-generating chunks consisting of statements. + Creates new policies (if needed) and then updates existing policies with statement chunks. Based on target_sid: 1. This method merges all the S3 statments 2. Splits the policy into policy chunks, where each chunk is <= size of the policy ( this is approximately true ) @@ -493,6 +493,10 @@ def add_resources_and_generate_split_statements(self, statements, target_resourc for target_resource in target_resources: if target_resource not in s3_statements_resources: s3_statements_resources.append(target_resource) + + if len(s3_statements_resources) == 0: + return [] + statement_chunks = split_policy_with_resources_in_statements( base_sid=sid, effect='Allow', diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py index 6ffccaf96..644591fec 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_validator.py @@ -140,9 +140,10 @@ def _validate_iam_role_and_policy( # In this case it could also happen that the role is the Admin of the environment old_managed_policy_name = policy_manager.generate_old_policy_name() old_managed_policy_exists = policy_manager.check_if_policy_exists(policy_name=old_managed_policy_name) + share_managed_policies_exist = True if policy_manager.get_managed_policies() else False # If old managed policy doesn't exist and also new managed policies do not exist. # Then there might be inline policies, convert them to managed indexed policies - if not old_managed_policy_exists and not policy_manager.check_if_managed_policies_exists(): + if not old_managed_policy_exists and not share_managed_policies_exist: policy_manager.create_managed_policy_from_inline_and_delete_inline() # End of backwards compatibility @@ -154,11 +155,11 @@ def _validate_iam_role_and_policy( policy_manager.create_managed_indexed_policy_from_managed_policy_delete_old_policy() # End of backwards compatibility - attached = policy_manager.check_if_policies_attached() - if not attached and not managed and not attachMissingPolicies: + unattached = policy_manager.get_policies_unattached_to_role() + if unattached and not managed and not attachMissingPolicies: raise Exception( f'Required customer managed policies {policy_manager.get_policies_unattached_to_role()} are not attached to role {principal_role_name}' ) - elif not attached: + elif unattached: managed_policy_list = policy_manager.get_policies_unattached_to_role() policy_manager.attach_policies(managed_policy_list) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index fa8e7f508..0ebd513b5 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -351,21 +351,6 @@ def grant_target_role_access_policy(self): if kms_key_id: kms_target_resources = [f'arn:aws:kms:{self.dataset_region}:{self.dataset_account_id}:key/{kms_key_id}'] - managed_policy_exists = True if share_policy_service.get_managed_policies() else False - - if not managed_policy_exists: - logger.info('Managed policies do not exist. Creating one') - # Create a managed policy with naming convention and index - share_resource_policy_name = share_policy_service.generate_indexed_policy_name(index=0) - empty_policy = share_policy_service.generate_empty_policy() - IAM.create_managed_policy( - self.target_account_id, - self.target_environment.region, - share_resource_policy_name, - json.dumps(empty_policy), - ) - - s3_kms_statement_chunks = [] s3_statements = share_policy_service.total_s3_access_point_stmts s3_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( statements=s3_statements, @@ -376,16 +361,16 @@ def grant_target_role_access_policy(self): logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') - if kms_target_resources: - s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts - s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( - statements=s3_kms_statements, - target_resources=kms_target_resources, - sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', - resource_type='kms', - ) - logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') - logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') + s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts + s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( + statements=s3_kms_statements, + target_resources=kms_target_resources, + sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + resource_type='kms', + ) + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') + try: share_policy_service.merge_statements_and_update_policies( target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, @@ -737,9 +722,7 @@ def revoke_target_role_access_policy(self): logger.info(f'Managed policies for share with uri: {self.share.shareUri} are not found') return - s3_kms_statement_chunks = [] s3_statements = share_policy_service.total_s3_access_point_stmts - s3_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( statements=s3_statements, target_resources=s3_target_resources, @@ -749,17 +732,15 @@ def revoke_target_role_access_policy(self): logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') - if kms_target_resources: - s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts - - s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( - statements=s3_kms_statements, - target_resources=kms_target_resources, - sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', - resource_type='kms', - ) - logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') - logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') + s3_kms_statements = share_policy_service.total_s3_access_point_kms_stmts + s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( + statements=s3_kms_statements, + target_resources=kms_target_resources, + sid=f'{IAM_S3_ACCESS_POINTS_STATEMENT_SID}KMS', + resource_type='kms', + ) + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') share_policy_service.merge_statements_and_update_policies( target_sid=IAM_S3_ACCESS_POINTS_STATEMENT_SID, diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py index 5d7640ac9..aac6fea78 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_bucket_share_manager.py @@ -5,6 +5,7 @@ from warnings import warn from dataall.base.aws.iam import IAM from dataall.base.aws.sts import SessionHelper +from dataall.base.db.exceptions import AWSServiceQuotaExceeded from dataall.core.environment.db.environment_models import Environment from dataall.core.environment.services.environment_service import EnvironmentService from dataall.modules.s3_datasets.db.dataset_models import DatasetBucket @@ -29,6 +30,7 @@ from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository from dataall.modules.shares_base.services.share_exceptions import PrincipalRoleNotFound from dataall.modules.shares_base.services.share_manager_utils import ShareErrorFormatter +from dataall.modules.shares_base.services.share_notification_service import ShareNotificationService from dataall.modules.shares_base.services.shares_enums import PrincipalType from dataall.modules.shares_base.services.sharing_service import ShareData @@ -251,7 +253,6 @@ def grant_s3_iam_access(self): if kms_key_id: kms_target_resources = [f'arn:aws:kms:{self.bucket_region}:{self.source_account_id}:key/{kms_key_id}'] - s3_kms_statement_chunks = [] s3_statements = share_policy_service.total_s3_stmts s3_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( statements=s3_statements, @@ -262,22 +263,34 @@ def grant_s3_iam_access(self): logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') - if kms_target_resources: - s3_kms_statements = share_policy_service.total_s3_kms_stmts - s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( - statements=s3_kms_statements, - target_resources=kms_target_resources, - sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', - resource_type='kms', - ) - logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') - logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') - - share_policy_service.merge_statements_and_update_policies( - target_sid=IAM_S3_BUCKETS_STATEMENT_SID, - target_s3_statements=s3_statement_chunks, - target_s3_kms_statements=s3_kms_statement_chunks, + s3_kms_statements = share_policy_service.total_s3_kms_stmts + s3_kms_statement_chunks = share_policy_service.add_resources_and_generate_split_statements( + statements=s3_kms_statements, + target_resources=kms_target_resources, + sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + resource_type='kms', ) + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') + + try: + share_policy_service.merge_statements_and_update_policies( + target_sid=IAM_S3_BUCKETS_STATEMENT_SID, + target_s3_statements=s3_statement_chunks, + target_s3_kms_statements=s3_kms_statement_chunks, + ) + except AWSServiceQuotaExceeded as e: + error_message = e.message + try: + ShareNotificationService( + session=None, dataset=self.dataset, share=self.share + ).notify_managed_policy_limit_exceeded_action(email_id=self.share.owner) + except Exception as e: + logger.error( + f'Error sending email for notifying that managed policy limit exceeded on role due to: {e}' + ) + finally: + raise Exception(error_message) is_unattached_policies = share_policy_service.get_policies_unattached_to_role() if is_unattached_policies: @@ -554,9 +567,7 @@ def delete_target_role_access_policy( if kms_key_id: kms_target_resources = [f'arn:aws:kms:{target_bucket.region}:{target_bucket.AwsAccountId}:key/{kms_key_id}'] - s3_kms_statement_chunks = [] s3_statements = share_policy_service.total_s3_stmts - s3_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( statements=s3_statements, target_resources=s3_target_resources, @@ -566,16 +577,15 @@ def delete_target_role_access_policy( logger.info(f'Number of S3 statements created after splitting: {len(s3_statement_chunks)}') logger.debug(f'S3 statements after adding resources and splitting: {s3_statement_chunks}') - if kms_target_resources: - s3_kms_statements = share_policy_service.total_s3_kms_stmts - s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( - statements=s3_kms_statements, - target_resources=kms_target_resources, - sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', - resource_type='kms', - ) - logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') - logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') + s3_kms_statements = share_policy_service.total_s3_kms_stmts + s3_kms_statement_chunks = share_policy_service.remove_resources_and_generate_split_statements( + statements=s3_kms_statements, + target_resources=kms_target_resources, + sid=f'{IAM_S3_BUCKETS_STATEMENT_SID}KMS', + resource_type='kms', + ) + logger.info(f'Number of S3 KMS statements created after splitting: {len(s3_kms_statement_chunks)}') + logger.debug(f'S3 KMS statements after adding resources and splitting: {s3_kms_statement_chunks}') share_policy_service.merge_statements_and_update_policies( target_sid=IAM_S3_BUCKETS_STATEMENT_SID, diff --git a/backend/dataall/modules/shares_base/tasks/share_manager_task.py b/backend/dataall/modules/shares_base/tasks/share_manager_task.py index 9b8b8c70d..f69309042 100644 --- a/backend/dataall/modules/shares_base/tasks/share_manager_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_manager_task.py @@ -7,10 +7,11 @@ from dataall.base.loader import load_modules, ImportMode root = logging.getLogger() +root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) +# log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) if __name__ == '__main__': From 61f616b27498b85e3daffe89100078ee2154364b Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 4 Nov 2024 15:12:14 -0600 Subject: [PATCH 27/30] Fixing unit tests --- .../test_s3_access_point_share_manager.py | 116 ++-------- .../tasks/test_s3_bucket_share_manager.py | 202 +++++------------- .../modules/s3_datasets_shares/test_share.py | 86 ++++---- 3 files changed, 104 insertions(+), 300 deletions(-) diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py index ff1d3caa2..8cdabc4cc 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py @@ -353,22 +353,13 @@ def test_grant_target_role_access_policy_test_empty_policy( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=False, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], ) access_point_name = share_manager.build_access_point_name(share1) @@ -441,10 +432,6 @@ def test_grant_target_role_access_policy_existing_policy_bucket_not_included( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=False, @@ -454,8 +441,8 @@ def test_grant_target_role_access_policy_existing_policy_bucket_not_included( return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], ) iam_update_role_policy_mock = mocker.patch( @@ -522,17 +509,13 @@ def test_grant_target_role_access_policy_existing_policy_bucket_included(mocker, 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], ) iam_update_role_policy_mock = mocker.patch( @@ -983,23 +966,15 @@ def test_delete_target_role_access_policy_no_remaining_statement( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) @@ -1111,22 +1086,13 @@ def test_delete_target_role_access_policy_with_remaining_statement( return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], ) mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) @@ -1326,10 +1292,9 @@ def test_check_target_role_access_policy(mocker, share_manager): 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=[], ) iam_get_policy_mock = mocker.patch( @@ -1339,15 +1304,6 @@ def test_check_target_role_access_policy(mocker, share_manager): _create_target_dataset_access_control_policy(share_manager.bucket_name, share_manager.access_point_name), ), ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', - return_value=[], - ) mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) @@ -1375,17 +1331,6 @@ def test_check_target_role_access_policy_wrong_permissions(mocker, share_manager 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=[], @@ -1436,17 +1381,6 @@ def test_check_target_role_access_policy_existing_policy_bucket_and_key_not_incl 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=[], @@ -1488,11 +1422,6 @@ def test_check_target_role_access_policy_test_no_policy(mocker, share_manager): return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=False, - ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=[], @@ -1502,11 +1431,6 @@ def test_check_target_role_access_policy_test_no_policy(mocker, share_manager): mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=[]) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) - kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' @@ -1518,12 +1442,6 @@ def test_check_target_role_access_policy_test_no_policy(mocker, share_manager): def test_check_target_role_access_policy_test_policy_not_attached(mocker, share_manager): # Given - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=['policy-0'], @@ -1542,16 +1460,6 @@ def test_check_target_role_access_policy_test_policy_not_attached(mocker, share_ ), ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=False, - ) - # Policy is not attached - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=False, - ) - kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py index 1b6f604f0..420859204 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py @@ -416,7 +416,7 @@ def test_grant_role_bucket_policy_with_another_read_only_role( def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): # Given - # The IAM Policy for sharing for the IAM role does not exist (check_if_policy_exists returns False) + # The IAM Policy for sharing on the IAM role does not exist (check_if_policy_exists returns False) # Backwards compatibility: check that the create_managed_policy_from_inline_and_delete_inline is called # Check if the get and update_role_policy func are called and policy statements are added @@ -424,23 +424,17 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=['policy-0']) - mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) - mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_non_default_versions', return_value=True) mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=False, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=False, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) mocker.patch( @@ -458,14 +452,17 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.create_managed_policy_from_inline_and_delete_inline', return_value='arn:iam::someArn', ) + + # Return [] when first called, indicating that managed indexed policies don't exist, Once share_policy_service_mock_1.called is called then return some indexed managed policy + mocker.patch( + 'dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', + side_effect=lambda account_id, region, policy_filter_pattern : [] if not share_policy_service_mock_1.called else ['policy-0'] + ) + share_policy_service_mock_2 = mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) - share_policy_service_mock_3 = mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) iam_update_role_policy_mock_1 = mocker.patch( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', empty_policy_document) ) @@ -477,7 +474,6 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): # Assert IAM and service calls called share_policy_service_mock_1.assert_called_once() share_policy_service_mock_2.assert_called() - share_policy_service_mock_3.assert_called_once() iam_update_role_policy_mock_1.assert_called_once() iam_update_role_policy_mock_2.assert_called_once() iam_policy = json.loads(iam_update_role_policy_mock_2.call_args.args[4]) @@ -518,20 +514,17 @@ def test_grant_s3_iam_access_with_empty_policy(mocker, dataset2, share2_manager) mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) @@ -615,29 +608,21 @@ def test_grant_s3_iam_access_with_policy_and_target_resources_not_present(mocker mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=False, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) @@ -678,7 +663,7 @@ def test_grant_s3_iam_access_with_policy_and_target_resources_not_present(mocker def test_grant_s3_iam_access_with_complete_policy_present(mocker, dataset2, share2_manager): # Given - # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns False but check_if_managed_policies_exists returns True, indicating that indexed IAM policies exist) + # The IAM Policy for sharing for the IAM role exists (check_if_policy_exists returns False but get_managed_policies returns ['policy-0'], indicating that indexed IAM policies exist) # And the IAM Policy is NOT empty and already contains all target resources (get_managed_policy_default_version returns policy) # Check if policy created after calling function and the existing Policy are the same @@ -715,31 +700,24 @@ def test_grant_s3_iam_access_with_complete_policy_present(mocker, dataset2, shar mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=False, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) + kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' @@ -1036,26 +1014,19 @@ def test_delete_target_role_access_no_policy_no_other_resources_shared( mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=False, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) kms_client = mock_kms_client(mocker) @@ -1066,6 +1037,12 @@ def test_delete_target_role_access_no_policy_no_other_resources_shared( return_value='arn:iam::someArn', ) + # Return [] when first called, indicating that managed indexed policies don't exist, Once share_policy_service_mock_1.called is called then return some indexed managed policy + mocker.patch( + 'dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', + side_effect=lambda account_id, region, policy_filter_pattern : [] if not share_policy_service_mock_1.called else ['policy-0'] + ) + share_policy_service_mock_2 = mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, @@ -1130,31 +1107,21 @@ def test_delete_target_role_access_policy_no_resource_of_datasets_s3_bucket( mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) + kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' @@ -1231,30 +1198,20 @@ def test_delete_target_role_access_policy_with_multiple_s3_buckets_in_policy( mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) + mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) + mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) iam_update_role_policy_mock_1 = mocker.patch( @@ -1332,22 +1289,10 @@ def test_delete_target_role_access_policy_with_one_s3_bucket_and_one_kms_resourc mocker.patch('dataall.base.aws.iam.IAM.delete_managed_policy_by_name', return_value=True) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService._get_managed_policy_quota', return_value=10, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', @@ -1355,8 +1300,8 @@ def test_delete_target_role_access_policy_with_one_s3_bucket_and_one_kms_resourc ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value=['policy-0'], ) kms_client = mock_kms_client(mocker) @@ -1606,10 +1551,6 @@ def test_check_s3_iam_access(mocker, dataset2, share2_manager): }, ], } - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', @@ -1624,10 +1565,7 @@ def test_check_s3_iam_access(mocker, dataset2, share2_manager): 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) + # Gets policy with S3 and KMS iam_update_role_policy_mock_1 = mocker.patch( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', policy) @@ -1664,11 +1602,6 @@ def test_check_s3_iam_access_wrong_actions(mocker, dataset2, share2_manager): ], } - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=[], @@ -1681,10 +1614,7 @@ def test_check_s3_iam_access_wrong_actions(mocker, dataset2, share2_manager): 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) + # Gets policy with S3 and KMS iam_update_role_policy_mock_1 = mocker.patch( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', policy) @@ -1710,11 +1640,6 @@ def test_check_s3_iam_access_no_policy(mocker, dataset2, share2_manager): # There is not existing IAM policy in the requesters account for the dataset's S3bucket # Check if the update_role_policy func is called and policy statements are added - share_managerpolicy_mock_1 = mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=False, - ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=[], @@ -1734,7 +1659,6 @@ def test_check_s3_iam_access_no_policy(mocker, dataset2, share2_manager): # When share2_manager.check_s3_iam_access() # Then - share_managerpolicy_mock_1.assert_called_once() assert (len(share2_manager.bucket_errors)) == 1 assert 'IAM Policy Target Resource' in share2_manager.bucket_errors[0] @@ -1769,25 +1693,15 @@ def test_check_s3_iam_access_policy_not_attached(mocker, dataset2, share2_manage 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=False, - ) - - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) iam_update_role_policy_mock_1 = mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=['policy-0'], ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) mocker.patch('dataall.base.aws.iam.IAM.get_attached_managed_policies_to_role', return_value=[]) - mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=[]) - kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' # When @@ -1818,14 +1732,6 @@ def test_check_s3_iam_access_missing_policy_statement(mocker, dataset2, share2_m 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', @@ -1877,11 +1783,6 @@ def test_check_s3_iam_access_missing_target_resource(mocker, dataset2, share2_ma ], } - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', return_value=[], @@ -1894,10 +1795,7 @@ def test_check_s3_iam_access_missing_target_resource(mocker, dataset2, share2_ma 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=True, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', - return_value=True, - ) + # Gets policy with other S3 and KMS iam_update_role_policy_mock_1 = mocker.patch( 'dataall.base.aws.iam.IAM.get_managed_policy_default_version', return_value=('v1', policy) diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 9f0f475ca..00a46fac4 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -8,6 +8,7 @@ from assertpy import assert_that from dataall.base.utils.expiration_util import ExpirationUtils +from dataall.base.utils.naming_convention import NamingConventionPattern, NamingConventionService from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup, ConsumptionRole from dataall.core.organizations.db.organization_models import Organization from dataall.modules.shares_base.services.share_object_service import ShareObjectService @@ -1272,10 +1273,11 @@ def test_create_share_object_as_requester(mocker, client, user2, group2, env2gro 'dataall.base.aws.iam.IAM.get_role_arn_by_name', return_value='role_arn', ) - # When a user that belongs to environment and group creates request + + old_policy_exists = False mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - side_effect=lambda input: True if input == 'policy-0' else False, + return_value=old_policy_exists ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', @@ -1308,9 +1310,10 @@ def test_create_share_object_as_approver_and_requester(mocker, client, user, gro 'dataall.base.aws.iam.IAM.get_role_arn_by_name', return_value='role_arn', ) + old_policy_exists = False mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - side_effect=lambda input: True if input == 'policy-0' else False, + return_value=old_policy_exists ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', @@ -1344,6 +1347,17 @@ def test_create_share_object_invalid_account(mocker, client, user, group2, env2g 'dataall.base.aws.iam.IAM.get_role_arn_by_name', return_value='role_arn', ) + old_policy_exists = False + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', + return_value=old_policy_exists + ) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', + return_value=True, + ) + + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) create_share_object_response = create_share_object( mocker=mocker, @@ -1380,15 +1394,11 @@ def test_create_share_object_with_item_authorized( return_value=True, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', return_value=True, ) mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, - ) create_share_object_response = create_share_object( mocker=mocker, @@ -1435,22 +1445,16 @@ def test_create_share_object_share_policy_not_attached_attachMissingPolicies_ena 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=False, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', - return_value='some-policy', + return_value='policy-0', ) attach_mocker = mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1484,22 +1488,16 @@ def test_create_share_object_share_policy_not_attached_attachMissingPolicies_dis 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=False, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', - return_value='some-policy', + return_value='policy-0', ) attach_mocker = mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1533,18 +1531,12 @@ def test_create_share_object_share_policy_not_attached_attachMissingPolicies_dis 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, - ) - mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=False, - ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', - return_value='some-policy', + return_value='policy-0', ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + consumption_role = MagicMock(spec_set=ConsumptionRole) consumption_role.IAMRoleName = 'randomName' consumption_role.IAMRoleArn = 'randomArn' @@ -1582,13 +1574,15 @@ def test_create_share_object_with_share_expiration_added( return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_managed_policies_exists', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value='policy-0', ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) + create_share_object_response = create_share_object( mocker=mocker, client=client, @@ -1626,14 +1620,14 @@ def test_create_share_object_with_non_expiring_share( return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value='policy-0', ) + mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', return_value=True, ) - mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) create_share_object_response = create_share_object( mocker=mocker, @@ -1666,10 +1660,14 @@ def test_create_share_object_with_share_expiration_incorrect_share_expiration( return_value=False, ) mocker.patch( - 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policies_attached', - return_value=True, + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.get_policies_unattached_to_role', + return_value='policy-0', ) mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) + mocker.patch( + 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.attach_policies', + return_value=True, + ) create_share_object_response = create_share_object( mocker=mocker, From e84e52678d42e9e4a907e6c1cbeb3ea26481eaad Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 4 Nov 2024 17:34:29 -0600 Subject: [PATCH 28/30] Linting after correcting tests --- backend/dataall/base/utils/iam_cdk_utils.py | 26 +++++++++++-------- .../cdk/pivot_role_datasets_policy.py | 3 ++- .../test_s3_access_point_share_manager.py | 1 - .../tasks/test_s3_bucket_share_manager.py | 10 ++++--- .../modules/s3_datasets_shares/test_share.py | 7 +++-- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/backend/dataall/base/utils/iam_cdk_utils.py b/backend/dataall/base/utils/iam_cdk_utils.py index 5881033d7..041b4d129 100644 --- a/backend/dataall/base/utils/iam_cdk_utils.py +++ b/backend/dataall/base/utils/iam_cdk_utils.py @@ -1,8 +1,11 @@ from typing import Dict, Any, List from aws_cdk import aws_iam as iam -from dataall.base.utils.iam_policy_utils import split_policy_statements_in_chunks, \ - split_policy_with_resources_in_statements, split_policy_with_mutiple_value_condition_in_statements +from dataall.base.utils.iam_policy_utils import ( + split_policy_statements_in_chunks, + split_policy_with_resources_in_statements, + split_policy_with_mutiple_value_condition_in_statements, +) def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[Any, Any]): @@ -38,17 +41,18 @@ def process_and_split_statements_in_chunks(statements: List[Dict]): return statements_chunks -def process_and_split_policy_with_resources_in_statements(base_sid: str, effect: str, actions: List[str], - resources: List[str], condition_dict: Dict = None): +def process_and_split_policy_with_resources_in_statements( + base_sid: str, effect: str, actions: List[str], resources: List[str], condition_dict: Dict = None +): if condition_dict is not None: - print(f"Condition dictionary is: {condition_dict}") - json_statements = split_policy_with_mutiple_value_condition_in_statements(base_sid=base_sid, effect=effect, - actions=actions, - resources=resources, - condition_dict=condition_dict) + print(f'Condition dictionary is: {condition_dict}') + json_statements = split_policy_with_mutiple_value_condition_in_statements( + base_sid=base_sid, effect=effect, actions=actions, resources=resources, condition_dict=condition_dict + ) else: - json_statements = split_policy_with_resources_in_statements(base_sid=base_sid, effect=effect, actions=actions, - resources=resources) + json_statements = split_policy_with_resources_in_statements( + base_sid=base_sid, effect=effect, actions=actions, resources=resources + ) iam_statements: [iam.PolicyStatement] = [] for json_statement in json_statements: if json_statement.get('Condition', None): diff --git a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py index 4fd325f4b..1ebb9e92c 100644 --- a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py @@ -1,7 +1,8 @@ import os from dataall.base import db from dataall.base.utils.iam_cdk_utils import ( - convert_from_json_to_iam_policy_statement_with_conditions, process_and_split_policy_with_resources_in_statements, + convert_from_json_to_iam_policy_statement_with_conditions, + process_and_split_policy_with_resources_in_statements, ) from dataall.base.utils.iam_policy_utils import ( split_policy_with_mutiple_value_condition_in_statements, diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py index 8cdabc4cc..f2283deae 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_access_point_share_manager.py @@ -975,7 +975,6 @@ def test_delete_target_role_access_policy_no_remaining_statement( return_value=[], ) - mocker.patch('dataall.base.aws.iam.IAM.create_managed_policy', return_value=True) mocker.patch( diff --git a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py index 420859204..b0db7668f 100644 --- a/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py +++ b/tests/modules/s3_datasets_shares/tasks/test_s3_bucket_share_manager.py @@ -456,7 +456,9 @@ def test_grant_s3_iam_access_with_no_policy(mocker, dataset2, share2_manager): # Return [] when first called, indicating that managed indexed policies don't exist, Once share_policy_service_mock_1.called is called then return some indexed managed policy mocker.patch( 'dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', - side_effect=lambda account_id, region, policy_filter_pattern : [] if not share_policy_service_mock_1.called else ['policy-0'] + side_effect=lambda account_id, region, policy_filter_pattern: [] + if not share_policy_service_mock_1.called + else ['policy-0'], ) share_policy_service_mock_2 = mocker.patch( @@ -1040,7 +1042,9 @@ def test_delete_target_role_access_no_policy_no_other_resources_shared( # Return [] when first called, indicating that managed indexed policies don't exist, Once share_policy_service_mock_1.called is called then return some indexed managed policy mocker.patch( 'dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', - side_effect=lambda account_id, region, policy_filter_pattern : [] if not share_policy_service_mock_1.called else ['policy-0'] + side_effect=lambda account_id, region, policy_filter_pattern: [] + if not share_policy_service_mock_1.called + else ['policy-0'], ) share_policy_service_mock_2 = mocker.patch( @@ -1122,7 +1126,6 @@ def test_delete_target_role_access_policy_no_resource_of_datasets_s3_bucket( return_value=False, ) - kms_client = mock_kms_client(mocker) kms_client().get_key_id.return_value = 'kms-key' @@ -1203,7 +1206,6 @@ def test_delete_target_role_access_policy_with_multiple_s3_buckets_in_policy( return_value=10, ) - mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', return_value=False, diff --git a/tests/modules/s3_datasets_shares/test_share.py b/tests/modules/s3_datasets_shares/test_share.py index 00a46fac4..1a1892333 100644 --- a/tests/modules/s3_datasets_shares/test_share.py +++ b/tests/modules/s3_datasets_shares/test_share.py @@ -1277,7 +1277,7 @@ def test_create_share_object_as_requester(mocker, client, user2, group2, env2gro old_policy_exists = False mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=old_policy_exists + return_value=old_policy_exists, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', @@ -1313,7 +1313,7 @@ def test_create_share_object_as_approver_and_requester(mocker, client, user, gro old_policy_exists = False mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=old_policy_exists + return_value=old_policy_exists, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', @@ -1350,7 +1350,7 @@ def test_create_share_object_invalid_account(mocker, client, user, group2, env2g old_policy_exists = False mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_exists', - return_value=old_policy_exists + return_value=old_policy_exists, ) mocker.patch( 'dataall.modules.s3_datasets_shares.services.s3_share_managed_policy_service.S3SharePolicyService.check_if_policy_attached', @@ -1399,7 +1399,6 @@ def test_create_share_object_with_item_authorized( ) mocker.patch('dataall.base.aws.iam.IAM.list_policy_names_by_policy_pattern', return_value=['policy-0']) - create_share_object_response = create_share_object( mocker=mocker, client=client, From f5825efe4f17ad19608a4087531cd5699e986483 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 4 Nov 2024 17:43:52 -0600 Subject: [PATCH 29/30] Removing parts not part of this PR --- .../s3_access_point_share_manager.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py index 0ebd513b5..772b409c6 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py @@ -42,9 +42,6 @@ from dataall.modules.shares_base.services.sharing_service import ShareData logger = logging.getLogger(__name__) -ACCESS_POINT_CREATION_TIME = 30 -ACCESS_POINT_CREATION_RETRIES = 10 -ACCESS_POINT_BACKOFF_COEFFICIENT = 1.1 # every time increase retry delay by 10% class S3AccessPointShareManager: @@ -476,21 +473,9 @@ def manage_access_point_and_policy(self): """ s3_client = S3ControlClient(self.source_account_id, self.source_environment.region) - access_point_arn = s3_client.get_bucket_access_point_arn(self.access_point_name) + access_point_arn = s3_client.create_bucket_access_point(self.bucket_name, self.access_point_name) if not access_point_arn: - logger.info(f'Access point {self.access_point_name} does not exists, creating...') - access_point_arn = s3_client.create_bucket_access_point(self.bucket_name, self.access_point_name) - # Access point creation is slow - retries = 1 - sleep_coeff = 1 - while ( - not s3_client.get_bucket_access_point_arn(self.access_point_name) - and retries < ACCESS_POINT_CREATION_RETRIES - ): - logger.info('Waiting 30s for access point creation to complete..') - time.sleep(ACCESS_POINT_CREATION_TIME * sleep_coeff) - sleep_coeff = sleep_coeff * ACCESS_POINT_BACKOFF_COEFFICIENT - retries += 1 + raise Exception('Failed to create access point') existing_policy = s3_client.get_access_point_policy(self.access_point_name) # requester will use this role to access resources target_requester_id = SessionHelper.get_role_id( From 50c906d02acfab3027445e2cc89feda10ec50fc1 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 12 Nov 2024 09:35:08 -0600 Subject: [PATCH 30/30] Corrections --- backend/dataall/base/utils/iam_cdk_utils.py | 37 +++++++++++-------- .../cdk/pivot_role_datasets_policy.py | 7 +--- .../shares_base/tasks/share_manager_task.py | 3 +- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/backend/dataall/base/utils/iam_cdk_utils.py b/backend/dataall/base/utils/iam_cdk_utils.py index 041b4d129..c87af5204 100644 --- a/backend/dataall/base/utils/iam_cdk_utils.py +++ b/backend/dataall/base/utils/iam_cdk_utils.py @@ -18,7 +18,7 @@ def convert_from_json_to_iam_policy_statement_with_conditions(iam_policy: Dict[A ) -def convert_from_json_to_iam_policy_statement(iam_policy: Dict[Any, Any]): +def convert_from_json_to_iam_policy_statement_with_resources(iam_policy: Dict[Any, Any]): return iam.PolicyStatement( sid=iam_policy.get('Sid'), effect=iam.Effect.ALLOW if iam_policy.get('Effect').casefold() == 'Allow'.casefold() else iam.Effect.DENY, @@ -36,29 +36,34 @@ def process_and_split_statements_in_chunks(statements: List[Dict]): if statement.get('Condition', None): statements.append(convert_from_json_to_iam_policy_statement_with_conditions(statement)) else: - statements.append(convert_from_json_to_iam_policy_statement(statement)) + statements.append(convert_from_json_to_iam_policy_statement_with_resources(statement)) statements_chunks.append(statements) return statements_chunks -def process_and_split_policy_with_resources_in_statements( +def process_and_split_policy_with_conditions_in_statements( base_sid: str, effect: str, actions: List[str], resources: List[str], condition_dict: Dict = None ): - if condition_dict is not None: - print(f'Condition dictionary is: {condition_dict}') - json_statements = split_policy_with_mutiple_value_condition_in_statements( - base_sid=base_sid, effect=effect, actions=actions, resources=resources, condition_dict=condition_dict - ) - else: - json_statements = split_policy_with_resources_in_statements( - base_sid=base_sid, effect=effect, actions=actions, resources=resources - ) + json_statements = split_policy_with_mutiple_value_condition_in_statements( + base_sid=base_sid, effect=effect, actions=actions, resources=resources, condition_dict=condition_dict + ) + + iam_statements: [iam.PolicyStatement] = [] + for json_statement in json_statements: + iam_policy_statement = convert_from_json_to_iam_policy_statement_with_conditions(json_statement) + iam_statements.append(iam_policy_statement) + return iam_statements + + +def process_and_split_policy_with_resources_in_statements( + base_sid: str, effect: str, actions: List[str], resources: List[str] +): + json_statements = split_policy_with_resources_in_statements( + base_sid=base_sid, effect=effect, actions=actions, resources=resources + ) iam_statements: [iam.PolicyStatement] = [] for json_statement in json_statements: - if json_statement.get('Condition', None): - iam_policy_statement = convert_from_json_to_iam_policy_statement_with_conditions(json_statement) - else: - iam_policy_statement = convert_from_json_to_iam_policy_statement(json_statement) + iam_policy_statement = convert_from_json_to_iam_policy_statement_with_resources(json_statement) iam_statements.append(iam_policy_statement) return iam_statements diff --git a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py index 1ebb9e92c..b18fe1646 100644 --- a/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py @@ -1,11 +1,8 @@ import os from dataall.base import db from dataall.base.utils.iam_cdk_utils import ( - convert_from_json_to_iam_policy_statement_with_conditions, process_and_split_policy_with_resources_in_statements, -) -from dataall.base.utils.iam_policy_utils import ( - split_policy_with_mutiple_value_condition_in_statements, + process_and_split_policy_with_conditions_in_statements, ) from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository @@ -173,7 +170,7 @@ def get_statements(self): ) statements.extend(dataset_statements) if imported_kms_alias: - kms_statements = process_and_split_policy_with_resources_in_statements( + kms_statements = process_and_split_policy_with_conditions_in_statements( base_sid='KMSImportedDataset', effect=iam.Effect.ALLOW.value, actions=[ diff --git a/backend/dataall/modules/shares_base/tasks/share_manager_task.py b/backend/dataall/modules/shares_base/tasks/share_manager_task.py index f69309042..9b8b8c70d 100644 --- a/backend/dataall/modules/shares_base/tasks/share_manager_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_manager_task.py @@ -7,11 +7,10 @@ from dataall.base.loader import load_modules, ImportMode root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -# log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) if __name__ == '__main__':