diff --git a/awscli/clidriver.py b/awscli/clidriver.py index 7a5e95ebaeab..84c131e1bb7d 100644 --- a/awscli/clidriver.py +++ b/awscli/clidriver.py @@ -109,19 +109,21 @@ def main(): def create_clidriver(args=None): debug = None + parsed_args = None if args is not None: parser = FirstPassGlobalArgParser() - args, _ = parser.parse_known_args(args) - debug = args.debug + parsed_args, _ = parser.parse_known_args(args) + debug = parsed_args.debug session = botocore.session.Session() _set_user_agent_for_session(session) load_plugins( session.full_config.get('plugins', {}), event_hooks=session.get_component('event_emitter'), + args=args, ) error_handlers_chain = construct_cli_error_handlers_chain(session) driver = CLIDriver( - session=session, error_handler=error_handlers_chain, debug=debug + session=session, error_handler=error_handlers_chain, debug=parsed_args.debug if parsed_args else False ) return driver diff --git a/awscli/handlers.py b/awscli/handlers.py index 9c605dc37c33..d2a015004039 100644 --- a/awscli/handlers.py +++ b/awscli/handlers.py @@ -15,227 +15,107 @@ This is a collection of built in CLI extensions that can be automatically registered with the event system. +Plugin initialization is lazy: only the plugins relevant to the command +being invoked are imported and registered. The mapping from command names +to plugins is maintained in awscli/handlers_registry.py, which is +auto-generated by scripts/generate_plugin_registry. + """ +import importlib + +from awscli.handlers_registry import PLUGIN_REGISTRY + +_SENTINEL_ALWAYS = '__always__' +_SENTINEL_MAIN = '__main__' + + +def _get_top_level_command(args): + """Return the first non-flag, non-option-value token from args. + + args is the raw CLI argument list as passed to create_clidriver — + i.e. sys.argv[1:] with the program name already stripped. + + Returns None if no positional argument is found (e.g. ['--version']). + """ + skip_next = False + for token in args: + if skip_next: + skip_next = False + continue + if token.startswith('--'): + if '=' not in token: + skip_next = True + continue + if token.startswith('-'): + continue + return token + return None + + +def _get_top_level_command(args): + """Return the first non-flag, non-option-value token from args.""" + skip_next = False + for token in args: + if skip_next: + skip_next = False + continue + if token.startswith('--'): + if '=' not in token: + skip_next = True + continue + if token.startswith('-'): + continue + return token + return None + + +def _call(entry, event_handlers): + """Dispatch a single registry entry. + + Entry formats: + (module, fn_name) -- call fn(event_handlers) + (module, fn_name, event_name) -- event_handlers.register(event_name, fn) + """ + module_path, fn_name = entry[0], entry[1] + mod = importlib.import_module(module_path) + fn = getattr(mod, fn_name) + if len(entry) == 3: + # Direct registration. If fn is a class, instantiate it first + # (e.g. ParamShorthandParser). + handler = fn() if isinstance(fn, type) else fn + event_handlers.register(entry[2], handler) + else: + fn(event_handlers) + + +def awscli_initialize(event_handlers, args=None): + command = _get_top_level_command(args) if args is not None else None + + # Collect the set of entries to call, preserving order and deduplicating + # so that entries shared across sentinels are only called once. + seen = set() + to_call = [] + + def _enqueue(entries): + for entry in entries: + if entry not in seen: + seen.add(entry) + to_call.append(entry) + + _enqueue(PLUGIN_REGISTRY.get(_SENTINEL_ALWAYS, [])) + _enqueue(PLUGIN_REGISTRY.get(_SENTINEL_MAIN, [])) + + if command is not None: + # Load only the service-scoped plugins for the given command. + _enqueue(PLUGIN_REGISTRY.get(command, [])) + else: + # args was not provided — load all service-scoped plugins to ensure + # nothing is missing. In production args is always provided via + # create_clidriver(args); this path is hit only when a driver is + # pre-created without args (e.g. in tests). + for sentinel, entries in PLUGIN_REGISTRY.items(): + if sentinel not in (_SENTINEL_ALWAYS, _SENTINEL_MAIN): + _enqueue(entries) -from awscli.alias import register_alias_commands -from awscli.argprocess import ParamShorthandParser -from awscli.clidriver import no_pager_handler -from awscli.customizations import datapipeline -from awscli.customizations.addexamples import add_examples -from awscli.customizations.argrename import register_arg_renames -from awscli.customizations.assumerole import register_assume_role_provider -from awscli.customizations.awslambda import register_lambda_create_function -from awscli.customizations.binaryformat import add_binary_formatter -from awscli.customizations.cliinput import register_cli_input_args -from awscli.customizations.cloudformation import ( - initialize as cloudformation_init, -) -from awscli.customizations.cloudfront import register as register_cloudfront -from awscli.customizations.cloudsearch import initialize as cloudsearch_init -from awscli.customizations.cloudsearchdomain import register_cloudsearchdomain -from awscli.customizations.cloudtrail import initialize as cloudtrail_init -from awscli.customizations.codeartifact import register_codeartifact_commands -from awscli.customizations.codecommit import initialize as codecommit_init -from awscli.customizations.codedeploy.codedeploy import ( - initialize as codedeploy_init, -) -from awscli.customizations.configservice.getstatus import register_get_status -from awscli.customizations.configservice.putconfigurationrecorder import ( - register_modify_put_configuration_recorder, -) -from awscli.customizations.configservice.rename_cmd import ( - register_rename_config, -) -from awscli.customizations.configservice.subscribe import register_subscribe -from awscli.customizations.configure.configure import register_configure_cmd -from awscli.customizations.devcommands import register_dev_commands -from awscli.customizations.dlm.dlm import dlm_initialize -from awscli.customizations.dsql import register_dsql_customizations -from awscli.customizations.dynamodb.ddb import register_ddb -from awscli.customizations.dynamodb.paginatorfix import ( - register_dynamodb_paginator_fix, -) -from awscli.customizations.ec2.addcount import register_count_events -from awscli.customizations.ec2.bundleinstance import register_bundleinstance -from awscli.customizations.ec2.decryptpassword import ec2_add_priv_launch_key -from awscli.customizations.ec2.paginate import register_ec2_page_size_injector -from awscli.customizations.ec2.protocolarg import register_protocol_args -from awscli.customizations.ec2.runinstances import register_runinstances -from awscli.customizations.ec2.secgroupsimplify import register_secgroup -from awscli.customizations.ec2instanceconnect import ( - register_ec2_instance_connect_commands, -) -from awscli.customizations.ecr import register_ecr_commands -from awscli.customizations.ecr_public import register_ecr_public_commands -from awscli.customizations.ecs import initialize as ecs_initialize -from awscli.customizations.ecs.monitormutatinggatewayservice import ( - register_monitor_mutating_gateway_service, -) -from awscli.customizations.eks import initialize as eks_initialize -from awscli.customizations.emr.emr import emr_initialize -from awscli.customizations.emrcontainers import ( - initialize as emrcontainers_initialize, -) -from awscli.customizations.gamelift import register_gamelift_commands -from awscli.customizations.generatecliskeleton import ( - register_generate_cli_skeleton, -) -from awscli.customizations.globalargs import register_parse_global_args -from awscli.customizations.history import ( - register_history_commands, - register_history_mode, -) -from awscli.customizations.iamvirtmfa import IAMVMFAWrapper -from awscli.customizations.iot import ( - register_create_keys_and_cert_arguments, - register_create_keys_from_csr_arguments, -) -from awscli.customizations.iot_data import register_custom_endpoint_note -from awscli.customizations.kinesis import ( - register_kinesis_list_streams_pagination_backcompat, -) -from awscli.customizations.kms import register_fix_kms_create_grant_docs -from awscli.customizations.lightsail import initialize as lightsail_initialize -from awscli.customizations.login import register_login_cmds -from awscli.customizations.logs import register_logs_commands -from awscli.customizations.paginate import register_pagination -from awscli.customizations.putmetricdata import register_put_metric_data -from awscli.customizations.quicksight import ( - register_quicksight_asset_bundle_customizations, -) -from awscli.customizations.rds import ( - register_add_generate_db_auth_token, - register_rds_modify_split, -) -from awscli.customizations.rekognition import ( - register_rekognition_detect_labels, -) -from awscli.customizations.removals import register_removals -from awscli.customizations.route53 import register_create_hosted_zone_doc_fix -from awscli.customizations.s3.s3 import s3_plugin_initialize -from awscli.customizations.s3errormsg import register_s3_error_msg -from awscli.customizations.s3events import ( - register_document_expires_string, - register_event_stream_arg, -) -from awscli.customizations.servicecatalog import ( - register_servicecatalog_commands, -) -from awscli.customizations.sessendemail import register_ses_send_email -from awscli.customizations.sessionmanager import register_ssm_session -from awscli.customizations.sso import register_sso_commands -from awscli.customizations.streamingoutputarg import add_streaming_output_arg -from awscli.customizations.timestampformat import register_timestamp_format -from awscli.customizations.toplevelbool import register_bool_params -from awscli.customizations.translate import ( - register_translate_import_terminology, -) -from awscli.customizations.waiters import register_add_waiters -from awscli.customizations.wizard.commands import register_wizard_commands -from awscli.paramfile import register_uri_param_handler - - -def awscli_initialize(event_handlers): - event_handlers.register('session-initialized', register_uri_param_handler) - event_handlers.register('session-initialized', add_binary_formatter) - event_handlers.register('session-initialized', no_pager_handler) - param_shorthand = ParamShorthandParser() - event_handlers.register('process-cli-arg', param_shorthand) - # The s3 error mesage needs to registered before the - # generic error handler. - register_s3_error_msg(event_handlers) - # # The following will get fired for every option we are - # # documenting. It will attempt to add an example_fn on to - # # the parameter object if the parameter supports shorthand - # # syntax. The documentation event handlers will then use - # # the examplefn to generate the sample shorthand syntax - # # in the docs. Registering here should ensure that this - # # handler gets called first but it still feels a bit brittle. - # event_handlers.register('doc-option-example.*.*.*', - # param_shorthand.add_example_fn) - event_handlers.register('doc-examples.*.*', add_examples) - register_cli_input_args(event_handlers) - event_handlers.register( - 'building-argument-table.*', add_streaming_output_arg - ) - register_count_events(event_handlers) - event_handlers.register( - 'building-argument-table.ec2.get-password-data', - ec2_add_priv_launch_key, - ) - register_parse_global_args(event_handlers) - register_pagination(event_handlers) - register_secgroup(event_handlers) - register_bundleinstance(event_handlers) - s3_plugin_initialize(event_handlers) - register_ddb(event_handlers) - register_runinstances(event_handlers) - register_removals(event_handlers) - register_rds_modify_split(event_handlers) - register_rekognition_detect_labels(event_handlers) - register_add_generate_db_auth_token(event_handlers) - register_dsql_customizations(event_handlers) - register_put_metric_data(event_handlers) - register_ses_send_email(event_handlers) - IAMVMFAWrapper(event_handlers) - register_arg_renames(event_handlers) - register_configure_cmd(event_handlers) - cloudtrail_init(event_handlers) - register_ecr_commands(event_handlers) - register_ecr_public_commands(event_handlers) - register_bool_params(event_handlers) - register_protocol_args(event_handlers) - datapipeline.register_customizations(event_handlers) - cloudsearch_init(event_handlers) - emr_initialize(event_handlers) - emrcontainers_initialize(event_handlers) - eks_initialize(event_handlers) - ecs_initialize(event_handlers) - register_monitor_mutating_gateway_service(event_handlers) - lightsail_initialize(event_handlers) - register_cloudsearchdomain(event_handlers) - register_generate_cli_skeleton(event_handlers) - register_assume_role_provider(event_handlers) - register_add_waiters(event_handlers) - codedeploy_init(event_handlers) - register_subscribe(event_handlers) - register_get_status(event_handlers) - register_rename_config(event_handlers) - register_timestamp_format(event_handlers) - register_lambda_create_function(event_handlers) - register_fix_kms_create_grant_docs(event_handlers) - register_create_hosted_zone_doc_fix(event_handlers) - register_modify_put_configuration_recorder(event_handlers) - register_codeartifact_commands(event_handlers) - codecommit_init(event_handlers) - register_custom_endpoint_note(event_handlers) - event_handlers.register( - 'building-argument-table.iot.create-keys-and-certificate', - register_create_keys_and_cert_arguments, - ) - event_handlers.register( - 'building-argument-table.iot.create-certificate-from-csr', - register_create_keys_from_csr_arguments, - ) - register_cloudfront(event_handlers) - register_gamelift_commands(event_handlers) - register_ec2_page_size_injector(event_handlers) - cloudformation_init(event_handlers) - register_servicecatalog_commands(event_handlers) - register_translate_import_terminology(event_handlers) - register_history_mode(event_handlers) - register_history_commands(event_handlers) - register_event_stream_arg(event_handlers) - register_document_expires_string(event_handlers) - dlm_initialize(event_handlers) - register_ssm_session(event_handlers) - register_logs_commands(event_handlers) - register_dev_commands(event_handlers) - register_wizard_commands(event_handlers) - register_sso_commands(event_handlers) - register_dynamodb_paginator_fix(event_handlers) - register_alias_commands(event_handlers) - register_kinesis_list_streams_pagination_backcompat(event_handlers) - register_quicksight_asset_bundle_customizations(event_handlers) - register_ec2_instance_connect_commands(event_handlers) - register_login_cmds(event_handlers) + for entry in to_call: + _call(entry, event_handlers) diff --git a/awscli/handlers_registry.py b/awscli/handlers_registry.py new file mode 100644 index 000000000000..f079768442cf --- /dev/null +++ b/awscli/handlers_registry.py @@ -0,0 +1,314 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# AUTO-GENERATED by scripts/generate_plugin_registry. +# DO NOT EDIT MANUALLY. +# +# Re-generate by running: +# scripts/generate_plugin_registry +# +# Sentinels: +# __always__ registered for every CLI invocation +# __main__ registered when building the top-level command table +# registered only when that service/command is invoked +# +# Entry formats: +# (module, fn_name) call fn(event_handlers) +# (module, fn_name, event_name) event_handlers.register(event_name, fn) + +PLUGIN_REGISTRY = { + '__always__': [ + ('awscli.paramfile', 'register_uri_param_handler', 'session-initialized'), + ('awscli.customizations.binaryformat', 'add_binary_formatter', 'session-initialized'), + ('awscli.clidriver', 'no_pager_handler', 'session-initialized'), + ('awscli.argprocess', 'ParamShorthandParser', 'process-cli-arg'), + ('awscli.customizations.s3errormsg', 'register_s3_error_msg'), + ('awscli.customizations.addexamples', 'add_examples', 'doc-examples.*.*'), + ('awscli.customizations.cliinput', 'register_cli_input_args'), + ('awscli.customizations.streamingoutputarg', 'add_streaming_output_arg', 'building-argument-table.*'), + ('awscli.customizations.globalargs', 'register_parse_global_args'), + ('awscli.customizations.paginate', 'register_pagination'), + ('awscli.customizations.ec2.secgroupsimplify', 'register_secgroup'), + ('awscli.customizations.iamvirtmfa', 'IAMVMFAWrapper'), + ('awscli.customizations.datapipeline', 'register_customizations'), + ('awscli.customizations.ecs.monitormutatinggatewayservice', 'register_monitor_mutating_gateway_service'), + ('awscli.customizations.cloudsearchdomain', 'register_cloudsearchdomain'), + ('awscli.customizations.generatecliskeleton', 'register_generate_cli_skeleton'), + ('awscli.customizations.assumerole', 'register_assume_role_provider'), + ('awscli.customizations.waiters', 'register_add_waiters'), + ('awscli.customizations.timestampformat', 'register_timestamp_format'), + ('awscli.customizations.awslambda', 'register_lambda_create_function'), + ('awscli.customizations.kms', 'register_fix_kms_create_grant_docs'), + ('awscli.customizations.route53', 'register_create_hosted_zone_doc_fix'), + ('awscli.customizations.iot_data', 'register_custom_endpoint_note'), + ('awscli.customizations.cloudfront', 'register'), + ('awscli.customizations.ec2.paginate', 'register_ec2_page_size_injector'), + ('awscli.customizations.history', 'register_history_mode'), + ('awscli.customizations.s3events', 'register_event_stream_arg'), + ('awscli.customizations.s3events', 'register_document_expires_string'), + ('awscli.customizations.sso', 'register_sso_commands'), + ('awscli.customizations.dynamodb.paginatorfix', 'register_dynamodb_paginator_fix'), + ('awscli.alias', 'register_alias_commands') + ], + '__main__': [ + ('awscli.customizations.s3.s3', 's3_plugin_initialize'), + ('awscli.customizations.dynamodb.ddb', 'register_ddb'), + ('awscli.customizations.configure.configure', 'register_configure_cmd'), + ('awscli.customizations.codedeploy.codedeploy', 'initialize'), + ('awscli.customizations.configservice.rename_cmd', 'register_rename_config'), + ('awscli.customizations.history', 'register_history_commands'), + ('awscli.customizations.devcommands', 'register_dev_commands'), + ('awscli.customizations.login', 'register_login_cmds') + ], + 'apigateway': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'apigatewayv2': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'bedrock-agent-runtime': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'bedrock-agentcore': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'bedrock-runtime': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'cli-dev': [ + ('awscli.customizations.wizard.commands', 'register_wizard_commands') + ], + 'clouddirectory': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'cloudformation': [ + ('awscli.customizations.cloudformation', 'initialize') + ], + 'cloudfront': [ + ('awscli.customizations.cloudfront', 'register') + ], + 'cloudsearch': [ + ('awscli.customizations.cloudsearch', 'initialize') + ], + 'cloudsearchdomain': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'cloudtrail': [ + ('awscli.customizations.cloudtrail', 'initialize') + ], + 'cloudwatch': [ + ('awscli.customizations.putmetricdata', 'register_put_metric_data') + ], + 'codeartifact': [ + ('awscli.customizations.codeartifact', 'register_codeartifact_commands') + ], + 'codecommit': [ + ('awscli.customizations.codecommit', 'initialize') + ], + 'codepipeline': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'configservice': [ + ('awscli.customizations.configservice.subscribe', 'register_subscribe'), + ('awscli.customizations.configservice.getstatus', 'register_get_status'), + ('awscli.customizations.configservice.putconfigurationrecorder', 'register_modify_put_configuration_recorder') + ], + 'configure': [ + ('awscli.customizations.wizard.commands', 'register_wizard_commands') + ], + 'connecthealth': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'controltower': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'datapipeline': [ + ('awscli.customizations.argrename', 'register_arg_renames'), + ('awscli.customizations.datapipeline', 'register_customizations') + ], + 'deploy': [ + ('awscli.customizations.argrename', 'register_arg_renames'), + ('awscli.customizations.codedeploy.codedeploy', 'initialize') + ], + 'devops-agent': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'dlm': [ + ('awscli.customizations.dlm.dlm', 'dlm_initialize') + ], + 'dsql': [ + ('awscli.customizations.dsql', 'register_dsql_customizations') + ], + 'dynamodb': [ + ('awscli.customizations.wizard.commands', 'register_wizard_commands') + ], + 'ec2': [ + ('awscli.customizations.ec2.decryptpassword', 'ec2_add_priv_launch_key', 'building-argument-table.ec2.get-password-data'), + ('awscli.customizations.ec2.addcount', 'register_count_events'), + ('awscli.customizations.ec2.secgroupsimplify', 'register_secgroup'), + ('awscli.customizations.ec2.bundleinstance', 'register_bundleinstance'), + ('awscli.customizations.ec2.runinstances', 'register_runinstances'), + ('awscli.customizations.removals', 'register_removals'), + ('awscli.customizations.argrename', 'register_arg_renames'), + ('awscli.customizations.toplevelbool', 'register_bool_params'), + ('awscli.customizations.ec2.protocolarg', 'register_protocol_args') + ], + 'ec2-instance-connect': [ + ('awscli.customizations.ec2instanceconnect', 'register_ec2_instance_connect_commands') + ], + 'ecr': [ + ('awscli.customizations.ecr', 'register_ecr_commands') + ], + 'ecr-public': [ + ('awscli.customizations.ecr_public', 'register_ecr_public_commands') + ], + 'ecs': [ + ('awscli.customizations.argrename', 'register_arg_renames'), + ('awscli.customizations.ecs', 'initialize'), + ('awscli.customizations.ecs.monitormutatinggatewayservice', 'register_monitor_mutating_gateway_service') + ], + 'eks': [ + ('awscli.customizations.argrename', 'register_arg_renames'), + ('awscli.customizations.eks', 'initialize') + ], + 'elasticache': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'emr': [ + ('awscli.customizations.removals', 'register_removals'), + ('awscli.customizations.argrename', 'register_arg_renames'), + ('awscli.customizations.emr.emr', 'emr_initialize') + ], + 'emr-containers': [ + ('awscli.customizations.emrcontainers', 'initialize') + ], + 'events': [ + ('awscli.customizations.wizard.commands', 'register_wizard_commands') + ], + 'gamelift': [ + ('awscli.customizations.argrename', 'register_arg_renames'), + ('awscli.customizations.gamelift', 'register_gamelift_commands') + ], + 'glue': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'iam': [ + ('awscli.customizations.iamvirtmfa', 'IAMVMFAWrapper'), + ('awscli.customizations.wizard.commands', 'register_wizard_commands') + ], + 'iot': [ + ('awscli.customizations.iot', 'register_create_keys_and_cert_arguments', 'building-argument-table.iot.create-keys-and-certificate'), + ('awscli.customizations.iot', 'register_create_keys_from_csr_arguments', 'building-argument-table.iot.create-certificate-from-csr') + ], + 'iotsitewise': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'iotwireless': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'kinesis': [ + ('awscli.customizations.removals', 'register_removals'), + ('awscli.customizations.kinesis', 'register_kinesis_list_streams_pagination_backcompat') + ], + 'kinesisanalytics': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'kinesisanalyticsv2': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'lambda': [ + ('awscli.customizations.removals', 'register_removals'), + ('awscli.customizations.awslambda', 'register_lambda_create_function'), + ('awscli.customizations.wizard.commands', 'register_wizard_commands') + ], + 'lex-models': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'lexv2-runtime': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'license-manager': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'lightsail': [ + ('awscli.customizations.lightsail', 'initialize') + ], + 'logs': [ + ('awscli.customizations.removals', 'register_removals'), + ('awscli.customizations.logs', 'register_logs_commands') + ], + 'mgn': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'mturk': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'pinpoint': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'polly': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'qbusiness': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'quicksight': [ + ('awscli.customizations.quicksight', 'register_quicksight_asset_bundle_customizations') + ], + 'rds': [ + ('awscli.customizations.rds', 'register_rds_modify_split'), + ('awscli.customizations.rds', 'register_add_generate_db_auth_token') + ], + 'rekognition': [ + ('awscli.customizations.rekognition', 'register_rekognition_detect_labels'), + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'route53': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'route53domains': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 's3_sync': [ + ('awscli.customizations.s3.s3', 's3_plugin_initialize') + ], + 's3api': [ + ('awscli.customizations.s3events', 'register_event_stream_arg') + ], + 'sagemaker': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'sagemaker-runtime': [ + ('awscli.customizations.removals', 'register_removals') + ], + 'schemas': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'servicecatalog': [ + ('awscli.customizations.servicecatalog', 'register_servicecatalog_commands') + ], + 'ses': [ + ('awscli.customizations.removals', 'register_removals'), + ('awscli.customizations.sessendemail', 'register_ses_send_email') + ], + 'sns': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'ssm': [ + ('awscli.customizations.sessionmanager', 'register_ssm_session') + ], + 'sso': [ + ('awscli.customizations.sso', 'register_sso_commands') + ], + 'stepfunctions': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'swf': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ], + 'translate': [ + ('awscli.customizations.translate', 'register_translate_import_terminology') + ], + 'workdocs': [ + ('awscli.customizations.argrename', 'register_arg_renames') + ] +} diff --git a/awscli/plugin.py b/awscli/plugin.py index 46a26a4fc1a7..c340c1063917 100644 --- a/awscli/plugin.py +++ b/awscli/plugin.py @@ -22,7 +22,8 @@ CLI_LEGACY_PLUGIN_PATH = 'cli_legacy_plugin_path' -def load_plugins(plugin_mapping, event_hooks=None, include_builtins=True): +def load_plugins(plugin_mapping, event_hooks=None, include_builtins=True, + args=None): """ :type plugin_mapping: dict @@ -38,6 +39,11 @@ def load_plugins(plugin_mapping, event_hooks=None, include_builtins=True): :param include_builtins: If True, the builtin awscli plugins (specified in ``BUILTIN_PLUGINS``) will be included in the list of plugins to load. + :type args: list + :param args: The raw CLI argument list (without the program name), used + to determine which service is being invoked so that only relevant + plugins are initialized. + :rtype: HierarchicalEmitter :return: An event emitter object. @@ -45,7 +51,7 @@ def load_plugins(plugin_mapping, event_hooks=None, include_builtins=True): if event_hooks is None: event_hooks = HierarchicalEmitter() if include_builtins: - _load_plugins(BUILTIN_PLUGINS, event_hooks) + _load_plugins(BUILTIN_PLUGINS, event_hooks, args=args) plugin_path = plugin_mapping.pop(CLI_LEGACY_PLUGIN_PATH, None) if plugin_path is not None: _add_plugin_path_to_sys_path(plugin_path) @@ -58,10 +64,16 @@ def load_plugins(plugin_mapping, event_hooks=None, include_builtins=True): return event_hooks -def _load_plugins(plugin_mapping, event_hooks): +def _load_plugins(plugin_mapping, event_hooks, args=None): modules = _import_plugins(plugin_mapping) for name, plugin in zip(plugin_mapping.keys(), modules): log.debug("Initializing plugin %s: %s", name, plugin) + if args is not None and hasattr(plugin, 'awscli_initialize'): + import inspect + sig = inspect.signature(plugin.awscli_initialize) + if 'args' in sig.parameters: + plugin.awscli_initialize(event_hooks, args=args) + continue plugin.awscli_initialize(event_hooks) diff --git a/exe/pyinstaller/hook-awscli.py b/exe/pyinstaller/hook-awscli.py index a0767694019a..4b4646907eed 100644 --- a/exe/pyinstaller/hook-awscli.py +++ b/exe/pyinstaller/hook-awscli.py @@ -28,6 +28,18 @@ ) + hooks.collect_submodules('awscli.s3transfer') hiddenimports += alias_packages_plugins +# handlers.py uses importlib.import_module at runtime to load customization +# modules, so PyInstaller cannot discover them statically. Collect all module +# paths referenced in handlers_registry.py as hidden imports. +from awscli.handlers_registry import PLUGIN_REGISTRY +_registry_modules = { + module_path + for entries in PLUGIN_REGISTRY.values() + for entry in entries + for module_path in [entry[0]] +} +hiddenimports += sorted(_registry_modules) + # Completion model files are only used at build time to generate the # ac.index SQLite database. They are not needed at runtime and can be diff --git a/scripts/generate_plugin_registry b/scripts/generate_plugin_registry new file mode 100755 index 000000000000..91b2a66a6d51 --- /dev/null +++ b/scripts/generate_plugin_registry @@ -0,0 +1,424 @@ +#!/usr/bin/env python +"""Generate the lazy plugin registry for awscli_initialize. + +This script is the authoritative source of truth for which customizations +exist and in what order they are registered. It mirrors what the old +monolithic awscli_initialize used to do, but instead of actually importing +and calling everything, it runs each registration call against a +RecordingEmitter that captures (module_path, fn_name, event_name) triples +without importing any customization modules. + +The output is written to awscli/handlers_registry.py as a plain Python dict +PLUGIN_REGISTRY mapping each sentinel to a list of entries: + + (module, fn_name) -- call fn(event_handlers) + (module, fn_name, event_name) -- event_handlers.register(event_name, fn) + +Sentinels: + __always__ registered for every CLI invocation + __main__ registered when building the top-level command table + registered only when that service/command is invoked + +Usage: + scripts/generate_plugin_registry [--output PATH] [--check] [--print] + +Run this script whenever a customization is added, removed, or changed, and +commit the updated registry. A CI check should verify the committed registry +matches a fresh run of this script. + +To add a new customization: + 1. Add a call to _register() or _register_direct() in _all_registrations() + below, in the same position it would appear in awscli_initialize. + 2. Re-run this script. + 3. Commit both this script and the updated handlers_registry.py. +""" +import argparse +import collections +import os +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) + +DEFAULT_OUTPUT = os.path.join(REPO_ROOT, 'awscli', 'handlers_registry.py') + +SENTINEL_ALWAYS = '__always__' +SENTINEL_MAIN = '__main__' + +ALWAYS_PREFIXES = ( + 'session-initialized', + 'top-level-args-parsed', + 'process-cli-arg', + 'provide-client-params', + 'calling-command', + 'doc-', + 'after-call', +) + +ALWAYS_EXACT = { + 'building-argument-table', + 'building-command-table', +} + + +# --------------------------------------------------------------------------- +# Recording infrastructure +# --------------------------------------------------------------------------- + +class _Entry: + """A single pending registration.""" + __slots__ = ('module', 'fn_name', 'event_name') + + def __init__(self, module, fn_name, event_name=None): + self.module = module + self.fn_name = fn_name + self.event_name = event_name # None means "call fn(event_handlers)" + + def as_tuple(self): + if self.event_name is not None: + return (self.module, self.fn_name, self.event_name) + return (self.module, self.fn_name) + + +class _Recorder: + """Collects registration calls without importing any modules.""" + + def __init__(self): + self._entries = [] + + def register_fn(self, module, fn_name): + """Record a call of the form fn(event_handlers).""" + self._entries.append(_Entry(module, fn_name)) + + def register_direct(self, module, fn_name, event_name): + """Record a call of the form event_handlers.register(event_name, fn).""" + self._entries.append(_Entry(module, fn_name, event_name)) + + @property + def entries(self): + return list(self._entries) + + +# --------------------------------------------------------------------------- +# Classification +# --------------------------------------------------------------------------- + +def _classify_event(event_name): + if event_name in ALWAYS_EXACT: + return SENTINEL_ALWAYS + for prefix in ALWAYS_PREFIXES: + if event_name.startswith(prefix): + return SENTINEL_ALWAYS + parts = event_name.split('.') + if len(parts) >= 2: + namespace = parts[0] + qualifier = parts[1] + if qualifier == '*': + return SENTINEL_ALWAYS + if namespace == 'building-argument-table': + return qualifier + if namespace == 'building-command-table': + if qualifier == 'main': + return SENTINEL_MAIN + return qualifier + return qualifier + return SENTINEL_ALWAYS + + +# --------------------------------------------------------------------------- +# The authoritative list of all registrations. +# This is the source of truth — update this when adding/removing customizations. +# --------------------------------------------------------------------------- + +def _all_registrations(r): + """Populate r with every registration in the same order as the original + awscli_initialize. + + r.register_fn(module, fn_name) + -- will call fn(event_handlers) at runtime + + r.register_direct(module, fn_name, event_name) + -- will call event_handlers.register(event_name, fn) at runtime + """ + r.register_direct('awscli.paramfile', 'register_uri_param_handler', 'session-initialized') + r.register_direct('awscli.customizations.binaryformat', 'add_binary_formatter', 'session-initialized') + r.register_direct('awscli.clidriver', 'no_pager_handler', 'session-initialized') + r.register_direct('awscli.argprocess', 'ParamShorthandParser', 'process-cli-arg') + r.register_fn('awscli.customizations.s3errormsg', 'register_s3_error_msg') + r.register_direct('awscli.customizations.addexamples', 'add_examples', 'doc-examples.*.*') + r.register_fn('awscli.customizations.cliinput', 'register_cli_input_args') + r.register_direct('awscli.customizations.streamingoutputarg', 'add_streaming_output_arg', 'building-argument-table.*') + r.register_direct('awscli.customizations.ec2.decryptpassword', 'ec2_add_priv_launch_key', 'building-argument-table.ec2.get-password-data') + r.register_fn('awscli.customizations.globalargs', 'register_parse_global_args') + r.register_fn('awscli.customizations.paginate', 'register_pagination') + r.register_fn('awscli.customizations.ec2.addcount', 'register_count_events') + r.register_fn('awscli.customizations.ec2.secgroupsimplify', 'register_secgroup') + r.register_fn('awscli.customizations.ec2.bundleinstance', 'register_bundleinstance') + r.register_fn('awscli.customizations.s3.s3', 's3_plugin_initialize') + r.register_fn('awscli.customizations.dynamodb.ddb', 'register_ddb') + r.register_fn('awscli.customizations.ec2.runinstances', 'register_runinstances') + r.register_fn('awscli.customizations.removals', 'register_removals') + r.register_fn('awscli.customizations.rds', 'register_rds_modify_split') + r.register_fn('awscli.customizations.rekognition', 'register_rekognition_detect_labels') + r.register_fn('awscli.customizations.rds', 'register_add_generate_db_auth_token') + r.register_fn('awscli.customizations.dsql', 'register_dsql_customizations') + r.register_fn('awscli.customizations.putmetricdata', 'register_put_metric_data') + r.register_fn('awscli.customizations.sessendemail', 'register_ses_send_email') + r.register_fn('awscli.customizations.iamvirtmfa', 'IAMVMFAWrapper') + r.register_fn('awscli.customizations.argrename', 'register_arg_renames') + r.register_fn('awscli.customizations.configure.configure', 'register_configure_cmd') + r.register_fn('awscli.customizations.cloudtrail', 'initialize') + r.register_fn('awscli.customizations.ecr', 'register_ecr_commands') + r.register_fn('awscli.customizations.ecr_public', 'register_ecr_public_commands') + r.register_fn('awscli.customizations.toplevelbool', 'register_bool_params') + r.register_fn('awscli.customizations.ec2.protocolarg', 'register_protocol_args') + r.register_fn('awscli.customizations.datapipeline', 'register_customizations') + r.register_fn('awscli.customizations.cloudsearch', 'initialize') + r.register_fn('awscli.customizations.emr.emr', 'emr_initialize') + r.register_fn('awscli.customizations.emrcontainers', 'initialize') + r.register_fn('awscli.customizations.eks', 'initialize') + r.register_fn('awscli.customizations.ecs', 'initialize') + r.register_fn('awscli.customizations.ecs.monitormutatinggatewayservice', 'register_monitor_mutating_gateway_service') + r.register_fn('awscli.customizations.lightsail', 'initialize') + r.register_fn('awscli.customizations.cloudsearchdomain', 'register_cloudsearchdomain') + r.register_fn('awscli.customizations.generatecliskeleton', 'register_generate_cli_skeleton') + r.register_fn('awscli.customizations.assumerole', 'register_assume_role_provider') + r.register_fn('awscli.customizations.waiters', 'register_add_waiters') + r.register_fn('awscli.customizations.codedeploy.codedeploy', 'initialize') + r.register_fn('awscli.customizations.configservice.subscribe', 'register_subscribe') + r.register_fn('awscli.customizations.configservice.getstatus', 'register_get_status') + r.register_fn('awscli.customizations.configservice.rename_cmd', 'register_rename_config') + r.register_fn('awscli.customizations.timestampformat', 'register_timestamp_format') + r.register_fn('awscli.customizations.awslambda', 'register_lambda_create_function') + r.register_fn('awscli.customizations.kms', 'register_fix_kms_create_grant_docs') + r.register_fn('awscli.customizations.route53', 'register_create_hosted_zone_doc_fix') + r.register_fn('awscli.customizations.configservice.putconfigurationrecorder', 'register_modify_put_configuration_recorder') + r.register_fn('awscli.customizations.codeartifact', 'register_codeartifact_commands') + r.register_fn('awscli.customizations.codecommit', 'initialize') + r.register_fn('awscli.customizations.iot_data', 'register_custom_endpoint_note') + r.register_direct('awscli.customizations.iot', 'register_create_keys_and_cert_arguments', 'building-argument-table.iot.create-keys-and-certificate') + r.register_direct('awscli.customizations.iot', 'register_create_keys_from_csr_arguments', 'building-argument-table.iot.create-certificate-from-csr') + r.register_fn('awscli.customizations.cloudfront', 'register') + r.register_fn('awscli.customizations.gamelift', 'register_gamelift_commands') + r.register_fn('awscli.customizations.ec2.paginate', 'register_ec2_page_size_injector') + r.register_fn('awscli.customizations.cloudformation', 'initialize') + r.register_fn('awscli.customizations.servicecatalog', 'register_servicecatalog_commands') + r.register_fn('awscli.customizations.translate', 'register_translate_import_terminology') + r.register_fn('awscli.customizations.history', 'register_history_mode') + r.register_fn('awscli.customizations.history', 'register_history_commands') + r.register_fn('awscli.customizations.s3events', 'register_event_stream_arg') + r.register_fn('awscli.customizations.s3events', 'register_document_expires_string') + r.register_fn('awscli.customizations.dlm.dlm', 'dlm_initialize') + r.register_fn('awscli.customizations.sessionmanager', 'register_ssm_session') + r.register_fn('awscli.customizations.logs', 'register_logs_commands') + r.register_fn('awscli.customizations.devcommands', 'register_dev_commands') + r.register_fn('awscli.customizations.wizard.commands', 'register_wizard_commands') + r.register_fn('awscli.customizations.sso', 'register_sso_commands') + r.register_fn('awscli.customizations.dynamodb.paginatorfix', 'register_dynamodb_paginator_fix') + r.register_fn('awscli.alias', 'register_alias_commands') + r.register_fn('awscli.customizations.kinesis', 'register_kinesis_list_streams_pagination_backcompat') + r.register_fn('awscli.customizations.quicksight', 'register_quicksight_asset_bundle_customizations') + r.register_fn('awscli.customizations.ec2instanceconnect', 'register_ec2_instance_connect_commands') + r.register_fn('awscli.customizations.login', 'register_login_cmds') + + +# --------------------------------------------------------------------------- +# Registry construction +# --------------------------------------------------------------------------- + +def _build_registry(entries): + """Group entries by sentinel, deduplicating while preserving order.""" + registry = collections.OrderedDict() + seen = set() + + for entry in entries: + # Determine the sentinel from the event name. For register_fn entries + # we don't have an event name yet — we need to import the module to + # find out what events it subscribes to. Instead we use a lightweight + # RecordingEmitter to run each register_fn against and capture the + # events it subscribes to. + if entry.event_name is not None: + # Direct registration — sentinel comes from the event name. + sentinel = _classify_event(entry.event_name) + key = (sentinel,) + entry.as_tuple() + if key not in seen: + seen.add(key) + registry.setdefault(sentinel, []).append(entry.as_tuple()) + else: + # register_fn — we need to run it against a mini RecordingEmitter + # to discover what events it subscribes to. + mini = _MiniEmitter() + import importlib + mod = importlib.import_module(entry.module) + fn = getattr(mod, entry.fn_name) + fn(mini) + + if not mini.registrations: + # No events recorded — treat as __always__ so it still runs. + sentinel = SENTINEL_ALWAYS + key = (sentinel,) + entry.as_tuple() + if key not in seen: + seen.add(key) + registry.setdefault(sentinel, []).append(entry.as_tuple()) + else: + sentinels_seen_for_fn = set() + for event_name in mini.registrations: + sentinel = _classify_event(event_name) + if sentinel in sentinels_seen_for_fn: + continue + sentinels_seen_for_fn.add(sentinel) + key = (sentinel,) + entry.as_tuple() + if key not in seen: + seen.add(key) + registry.setdefault(sentinel, []).append(entry.as_tuple()) + + return registry + + +class _MiniEmitter: + """Minimal emitter that just records event names.""" + + def __init__(self): + self.registrations = [] + + def register(self, event_name, *args, **kwargs): + self.registrations.append(event_name) + + register_first = register + register_last = register + + # Some register functions call session.register() — support that too. + def emit(self, *a, **k): pass + def emit_first_non_none_response(self, *a, **k): return None + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + +def _render_registry(registry): + lines = [ + '# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.', + '#', + '# AUTO-GENERATED by scripts/generate_plugin_registry.', + '# DO NOT EDIT MANUALLY.', + '#', + '# Re-generate by running:', + '# scripts/generate_plugin_registry', + '#', + '# Sentinels:', + '# __always__ registered for every CLI invocation', + '# __main__ registered when building the top-level command table', + '# registered only when that service/command is invoked', + '#', + '# Entry formats:', + '# (module, fn_name) call fn(event_handlers)', + '# (module, fn_name, event_name) event_handlers.register(event_name, fn)', + '', + 'PLUGIN_REGISTRY = {', + ] + + # Sort sentinels so output is stable across platforms (e.g. os.listdir order). + # Pin __always__ and __main__ first, then alphabetical. + def _sentinel_key(s): + if s == SENTINEL_ALWAYS: + return (0, s) + if s == SENTINEL_MAIN: + return (1, s) + return (2, s) + + sentinels = sorted(registry.keys(), key=_sentinel_key) + for i, sentinel in enumerate(sentinels): + entries = registry[sentinel] + is_last = (i == len(sentinels) - 1) + lines.append(f' {sentinel!r}: [') + for j, entry in enumerate(entries): + comma = '' if j == len(entries) - 1 else ',' + lines.append(f' {entry!r}{comma}') + lines.append(f' ]{("" if is_last else ",")}') + + lines.append('}') + lines.append('') + return '\n'.join(lines) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + '--output', + default=DEFAULT_OUTPUT, + help=f'Path to write the generated registry (default: {DEFAULT_OUTPUT})', + ) + parser.add_argument( + '--check', + action='store_true', + help=( + 'Check that the committed registry matches a fresh generation. ' + 'Exits with code 1 if they differ (for CI).' + ), + ) + parser.add_argument( + '--print', + dest='print_only', + action='store_true', + help='Print the generated registry to stdout instead of writing a file.', + ) + args = parser.parse_args() + + recorder = _Recorder() + _all_registrations(recorder) + registry = _build_registry(recorder.entries) + source = _render_registry(registry) + + if args.print_only: + print(source) + return + + if args.check: + if not os.path.exists(args.output): + print( + f'ERROR: {args.output} does not exist. ' + 'Run without --check to generate it.', + file=sys.stderr, + ) + sys.exit(1) + with open(args.output) as f: + committed = f.read() + if committed == source: + print('OK: committed registry matches fresh generation.') + else: + import difflib + diff = difflib.unified_diff( + committed.splitlines(keepends=True), + source.splitlines(keepends=True), + fromfile=f'{args.output} (committed)', + tofile=f'{args.output} (generated)', + ) + sys.stderr.write(''.join(diff)) + print( + f'ERROR: {args.output} is out of date.\n' + 'Run `scripts/generate_plugin_registry` and commit the result.', + file=sys.stderr, + ) + sys.exit(1) + return + + with open(args.output, 'w') as f: + f.write(source) + print(f'Written {args.output}') + + total = sum(len(v) for v in registry.values()) + print(f'\nRegistry summary ({total} entries across {len(registry)} sentinels):') + for sentinel, entries in registry.items(): + print(f' {sentinel!r:30s} {len(entries):3d} entries') + + +if __name__ == '__main__': + main() diff --git a/tests/functional/test_handlers_registry.py b/tests/functional/test_handlers_registry.py new file mode 100644 index 000000000000..8d392aa3dd42 --- /dev/null +++ b/tests/functional/test_handlers_registry.py @@ -0,0 +1,40 @@ +# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Verify that awscli/handlers_registry.py is up to date. + +If this test fails, re-run the generation script and commit the result: + + scripts/generate_plugin_registry +""" +import os +import subprocess +import sys + +REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +) +SCRIPT = os.path.join(REPO_ROOT, 'scripts', 'generate_plugin_registry') + + +def test_handlers_registry_matches_generation_script(): + result = subprocess.run( + [sys.executable, SCRIPT, '--check'], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f'awscli/handlers_registry.py is out of date.\n' + f'Re-generate it by running:\n\n' + f' scripts/generate_plugin_registry\n\n' + f'Diff (committed vs generated):\n{result.stderr}' + )