Fix/1698 (#1699)

* Fix of 1698

Turned into a major usage clean-up as well, as the code to address #1698 was merged upstream into es_client.

As such, other redundant code shared with es_client has been removed. That should clear up some of the cruft and code sprawl that's been in Curator for a while.

Tests have all been verified clean.

Copyright years bumped where applicable.

* A few changes noticed while going over the code in the last commit.

Author: untergeek

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
-ARG PYVER=3.11.4
-ARG ALPTAG=3.17
+ARG PYVER=3.11.7
+ARG ALPTAG=3.18
 FROM python:${PYVER}-alpine${ALPTAG} as builder
 
 # Add the community repo for access to patchelf binary package

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2011–2023 Elasticsearch <http://elastic.co> and contributors.
+Copyright 2011–2024 Elasticsearch <http://elastic.co> and contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

curator/__init__.py

@@ -4,7 +4,6 @@
 from curator.exceptions import *
 from curator.defaults import *
 from curator.validators import *
-from curator.logtools import *
 from curator.indexlist import IndexList
 from curator.snapshotlist import SnapshotList
 from curator.actions import *

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.8'
+__version__ = '8.0.9'

curator/classdef.py

@@ -1,11 +1,11 @@
 """Other Classes"""
 import logging
-from es_client.exceptions import ConfigurationError as ESclient_ConfigError
+from es_client.exceptions import FailedValidation
+from es_client.helpers.schemacheck import password_filter
 from es_client.helpers.utils import get_yaml
 from curator import IndexList, SnapshotList
 from curator.actions import CLASS_MAP
 from curator.exceptions import ConfigurationError
-from curator.config_utils import password_filter
 from curator.helpers.testers import validate_actions
 
 # Let me tell you the story of the nearly wasted afternoon and the research that went into this
@@ -62,7 +62,7 @@ def get_validated(self, action_file):
         """
         try:
             return validate_actions(get_yaml(action_file))
-        except (ESclient_ConfigError, UnboundLocalError) as err:
+        except (FailedValidation, UnboundLocalError) as err:
             self.logger.critical('Configuration Error: %s', err)
             raise ConfigurationError from err
 

curator/cli.py

@@ -1,87 +1,20 @@
 """Main CLI for Curator"""
 import sys
 import logging
-import pathlib
 import click
-from es_client.builder import ClientArgs, OtherArgs
-from es_client.helpers.utils import get_yaml, check_config, prune_nones, verify_url_schema
+from es_client.defaults import LOGGING_SETTINGS
+from es_client.helpers.config import cli_opts, context_settings, get_args, get_client, get_config
+from es_client.helpers.logging import configure_logging
+from es_client.helpers.utils import option_wrapper, prune_nones
 from curator.exceptions import ClientException
 from curator.classdef import ActionsFile
-from curator.config_utils import check_logging_config, set_logging
-from curator.defaults import settings
+from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer, snapshot_actions
 from curator.exceptions import NoIndices, NoSnapshots
-from curator.helpers.getters import get_client
 from curator.helpers.testers import ilm_policy_check
-from curator.cli_singletons.utils import get_width
 from curator._version import __version__
 
-def configfile_callback(ctx, param, value):
-    """Callback to validate whether the provided config file exists and is writeable
-
-    :param ctx: The click context
-    :param param: The click parameter object
-    :param value: The value of the parameter
-
-    :type ctx: Click context
-    :type param: Click object
-    :type value: Any
-
-    :returns: Config file path or None
-    :rtype: str
-    """
-    logger = logging.getLogger(__name__)
-    logger.debug('Click ctx = %s', ctx)
-    logger.debug('Click param = %s', param)
-    logger.debug('Click value = %s', value)
-    path = pathlib.Path(value)
-    if path.is_file():
-        return value
-    logger.warning('Config file not found: %s', value)
-    return None
-
-def override_logging(config, loglevel, logfile, logformat):
-    """Get logging config and override from command-line options
-
-    :param config: The configuration from file
-    :param loglevel: The log level
-    :param logfile: The log file to write
-    :param logformat: Which log format to use
-
-    :type config: dict
-    :type loglevel: str
-    :type logfile: str
-    :type logformat: str
-
-    :returns: Log configuration ready for validation
-    :rtype: dict
-    """
-    # Check for log settings from config file
-    init_logcfg = check_logging_config(config)
-
-    # Override anything with options from the command-line
-    if loglevel:
-        init_logcfg['loglevel'] = loglevel
-    if logfile:
-        init_logcfg['logfile'] = logfile
-    if logformat:
-        init_logcfg['logformat'] = logformat
-    return init_logcfg
-
-def cli_hostslist(hosts):
-    """
-    :param hosts: One or more hosts.
-    :type hosts: str or list
-
-    :returns: A list of hosts that came in from the command-line, or ``None``
-    :rtype: list or ``None``
-    """
-    hostslist = []
-    if hosts:
-        for host in list(hosts):
-            hostslist.append(verify_url_schema(host))
-    else:
-        hostslist = None
-    return hostslist
+ONOFF = {'on': '', 'off': 'no-'}
+click_opt_wrap = option_wrapper()
 
 def ilm_action_skip(client, action_def):
     """
@@ -97,7 +30,7 @@ def ilm_action_skip(client, action_def):
     :rtype: bool
     """
     logger = logging.getLogger(__name__)
-    if not action_def.allow_ilm and action_def.action not in settings.snapshot_actions():
+    if not action_def.allow_ilm and action_def.action not in snapshot_actions():
         if action_def.action == 'rollover':
             if ilm_policy_check(client, action_def.options['name']):
                 logger.info('Alias %s is associated with ILM policy.', action_def.options['name'])
@@ -261,35 +194,35 @@ def run(client_args, other_args, action_file, dry_run=False):
         logger.info('Action ID: %s, "%s" completed.', idx, action_def.action)
     logger.info('All actions completed.')
 
-# pylint: disable=unused-argument, redefined-builtin
-@click.command(context_settings=get_width())
-@click.option('--config', help='Path to configuration file.', type=str, default=settings.config_file(), callback=configfile_callback)
-@click.option('--hosts', help='Elasticsearch URL to connect to', multiple=True)
-@click.option('--cloud_id', help='Shorthand to connect to Elastic Cloud instance')
-@click.option('--api_token', help='The base64 encoded API Key token', type=str)
-@click.option('--id', help='API Key "id" value', type=str)
-@click.option('--api_key', help='API Key "api_key" value', type=str)
-@click.option('--username', help='Username used to create "basic_auth" tuple')
-@click.option('--password', help='Password used to create "basic_auth" tuple')
-@click.option('--bearer_auth', type=str)
-@click.option('--opaque_id', type=str)
-@click.option('--request_timeout', help='Request timeout in seconds', type=float)
-@click.option('--http_compress', help='Enable HTTP compression', is_flag=True, default=None)
-@click.option('--verify_certs', help='Verify SSL/TLS certificate(s)', is_flag=True, default=None)
-@click.option('--ca_certs', help='Path to CA certificate file or directory')
-@click.option('--client_cert', help='Path to client certificate file')
-@click.option('--client_key', help='Path to client certificate key')
-@click.option('--ssl_assert_hostname', help='Hostname or IP address to verify on the node\'s certificate.', type=str)
-@click.option('--ssl_assert_fingerprint', help='SHA-256 fingerprint of the node\'s certificate. If this value is given then root-of-trust verification isn\'t done and only the node\'s certificate fingerprint is verified.', type=str)
-@click.option('--ssl_version', help='Minimum acceptable TLS/SSL version', type=str)
-@click.option('--master-only', help='Only run if the single host provided is the elected master', is_flag=True, default=None)
-@click.option('--skip_version_test', help='Do not check the host version', is_flag=True, default=None)
-@click.option('--dry-run', is_flag=True, help='Do not perform any changes.')
-@click.option('--loglevel', help='Log level', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']))
-@click.option('--logfile', help='log file')
-@click.option('--logformat', help='Log output format', type=click.Choice(['default', 'logstash', 'json', 'ecs']))
+# pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals, line-too-long
+@click.command(context_settings=context_settings(), epilog=footer(__version__, tail='command-line.html'))
+@click_opt_wrap(*cli_opts('config'))
+@click_opt_wrap(*cli_opts('hosts'))
+@click_opt_wrap(*cli_opts('cloud_id'))
+@click_opt_wrap(*cli_opts('api_token'))
+@click_opt_wrap(*cli_opts('id'))
+@click_opt_wrap(*cli_opts('api_key'))
+@click_opt_wrap(*cli_opts('username'))
+@click_opt_wrap(*cli_opts('password'))
+@click_opt_wrap(*cli_opts('bearer_auth'))
+@click_opt_wrap(*cli_opts('opaque_id'))
+@click_opt_wrap(*cli_opts('request_timeout'))
+@click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF))
+@click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF))
+@click_opt_wrap(*cli_opts('ca_certs'))
+@click_opt_wrap(*cli_opts('client_cert'))
+@click_opt_wrap(*cli_opts('client_key'))
+@click_opt_wrap(*cli_opts('ssl_assert_hostname'))
+@click_opt_wrap(*cli_opts('ssl_assert_fingerprint'))
+@click_opt_wrap(*cli_opts('ssl_version'))
+@click_opt_wrap(*cli_opts('master-only', onoff=ONOFF))
+@click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF))
+@click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN))
+@click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS))
+@click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS))
+@click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS))
 @click.argument('action_file', type=click.Path(exists=True), nargs=1)
-@click.version_option(version=__version__)
+@click.version_option(__version__, '-v', '--version', prog_name="curator")
 @click.pass_context
 def cli(
     ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth,
@@ -298,75 +231,21 @@ def cli(
     dry_run, loglevel, logfile, logformat, action_file
 ):
     """
-    Curator for Elasticsearch indices.
-
-    See http://elastic.co/guide/en/elasticsearch/client/curator/current
-    """
-    client_args = ClientArgs()
-    other_args = OtherArgs()
-    if config:
-        from_yaml = get_yaml(config)
-    else:
-        # Use empty defaults.
-        from_yaml = {'elasticsearch': {'client': {}, 'other_settings': {}}, 'logging': {}}
-    raw_config = check_config(from_yaml)
-    client_args.update_settings(raw_config['client'])
-    other_args.update_settings(raw_config['other_settings'])
-
-    set_logging(check_logging_config(
-        {'logging': override_logging(from_yaml, loglevel, logfile, logformat)}))
-
-    hostslist = cli_hostslist(hosts)
+    Curator for Elasticsearch indices
 
-    cli_client = prune_nones({
-        'hosts': hostslist,
-        'cloud_id': cloud_id,
-        'bearer_auth': bearer_auth,
-        'opaque_id': opaque_id,
-        'request_timeout': request_timeout,
-        'http_compress': http_compress,
-        'verify_certs': verify_certs,
-        'ca_certs': ca_certs,
-        'client_cert': client_cert,
-        'client_key': client_key,
-        'ssl_assert_hostname': ssl_assert_hostname,
-        'ssl_assert_fingerprint': ssl_assert_fingerprint,
-        'ssl_version': ssl_version
-    })
+    The default $HOME/.curator/curator.yml configuration file (--config)
+    can be used but is not needed.
+    
+    Command-line settings will always override YAML configuration settings.
 
-    cli_other = prune_nones({
-        'master_only': master_only,
-        'skip_version_test': skip_version_test,
-        'username': username,
-        'password': password,
-        'api_key': {
-            'id': id,
-            'api_key': api_key,
-            'token': api_token,
-        }
-    })
-    # Remove `api_key` root key if `id` and `api_key` and `token` are all None
-    if id is None and api_key is None and api_token is None:
-        del cli_other['api_key']
+    Some less-frequently used client configuration options are now hidden. To see the full list,
+    run:
 
-    # If hosts are in the config file, but cloud_id is specified at the command-line,
-    # we need to remove the hosts parameter as cloud_id and hosts are mutually exclusive
-    if cloud_id:
-        click.echo('cloud_id provided at CLI, superseding any other configured hosts')
-        client_args.hosts = None
-        cli_client.pop('hosts', None)
-
-    # Likewise, if hosts are provided at the command-line, but cloud_id was in the config file,
-    # we need to remove the cloud_id parameter from the config file-based dictionary before merging
-    if hosts:
-        click.echo('hosts specified manually, superseding any other cloud_id or hosts')
-        client_args.hosts = None
-        client_args.cloud_id = None
-        cli_client.pop('cloud_id', None)
-
-    # Update the objects if we have settings after pruning None values
-    if cli_client:
-        client_args.update_settings(cli_client)
-    if cli_other:
-        other_args.update_settings(cli_other)
+        curator_cli -h
+    """
+    ctx.obj = {}
+    ctx.obj['dry_run'] = dry_run
+    cfg = get_config(ctx.params, default_config_file())
+    configure_logging(cfg, ctx.params)
+    client_args, other_args = get_args(ctx.params, cfg)
     run(client_args, other_args, action_file, dry_run)

curator/cli_singletons/alias.py

@@ -1,10 +1,11 @@
 """Alias Singleton"""
 import logging
 import click
+from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
-from curator.cli_singletons.utils import get_width, json_to_dict, validate_filter_json
+from curator.cli_singletons.utils import json_to_dict, validate_filter_json
 
-@click.command(context_settings=get_width())
+@click.command(context_settings=context_settings())
 @click.option('--name', type=str, help='Alias name', required=True)
 @click.option(
     '--add',
@@ -48,12 +49,6 @@ def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_
     logger.debug('manual_options %s', manual_options)
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     ignore_empty_list = warn_if_no_indices
-    logger.debug('ctx.info_name %s', ctx.info_name)
-    logger.debug('ignore_empty_list %s', ignore_empty_list)
-    logger.debug('add %s', add)
-    logger.debug('remove %s', remove)
-    logger.debug('warn_if_no_indices %s', warn_if_no_indices)
-    logger.debug("ctx.obj['dry_run'] %s", ctx.obj['dry_run'])
     action = CLIAction(
         ctx.info_name,
         ctx.obj['config'],
@@ -62,5 +57,4 @@ def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_
         ignore_empty_list,
         add=add, remove=remove, warn_if_no_indices=warn_if_no_indices, # alias specific kwargs
     )
-    logger.debug('We did not get here, did we?')
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])

curator/cli_singletons/allocation.py

@@ -1,9 +1,10 @@
 """Allocation Singleton"""
 import click
+from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
-from curator.cli_singletons.utils import get_width, validate_filter_json
+from curator.cli_singletons.utils import validate_filter_json
 
-@click.command(context_settings=get_width())
+@click.command(context_settings=context_settings())
 @click.option('--key', type=str, required=True, help='Node identification tag')
 @click.option('--value', type=str, default=None, help='Value associated with --key')
 @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude']))

curator/cli_singletons/close.py

@@ -1,9 +1,10 @@
 """Close Singleton"""
 import click
+from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
-from curator.cli_singletons.utils import get_width, validate_filter_json
+from curator.cli_singletons.utils import validate_filter_json
 
-@click.command(context_settings=get_width())
+@click.command(context_settings=context_settings())
 @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed')
 @click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed')
 @click.option(

curator/cli_singletons/delete.py

@@ -1,10 +1,11 @@
 """Delete Index and Delete Snapshot Singletons"""
 import click
+from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
-from curator.cli_singletons.utils import get_width, validate_filter_json
+from curator.cli_singletons.utils import validate_filter_json
 
 #### Indices ####
-@click.command(context_settings=get_width())
+@click.command(context_settings=context_settings())
 @click.option(
     '--ignore_empty_list',
     is_flag=True,
@@ -38,7 +39,7 @@ def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])
 
 #### Snapshots ####
-@click.command(context_settings=get_width())
+@click.command(context_settings=context_settings())
 @click.option('--repository', type=str, required=True, help='Snapshot repository name')
 @click.option('--retry_count', type=int, help='Number of times to retry (max 3)')
 @click.option('--retry_interval', type=int, help='Time in seconds between retries')