From 19c2098ac58d80402f5ad66354009beb1f194086 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Wed, 16 May 2018 23:57:27 +0300 Subject: [PATCH 01/16] initialize log file and log level in ovirt-dr --- README.md | 7 ++- files/ovirt-dr | 117 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index afa56a5..3befe01 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,15 @@ There are four actions which the user can execute: failover Start a failover process to the target setup failback Start a failback process from the target setup to the source setup -Each of those actions is using a configuration file which its default location is oVirt.disaster-recovery/files/dr.conf +Each of those actions are using a configuration file which its default location is oVirt.disaster-recovery/files/dr.conf +The configuration file location can be changed using --conf-file flag in the ovirt-dr script. +Log file and log level can be configured as well through the ovirt-dr script using the flags --log-file and --log-level + Example Script -------------- For mapping file generation: - ./ovirt-dr generate + ./ovirt-dr generate --log-file=ovirt-dr.log --log-level=DEBUG For mapping file validation: ./ovirt-dr validate For fail-over operation: diff --git a/files/ovirt-dr b/files/ovirt-dr index fa3329b..36b18a5 100755 --- a/files/ovirt-dr +++ b/files/ovirt-dr @@ -1,9 +1,15 @@ #!/usr/bin/python +try: + import configparser +except ImportError: + import ConfigParser as configparser import fail_back import fail_over import errno +import logging as logg import os import sys +import time import generate_vars import getopt import validator @@ -12,26 +18,45 @@ VALIDATE = 'validate' GENERATE = 'generate' FAILOVER = 'failover' FAILBACK = 'failback' -# TODO: Use build.sh for those files -DIR = '/var/log/ovirt-dr' -LOG_FILE = DIR + '/ovirt-dr.log' +LOG_FILE = 'log-file' +LOG_LEVEL = 'log-level' +DEF_LOG_FILE = "" +DEF_DEBUG_LEVEL = 'DEBUG' DEF_CONF_FILE = 'dr.conf' def main(argv): - action, conf_file = _init_vars(argv) + action, conf_file, log_file, log_level = _init_vars(argv) + log_file = log_file.format(int(round(time.time() * 1000))) + if log_level not in ['DEBUG', 'INFO', 'WARNING', 'ERROR']: + print("ovirt-dr: log level must be 'DEBUG' 'INFO' 'WARNING' 'ERROR'\n" + "Use 'ovirt-dr --help' for more information.") + sys.exit(2) + while not os.path.isfile(conf_file): conf_file = raw_input( "Conf file '" + conf_file + "' does not exist." " Please provide the configuration file location: ") + + create_log_dir(log_file) + if log_file is not None and log_file != '': + print("Log file: '%s'" % log_file) if action == 'validate': - validator.ValidateMappingFile().run(conf_file, LOG_FILE) + validator.ValidateMappingFile().run(conf_file) elif action == 'generate': - generate_vars.GenerateMappingFile().run(conf_file, LOG_FILE) + generate_vars.GenerateMappingFile().run(conf_file, + log_file, + logg.getLevelName(log_level)) elif action == 'failover': - fail_over.FailOver().run(conf_file, LOG_FILE) + fail_over.FailOver().run(conf_file, + log_file, + logg.getLevelName(log_level)) elif action == 'failback': - fail_back.FailBack().run(conf_file, LOG_FILE) + fail_back.FailBack().run(conf_file, + log_file, + logg.getLevelName(log_level)) + elif action == '--help': + help_log() else: print "\tError: action '%s' is not defined" % action help_log() @@ -39,45 +64,85 @@ def main(argv): def _init_vars(argv): conf_file = DEF_CONF_FILE + log_file = '' + log_level = '' + if len(argv) == 0: print("ovirt-dr: missing action operand\n" - "Try 'ovirt-dr --help' for more information.") + "Use 'ovirt-dr --help' for more information.") sys.exit(2) action = argv[0] + try: opts, args = \ - getopt.getopt(argv, "f:", ["file="]) + getopt.getopt(argv[1:], "f:log:level:", + ["conf-file=", "log-file=", "log-level="]) except getopt.GetoptError: help_log() sys.exit(2) - if not os.path.exists(DIR): - print("Create dir file '%s'" % DIR) - os.makedirs(DIR) - for opt, arg in opts: - if opt in ("-f", "--filename"): + if opt in ("-f", "--conf-file"): conf_file = arg - return (action, conf_file) + if opt in ("-log", "--log-file"): + log_file = arg + if opt in ("-level", "--log-level"): + log_level = arg + + log_file, log_level = _get_log_conf(conf_file, log_file, log_level) + return (action, conf_file, log_file, log_level.upper()) + + +def _get_log_conf(conf_file, log_file, log_level): + log_section = "log" + log_file_conf = "log_file" + log_level_conf = "log_level" + while not os.path.isfile(conf_file): + conf_file = raw_input( + "Conf file '" + conf_file + "' does not exist." + " Please provide the configuration file location: ") + settings = configparser.ConfigParser() + settings._interpolation = configparser.ExtendedInterpolation() + settings.read(conf_file) + if log_section not in settings.sections(): + settings.add_section(log_section) + if settings.has_option(log_section, log_file_conf) and \ + (log_file is None or log_file == ''): + log_file = settings.get(log_section, log_file_conf) + if settings.has_option(log_section, log_level_conf) and \ + (log_level is None or log_level == ''): + log_level = settings.get(log_section, log_level_conf) + else: + log_level = "DEBUG" + return (log_file, log_level) + + +def create_log_dir(fname): + _dir = os.path.dirname(fname) + if _dir != '' and not os.path.exists(_dir): + os.makedirs(_dir) def help_log(): print( - ''' - \tusage: ovirt-dr <%s/%s/%s/%s> [-f ]\n + ''' + \tusage: ovirt-dr <%s/%s/%s/%s> + [--conf-file=dr.conf] + [--log-file=log_file.log] + [--log-level=DEBUG/INFO/WARNING/ERROR]\n \tHere is a description of the following actions:\n \t\t%s\tGenerate the mapping var file based on primary setup \t\t%s\tValidate the var file mapping \t\t%s\tStart a failover process to the target setup \t\t%s\tStart a failback process to the source setup - ''' % (GENERATE, - VALIDATE, - FAILOVER, - FAILBACK, - GENERATE, - VALIDATE, - FAILOVER, - FAILBACK)) + ''' % (GENERATE, + VALIDATE, + FAILOVER, + FAILBACK, + GENERATE, + VALIDATE, + FAILOVER, + FAILBACK)) if __name__ == "__main__": From 70eabebe01ed74ce632bcd762a32f73502001111 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Wed, 16 May 2018 23:57:39 +0300 Subject: [PATCH 02/16] Change default value in dr.conf Change dr-play location in dr.conf and change vault location --- files/dr.conf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/files/dr.conf b/files/dr.conf index 7855f05..924fd9c 100644 --- a/files/dr.conf +++ b/files/dr.conf @@ -1,18 +1,22 @@ +[log] +log_file=/tmp/ovirt-dr-{}.log +log_level=DEBUG + [generate_vars] site=http://engine.example.com/ovirt-engine/api username=admin@internal password= ca_file=/etc/pki/ovirt-engine/ca.pem output_file=/var/lib/ovirt-ansible-disaster-recovery/mapping_vars.yml -ansible_play=../examples/dr_play.yml +ansible_play=/usr/share/doc/ovirt-ansible-disaster-recovery/examples/dr_play.yml [validate_vars] var_file=/var/lib/ovirt-ansible-disaster-recovery/mapping_vars.yml -vault=../examples/ovirt_passwords.yml +vault=/usr/share/doc/ovirt-ansible-disaster-recovery/examples/ovirt_passwords.yml [failover_failback] dr_target_host=secondary dr_source_map=primary -vault=passwords.yml +vault=/usr/share/doc/ovirt-ansible-disaster-recovery/examples/ovirt_passwords.yml var_file=/var/lib/ovirt-ansible-disaster-recovery/mapping_vars.yml -ansible_play=../examples/dr_play.yml +ansible_play=/usr/share/doc/ovirt-ansible-disaster-recovery/examples/dr_play.yml From 1e77afb0fc9405859aa099942fdf5e285f26b0cf Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Wed, 9 May 2018 12:09:52 +0300 Subject: [PATCH 03/16] Use logging in generate vars --- files/generate_vars.py | 194 ++++++++++++++++++++++------------------- 1 file changed, 105 insertions(+), 89 deletions(-) diff --git a/files/generate_vars.py b/files/generate_vars.py index df6fd0f..20c0e66 100755 --- a/files/generate_vars.py +++ b/files/generate_vars.py @@ -2,6 +2,7 @@ from bcolors import bcolors from ConfigParser import SafeConfigParser +import logging import os.path import ovirtsdk4 as sdk import shlex @@ -25,36 +26,30 @@ class GenerateMappingFile(): - def run(self, conf_file, log_file): - print("\n%s%sStart generate variable mapping file " - "for oVirt ansible disaster recovery%s" - % (INFO, - PREFIX, - END)) + def run(self, conf_file, log_file, log_level): + log = self._set_log(log_file, log_level) + log.info("Start generate variable mapping file " + "for oVirt ansible disaster recovery") dr_tag = "generate_mapping" site, username, password, ca_file, var_file_path, _ansible_play = \ - self._init_vars(conf_file) - print("\n%s%sSite address: %s \n" - "%susername: %s \n" - "%spassword: *******\n" - "%sca file location: %s \n" - "%soutput file location: %s \n" - "%sansible play location: %s \n%s" - % (INFO, - PREFIX, - site, - PREFIX, - username, - PREFIX, - PREFIX, - ca_file, - PREFIX, - var_file_path, - PREFIX, - _ansible_play, - END)) - if not self._validate_connection(site, username, password, ca_file): - self._print_error(log_file) + self._init_vars(conf_file, log) + log.info("Site address: %s \n" + "username: %s \n" + "password: *******\n" + "ca file location: %s \n" + "output file location: %s \n" + "ansible play location: %s " + % (site, + username, + ca_file, + var_file_path, + _ansible_play)) + if not self._validate_connection(log, + site, + username, + password, + ca_file): + self._print_error(log) exit() command = "site=" + site + " username=" + username + " password=" + \ password + " ca=" + ca_file + " var_file=" + var_file_path @@ -66,38 +61,65 @@ def run(self, conf_file, log_file): cmd.append("-e") cmd.append(command) cmd.append("-vvvvv") - with open(log_file, "w") as f: - f.write("Executing command %s" % ' '.join(map(str, cmd))) - call(cmd, stdout=f) + log.info("Executing command %s" % ' '.join(map(str, cmd))) + if log_file is not None and log_file != '': + self._log_to_file(log_file, cmd) + else: + self._log_to_console(cmd, log) + if not os.path.isfile(var_file_path): - print("%s%scan not find output file in '%s'.%s" - % (FAIL, - PREFIX, - var_file_path, - END)) - self._print_error(log_file) + log.error("Can not find output file in '%s'." % var_file_path) + self._print_error(log) exit() - print("\n%s%sVar file location: '%s'%s" - % (INFO, - PREFIX, - var_file_path, - END)) - self._print_success() + log.info("Var file location: '%s'" % var_file_path) + self._print_success(log) + + def _log_to_file(self, log_file, cmd): + with open(log_file, "a") as f: + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + for line in iter(proc.stdout.readline, ''): + f.write(line) + for line in iter(proc.stderr.readline, ''): + f.write(line) + print("%s%s%s" % (FAIL, + line, + END)) + + def _log_to_console(self, cmd, log): + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + for line in iter(proc.stdout.readline, ''): + log.debug(line) + for line in iter(proc.stderr.readline, ''): + log.error(line) + + def _set_log(self, log_file, log_level): + logger = logging.getLogger(PREFIX) + formatter = logging.Formatter( + '%(asctime)s %(levelname)s %(message)s') + if log_file is not None and log_file != '': + hdlr = logging.FileHandler(log_file) + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + else: + ch = logging.StreamHandler(sys.stdout) + logger.addHandler(ch) + logger.setLevel(log_level) + return logger - def _print_success(self): - print("%s%sFinished generating variable mapping file " - "for oVirt ansible disaster recovery%s" - % (INFO, - PREFIX, - END)) + def _print_success(self, log): + msg = "Finished generating variable mapping file " \ + "for oVirt ansible disaster recovery." + log.info(msg) + print("%s%s%s%s" % (INFO, PREFIX, msg, END)) - def _print_error(self, log_file): - print("%s%sFailed to generate var file." - " See log file '%s' for further details%s" - % (FAIL, - PREFIX, - log_file, - END)) + def _print_error(self, log): + msg = "Failed to generate var file." + log.error(msg) + print("%s%s%s%s" % (FAIL, PREFIX, msg, END)) def _connect_sdk(self, url, username, password, ca): connection = sdk.Connection( @@ -109,6 +131,7 @@ def _connect_sdk(self, url, username, password, ca): return connection def _validate_connection(self, + log, url, username, password, @@ -122,35 +145,24 @@ def _validate_connection(self, dcs_service = conn.system_service().data_centers_service() dcs_service.list() except Exception as e: - print( - "%s%sConnection to setup has failed." - " Please check your cradentials: " - "\n%s URL: %s" - "\n%s USER: %s" - "\n%s CA file: %s%s" % - (FAIL, - PREFIX, - PREFIX, - url, - PREFIX, - username, - PREFIX, - ca, - END)) - print("Error: %s" % e) + msg = "Connection to setup has failed. " \ + "Please check your cradentials: " \ + "\n URL: " + url \ + + "\n USER: " + username \ + + "\n CA file: " + ca + log.error(msg) + print("%s%s%s%s" % (FAIL, PREFIX, msg, END)) + log.error("Error: %s" % e) if conn: conn.close() return False return True - def _validate_output_file_exists(self, fname): + def _validate_output_file_exists(self, fname, log): _dir = os.path.dirname(fname) if _dir != '' and not os.path.exists(_dir): - print("%s%sPath '%s' does not exists. Create folder%s" - % (WARN, - PREFIX, - _dir, - END)) + log.warn("Path '%s' does not exists. Create folder '%s'" + % _dir) os.makedirs(_dir) if os.path.isfile(fname): valid = {"yes": True, "y": True, "ye": True, @@ -166,11 +178,10 @@ def _validate_output_file_exists(self, fname): ans = ans.lower() if ans in valid: if not valid[ans]: - print("%s%sFailed to create output file. " - "File could not be overriden.%s" - % (WARN, - PREFIX, - END)) + msg = "Failed to create output file. " \ + "File could not be overriden." + log.error(msg) + print("%s%s%s%s" % (FAIL, PREFIX, msg, END)) sys.exit(0) break else: @@ -181,14 +192,15 @@ def _validate_output_file_exists(self, fname): try: os.remove(fname) except OSError: - print("\n\n%s%SFile %s could not be replaced.%s" - % (WARN, + log.error("File %s could not be replaced." % fname) + print("%s%sFile %s could not be replaced.%s" + % (FAIL, PREFIX, fname, END)) sys.exit(0) - def _init_vars(self, conf_file): + def _init_vars(self, conf_file, log): """ Declare constants """ _SECTION = "generate_vars" _SITE = 'site' @@ -196,7 +208,8 @@ def _init_vars(self, conf_file): _PASSWORD = 'password' _CA_FILE = 'ca_file' # TODO: Must have full path, should add relative path - _OUTPUT_FILE = '/var/lib/ovirt-ansible-disaster-recovery/mapping_vars.yml' + _OUTPUT_FILE = "/usr/share/ansible/roles/oVirt.disaster-recovery" \ + "/mapping_vars.yml" _ANSIBLE_PLAY = 'ansible_play' """ Declare varialbles """ @@ -283,7 +296,7 @@ def _init_vars(self, conf_file): PREFIX, _OUTPUT_FILE, END)) or _OUTPUT_FILE - self._validate_output_file_exists(output_file) + self._validate_output_file_exists(output_file, log) while (not ansible_play) or (not os.path.isfile(ansible_play)): ansible_play = raw_input("%s%sAnsible play '%s' is not " "initialized. Please provide the ansible " @@ -316,4 +329,7 @@ def items(self): if __name__ == "__main__": - GenerateMappingFile().run('dr.conf', '/var/log/ovirt-dr/ovirt-dr.log') + level = logging.getLevelName("DEBUG") + conf = 'dr.conf' + log = '/tmp/ovirt-dr.log' + GenerateMappingFile().run(conf, log, level) From 4e635e439de8a397add4865e2308d4cdf34fa78c Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Wed, 9 May 2018 17:14:48 +0300 Subject: [PATCH 04/16] Remove unused log file in validator --- files/validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/validator.py b/files/validator.py index a27e545..f3e96b4 100755 --- a/files/validator.py +++ b/files/validator.py @@ -34,7 +34,7 @@ class ValidateMappingFile(): aff_label_map = 'dr_affinity_label_mappings' network_map = 'dr_network_mappings' - def run(self, conf_file, log_file): + def run(self, conf_file): print("%s%sValidate variable mapping file " "for oVirt ansible disaster recovery%s" % (INFO, @@ -788,4 +788,4 @@ def _connect_sdk(self, url, username, password, ca): if __name__ == "__main__": - ValidateMappingFile().run('dr.conf', '/var/log/ovirt-dr/ovirt-dr.log') + ValidateMappingFile().run('dr.conf') From 26a50440d5fb579e2a2ffdcb73a8ae9285f93e31 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Wed, 9 May 2018 17:57:57 +0300 Subject: [PATCH 05/16] add logging support for fail over --- files/fail_over.py | 100 +++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 36 deletions(-) diff --git a/files/fail_over.py b/files/fail_over.py index 87640d1..07c0770 100755 --- a/files/fail_over.py +++ b/files/fail_over.py @@ -1,6 +1,7 @@ #!/usr/bin/python from bcolors import bcolors from ConfigParser import SafeConfigParser +import logging import os.path import shlex import subprocess @@ -19,31 +20,22 @@ class FailOver(): - def run(self, conf_file, log_file): - print("\n%s%sStart failover operation...%s" - % (INFO, - PREFIX, - END)) + def run(self, conf_file, log_file, log_level): + log = self._set_log(log_file, log_level) + log.info("Start failover operation...") dr_tag = "fail_over" target_host, source_map, var_file, vault, ansible_play = \ self._init_vars(conf_file) - print("\n%s%starget_host: %s \n" - "%ssource_map: %s \n" - "%svar_file: %s \n" - "%svault: %s \n" - "%sansible_play: %s%s \n" - % (INFO, - PREFIX, - target_host, - PREFIX, - source_map, - PREFIX, - var_file, - PREFIX, - vault, - PREFIX, - ansible_play, - END)) + log.info("target_host: %s \n" + "source_map: %s \n" + "var_file: %s \n" + "vault: %s \n" + "ansible_play: %s " + % (target_host, + source_map, + var_file, + vault, + ansible_play)) cmd = [] cmd.append("ansible-playbook") @@ -59,25 +51,44 @@ def run(self, conf_file, log_file): " dr_target_host=" + target_host + " dr_source_map=" + source_map) cmd.append("--ask-vault-pass") cmd.append("-vvvvv") - with open(log_file, "w") as f: - f.write("Executing failover command: %s" % ' '.join(map(str, cmd))) + log.info("Executing failover command: %s" % ' '.join(map(str, cmd))) + if log_file is not None and log_file != '': + self._log_to_file(log_file, cmd) + else: + self._log_to_console(cmd, log) + call(['cat', 'report.log']) + print("\n%s%sFinished failover operation" + " for oVirt ansible disaster recovery%s" + % (INFO, + PREFIX, + END)) + + def _log_to_file(self, log_file, cmd): + with open(log_file, "a") as f: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in iter(proc.stdout.readline, ''): - # TODO: since we dont want to have log and print the - # progress in the stdout, we only filter the task names - # We should find a better way to do so. - if 'TASK [ovirt-ansible-disaster-recovery : ' in line: - sys.stdout.write("\n" + line + "\n") + if 'TASK [' in line: + print("\n%s%s%s\n" % (INFO, + line, + END)) f.write(line) + for line in iter(proc.stderr.readline, ''): + f.write(line) + print("%s%s%s" % (WARN, + line, + END)) call(['cat', '/tmp/report.log']) - print("\n%s%sFinished failover operation" - " for oVirt ansible disaster recovery%s" - % (INFO, - PREFIX, - END)) + def _log_to_console(self, cmd, log): + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + for line in iter(proc.stdout.readline, ''): + log.debug(line) + for line in iter(proc.stderr.readline, ''): + log.warn(line) def _init_vars(self, conf_file): """ Declare constants """ @@ -144,7 +155,7 @@ def _init_vars(self, conf_file): var_file, END)) while not os.path.isfile(vault): - vault = raw_input("%s%spassword file '%s' does not exist." + vault = raw_input("%s%spassword file '%s' does not exist. " "Please provide a valid password file:%s " % (INPUT, PREFIX, @@ -163,6 +174,20 @@ def _init_vars(self, conf_file): END) or PLAY_DEF) return (target_host, source_map, var_file, vault, ansible_play) + def _set_log(self, log_file, log_level): + logger = logging.getLogger(PREFIX) + formatter = logging.Formatter( + '%(asctime)s %(levelname)s %(message)s') + if log_file is not None and log_file != '': + hdlr = logging.FileHandler(log_file) + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + else: + ch = logging.StreamHandler(sys.stdout) + logger.addHandler(ch) + logger.setLevel(log_level) + return logger + class DefaultOption(dict): @@ -183,4 +208,7 @@ def items(self): if __name__ == "__main__": - FailOver().run('dr.conf', '/var/log/ovirt-dr/ovirt-dr.log') + level = logging.getLevelName("DEBUG") + conf = 'dr.conf' + log = '/tmp/ovirt-dr.log' + FailOver().run(conf, log, level) From 010eff30190a8327494a324b216a9fed8acadd60 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Wed, 9 May 2018 18:09:37 +0300 Subject: [PATCH 06/16] add logging support for fail back --- files/fail_back.py | 158 +++++++++++++++++++++++++++------------------ 1 file changed, 95 insertions(+), 63 deletions(-) diff --git a/files/fail_back.py b/files/fail_back.py index 7fafdf1..c9dfb62 100755 --- a/files/fail_back.py +++ b/files/fail_back.py @@ -1,6 +1,7 @@ #!/usr/bin/python from bcolors import bcolors from ConfigParser import SafeConfigParser +import logging import os.path import shlex import subprocess @@ -19,32 +20,23 @@ class FailBack(): - def run(self, conf_file, log_file): - print("\n%s%sStart failback operation...%s" - % (INFO, - PREFIX, - END)) + def run(self, conf_file, log_file, log_level): + log = self._set_log(log_file, log_level) + log.info("Start failback operation...") dr_tag = "fail_back" dr_clean_tag = "clean_engine" target_host, source_map, var_file, vault, ansible_play = \ self._init_vars(conf_file) - print("\n%s%starget_host: %s \n" - "%ssource_map: %s \n" - "%svar_file: %s \n" - "%svault: %s \n" - "%sansible_play: %s%s \n" - % (INFO, - PREFIX, - target_host, - PREFIX, - source_map, - PREFIX, - var_file, - PREFIX, - vault, - PREFIX, - ansible_play, - END)) + log.info("\ntarget_host: %s \n" + "source_map: %s \n" + "var_file: %s \n" + "vault: %s \n" + "ansible_play: %s \n" + % (target_host, + source_map, + var_file, + vault, + ansible_play)) cmd = [] cmd.append("ansible-playbook") @@ -81,52 +73,44 @@ def run(self, conf_file, log_file): vault_pass = raw_input( INPUT + PREFIX + "Please enter the vault password: " + END) os.system("export vault_password=\"" + vault_pass + "\"") + log.info("Starting cleanup process of setup %s" + " for oVirt ansible disaster recovery" % target_host) print("\n%s%sStarting cleanup process of setup '%s'" " for oVirt ansible disaster recovery%s" % (INFO, PREFIX, target_host, END)) - with open(log_file, "w") as f: - f.write("Executing cleanup command: %s" % ' '.join(map(str, cmd))) - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - for line in iter(proc.stdout.readline, ''): - # TODO: since we dont want to have log and print the - # progress in the stdout, we only filter the task names - # We should find a better way to do so. - if 'TASK [ovirt-ansible-disaster-recovery : ' in line: - sys.stdout.write("\n" + line + "\n") - f.write(line) + log.info("Executing cleanup command: %s" % ' '.join(map(str, cmd))) + if log_file is not None and log_file != '': + self._log_to_file(log_file, cmd) + else: + self._log_to_console(cmd, log) - print("\n%s%sFinished cleanup of setup '%s'" - " for oVirt ansible disaster recovery%s" - % (INFO, - PREFIX, - source_map, - END)) - - print("\n%s%sStarting fail-back process to setup '%s'" - " from setup '%s' for oVirt ansible disaster recovery" - % (INFO, - PREFIX, - target_host, - source_map)) - - f.write("Executing command %s" % ' '.join(map(str, cmd_fb))) - proc_fb = subprocess.Popen(cmd_fb, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - for line in iter(proc_fb.stdout.readline, ''): - # TODO: since we dont want to have log and print the - # progress in the stdout, we only filter the task names - # We should find a better way to do so. - if 'TASK [ovirt-ansible-disaster-recovery :' in line: - sys.stdout.write("\n" + line + "\n") - if "[Failback Replication Sync]" in line: - sys.stdout.write("\n" + INPUT + line + END) - f.write(line) + log.info("Finished cleanup of setup %s" + " for oVirt ansible disaster recovery" % source_map) + print("\n%s%sFinished cleanup of setup '%s'" + " for oVirt ansible disaster recovery%s" + % (INFO, + PREFIX, + source_map, + END)) + + log.info("Start failback DR from setup '%s'" % target_host) + print("\n%s%sStarting fail-back process to setup '%s'" + " from setup '%s' for oVirt ansible disaster recovery%s" + % (INFO, + PREFIX, + target_host, + source_map, + END)) + + log.info("Executing failback command: %s" + % ' '.join(map(str, cmd_fb))) + if log_file is not None and log_file != '': + self._log_to_file(log_file, cmd_fb) + else: + self._log_to_console(cmd_fb, log) call(["cat", "/tmp/report.log"]) print("\n%s%sFinished failback operation" @@ -135,6 +119,37 @@ def run(self, conf_file, log_file): PREFIX, END)) + def _log_to_file(self, log_file, cmd): + with open(log_file, "a") as f: + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + for line in iter(proc.stdout.readline, ''): + if 'TASK [' in line: + print("\n%s%s%s\n" % (INFO, + line, + END)) + if "[Failback Replication Sync]" in line: + print("%s%s%s" % (INFO, line, END)) + f.write(line) + for line in iter(proc.stderr.readline, ''): + f.write(line) + print("%s%s%s" % (WARN, + line, + END)) + + def _log_to_console(self, cmd, log): + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + for line in iter(proc.stdout.readline, ''): + if "[Failback Replication Sync]" in line: + print("%s%s%s" % (INFO, line, END)) + else: + log.debug(line) + for line in iter(proc.stderr.readline, ''): + log.warn(line) + def _init_vars(self, conf_file): """ Declare constants """ _SECTION = "failover_failback" @@ -205,7 +220,7 @@ def _init_vars(self, conf_file): END)) while not os.path.isfile(vault): vault = raw_input("%s%spassword file '%s' does not exist." - "Please provide a valid password file:%s " + " Please provide a valid password file:%s " % (INPUT, PREFIX, vault, @@ -223,6 +238,20 @@ def _init_vars(self, conf_file): END) or PLAY_DEF) return (target_host, source_map, var_file, vault, ansible_play) + def _set_log(self, log_file, log_level): + logger = logging.getLogger(PREFIX) + formatter = logging.Formatter( + '%(asctime)s %(levelname)s %(message)s') + if log_file is not None and log_file != '': + hdlr = logging.FileHandler(log_file) + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + else: + ch = logging.StreamHandler(sys.stdout) + logger.addHandler(ch) + logger.setLevel(log_level) + return logger + class DefaultOption(dict): @@ -243,4 +272,7 @@ def items(self): if __name__ == "__main__": - FailBack().run('dr.conf', '/var/log/ovirt-dr/ovirt-dr.log') + level = logging.getLevelName("DEBUG") + conf = 'dr.conf' + log = '/tmp/ovirt-dr.log' + FailBack().run(conf, log, level) From 6ef2cd5a033076900824dc642ff5f11f09a0b7f7 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 10 May 2018 01:28:19 +0300 Subject: [PATCH 07/16] Use SafeConfigParser from ConfigParser Using attribute 'ExtendedInterpolation' is not compatible for configparser. If it is being used from RHEL 7.4 the following exception is being thrown: Traceback (most recent call last): File "./ovirt-dr", line 145, in main(sys.argv[1:]) File "./ovirt-dr", line 28, in main action, conf_file, log_file, log_level = _init_vars(argv) File "./ovirt-dr", line 88, in _init_vars log_file, log_level = _get_log_conf(conf_file, log_file, log_level) File "./ovirt-dr", line 101, in _get_log_conf settings._interpolation = configparser.ExtendedInterpolation() AttributeError: 'module' object has no attribute 'ExtendedInterpolation' Therefore to make the code compatible for RHEL as well, instead of using configparser, we use SafeConfigParser from ConfigParser. --- files/ovirt-dr | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/files/ovirt-dr b/files/ovirt-dr index 36b18a5..019c91f 100755 --- a/files/ovirt-dr +++ b/files/ovirt-dr @@ -1,8 +1,5 @@ #!/usr/bin/python -try: - import configparser -except ImportError: - import ConfigParser as configparser +from ConfigParser import SafeConfigParser import fail_back import fail_over import errno @@ -101,8 +98,7 @@ def _get_log_conf(conf_file, log_file, log_level): conf_file = raw_input( "Conf file '" + conf_file + "' does not exist." " Please provide the configuration file location: ") - settings = configparser.ConfigParser() - settings._interpolation = configparser.ExtendedInterpolation() + settings = SafeConfigParser() settings.read(conf_file) if log_section not in settings.sections(): settings.add_section(log_section) From a8801a9a9966e0fe26e97b68361e1741aaa9bc3e Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 10 May 2018 00:04:53 +0300 Subject: [PATCH 08/16] Add missing file of vault secret to be used for fail_back.py --- files/vault_secret.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 files/vault_secret.sh diff --git a/files/vault_secret.sh b/files/vault_secret.sh new file mode 100644 index 0000000..52e31c6 --- /dev/null +++ b/files/vault_secret.sh @@ -0,0 +1 @@ +echo $vault_password From d31f07b1717518f91913d7121b60659ba297a2b2 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 10 May 2018 21:45:41 +0300 Subject: [PATCH 09/16] Use vault password as input for fail_over --- files/fail_over.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/files/fail_over.py b/files/fail_over.py index 07c0770..b5530d0 100755 --- a/files/fail_over.py +++ b/files/fail_over.py @@ -49,8 +49,12 @@ def run(self, conf_file, log_file, log_level): cmd.append("-e") cmd.append( " dr_target_host=" + target_host + " dr_source_map=" + source_map) - cmd.append("--ask-vault-pass") - cmd.append("-vvvvv") + cmd.append("--vault-password-file") + cmd.append("vault_secret.sh") + cmd.append("-vvv") + vault_pass = raw_input( + INPUT + PREFIX + "Please enter the vault password: " + END) + os.system("export vault_password=\"" + vault_pass + "\"") log.info("Executing failover command: %s" % ' '.join(map(str, cmd))) if log_file is not None and log_file != '': self._log_to_file(log_file, cmd) From 80e9efccc59d044cff0e4e4f15975c557b69210a Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 10 May 2018 12:01:25 +0300 Subject: [PATCH 10/16] Fix redundant var in generate_vars --- files/generate_vars.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/files/generate_vars.py b/files/generate_vars.py index 20c0e66..15fd43e 100755 --- a/files/generate_vars.py +++ b/files/generate_vars.py @@ -161,7 +161,7 @@ def _validate_connection(self, def _validate_output_file_exists(self, fname, log): _dir = os.path.dirname(fname) if _dir != '' and not os.path.exists(_dir): - log.warn("Path '%s' does not exists. Create folder '%s'" + log.warn("Path '%s' does not exists. Create folder" % _dir) os.makedirs(_dir) if os.path.isfile(fname): @@ -202,14 +202,13 @@ def _validate_output_file_exists(self, fname, log): def _init_vars(self, conf_file, log): """ Declare constants """ - _SECTION = "generate_vars" + _SECTION = 'generate_vars' _SITE = 'site' _USERNAME = 'username' _PASSWORD = 'password' _CA_FILE = 'ca_file' # TODO: Must have full path, should add relative path - _OUTPUT_FILE = "/usr/share/ansible/roles/oVirt.disaster-recovery" \ - "/mapping_vars.yml" + _OUTPUT_FILE = 'output_file' _ANSIBLE_PLAY = 'ansible_play' """ Declare varialbles """ From 4147686361581cce953c13d7689ed1d20c9669e3 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 10 May 2018 21:45:06 +0300 Subject: [PATCH 11/16] Print failure message for failover and failback --- files/fail_back.py | 14 ++++++++++++++ files/fail_over.py | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/files/fail_back.py b/files/fail_back.py index c9dfb62..0727bc6 100755 --- a/files/fail_back.py +++ b/files/fail_back.py @@ -149,6 +149,20 @@ def _log_to_console(self, cmd, log): log.debug(line) for line in iter(proc.stderr.readline, ''): log.warn(line) + self._handle_result(subprocess, cmd) + + def _handle_result(self, subprocess, cmd): + try: + proc = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + # do something with output + except subprocess.CalledProcessError, e: + print("%sException: %s\n\n" + "failback operation failed, please check log file for " + "further details.%s" + % (FAIL, + e, + END)) + exit() def _init_vars(self, conf_file): """ Declare constants """ diff --git a/files/fail_over.py b/files/fail_over.py index b5530d0..4e4bd6c 100755 --- a/files/fail_over.py +++ b/files/fail_over.py @@ -83,8 +83,21 @@ def _log_to_file(self, log_file, cmd): print("%s%s%s" % (WARN, line, END)) + self._handle_result(subprocess, cmd) call(['cat', '/tmp/report.log']) + def _handle_result(self, subprocess, cmd): + try: + proc = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError, e: + print("%sException: %s\n\n" + "failover operation failed, please check log file for " + "further details.%s" + % (FAIL, + e, + END)) + exit() + def _log_to_console(self, cmd, log): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, From 5f43a012a76d168750e5b27ea317b1c60327b4a4 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Tue, 15 May 2018 17:43:19 +0300 Subject: [PATCH 12/16] Configure report file name on failover and failback Configure the report file name to be passed as an argument when running the ovirt_dr script on failover and failback. --- files/fail_back.py | 12 +++++++++--- files/fail_over.py | 19 ++++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/files/fail_back.py b/files/fail_back.py index 0727bc6..3d309d1 100755 --- a/files/fail_back.py +++ b/files/fail_back.py @@ -6,6 +6,7 @@ import shlex import subprocess import sys +import time from subprocess import call @@ -16,6 +17,7 @@ END = bcolors.ENDC PREFIX = "[Failback] " PLAY_DEF = "../examples/dr_play.yml" +report_name = "report-{}.log" class FailBack(): @@ -27,16 +29,19 @@ def run(self, conf_file, log_file, log_level): dr_clean_tag = "clean_engine" target_host, source_map, var_file, vault, ansible_play = \ self._init_vars(conf_file) + report = report_name.format(int(round(time.time() * 1000))) log.info("\ntarget_host: %s \n" "source_map: %s \n" "var_file: %s \n" "vault: %s \n" "ansible_play: %s \n" + "report log: /tmp/%s \n" % (target_host, source_map, var_file, vault, - ansible_play)) + ansible_play, + report)) cmd = [] cmd.append("ansible-playbook") @@ -64,7 +69,8 @@ def run(self, conf_file, log_file, log_level): cmd_fb.append("@" + vault) cmd_fb.append("-e") cmd_fb.append(" dr_target_host=" + target_host + - " dr_source_map=" + source_map) + " dr_source_map=" + source_map + + " dr_report_file=" + report) cmd_fb.append("--vault-password-file") cmd_fb.append("vault_secret.sh") cmd_fb.append("-vvv") @@ -112,7 +118,7 @@ def run(self, conf_file, log_file, log_level): else: self._log_to_console(cmd_fb, log) - call(["cat", "/tmp/report.log"]) + call(["cat", "/tmp/" + report]) print("\n%s%sFinished failback operation" " for oVirt ansible disaster recovery%s" % (INFO, diff --git a/files/fail_over.py b/files/fail_over.py index 4e4bd6c..367b093 100755 --- a/files/fail_over.py +++ b/files/fail_over.py @@ -6,6 +6,7 @@ import shlex import subprocess import sys +import time from subprocess import call @@ -16,6 +17,7 @@ END = bcolors.ENDC PREFIX = "[Failover] " PLAY_DEF = "../examples/dr_play.yml" +report_name = "report-{}.log" class FailOver(): @@ -26,16 +28,19 @@ def run(self, conf_file, log_file, log_level): dr_tag = "fail_over" target_host, source_map, var_file, vault, ansible_play = \ self._init_vars(conf_file) + report = report_name.format(int(round(time.time() * 1000))) log.info("target_host: %s \n" "source_map: %s \n" "var_file: %s \n" "vault: %s \n" - "ansible_play: %s " + "ansible_play: %s \n" + "report file: /tmp/%s " % (target_host, source_map, var_file, vault, - ansible_play)) + ansible_play, + report)) cmd = [] cmd.append("ansible-playbook") @@ -48,7 +53,8 @@ def run(self, conf_file, log_file, log_level): cmd.append("@" + vault) cmd.append("-e") cmd.append( - " dr_target_host=" + target_host + " dr_source_map=" + source_map) + " dr_target_host=" + target_host + " dr_source_map=" + source_map + + " dr_report_file=" + report) cmd.append("--vault-password-file") cmd.append("vault_secret.sh") cmd.append("-vvv") @@ -60,7 +66,7 @@ def run(self, conf_file, log_file, log_level): self._log_to_file(log_file, cmd) else: self._log_to_console(cmd, log) - call(['cat', 'report.log']) + call(['cat', "/tmp/" + report]) print("\n%s%sFinished failover operation" " for oVirt ansible disaster recovery%s" % (INFO, @@ -83,10 +89,9 @@ def _log_to_file(self, log_file, cmd): print("%s%s%s" % (WARN, line, END)) - self._handle_result(subprocess, cmd) - call(['cat', '/tmp/report.log']) + self._handle_result(cmd) - def _handle_result(self, subprocess, cmd): + def _handle_result(self, cmd): try: proc = subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError, e: From 95748878586f4e33abe0c91458866cf5ac2a497f Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Mon, 14 May 2018 15:38:11 +0300 Subject: [PATCH 13/16] Add stdout callback plugin to print appropriate output --- callback_plugins/stdout.py | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 callback_plugins/stdout.py diff --git a/callback_plugins/stdout.py b/callback_plugins/stdout.py new file mode 100644 index 0000000..3c6139e --- /dev/null +++ b/callback_plugins/stdout.py @@ -0,0 +1,54 @@ +# Make coding more python3-ish, this is required for contributions to Ansible +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# not only visible to ansible-doc, it also 'declares' the options the plugin requires and how to configure them. +DOCUMENTATION = ''' + callback: stdout + callback_type: aggregate + short_description: Output the log of ansible + version_added: "2.0" + description: + - This callback output the log of ansible play tasks. +''' +from datetime import datetime + +from ansible.plugins.callback import CallbackBase + + +class CallbackModule(CallbackBase): + """ + This callback module output the inforamtion with a specific style. + """ + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'stdout' + + # only needed if you ship it and don't want to enable by default + CALLBACK_NEEDS_WHITELIST = False + + def __init__(self): + + # make sure the expected objects are present, calling the base's __init__ + super(CallbackModule, self).__init__() + + def runner_on_failed(self, host, res, ignore_errors=False): + self._display.display('FAILED: %s %s' % (host, res)) + + def runner_on_ok(self, host, res): + self._display.display('OK: %s %s' % (host, res)) + + def runner_on_skipped(self, host, item=None): + self._display.display('SKIPPED: %s' % host) + + def runner_on_unreachable(self, host, res): + self._display.display('UNREACHABLE: %s %s' % (host, res)) + + def runner_on_async_failed(self, host, res, jid): + self._display.display('ASYNC_FAILED: %s %s %s' % (host, res, jid)) + + def playbook_on_import_for_host(self, host, imported_file): + self._display.display('IMPORTED: %s %s' % (host, imported_file)) + + def playbook_on_not_import_for_host(self, host, missing_file): + self._display.display('NOTIMPORTED: %s %s' % (host, missing_file)) From 12a68c2d96513a2aca40315be8b88b222aede657 Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 17 May 2018 02:29:45 +0300 Subject: [PATCH 14/16] Rename tasks of adding storage domains --- tasks/recover/add_domain.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks/recover/add_domain.yml b/tasks/recover/add_domain.yml index 2e57855..849afb1 100644 --- a/tasks/recover/add_domain.yml +++ b/tasks/recover/add_domain.yml @@ -8,7 +8,7 @@ fail: msg="No hosts available" when: ovirt_hosts.0 is undefined - block: - - name: Add NFS storage domain + - name: Add storage domain if NFS include_tasks: tasks/recover/add_nfs_domain.yml with_items: - "{{ storage }}" @@ -16,7 +16,7 @@ loop_control: loop_var: nfs_storage - - name: Add Gluster storage domain + - name: Add storage domain if Gluster include_tasks: tasks/recover/add_glusterfs_domain.yml with_items: - "{{ storage }}" @@ -24,7 +24,7 @@ loop_control: loop_var: gluster_storage - - name: Add posix storage domain + - name: Add storage domain if Posix include_tasks: tasks/recover/add_posixfs_domain.yml with_items: - "{{ storage }}" @@ -32,7 +32,7 @@ loop_control: loop_var: posix_storage - - name: Add iSCSI storage domain + - name: Add storage domain is scsi include_tasks: tasks/recover/add_iscsi_domain.yml with_items: - "{{ storage }}" @@ -40,7 +40,7 @@ loop_control: loop_var: iscsi_storage - - name: Add FCP storage domain + - name: Add storage domain if fcp include_tasks: tasks/recover/add_fcp_domain.yml with_items: - "{{ storage }}" From 6b27501f8eb9f2a1c9c4ad1dc21d5bb1d1b9087b Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 17 May 2018 11:10:37 +0300 Subject: [PATCH 15/16] Declare all facts in a unified way --- tasks/clean_engine.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tasks/clean_engine.yml b/tasks/clean_engine.yml index c48c23d..a99edb8 100755 --- a/tasks/clean_engine.yml +++ b/tasks/clean_engine.yml @@ -25,17 +25,12 @@ # Set all the queries suffix to fetch a storage domain in a specific status. # Note: Export storage domain is not supported and should not be part of storage mapping - - name: Set query suffix for active storage domains - set_fact: dr_active_domain_search='status = active and type != cinder' - - - name: Set query suffix for maintenance storage domains - set_fact: dr_maintenance_domain_search='status = maintenance and type != cinder' - - - name: Set query suffix for unattached storage domains - set_fact: dr_unattached_domain_search='status = unattached and type != cinder and type != glance' - - - name: Set query for remove invalid storage domains - set_fact: dr_inactive_domain_search='type != glance and type != cinder and status != active' + - name: Setup queries for storage domains + set_fact: + dr_active_domain_search='status = active and type != cinder' + dr_maintenance_domain_search='status = maintenance and type != cinder' + dr_unattached_domain_search='status = unattached and type != cinder and type != glance' + dr_inactive_domain_search='type != glance and type != cinder and status != active' - name: Set master storage domain filter set_fact: only_master=False From 4222c49da621184aca9571b740caaa5815d4c8ac Mon Sep 17 00:00:00 2001 From: Maor Lipchuk Date: Thu, 17 May 2018 11:54:42 +0300 Subject: [PATCH 16/16] Remove redundant minus mark in include_tasks --- tasks/clean_engine.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/clean_engine.yml b/tasks/clean_engine.yml index a99edb8..ece80f6 100755 --- a/tasks/clean_engine.yml +++ b/tasks/clean_engine.yml @@ -36,7 +36,7 @@ set_fact: only_master=False - name: Remove non master storage domains with valid statuses - - include_tasks: tasks/clean/remove_valid_filtered_master_domains.yml storage={{ item }} + include_tasks: tasks/clean/remove_valid_filtered_master_domains.yml storage={{ item }} with_items: - "{{ dr_import_storages }}" loop_control: @@ -48,7 +48,7 @@ set_fact: dr_force=True - name: Remove non master storage domains with invalid statuses using force remove - - include_tasks: tasks/clean/remove_invalid_filtered_master_domains.yml storage={{ item }} + include_tasks: tasks/clean/remove_invalid_filtered_master_domains.yml storage={{ item }} with_items: - "{{ dr_import_storages }}" loop_control: @@ -61,7 +61,7 @@ set_fact: dr_force=False - name: Remove master storage domains with valid statuses - - include_tasks: tasks/clean/remove_valid_filtered_master_domains.yml storage={{ item }} + include_tasks: tasks/clean/remove_valid_filtered_master_domains.yml storage={{ item }} with_items: - "{{ dr_import_storages }}" loop_control: @@ -71,7 +71,7 @@ set_fact: dr_force=True - name: Remove master storage domains with invalid statuses using force remove - - include_tasks: tasks/clean/remove_invalid_filtered_master_domains.yml storage={{ item }} + include_tasks: tasks/clean/remove_invalid_filtered_master_domains.yml storage={{ item }} with_items: - "{{ dr_import_storages }}" loop_control: