diff --git a/src/acc/opencl/smm/opencl_libsmm.c b/src/acc/opencl/smm/opencl_libsmm.c index 82141de768b..814c80ce21d 100644 --- a/src/acc/opencl/smm/opencl_libsmm.c +++ b/src/acc/opencl/smm/opencl_libsmm.c @@ -198,14 +198,16 @@ int opencl_libsmm_write_smm_params(FILE* stream, int only_key, const opencl_libs } -int opencl_libsmm_read_smm_params( - char* parambuf, opencl_libsmm_smmkey_t* key, opencl_libsmm_smm_t* value, opencl_libsmm_perfest_t* perfest, char* device) { +int opencl_libsmm_read_smm_params(char* parambuf, opencl_libsmm_smmkey_t* key, opencl_libsmm_smm_t* value, + opencl_libsmm_perfest_t* perfest, char* device, int* key_ok) { const char* const end = parambuf + strlen(parambuf); /* before strtok */ char* s = strtok(parambuf, ACC_OPENCL_DELIMS); const int opt_consumed = (NULL != perfest ? 2 : 0) + (NULL != device ? 1 : 0); int result = EXIT_SUCCESS, i = 0, ivalue, consumed = 0, c = 0, max_consumed = opt_consumed + 19; double gflops; assert(NULL != key && NULL != value); + LIBXSMM_MEMZERO127(key); /* potentially heterogeneous key-data (alignment gaps) */ + memset(value, 0, sizeof(opencl_libsmm_smm_t)); for (; NULL != s; ++i, s = (c != consumed ? ((s + 1) < end ? strtok((s + 1) + strlen(s), ACC_OPENCL_DELIMS) : NULL) : s), c = consumed) { @@ -352,9 +354,9 @@ int opencl_libsmm_read_smm_params( } } if (max_consumed == consumed) { - if (NULL != perfest) { - switch (key->type) { - case dbcsr_type_real_8: { + switch (key->type) { + case dbcsr_type_real_8: + if (NULL != perfest) { const double ratio = gflops / OPENCL_LIBSMM_AI(key->m, key->n, key->k, sizeof(double)); # if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER libxsmm_kahan_sum(log(ratio), &perfest->gf_ai_dratio_sumlog, &perfest->gf_ai_dratio_kahan); @@ -363,8 +365,10 @@ int opencl_libsmm_read_smm_params( # endif if (perfest->gf_ai_dratio_max < ratio) perfest->gf_ai_dratio_max = ratio; ++perfest->dcount; - } break; - case dbcsr_type_real_4: { + } + break; + case dbcsr_type_real_4: + if (NULL != perfest) { const double ratio = gflops / OPENCL_LIBSMM_AI(key->m, key->n, key->k, sizeof(float)); # if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER libxsmm_kahan_sum(log(ratio), &perfest->gf_ai_sratio_sumlog, &perfest->gf_ai_sratio_kahan); @@ -373,12 +377,13 @@ int opencl_libsmm_read_smm_params( # endif if (perfest->gf_ai_sratio_max < ratio) perfest->gf_ai_sratio_max = ratio; ++perfest->scount; - } break; - default: result = EXIT_FAILURE; - } + } + break; + default: result = EXIT_FAILURE; } } else result = EXIT_FAILURE; + if (NULL != key_ok && 4 <= consumed) *key_ok = 1; return result; } @@ -406,14 +411,13 @@ int libsmm_acc_init(void) { char* const env_params = getenv("OPENCL_LIBSMM_SMM_PARAMS"); memset(&perfest, 0, sizeof(perfest)); if (NULL == env_params || '0' != *env_params) { - char buffer[ACC_OPENCL_BUFFERSIZE], bufname[ACC_OPENCL_BUFFERSIZE], control = '0'; + char buffer[ACC_OPENCL_BUFFERSIZE], bufname[ACC_OPENCL_BUFFERSIZE]; # if defined(OPENCL_KERNELS_DEVICES) const int ndevices_params = (int)(sizeof(OPENCL_KERNELS_DEVICES) / sizeof(*OPENCL_KERNELS_DEVICES)); - unsigned int ntuned = 0; # endif opencl_libsmm_smm_t config; - opencl_libsmm_smmkey_t key; - LIBXSMM_MEMZERO127(&key); /* potentially heterogeneous key-data (alignment gaps) */ + opencl_libsmm_smmkey_t key, key_direct; + int key_direct_skip = 0, ntuned = 0; if (NULL != env_params && '\0' != *env_params) { /* filename */ FILE* const file = fopen(env_params, "r"); if (NULL != file) { @@ -422,8 +426,7 @@ int libsmm_acc_init(void) { char* const device = (NULL != LIBXSMM_STRISTR(buffer, "device") ? bufname : NULL); opencl_libsmm_perfest_t* const gflops = (NULL != LIBXSMM_STRISTR(buffer, "gflops") ? &perfest : NULL); while (NULL != fgets(buffer, ACC_OPENCL_BUFFERSIZE, file)) { /* read params from CSV-file */ - memset(&config, 0, sizeof(config)); - if (EXIT_SUCCESS == opencl_libsmm_read_smm_params(buffer, &key, &config, gflops, device)) { + if (EXIT_SUCCESS == opencl_libsmm_read_smm_params(buffer, &key, &config, gflops, device, NULL /*key_ok*/)) { opencl_libsmm_smm_t* config_init; c_dbcsr_acc_opencl_config.devmatch = 0; /* disable device-match */ key.devuid = 0; @@ -433,9 +436,7 @@ int libsmm_acc_init(void) { result = EXIT_FAILURE; break; } -# if defined(OPENCL_KERNELS_DEVICES) else ++ntuned; -# endif } else if (config_init->gflops < config.gflops) { /* update */ memcpy(config_init, &config, sizeof(config)); @@ -453,20 +454,29 @@ int libsmm_acc_init(void) { result = EXIT_FAILURE; } fclose(file); - control = '1'; } - else control = '2'; + else if (EXIT_SUCCESS == opencl_libsmm_read_smm_params( + env_params, &key_direct, &config, NULL /*perfest*/, NULL /*device*/, &key_direct_skip)) + { /* try OPENCL_LIBSMM_SMM_PARAMS as string of kernel parameters (not device-specific) */ + assert(0 == key_direct.devuid && 0 != key_direct_skip); + if (NULL != libxsmm_xregister(&key_direct, sizeof(key_direct), sizeof(config), &config)) { + c_dbcsr_acc_opencl_config.devmatch = 0; /* disable device-match */ + ntuned = 1; + } + else result = EXIT_FAILURE; + } + else if (0 == key_direct_skip && 0 != c_dbcsr_acc_opencl_config.verbosity) { /* soft-error */ + fprintf(stderr, "WARN LIBSMM: failed to open parameter file!\n"); + } } # if defined(OPENCL_KERNELS_PARAMS_SMM) && defined(OPENCL_KERNELS_DEVICES) - if (EXIT_SUCCESS == result && '1' != control) { + if (EXIT_SUCCESS == result && (0 == ntuned || 0 != key_direct_skip)) { const char *line = OPENCL_KERNELS_PARAMS_SMM, *next; # if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER - unsigned int active_uid; int active_match = -1; if (EXIT_SUCCESS == c_dbcsr_acc_opencl_device_name(c_dbcsr_acc_opencl_config.device.id, bufname, ACC_OPENCL_BUFFERSIZE, - NULL /*platform*/, 0 /*platform_maxlen*/, /*cleanup*/ 1) && - EXIT_SUCCESS == c_dbcsr_acc_opencl_device_uid(c_dbcsr_acc_opencl_config.device.id, bufname, &active_uid)) - { + NULL /*platform*/, 0 /*platform_maxlen*/, /*cleanup*/ 1)) + { /* determine best-matching parameters based on name of device */ int i = 0, best = 0; for (; i < ndevices_params; ++i) { const int score = libxsmm_strimatch(bufname, OPENCL_KERNELS_DEVICES[i], NULL); @@ -474,7 +484,7 @@ int libsmm_acc_init(void) { if (best < score || ((best == score) && EXIT_SUCCESS == c_dbcsr_acc_opencl_device_uid(NULL /*device*/, OPENCL_KERNELS_DEVICES[i], &uid) && - uid == active_uid)) + uid == c_dbcsr_acc_opencl_config.device.uid)) { active_match = i; best = score; @@ -488,44 +498,45 @@ int libsmm_acc_init(void) { const int len = next - line; memcpy(buffer, line, len); buffer[len] = '\0'; - memset(&config, 0, sizeof(config)); if (EXIT_SUCCESS == opencl_libsmm_read_smm_params(/* read params from embedded params */ - buffer, &key, &config, &perfest, bufname /*consume name/id*/)) + buffer, &key, &config, &perfest, bufname /*consume name/id*/, NULL /*key_ok*/)) { - opencl_libsmm_smm_t* config_init; - const int i = atoi(bufname); - if (0 >= ndevices_params || 0 == c_dbcsr_acc_opencl_config.devmatch || 0 > i || ndevices_params <= i || - EXIT_SUCCESS != c_dbcsr_acc_opencl_device_uid(NULL /*device*/, OPENCL_KERNELS_DEVICES[i], &key.devuid)) - { - key.devuid = 0; - } - config_init = (opencl_libsmm_smm_t*)libxsmm_xdispatch(&key, sizeof(key)); - if (NULL == config_init) { - if (NULL == libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) { - result = EXIT_FAILURE; - break; + if (0 == key_direct_skip || 0 != memcmp(&key_direct, &key, (const char*)&key.k - (const char*)&key)) { + opencl_libsmm_smm_t* config_init; + const int i = atoi(bufname); + if (0 >= ndevices_params || 0 == c_dbcsr_acc_opencl_config.devmatch || 0 > i || ndevices_params <= i || + EXIT_SUCCESS != c_dbcsr_acc_opencl_device_uid(NULL /*device*/, OPENCL_KERNELS_DEVICES[i], &key.devuid)) + { + key.devuid = 0; } - else ++ntuned; - } - else if (config_init->gflops < config.gflops) { /* update */ - memcpy(config_init, &config, sizeof(config)); - } -# if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER - if (active_match == i && active_uid != key.devuid) { - key.devuid = active_uid; config_init = (opencl_libsmm_smm_t*)libxsmm_xdispatch(&key, sizeof(key)); - if (NULL == config_init && NULL != libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) { - static int info = 0; - if (0 == info && 0 != c_dbcsr_acc_opencl_config.verbosity && - EXIT_SUCCESS == c_dbcsr_acc_opencl_device_name(c_dbcsr_acc_opencl_config.device.id, bufname, - ACC_OPENCL_BUFFERSIZE, NULL /*platform*/, 0 /*platform_maxlen*/, /*cleanup*/ 0)) - { - fprintf(stderr, "INFO ACC/LIBSMM: PARAMS of \"%s\" used for \"%s\"\n", OPENCL_KERNELS_DEVICES[i], bufname); - info = 1; + if (NULL == config_init) { + if (NULL == libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) { + result = EXIT_FAILURE; + break; + } + else ++ntuned; + } + else if (config_init->gflops < config.gflops) { /* update */ + memcpy(config_init, &config, sizeof(config)); + } +# if LIBXSMM_VERSION4(1, 17, 0, 0) < LIBXSMM_VERSION_NUMBER + if (active_match == i && c_dbcsr_acc_opencl_config.device.uid != key.devuid) { + key.devuid = c_dbcsr_acc_opencl_config.device.uid; + config_init = (opencl_libsmm_smm_t*)libxsmm_xdispatch(&key, sizeof(key)); + if (NULL == config_init && NULL != libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) { + static int info = 0; + if (0 == info && 0 != c_dbcsr_acc_opencl_config.verbosity && + EXIT_SUCCESS == c_dbcsr_acc_opencl_device_name(c_dbcsr_acc_opencl_config.device.id, bufname, + ACC_OPENCL_BUFFERSIZE, NULL /*platform*/, 0 /*platform_maxlen*/, /*cleanup*/ 0)) + { + fprintf(stderr, "INFO ACC/LIBSMM: PARAMS of \"%s\" used for \"%s\"\n", OPENCL_KERNELS_DEVICES[i], bufname); + info = 1; + } } } - } # endif + } } else { if (0 != c_dbcsr_acc_opencl_config.verbosity) { @@ -538,40 +549,27 @@ int libsmm_acc_init(void) { } while (NULL != next); } # endif - if (EXIT_SUCCESS == result) { - if ('2' == control) { /* try interpreting OPENCL_LIBSMM_SMM_PARAMS as kernel parameters (not device-specific) */ - memset(&config, 0, sizeof(config)); - if (EXIT_SUCCESS == opencl_libsmm_read_smm_params(env_params, &key, &config, NULL /*perfest*/, NULL /*device*/)) { - key.devuid = 0; - if (NULL != libxsmm_xregister(&key, sizeof(key), sizeof(config), &config)) { - c_dbcsr_acc_opencl_config.devmatch = 0; /* disable device-match */ -# if defined(OPENCL_KERNELS_DEVICES) - ntuned = LIBXSMM_MAX(ntuned, 1); /* no destinction of overridden or new */ -# endif - } - else result = EXIT_FAILURE; - } - else if (0 != c_dbcsr_acc_opencl_config.verbosity) { /* soft-error */ - fprintf(stderr, "WARN LIBSMM: failed to open parameter file!\n"); - } - } # if defined(OPENCL_KERNELS_DEVICES) - if (0 != ntuned && (2 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity)) { - fprintf(stderr, "INFO ACC/LIBSMM: PARAMS in %u set%s loaded targeting ", ntuned, 1 != ntuned ? "s" : ""); - if (0 != c_dbcsr_acc_opencl_config.devmatch) { - fprintf(stderr, "%i device%s\n", ndevices_params, 1 != ndevices_params ? "s" : ""); - if (3 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity) { - unsigned int i = 0; - for (; i < (unsigned int)ndevices_params; ++i) { - fprintf(stderr, "INFO ACC/LIBSMM: PARAMS -> \"%s\"\n", OPENCL_KERNELS_DEVICES[i]); - } + if (EXIT_SUCCESS == result && 0 != ntuned && + (2 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity)) + { + fprintf(stderr, "INFO ACC/LIBSMM: PARAMS in %i set%s loaded targeting ", ntuned, 1 != ntuned ? "s" : ""); + if (0 != c_dbcsr_acc_opencl_config.devmatch) { + fprintf(stderr, "%i device%s\n", ndevices_params, 1 != ndevices_params ? "s" : ""); + if (3 <= c_dbcsr_acc_opencl_config.verbosity || 0 > c_dbcsr_acc_opencl_config.verbosity) { + unsigned int i = 0; + for (; i < (unsigned int)ndevices_params; ++i) { + fprintf(stderr, "INFO ACC/LIBSMM: PARAMS -> \"%s\"\n", OPENCL_KERNELS_DEVICES[i]); } } - else fprintf(stderr, "any device\n"); } -# endif + else fprintf(stderr, "any device\n"); } +# endif } +# if defined(OPENCL_LIBSMM_VALIDATE) + c_dbcsr_acc_opencl_config.xhints &= ~1; /* disable USM */ +# endif } } ACC_OPENCL_RETURN(result); diff --git a/src/acc/opencl/smm/opencl_libsmm.h b/src/acc/opencl/smm/opencl_libsmm.h index 8e03b2e4565..0195da0a001 100644 --- a/src/acc/opencl/smm/opencl_libsmm.h +++ b/src/acc/opencl/smm/opencl_libsmm.h @@ -99,8 +99,8 @@ int opencl_libsmm_write_smm_params(FILE* stream, int only_key, const opencl_libs const char* delim, const char* begin, const char* close); /** Tokenize parambuf and initialize key/value pair. */ -int opencl_libsmm_read_smm_params( - char* parambuf, opencl_libsmm_smmkey_t* key, opencl_libsmm_smm_t* value, opencl_libsmm_perfest_t* perfest, char* device); +int opencl_libsmm_read_smm_params(char* parambuf, opencl_libsmm_smmkey_t* key, opencl_libsmm_smm_t* value, + opencl_libsmm_perfest_t* perfest, char* device, int* key_ok); #if defined(OPENCL_LIBSMM_VALIDATE) && defined(_DEBUG) void opencl_libsmm_print_matrix(FILE* ostream, const char* label, libsmm_acc_data_t type, const void* mat, int m, int n); diff --git a/src/acc/opencl/smm/tune_multiply.py b/src/acc/opencl/smm/tune_multiply.py index a2b81bc6486..0329dbb02bf 100755 --- a/src/acc/opencl/smm/tune_multiply.py +++ b/src/acc/opencl/smm/tune_multiply.py @@ -76,9 +76,13 @@ def __init__(self, args): self.exepath = os.path.join( os.path.dirname(sys.argv[0]), "..", "..", self.exename ) + runcmd = self.launch(["ACC_OPENCL_VERBOSE=2"], 0, nrep=1) self.run_result = ( # verbosity to capture device name and tuned parameters - self.call_program(self.launch(["ACC_OPENCL_VERBOSE=2", "CHECK=0"], nrep=1)) - if (self.args.merge is None or 0 > self.args.merge) + self.call_program(" ".join(runcmd)) + if ( # consider validating parameters during merge + (self.args.merge is None or 0 > self.args.merge) + or (self.args.check is None or 0 != self.args.check) + ) and (self.args.update is None or "" == self.args.update) else None ) @@ -153,10 +157,13 @@ def __init__(self, args): for param in params + paramt: manipulator.add_parameter(param) if ( # consider to update and/or merge JSONS (update first) - (self.args.merge is not None and (0 <= self.args.merge or self.typeid)) - or self.args.update is None - or "" != self.args.update - ): + self.args.merge is not None + and (0 <= self.args.merge or self.typeid) + and ( + (self.args.check is not None and 0 == self.args.check) + or (self.run_result and 0 == self.run_result["returncode"]) + ) + ) or (self.args.update is None or "" != self.args.update): filepattern = "{}-*.json".format(default_basename) filenames = glob.glob( os.path.normpath(os.path.join(self.args.jsondir, filepattern)) @@ -235,21 +242,26 @@ def create_param( if attribute is None: setattr(self, name.lower(), value) - def launch(self, envs, nrep=None, verbose=None): + def launch(self, envs, check, nrep=None, verbose=None): """Launch executable supplying environment and arguments""" - envstrs = " ".join(map(str, envs)) + envlist = envs if isinstance(envs, list) else self.environment(envs) + mnk = (envs["M"], envs["N"], envs["K"]) if "M" in envs else self.mnk + env_exe = " ".join(map(str, envlist)) if verbose is not None and 0 != int(verbose): - print(envstrs.replace("OPENCL_LIBSMM_SMM_", "").replace(" CHECK=0", "")) - env_defaults = "OMP_PROC_BIND=TRUE OPENCL_LIBSMM_SMM_S=0 NEO_CACHE_PERSISTENT=0" - env_exe_args = "{} {} {} {} {} {}".format( # consider device-id + msg = env_exe.replace("OPENCL_LIBSMM_SMM_", "") + print("{}: {}".format("x".join(map(str, mnk)), msg)) + env_std = "OMP_PROC_BIND=TRUE OPENCL_LIBSMM_SMM_S=0 NEO_CACHE_PERSISTENT=0" + env_check = "CHECK={}".format(check if check is not None else 1) + env_intrn = "{} {}".format( # consider device-id "" if self.idevice is None else "ACC_OPENCL_DEVICE={}".format(self.idevice), - "{} {}".format(env_defaults, envstrs), # environment - self.exepath, # executable file + "{} {}".format(env_std, env_check), # environment + ).strip() + arg_exe = "{} {} {}".format( self.args.r if nrep is None else nrep, self.size if self.size else self.args.size, - " ".join(map(str, self.mnk)), - ) - return env_exe_args + " ".join(map(str, mnk)), + ).strip() + return [env_exe, env_intrn, self.exepath, arg_exe] def seed_configurations(self): return [ @@ -286,15 +298,19 @@ def environment(self, config): if 2 == len(key) ] - def run(self, desired_result, input, limit): + def run(self, desired_result, input=None, limit=None): """Run a configuration and return performance""" - config = desired_result.configuration.data - cfgenv = self.environment(config) - runcmd = self.launch( - cfgenv + ["CHECK={}".format(self.args.check)], verbose=self.args.verbose - ) - self.run_result = self.call_program(runcmd) - result = self.run_result["returncode"] + try: + config = desired_result.configuration.data + mnk = self.mnk + nrep = 0 # default + except AttributeError: + config = desired_result + mnk = (config["M"], config["N"], config["K"]) + nrep = 1 # limit + runcmd = self.launch(config, self.args.check, nrep, self.args.verbose) + self.run_result = self.call_program(" ".join(runcmd)) + result = self.run_result["returncode"] if self.run_result else 1 if 0 == result: performance = re.search( "device:\\s+([0-9]+[^ ]*) ms\\s+([0-9]+[^ ]*)", @@ -305,26 +321,31 @@ def run(self, desired_result, input, limit): if performance and performance.group(1) and performance.group(2): mseconds = float(performance.group(1)) gflops = float(performance.group(2)) - if self.gflops < gflops: - # keep best configuration in case of an early exit - self.config = desired_result.configuration - self.gflops = gflops - if 0 == self.gfbase: # seed configuration - self.gfbase = gflops - else: - self.save_final_config(desired_result.configuration, final=False) - kernelreq = round((100.0 * config["BM"] * config["BN"]) / self.wsx) - # gflops are reported as "accuracy" (console output) - return Result(time=mseconds, accuracy=gflops, size=kernelreq) + if config is not desired_result: + kernelreq = round((100.0 * config["BM"] * config["BN"]) / self.wsx) + # gflops are reported as "accuracy" (console output) + result = Result(time=mseconds, accuracy=gflops, size=kernelreq) + if self.gflops < gflops: + # keep best configuration in case of an early exit + self.config = desired_result.configuration + self.gflops = gflops + if 0 == self.gfbase: # seed configuration + self.gfbase = gflops + else: + self.save_final_config( + desired_result.configuration, final=False + ) + elif not self.args.verbose: + print(".", end="", flush=True) else: # return non-competitive/bad result in case of an error - failed = ( - " ".join(map(str, cfgenv)).replace("OPENCL_LIBSMM_SMM_", "") - if not self.args.verbose - else runcmd - ) - mnk = "x".join(map(str, self.mnk)) - print("FAILED[{}] {}: {}".format(result, mnk, failed), flush=True) - return Result(time=float("inf"), accuracy=0.0, size=100.0) + if config is not desired_result: + result = Result(time=float("inf"), accuracy=0.0, size=100.0) + elif not self.args.verbose: + print("") + failed = runcmd[0].replace("OPENCL_LIBSMM_SMM_", "") + msg = "FAILED[{}] {}: {}".format(result, "x".join(map(str, mnk)), failed) + print(msg, flush=True) + return result def update_jsons(self, filenames): """Update device name of all JSONs""" @@ -357,8 +378,8 @@ def merge_jsons(self, filenames): return # early exit merged, worse = dict(), dict() for filename in filenames: + data = dict() try: - data = dict() with open(filename, "r") as file: data = json.load(file) if self.args.merge is not None and ( @@ -394,25 +415,28 @@ def merge_jsons(self, filenames): data["XF"] if "XF" in data else 0, filename, # last entry ) - if key not in merged: - merged[key] = value - else: - filename2 = merged[key][-1] - if merged[key][1] <= value[1]: # GFLOPS - merged[key] = value - else: - filename2 = filename - if key in worse: - worse[key].append(filename2) - else: - worse[key] = [filename2] except (json.JSONDecodeError, KeyError, TypeError): print("Failed to merge {} into CSV-file.".format(filename)) + data = dict() except: # noqa: E722 + data = dict() pass + if bool(data) and key in merged: + filename2 = merged[key][-1] + if value[1] < merged[key][1]: # GFLOPS + filename2, data = filename, dict() + if key in worse: + worse[key].append(filename2) + else: + worse[key] = [filename2] + if bool(data) and ( + (self.args.check is not None and 0 == self.args.check) + or 0 == self.run(data) + ): + merged[key] = value if bool(merged): - with open(self.args.csvfile, "w") as file: - file.write( # CSV header line with termination/newline + with open(self.args.csvfile, "w") as csvfile: + csvfile.write( # CSV header line with termination/newline "{}{}{}{}{}{}{}{}{}\n".format( # key-part self.args.csvsep.join(["DEVICE", "TYPEID", "M", "N", "K"]), self.args.csvsep, # separator for value-part @@ -428,7 +452,7 @@ def merge_jsons(self, filenames): for key, value in sorted(merged.items()): # CSV data lines strkey = self.args.csvsep.join([str(k) for k in key]) strval = self.args.csvsep.join([str(v) for v in value[:-1]]) - file.write("{}{}{}\n".format(strkey, self.args.csvsep, strval)) + csvfile.write("{}{}{}\n".format(strkey, self.args.csvsep, strval)) retsld, delsld = [0, 0, 0], [0, 0, 0] # [min, geo, max] retain, delete = [], [] # lists of filenames retcnt = delcnt = 0 # geo-counter @@ -494,6 +518,10 @@ def merge_jsons(self, filenames): msg = "Merged {} of {} JSONs into {}".format( len(merged), len(filenames), self.args.csvfile ) + if ( + self.args.check is None or 0 != self.args.check + ) and not self.args.verbose: + print("") print(msg) def save_final_config(self, configuration, final=True): @@ -502,10 +530,10 @@ def save_final_config(self, configuration, final=True): return # nothing to save config = configuration.data if configuration else None cfgenv = self.environment(config) if config else None - envchk = os.getenv("CHECK") # conside CHECKing result unless CHECK=0 + envchk = os.getenv("CHECK") # force CHECKing result unless CHECK=0 result = self.run_result["returncode"] if config and self.run_result else 1 if 0 == result and 0 == self.args.check and (envchk is None or "0" != envchk): - self.run_result = self.call_program(self.launch(cfgenv + ["CHECK=1"])) + self.run_result = self.call_program(" ".join(self.launch(cfgenv, 1))) result = self.run_result["returncode"] if self.run_result else 1 # extend result for easier reuse if config: