diff --git a/src/rda_python_miscs/bashqsub.py b/src/rda_python_miscs/bashqsub.py index b51acf3..12836b0 100644 --- a/src/rda_python_miscs/bashqsub.py +++ b/src/rda_python_miscs/bashqsub.py @@ -20,26 +20,25 @@ class BashQsub(PgLOG): def __init__(self): super().__init__() self.DEFMODS = { - 'default' : "ncarenv,netcdf,ncl,nco,cdo,conda,grib-util,wgrib2", + 'default': "ncarenv,netcdf,ncl,nco,cdo,conda,grib-util,wgrib2" } self.DEFLIBS = { - 'default' : "conda activate /glade/work/gdexdata/conda-envs/pg-gdex", - } - self.SWAPMODS = { + 'default': "conda activate /glade/work/gdexdata/conda-envs/pg-gdex" } + self.SWAPMODS = {} self.RESOURCES = { # resource list for option -l - 'walltime' : '6:00:00', # if this is changed, change defpbstime in PgCheck.py too - 'select' : '1:ncpus=1:mem=1gb' + 'walltime': '6:00:00', # if this is changed, change defpbstime in PgCheck.py too + 'select': '1:ncpus=1:mem=1gb' } self.SOPTIONS = { # single-dash option values - 'o' : None, # will set to default if not provided - 'e' : None, - 'A' : "P43713000", - 'q' : "gdex@casper-pbs", - # 'm' : 'a', - 'm' : 'n', + 'o': None, # will set to default if not provided + 'e': None, + 'A': "P43713000", + 'q': "gdex@casper-pbs", + # 'm': 'a', + 'm': 'n', } - self.coptions = {'cmd' : None, 'cwd' : None, 'env' : None, 'mod' : None, 'res' : 'default'} # customized options + self.coptions = {'cmd': None, 'cwd': None, 'env': None, 'mod': None, 'res': 'default'} # customized options self.gdexsub = self.BCHCMDS['PBS'] self.args = None @@ -50,7 +49,6 @@ def read_parameters(self): self.set_help_path(__file__) copts = '|'.join(self.coptions) option = None - dcount = 0 argv = sys.argv[1:] if not argv: self.show_usage(aname) self.PGLOG['LOGFILE'] = pname + ".log" @@ -92,7 +90,7 @@ def read_parameters(self): if self.coptions['cwd']: if 's' in self.coptions['cwd']: self.coptions['cwd'] = self.replace_environments(self.coptions['cwd'], '', self.LGWNEX) os.chdir(self.coptions['cwd']) - + # function to start actions def start_actions(self): cmd = self.valid_command(self.coptions['cmd']) @@ -129,7 +127,7 @@ def build_bash_script(self, cmd): buf += self.set_vm_libs(self.coptions['res']) buf += "\necho {}\n{}\n\ndate\n".format(cmd, cmd) return buf - + # check and add resource options def add_resources(self): for res in re.split(',', self.SOPTIONS['l']): @@ -139,7 +137,7 @@ def add_resources(self): else: self.pglog(res + ": use '=' to separate resource name & value", self.LGEREX) del self.SOPTIONS['l'] - + # add module loads for modules provided def add_modules(self, res, mods): mbuf = "\n" @@ -162,7 +160,7 @@ def add_modules(self, res, mods): if smod in self.SWAPMODS: mbuf += "module unload {}\n".format(self.SWAPMODS[smod]) mbuf += "module load {}\n".format(amod) return mbuf - + # set virtual machine libraries def set_vm_libs(self, res): deflibs = self.DEFLIBS[res] if res in self.DEFLIBS else self.DEFLIBS['default'] diff --git a/src/rda_python_miscs/gdexls.py b/src/rda_python_miscs/gdexls.py index e0d6ff1..f6cd640 100644 --- a/src/rda_python_miscs/gdexls.py +++ b/src/rda_python_miscs/gdexls.py @@ -27,24 +27,24 @@ def __init__(self): self.WIDTHS = [0, 0, 0] # WIDTHS for formated display self.ALIGNS = [0, 1, 1] # alignment, 0 - left; 1 - right self.GDEXLS = { - 'd' : 0, # 1 to list directory information only - 'f' : 0, # 1 to list file information only - 'N' : 0, # 1 to list files unformatted - 'r' : 0, # 1 if recursive all - 'R' : 0, # > 0 to set recursive limit - 'D' : None, # specify delimiting symbols, default to ' ' + 'd': 0, # 1 to list directory information only + 'f': 0, # 1 to list file information only + 'N': 0, # 1 to list files unformatted + 'r': 0, # 1 if recursive all + 'R': 0, # > 0 to set recursive limit + 'D': None, # specify delimiting symbols, default to ' ' } self.LINFO = { - 'files' : [], - 'curdir' : None, - 'tpath' : None, - 'dhome' : None, - 'dsid' : None, - 'dcnt' : 0, - 'gcnt' : 0, - 'fcnt' : 0, - 'pcnt' : 0, - 'pgrecs' : [] + 'files': [], + 'curdir': None, + 'tpath': None, + 'dhome': None, + 'dsid': None, + 'dcnt': 0, + 'gcnt': 0, + 'fcnt': 0, + 'pcnt': 0, + 'pgrecs': [] } # function to read parameters @@ -75,7 +75,7 @@ def read_parameters(self): else: self.GDEXLS[option] = arg option = defopt - + # functio to start actions def start_actions(self): self.view_dbinfo() @@ -149,7 +149,7 @@ def display_line(self, file, isdir): getwfile = 0 if getwfile: self.LINFO['dsid'] = self.find_dataset_id(file) - if self.LINFO['dsid'] == None: return # skip for missing dsid + if self.LINFO['dsid'] is None: return # skip for missing dsid pgrec = self.pgget("dataset", "title, (dwebcnt + nwebcnt) nc, (dweb_size + nweb_size) ns", "dsid = '{}'".format(self.LINFO['dsid']), self.LGEREX) if not pgrec: return None self.LINFO['dhome'] = "{}/{}".format(self.PGLOG['DSDHOME'], self.LINFO['dsid']) diff --git a/src/rda_python_miscs/rdacp.py b/src/rda_python_miscs/rdacp.py index 2c875b5..d17159f 100644 --- a/src/rda_python_miscs/rdacp.py +++ b/src/rda_python_miscs/rdacp.py @@ -19,29 +19,29 @@ class RdaCp(PgFile): def __init__(self): super().__init__() self.RDACP = { - 'fh' : None, # from host name, default to localhost - 'th' : None, # to host name, defaul to localhost - 'fb' : None, # from bucket name for a from file in Object Store - 'tb' : None, # to bucket name for a to file in Object Store - 'fp' : None, # from Globus endpoint - 'tp' : None, # to Globus endpoint - 'f' : [], # from file names - 't' : None, # to file name - 'r' : 0, # 1 if recursive all - 'R' : 0, # > 0 to set recursive limit - 'F' : 0o664, # to file mode, default to 664 - 'D' : 0o775, # to directory mode, default to 775 + 'fh': None, # from host name, default to localhost + 'th': None, # to host name, defaul to localhost + 'fb': None, # from bucket name for a from file in Object Store + 'tb': None, # to bucket name for a to file in Object Store + 'fp': None, # from Globus endpoint + 'tp': None, # to Globus endpoint + 'f': [], # from file names + 't': None, # to file name + 'r': 0, # 1 if recursive all + 'R': 0, # > 0 to set recursive limit + 'F': 0o664, # to file mode, default to 664 + 'D': 0o775, # to directory mode, default to 775 } self.CINFO = { - 'tcnt' : 0, - 'htcnt' : 0, - 'cpflag' : 0, # 1 file only, 2 directory only, 3 both - 'cpstr' : ['', 'Files', 'Directories', 'Files/Directories'], - 'fpath' : None, - 'tpath' : None, - 'fhost' : '', - 'thost' : '', - 'curdir' : os.getcwd() + 'tcnt': 0, + 'htcnt': 0, + 'cpflag': 0, # 1 file only, 2 directory only, 3 both + 'cpstr': ['', 'Files', 'Directories', 'Files/Directories'], + 'fpath': None, + 'tpath': None, + 'fhost': '', + 'thost': '', + 'curdir': os.getcwd() } # function to read parameters diff --git a/src/rda_python_miscs/rdakill.py b/src/rda_python_miscs/rdakill.py index e758fbc..4691b72 100644 --- a/src/rda_python_miscs/rdakill.py +++ b/src/rda_python_miscs/rdakill.py @@ -19,14 +19,14 @@ class RdaKill(PgFile): def __init__(self): super().__init__() self.RDAKILL = { - 'a' : None, # application name - 'h' : None, # hostname - 'p' : 0, # process id to be killed - 'P' : 0, # parent pid - 'r' : 0, # 1 - reserved for exclusive, working with -s PEND only - 'u' : None, # login user name - 's' : None, # batch status to kill - 'q' : None # batch partition/queue for SLURM/PBS, rda for default + 'a': None, # application name + 'h': None, # hostname + 'p': 0, # process id to be killed + 'P': 0, # parent pid + 'r': 0, # 1 - reserved for exclusive, working with -s PEND only + 'u': None, # login user name + 's': None, # batch status to kill + 'q': None # batch partition/queue for SLURM/PBS, rda for default } # function to read parameters @@ -217,7 +217,7 @@ def rdakill_pbs_status(self, stat, queue, uname): def record_dscheck_interrupt(self, pid, host): pgrec = self.pgget("dscheck", "cindex", "pid = {} AND hostname = '{}'".format(pid, host), self.LOGERR) if pgrec: - record = {'chktime' : int(time.time()), 'status' : 'I', 'pid' : 0} # release lock + record = {'chktime': int(time.time()), 'status': 'I', 'pid': 0} # release lock self.pgupdt("dscheck", record, "cindex = {}".format(pgrec['cindex']), self.LGEREX) # main function to excecute this script diff --git a/src/rda_python_miscs/rdamod.py b/src/rda_python_miscs/rdamod.py index 1e527d0..ea8ae20 100644 --- a/src/rda_python_miscs/rdamod.py +++ b/src/rda_python_miscs/rdamod.py @@ -20,20 +20,20 @@ class RdaMod(PgFile): def __init__(self): super().__init__() self.RDAMOD = { - 'd' : 0, # 1 to change directory mode - 'f' : 0, # 1 to change file mode - 'h' : 0, # 1 to show help message - 'r' : 0, # 1 if recursive all - 'R' : 0, # > 0 to set recursive limit - 'F' : 0o664, # to chnage file mode, default to 664 - 'D' : 0o775, # to chnge directory mode, default to 775 + 'd': 0, # 1 to change directory mode + 'f': 0, # 1 to change file mode + 'h': 0, # 1 to show help message + 'r': 0, # 1 if recursive all + 'R': 0, # > 0 to set recursive limit + 'F': 0o664, # to chnage file mode, default to 664 + 'D': 0o775, # to chnge directory mode, default to 775 } self.MINFO = { - 'files' : [], - 'curdir' : os.getcwd(), - 'tpath' : None, - 'dcnt' : 0, - 'fcnt' : 0 + 'files': [], + 'curdir': os.getcwd(), + 'tpath': None, + 'dcnt': 0, + 'fcnt': 0 } # function to read parameters diff --git a/src/rda_python_miscs/rdaown.py b/src/rda_python_miscs/rdaown.py index 4f8aed8..2db9bc2 100644 --- a/src/rda_python_miscs/rdaown.py +++ b/src/rda_python_miscs/rdaown.py @@ -22,20 +22,20 @@ class RdaOwn(PgFile): def __init__(self): super().__init__() self.RDAOWN = { - 'd' : 0, # 1 to change directory owner - 'f' : 0, # 1 to change file owner - 'h' : 0, # 1 to show help message - 'r' : 0, # 1 if recursive all - 'R' : 0, # > 0 to set recursive limit - 'F' : 0o664, # to change file mode, default to 664 - 'D' : 0o775, # to change directory mode, default to 775 + 'd': 0, # 1 to change directory owner + 'f': 0, # 1 to change file owner + 'h': 0, # 1 to show help message + 'r': 0, # 1 if recursive all + 'R': 0, # > 0 to set recursive limit + 'F': 0o664, # to change file mode, default to 664 + 'D': 0o775, # to change directory mode, default to 775 } self.OINFO = { - 'files' : [], - 'curdir' : os.getcwd(), - 'tpath' : None, - 'dcnt' : 0, - 'fcnt' : 0 + 'files': [], + 'curdir': os.getcwd(), + 'tpath': None, + 'dcnt': 0, + 'fcnt': 0 } # function to read paramters diff --git a/src/rda_python_miscs/rdasub.py b/src/rda_python_miscs/rdasub.py index 37cb676..7622900 100644 --- a/src/rda_python_miscs/rdasub.py +++ b/src/rda_python_miscs/rdasub.py @@ -19,87 +19,87 @@ class RdaSub(PgFile): def __init__(self): super().__init__() - self.coptions = {'cmd' : None, 'cwd' : None, 'env' : None} # customized options + self.coptions = {'cmd': None, 'cwd': None, 'env': None} # customized options self.args = None -# function to read parameters -def read_parameters(self): - aname = 'rdasub' - self.set_help_path(__file__) - copts = '|'.join(self.coptions) - option = None - argv = sys.argv[1:] - if not argv: self.show_usage(aname) - self.PGLOG['LOGFILE'] = aname + ".log" - self.cmdlog("{} {}".format(aname, ' '.join(argv))) - while argv: - arg = argv.pop(0) - if arg == "-b": - self.PGLOG['BCKGRND'] = 1 - option = None - continue - ms = re.match(r'^-({})$'.format(copts), arg) - if ms: - option = ms.group(1) - continue - if not option: self.pglog("{}: Value passed in without leading option for {}".format(arg, aname), self.LGEREX) - if arg.find(' ') > -1 and not re.match(r'^[\'\"].*[\'\"]$', arg): # quote string with space but not quoted yet - if arg.find("'") > -1: - arg = '"{}"'.format(arg) - else: - arg = "'{}'".format(arg) - self.coptions[option] = arg - if option == "cmd": break + # function to read parameters + def read_parameters(self): + aname = 'rdasub' + self.set_help_path(__file__) + copts = '|'.join(self.coptions) option = None - if not self.coptions['cmd']: self.pglog(aname + ": specify command via option -cmd to run", self.LGWNEX) - self.args = self.argv_to_string(argv, 0) # append command options - -# function to start actions -def start_actions(self): - msg = "{}-{}{}".format(self.PGLOG['HOSTNAME'], self.PGLOG['CURUID'], self.current_datetime()) - if self.coptions['cwd']: - if self.coptions['cwd'].find('$'): self.coptions['cwd'] = self.replace_environments(self.coptions['cwd'], '', self.LGWNEX) - msg += "-" + self.coptions['cwd'] - self.change_local_directory(self.coptions['cwd'], self.LGEREX) - else: - self.coptions['cwd'] = self.PGLOG['CURDIR'] - cmd = self.valid_command(self.coptions['cmd']) - if not cmd and not re.match(r'^/', self.coptions['cmd']): cmd = self.valid_command('./' + self.coptions['cmd']) - if not cmd: self.pglog(self.coptions['cmd'] + ": Cannot find given command to run", self.LGWNEX) - if self.args: cmd += " " + self.args - msg += ": " + cmd - self.pglog(msg, self.LOGWRN) - os.system("nohup " + cmd + " > /dev/null 2>&1 &") - self.display_process_info(self.coptions['cmd'], cmd) + argv = sys.argv[1:] + if not argv: self.show_usage(aname) + self.PGLOG['LOGFILE'] = aname + ".log" + self.cmdlog("{} {}".format(aname, ' '.join(argv))) + while argv: + arg = argv.pop(0) + if arg == "-b": + self.PGLOG['BCKGRND'] = 1 + option = None + continue + ms = re.match(r'^-({})$'.format(copts), arg) + if ms: + option = ms.group(1) + continue + if not option: self.pglog("{}: Value passed in without leading option for {}".format(arg, aname), self.LGEREX) + if arg.find(' ') > -1 and not re.match(r'^[\'\"].*[\'\"]$', arg): # quote string with space but not quoted yet + if arg.find("'") > -1: + arg = '"{}"'.format(arg) + else: + arg = "'{}'".format(arg) + self.coptions[option] = arg + if option == "cmd": break + option = None + if not self.coptions['cmd']: self.pglog(aname + ": specify command via option -cmd to run", self.LGWNEX) + self.args = self.argv_to_string(argv, 0) # append command options -# display the the most recent matching process info -def display_process_info(self, cname, cmd): - ctime = time.time() - RTIME = PID = 0 - pscmd = "ps -u {},{} -f | grep {} | grep ' 1 ' | grep -v ' grep '".format(self.PGLOG['CURUID'], self.PGLOG['RDAUSER'], cname) - for i in range(2): - buf = self.pgsystem(pscmd, self.LOGWRN, 20) - if buf: - lines = buf.split("\n") - for line in lines: - mp = "\s+(\d+)\s+1\s+.*\s(\d+:\d+)\s.*{}\S*\s*(.*)$".format(cname) - ms = re.search(mp, line) - if ms: - pid = ms.group(1) - rtm = ms.group(2) - arg = ms.group(3) - if not arg or cmd.find(arg) > -1: - rtime = self.unixtime(rtm + ':00') - if rtime > ctime: rtime -= 24*60*60 - if rtime > RTIME: - PID = pid - RTIME = rtime - if PID: - return self.pglog("Job <{}> is submitted to background <{}>".format(PID, self.PgLOG['HOSTNAME']), self.LOGWRN) - elif i == 0: - time.sleep(2) + # function to start actions + def start_actions(self): + msg = "{}-{}{}".format(self.PGLOG['HOSTNAME'], self.PGLOG['CURUID'], self.current_datetime()) + if self.coptions['cwd']: + if self.coptions['cwd'].find('$'): self.coptions['cwd'] = self.replace_environments(self.coptions['cwd'], '', self.LGWNEX) + msg += "-" + self.coptions['cwd'] + self.change_local_directory(self.coptions['cwd'], self.LGEREX) else: - return self.pglog("{}: No job information found, It may have finished".format(cmd), self.LOGWRN) + self.coptions['cwd'] = self.PGLOG['CURDIR'] + cmd = self.valid_command(self.coptions['cmd']) + if not cmd and not re.match(r'^/', self.coptions['cmd']): cmd = self.valid_command('./' + self.coptions['cmd']) + if not cmd: self.pglog(self.coptions['cmd'] + ": Cannot find given command to run", self.LGWNEX) + if self.args: cmd += " " + self.args + msg += ": " + cmd + self.pglog(msg, self.LOGWRN) + os.system("nohup " + cmd + " > /dev/null 2>&1 &") + self.display_process_info(self.coptions['cmd'], cmd) + + # display the the most recent matching process info + def display_process_info(self, cname, cmd): + ctime = time.time() + RTIME = PID = 0 + pscmd = "ps -u {},{} -f | grep {} | grep ' 1 ' | grep -v ' grep '".format(self.PGLOG['CURUID'], self.PGLOG['RDAUSER'], cname) + for i in range(2): + buf = self.pgsystem(pscmd, self.LOGWRN, 20) + if buf: + lines = buf.split("\n") + for line in lines: + mp = "\s+(\d+)\s+1\s+.*\s(\d+:\d+)\s.*{}\S*\s*(.*)$".format(cname) + ms = re.search(mp, line) + if ms: + pid = ms.group(1) + rtm = ms.group(2) + arg = ms.group(3) + if not arg or cmd.find(arg) > -1: + rtime = self.unixtime(rtm + ':00') + if rtime > ctime: rtime -= 24*60*60 + if rtime > RTIME: + PID = pid + RTIME = rtime + if PID: + return self.pglog("Job <{}> is submitted to background <{}>".format(PID, self.PgLOG['HOSTNAME']), self.LOGWRN) + elif i == 0: + time.sleep(2) + else: + return self.pglog("{}: No job information found, It may have finished".format(cmd), self.LOGWRN) # main function to excecute this script def main(): diff --git a/src/rda_python_miscs/tcshqsub.py b/src/rda_python_miscs/tcshqsub.py index 5b9475c..a078b93 100644 --- a/src/rda_python_miscs/tcshqsub.py +++ b/src/rda_python_miscs/tcshqsub.py @@ -20,27 +20,26 @@ class TcshQsub(PgLOG): def __init__(self): super().__init__() self.DEFMODS = { - 'default' : "ncarenv,netcdf,ncl,nco,cdo,conda,grib-util,wgrib2", + 'default': "ncarenv,netcdf,ncl,nco,cdo,conda,grib-util,wgrib2" } self.DEFLIBS = { - 'default' : "conda activate /glade/work/gdexdata/conda-envs/pg-gdex", - } - self.SWAPMODS = { + 'default': "conda activate /glade/work/gdexdata/conda-envs/pg-gdex" } + self.SWAPMODS = {} self.RESOURCES = { # resource list for option -l - 'walltime' : '6:00:00', # if this is changed, change defpbstime in PgCheck.py too - 'select' : '1:ncpus=1:mem=1gb' + 'walltime': '6:00:00', # if this is changed, change defpbstime in PgCheck.py too + 'select': '1:ncpus=1:mem=1gb' } self.SOPTIONS = { # single-dash option values - 'o' : None, # will set to default if not provided - 'e' : None, - 'A' : "P43713000", - 'q' : "gdex@casper-pbs", - # 'm' : 'a', - 'm' : 'n', + 'o': None, # will set to default if not provided + 'e': None, + 'A': "P43713000", + 'q': "gdex@casper-pbs", + # 'm': 'a', + 'm': 'n', } self.gdexsub = self.BCHCMDS['PBS'] - self.coptions = {'cmd' : None, 'cwd' : None, 'env' : None, 'mod' : None, 'res' : 'default'} # customized options + self.coptions = {'cmd': None, 'cwd': None, 'env': None, 'mod': None, 'res': 'default'} # customized options self.args = None # function to read parameters @@ -50,7 +49,6 @@ def read_parameters(self): self.set_help_path(__file__) copts = '|'.join(self.coptions) option = None - dcount = 0 argv = sys.argv[1:] if not argv: self.show_usage(aname) self.PGLOG['LOGFILE'] = pname + ".log" @@ -92,7 +90,7 @@ def read_parameters(self): if self.coptions['cwd']: if '$' in self.coptions['cwd']: self.coptions['cwd'] = self.replace_environments(self.coptions['cwd'], '', self.LGWNEX) os.chdir(self.coptions['cwd']) - + # fnction to start actions def start_actions(self): cmd = self.valid_command(self.coptions['cmd']) @@ -129,7 +127,7 @@ def build_tcsh_script(self, cmd): buf += self.set_vm_libs(self.coptions['res']) buf += "\necho {}\n{}\n\ndate\n".format(cmd, cmd) return buf - + # check and add resource options def add_resources(self): for res in re.split(',', self.SOPTIONS['l']): @@ -139,7 +137,7 @@ def add_resources(self): else: self.pglog(res + ": use '=' to separate resource name & value", self.LGEREX) del self.SOPTIONS['l'] - + # add module loads for modules provided def add_modules(self, res, mods): mbuf = "\n" @@ -162,7 +160,7 @@ def add_modules(self, res, mods): if smod in self.SWAPMODS: mbuf += "module unload {}\n".format(self.SWAPMODS[smod]) mbuf += "module load {}\n".format(amod) return mbuf - + # set virtual machine libraries def set_vm_libs(self, res): deflibs = self.DEFLIBS[res] if res in self.DEFLIBS else self.DEFLIBS['default']