-
Notifications
You must be signed in to change notification settings - Fork 0
/
archive_run
executable file
·161 lines (131 loc) · 6.8 KB
/
archive_run
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
#!/usr/bin/env python
from __future__ import print_function
from argparse import ArgumentParser
import sys
import re
import os
import flo_utils as fu
from math import ceil
scriptpath = os.path.dirname(sys.argv[0])
def cerr(*objs):
print( *objs, file=sys.stderr)
def generate_blocks(options):
start=options.start
last=options.last
if last < start:
cerr("ERROR: BAD ARGUMENTS TO archive_run")
cerr("ERROR: Last year before first year! %d < %d"%(last, start))
sys.exit(64)
starts=range(int(min(ceil(start/blocksize.)*blocksize, last)), last+1, options.blocksize)
if len(starts) == 0:
starts = [start]
if (starts[0] != start):
starts = [start] + starts
ends=[x-1 for x in starts[1:]]+[last]
print (zip(starts, ends))
sys.exit(666)
return (zip(starts,ends))
def deactivate (switches, script):
if type (switches) is str:
switches = [switches]
for switch in switches:
script = re.sub(r"(#\s*)?%s=.*"%(switch),r"%s=false"%(switch), script)
return script
def activate (switches, script):
if type (switches) is str:
switches = [switches]
for switch in switches:
script = re.sub(r"(#\s*)?%s=.*"%(switch),r"%s=true"%(switch), script)
return script
def generate_scripts(options):
iterations = (0,)
if (options.restart_interval != 1) :
iterations = (0, 1, 2 )
for restart_backup_iteration in iterations:
blocks=generate_blocks(options)
infile = "%s/run_archive_esm.sh"%scriptpath
# if ((restart_backup_iteration == 0 and options.restart_interval != 1) or restart_backup_iteration == 2 ):
# infile = infile[:-3] + "_wo_restart.sh"
# read template
template = open(infile).read()
template = re.sub("EXPERIMENT_ID", options.run, template)
for b in blocks:
# transform template
# First year is different, when archiving strided restarts
if (restart_backup_iteration == 0 or restart_backup_iteration == 2 ):
script = re.sub("FIRST_YEAR", str(b[0]), template)
elif (restart_backup_iteration == 1):
script = re.sub("FIRST_YEAR", str(b[0]+options.restart_interval - 1), template)
if (b[0] + options.restart_interval - 1 > b[1]): # first restart file falls into next block
fu.cerr("""Start year of block %d and restart interval %d yield
first restart file of block %d outside block [%d, %d]
""" % (b[0], options.restart_interval, b[0]+options.restart_interval-1, b[0], b[1]))
sys.exit(64)
else:
fu.cerr("Wrong restart backup iteration number %d"%restart_backup_iteration)
sys.exit(1)
script = re.sub("LAST_YEAR", str(b[1]), script)
if options.no_hamocc :
script = deactivate (['logbc'], script)
if (restart_backup_iteration == 0) : # Output (possibly without MPIOM) and restarts
script = re.sub("--job-name=.*","--job-name=a_%s_%d-%d"%(options.run, b[0], b[1]), script)
script = re.sub(r"--output=archive_esm.o(.*)",r"--output=a_%s_%d-%d_\1"%(options.run, b[0], b[1]), script)
script = re.sub(r"--error=archive_esm.o(.*)",r"--error=a_%s_%d-%d_\1"%(options.run, b[0], b[1]), script)
if (options.restart_interval != 1):
script = deactivate(['locean', "larchive_restart", "larchive_log"], script)
if (restart_backup_iteration == 1 or options.restart_only): # Restarts // years ending on 9
script = re.sub("--job-name=.*","--job-name=ar_%s_%d-%d"%(options.run, b[0], b[1]), script)
script = re.sub(r"--output=archive_esm.o(.*)",r"--output=ar_%s_%d-%d_\1"%(options.run, b[0], b[1]), script)
script = re.sub(r"--error=archive_esm.o(.*)",r"--error=ar_%s_%d-%d_\1"%(options.run, b[0], b[1]), script)
script = deactivate (['lclimatologies', 'larchive_data', 'larchive_log', 'llogg'], script)
if (restart_backup_iteration == 2): # MPIOM (years ending on 0)
script = re.sub("--job-name=.*","--job-name=am_%s_%d-%d"%(options.run, b[0], b[1]), script)
script = re.sub(r"--output=archive_esm.o(.*)",r"--output=am_%s_%d-%d_\1"%(options.run, b[0], b[1]), script)
script = re.sub(r"--error=archive_esm.o(.*)",r"--error=am_%s_%d-%d_\1"%(options.run, b[0], b[1]), script)
switches = ["lcoupler", "llogg", "lland", "latmosphere", "larchive_restart", "larchive_log"]
script = deactivate(switches, script)
# only monthly mean JSBACH data avail?
if (options.jsbach_mm_only):
script = re.sub(r'(#?land_data_pat=".*\n?.*)',r'\1\n#!FLO! CHANGED JSBACH OUTPUT FILE PATTERN!=====================#\nland_data_pat="jsbach_mm_ land_mm_ surf_mm_ veg_mm_ yasso_mm_"\n#!FLO! CHANGED JSBACH OUTPUT FILE PATTERN!=====================#', script)
script = re.sub(r'(#?land_data?_type=".*\n?.*)',r'\1\n#!FLO! CHANGED JSBACH OUTPUT FILE PATTERN!=====================#\nland_dat_type="${mm} ${mm} ${mm} ${mm} ${mm}"\n#!FLO! CHANGED JSBACH OUTPUT FILE PATTERN!=====================#', script)
# write script
outfilename = "%s_%d-%d.job"%(options.run, b[0], b[1])
if (restart_backup_iteration > 0):
script = re.sub("#incrementyear=.*", "incrementyear=%d"%options.restart_interval , script)
script = re.sub("incrementyear=.*", "incrementyear=%d"%options.restart_interval , script)
if (restart_backup_iteration == 1):
outfilename = "%s_%d-%d_restarts.job"%(options.run, b[0], b[1])
elif (restart_backup_iteration == 2):
outfilename = "%s_%d-%d_mpiom.job"%(options.run, b[0], b[1])
outfile = open(outfilename, "w")
outfile.writelines(script)
outfile.close()
# submit script
print ("sbatch", outfilename)
def parse_args():
'''Parses the command line arguments'''
parser = ArgumentParser()
parser.description = "Archive an mpi-esm run"
parser.add_argument("run")
parser.add_argument("start", type=int)
parser.add_argument("last", type=int)
parser.add_argument("-v", "--verbose",
help='''Be verbose''', action="store_true")
parser.add_argument("-b", "--blocksize",
help='''Block size''', type=int, default=100)
parser.add_argument( "--restart_interval",
help='''Interval of restart files''', type=int, default=1)
parser.add_argument( "--restart_only",
help='''only archive restarts''', action="store_true")
parser.add_argument( "--no_hamocc",
help='''Be verbose''', action="store_true")
parser.add_argument("--jsbach_mm_only",
help='''only look for monthly mean jsbach files''', action="store_true")
options = parser.parse_args()
return options
def main():
'''Start the archiving scripts'''
options = parse_args()
generate_scripts(options)
if __name__ == "__main__":
main()