-
Notifications
You must be signed in to change notification settings - Fork 0
/
submit-job-Hyak.sh
executable file
·112 lines (86 loc) · 4.03 KB
/
submit-job-Hyak.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#!/bin/bash
## --------------------------------------------------------
## NOTE: to submit jobs to Hyak use
## qsub <script.sh>
##
## #PBS is a directive requesting job scheduling resources
## and ALL PBS directives must be at the top of the script,
## standard bash commands can follow afterwards.
## NOTE: Lines that begin with #PBS are commands to PBS,
## and they are not comment lines. To comment out
## use "# PBS".
## --------------------------------------------------------
## --------------------------------------------------------
## RENAME for your job
## --------------------------------------------------------
#PBS -N my-Hyak_jobname
## --------------------------------------------------------
## DIRECTORY where this job is run
## --------------------------------------------------------
#PBS -d /gscratch/stf/username/mycode/
## --------------------------------------------------------
## GROUP to run under, or run under BACKFILL
## --------------------------------------------------------
#PBS -q bf
## PBS -W group_list=hyak-stf
## PBS -W group_list=hyak-secretgroup
## --------------------------------------------------------
## NUMBER nodes, CPUs per node, and MEMORY
## --------------------------------------------------------
#PBS -l nodes=1:ppn=16,mem=12gb,feature=intel
## PBS -l nodes=2:ppn=16,mem=120gb,feature=intel
## PBS -l nodes=4:ppn=16,mem=250gb,feature=intel
## PBS -l nodes=8:ppn=16,mem=510gb,feature=intel
## PBS -l nodes=16:ppn=16,mem=1000gb,feature=intel
## --------------------------------------------------------
## WALLTIME (defaults to 1 hour as the minimum, specify > 1 hour longer jobs)
## --------------------------------------------------------
#PBS -l walltime=01:59:00
## --------------------------------------------------------
## LOG the (stderr and stdout) job output in the directory
## --------------------------------------------------------
## PBS -j oe -o /gscratch/motley/dsale/job_output/logs
#PBS -j oe -o /gscratch/stf/dsale/job_output/logs
## --------------------------------------------------------
## EMAIL to send when job is aborted, begins, and terminates
## --------------------------------------------------------
## PBS -m abe -M [email protected]
#PBS -m abe -M [email protected]
## --------------------------------------------------------
## END of PBS commmands ... only BASH from here and below
## --------------------------------------------------------
## --------------------------------------------------------
## LOAD any appropriate environment modules and variables
## --------------------------------------------------------
module load git_2.4.4
## --------------------------------------------------------
## DEBUGGING information (include jobs logs in any help requests)
## --------------------------------------------------------
## Total Number of nodes and processors (cores) to be used by the job
echo "== JOB DEBUGGING INFORMATION=========================="
HYAK_NNODES=$(uniq $PBS_NODEFILE | wc -l )
HYAK_NPE=$(wc -l < $PBS_NODEFILE)
echo "This job will run on $HYAK_NNODES nodes with $HYAK_NPE total CPU-cores"
echo ""
echo "Node:CPUs Used"
uniq -c $PBS_NODEFILE | awk '{print $2 ":" $1}'
echo ""
echo "ENVIRONMENT VARIABLES"
set
echo ""
echo "== END DEBUGGING INFORMATION ========================"
## --------------------------------------------------------
## RUN your specific applications/scripts/code here
## --------------------------------------------------------
## CHANGE directory to where job was submitted (careful, PBS defaults to user home directory)
cd $PBS_O_WORKDIR
## SPECIFY some input variables
myInputFile1="run-case-1"
myInputFile2="run-case-2"
## KEEP copy of the initial cleared solution (small file size),
## and RENAME file used for restart after CHECKPOINTING (this file gets big)
cp --no-clobber $myInputFile1 runs.$myInputFile1
cp --no-clobber $myInputFile2 runs.$myInputFile2
## RUN my simulation file in BATCH MODE (and make a log file!)
./my-cool-code myInputFile1 2>&1 |tee log.$myInputFile1
./my-cool-code myInputFile2 2>&1 |tee log.$myInputFile2