-
Notifications
You must be signed in to change notification settings - Fork 6
/
submit-job-Hyak.sh
executable file
·108 lines (81 loc) · 3.8 KB
/
submit-job-Hyak.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/bin/bash
## --------------------------------------------------------
## NOTE: to submit jobs to Hyak use
## qsub <script.sh>
##
## #PBS is a directive requesting job scheduling resources
## and ALL PBS directives must be at the top of the script,
## standard bash commands can follow afterwards.
## --------------------------------------------------------
## --------------------------------------------------------
## RENAME for your job
## --------------------------------------------------------
#PBS -N stf-ftle
## --------------------------------------------------------
## SPECIFY the working directory for this job
## --------------------------------------------------------
## PBS -d /gscratch/stf/dsale/OpenFOAM/dsale-2.4.x/fastFlume-branches/fastFlume-Coarse-e1
## --------------------------------------------------------
## GROUP to run under, or run under backfill
## --------------------------------------------------------
## PBS -W group_list=hyak-motley
#PBS -W group_list=hyak-stf
## PBS -q bf
## --------------------------------------------------------
## NUMBER nodes, CPUs per node, and MEMORY
## --------------------------------------------------------
## PBS -l nodes=1:ppn=16,mem=10gb,feature=intel
## PBS -l nodes=2:ppn=16,mem=30gb,feature=intel
## PBS -l nodes=3:ppn=16,mem=50gb,feature=intel
#PBS -l nodes=6:ppn=16,mem=300gb,feature=intel
## PBS -l nodes=8:ppn=16,mem=32gb,feature=intel
## PBS -l nodes=16:ppn=16,mem=32gb,feature=intel
## --------------------------------------------------------
## WALLTIME (defaults to 1 hour, always specify for longer jobs)
## --------------------------------------------------------
#PBS -l walltime=4:05:00
## --------------------------------------------------------
## LOG the (stderr and stdout) job output in the directory
## --------------------------------------------------------
#PBS -j oe -o /gscratch/motley/dsale/job_logs
## PBS -j oe -o /gscratch/stf/dsale/job_output/logs
## --------------------------------------------------------
## EMAIL to send when job is aborted, begins, and terminates
## --------------------------------------------------------
## PBS -m abe -M [email protected]
#PBS -m abe -M [email protected]
## --------------------------------------------------------
## END of PBS commmands ... BASH from here and below
## --------------------------------------------------------
## CHANGE directory to where job was submitted (careful, PBS defaults to user home directory)
cd $PBS_O_WORKDIR
## LOAD the appropriate environment modules and variables
module load icc_15.0.3-impi_5.0.3
source /sw/contrib/OpenFOAM/OpenFOAM-2.4.x/etc/bashrc
#source /sw/contrib/OpenFOAM/OpenFOAM-2.4.x/src/LEMOS-2.4.x/bashrc
#source /sw/contrib/OpenFOAM/OpenFOAM-2.3.x/etc/bashrc
#source /sw/contrib/OpenFOAM/OpenFOAM-2.3.x/src/LEMOS-2.3.x/bashrc
## Some applications (particularly FORTRAN) require a larger
## than usual data stack size. Uncomment if your job exits unexpectedly
ulimit -s unlimited
## DUBUGGING information (include jobs logs in any help requests)
## Total Number of nodes and processors (cores) to be used by the job
echo "** JOB DEBUGGING INFORMATION *************************"
HYAK_NNODES=$(uniq $PBS_NODEFILE | wc -l )
HYAK_NPE=$(wc -l < $PBS_NODEFILE)
echo "This job will run on $HYAK_NNODES nodes with $HYAK_NPE total CPU-cores"
echo ""
echo "Node:CPUs Used"
uniq -c $PBS_NODEFILE | awk '{print $2 ":" $1}'
echo ""
echo "ENVIRONMENT VARIABLES"
set
echo "** END DEBUGGING INFORMATION *************************"
echo ""
## --------------------------------------------------------
## Specify the applications to run here
## --------------------------------------------------------
## my script to run all OpenFOAM things
./run.all
## BE CAREFUL about CLEANING, make sure your scripts do not
## clean the job (delete files) if re-submitting this case to queue