Re-wrangle phoSim inputs and outputs to make greater use of local scr…

…atch and thereby reduce load on a strained Lustre file system

workflows/TW-phoSim-r3/config.py

@@ -57,7 +57,7 @@
 
 ## phoSim (persistent) scratch space directory path
 ##    SLAC:  /lustre/ki/pfs/fermi_scratch/lsst/<task>/<subtask>/<stream>/<substream>
-PHOSIMSCRATCH = os.path.join('/lustre/ki/pfs/fermi_scratch/lsst',os.environ['PIPELINE_TASKPATH'].replace('.','/'),os.environ['PIPELINE_STREAMPATH'].replace('.','/'))
+PHOSIMPSCRATCH = os.path.join('/lustre/ki/pfs/fermi_scratch/lsst',os.environ['PIPELINE_TASKPATH'].replace('.','/'),os.environ['PIPELINE_STREAMPATH'].replace('.','/'))
 
 PHOSIMCLEANUP = True
 

workflows/TW-phoSim-r3/phoSimPrep.py

@@ -4,26 +4,35 @@
 ##
 
 import os,sys,shutil
+import tarfile
 
 ## Insert task config area for python modules (insert as 2nd element in sys.path)
 sys.path.insert(1,os.getenv('TW_CONFIGDIR'))
 from config import *
+import scratch
 
 
 print '\n\nWelcome to phoSimPrep.py\n========================\n'
 
 ## Generate instanceCatalog on-the-fly
 
-## Create scratch directory in (persistent) /lustre, if necessary.
+## Create persistent scratch directory in /lustre, if necessary.
 ## File path = PHOSIMPSCRATCH (defined in config.py)
-
 if not os.path.exists(PHOSIMPSCRATCH): os.makedirs(PHOSIMPSCRATCH)
 
+
+## Create true scratch directory in /scratch
+scr = scratch.scratch()
+SCRATCH = scr.getScratch()
+scr.statScratch()
+
+
 ## generate instance catalog and SED files for phoSim
 
 #  generatePhosimInput.py obsHistID [options]
 destIC = os.path.join(PHOSIMPSCRATCH,'instanceCatalog.txt')
-destSEDdir = PHOSIMPSCRATCH
+#destSEDdir = PHOSIMPSCRATCH
+destSEDdir = SCRATCH
 obsHistID = os.getenv('TW_OBSHISTID')
 cacheDir = TW_CACHEDIR
 opssimDir = TW_OPSSIMDIR
@@ -57,9 +66,28 @@
     print 'Awkward return code, redefining rc = ',rc
     pass
 
+
+## tar up the sprinkled SED files
+print 'tar up the sprinkled SED files and store in /lustre'
+workingdir = os.getcwd()
+os.chdir(destSEDdir)       ## Move to directory containing SED dir to feed tar relative paths
+
+seddir = 'spectra_files'   ## This is where the sprinkled SEDs are generated
+tarinput = seddir
+tarname = seddir+'.tar.gz'
+taroutput = os.path.join(PHOSIMPSCRATCH,tarname)  ## output goes into /lustre
+
+tarobj = tarfile.open(name=taroutput,mode='w:gz')
+tarobj.add(tarinput,recursive=True)  ## add entire directory tree of SEDs to tar archive
+tarobj.close()
+
+os.chdir(workingdir)
+
+
+
 ## Protect scratch directory: rwxr-sr-t
 cmd = 'chmod -R 3755 '+PHOSIMPSCRATCH
-print 'Protect scratch directory\n',cmd
+print 'Protect scratch directory and its contents\n',cmd
 
 rc2 = os.system(cmd)
 if rc2 != 0:
@@ -70,10 +98,15 @@
 
 
 ## Confirm working directory contents
-cmd = 'ls -l '+destSEDdir
+cmd = 'ls -l '+PHOSIMPSCRATCH
 print cmd
 os.system(cmd)
 
+
+## Clean up the local scratch space
+scr.cleanScratch()
+scr.statScratch()
+
 ## Run a trial phoSim to ensure all inputs+code respond reasonably?
 ##      (not yet, if ever)
 

workflows/TW-phoSim-r3/runPhoSim.py

@@ -68,11 +68,12 @@
 #prep.inputRoot = PHOSIMIN
 prep.inputRoot = PHOSIMPSCRATCH
 prep.phosimInstDir = PHOSIMINST
-prep.SEDlib = PHOSIMSEDS
-prep.scratch = PHOSIMPSCRATCH
-prep.refCF = PHOSIMCF   ## cmd file template may require editing
+prep.SEDlib = PHOSIMSEDS   ## production SEDs
+prep.sedFile = 'spectra_files.tar.gz' ## sprinkled SEDs
+
+prep.refCF = PHOSIMCF   ## cmd file template (may require editing)
 prep.persistentScratch = True  ## dynamically generated instance catalog + SEDs
-prep.cleanupFlag = False  ## DEBUG - keep contents of scratch (/lustre)
+prep.cleanupFlag = False  ## DEBUG - keep contents of scratch
 
 (work1,ic,seds,cFile) = prep.run()
 

workflows/TW-phoSim-r3/scratch.py

@@ -0,0 +1,93 @@
+#!/nfs/farm/g/lsst/u1/software/redhat6-x86_64-64bit-gcc44/anaconda/2.3.0/bin/python
+
+## scratch.py - Manage local /scratch space
+
+
+import os,sys,shutil,socket
+
+class scratch(object):
+    def __init__(self):
+        self.scratch = None
+        self.prefix = 'LSSTSIM'
+        self.scratches = ['/scratch']
+        self.scrMode = 0o3755
+        self.cleanupFlag = True
+        self.host = socket.gethostname()
+        self.trace = False
+        return
+
+    def getScratch(self):
+    ## Set up scratch space
+        if self.trace: print 'Entering getScratch().'
+        if self.scratch == None:
+            scratchRoot = ''
+            for scratch in self.scratches:
+                if os.path.isdir(scratch):
+                    scratchRoot = scratch
+                    break
+                pass
+
+            if scratchRoot == '':
+                print "Unable to find suitable scratch space."
+                print self.scratches
+                sys.exit(1)
+
+            ## Define scratch directory
+            ##  Naming convention for scratch directories:
+            ##    Standalone job:     /scratch/PID-<processID>
+            ##    Ordinary batch job: /scratch/LSF-<jobID>
+            ##    Pipeline batch job: /scratch/LSSTSIM/<taskName>/<subtaskName>/<StreamNum>/<jobID>
+            ##  Naming convention for scratch directories:
+            jobid = os.path.join(self.prefix,'PID-'+str(os.getpid()))
+            if os.environ.get('LSB_JOBID') != None:
+                jobid=os.path.join(self.prefix,'LSF-'+os.environ['LSB_JOBID'])
+                if os.environ.get('PIPELINE_TASKPATH') != None:
+                    jobid = os.path.join(self.prefix,os.environ.get('PIPELINE_TASKPATH').replace('.','/'),os.environ.get('PIPELINE_STREAMPATH').split('.')[0],os.environ.get('LSB_JOBID'))
+                    pass
+                pass
+
+            self.scratch = os.path.join(scratchRoot,str(jobid))
+            print 'defined self.scratch = ',self.scratch
+
+            ## Create scratch directory
+            if os.path.exists(self.scratch):
+                print 'WARNING: scratch directory already exists.  Removing...'
+                shutil.rmtree(self.scratch)
+                pass
+            os.makedirs(self.scratch,self.scrMode)
+            pass
+        else:
+            print 'WARNING: scratch already defined - ',self.scratch
+            pass
+
+        return self.scratch
+
+
+    def cleanScratch(self):
+        ## Cleanup the 'scratch' space
+        if self.trace: print 'Entering cleanScratch().'
+
+        if self.scratch != None:
+            ## Now obliterate the scratch space
+            if self.cleanupFlag:
+                print "Cleaning up scratch area"
+                shutil.rmtree(self.scratch)
+                self.scratch = None
+            else:
+                print "Retaining scratch area"
+                pass
+            pass
+        return
+
+    def statScratch(self):
+        print '\n\n====================================================='
+        print 'Status of scratch space'
+        print ' scratch     = ',self.scratch
+        print ' prefix      = ',self.prefix
+        print ' scratches   = ',self.scratches
+        print ' scrMode     = ',oct(self.scrMode)
+        print ' host        = ',self.host
+        print ' cleanupFlag = ',self.cleanupFlag
+        print ' trace       = ',self.trace
+        print '=======================================================\n\n'
+        return

workflows/TW-phoSim-r3/setupPhoSimInput.py

@@ -12,9 +12,10 @@
 
 import os,sys,shutil
 import argparse
+import scratch
 trace = True
 
-class setupPhoSimInput:
+class setupPhoSimInput(object):
     def __init__(self,icFile=None):
         self.debug = True
         self.projectName = 'LSSTSIM'
@@ -29,9 +30,11 @@ def __init__(self,icFile=None):
         self.specialSEDdir = 'spectra_files'   ## name of directory to contain custom SEDs
         self.locSpecialSEDs = None  ## full path to (unpacked) custom SED files
 
+        self.scratchObj = scratch.scratch()  ## Create a scratch object
         self.scratch = None             ## high-performance I/O scratch space (if possible)
-#        self.scratches = ['/lustre/ki/pfs/fermi_scratch','/scratch','/tmp']  # list of scratch candidates
-        self.scratches = ['/lustre/ki/pfs/fermi_scratch']  # for TW-run3 insist upon semi-persistent only
+        self.scratches = ['/scratch']   ## list of scratch candidates
+#        self.scratches = ['/lustre/ki/pfs/fermi_scratch','/scratch','/tmp']
+
         self.persistentScratch = False  ## Will the scratch area persist after this task step?
         self.cleanupFlag = True         ## True => erase scratch directory at end of task
 
@@ -46,41 +49,24 @@ def __init__(self,icFile=None):
         self.nextCP = None  ## which checkpoint will be performed
         self.reqCP = None
         self.archiveWorkDir = None    ## location of checkpoint files
-
         return
 
-    def getScratch(self):
-        ## Set up scratch space
-        if trace: print 'Entering getScratch().'
-        if self.scratch == None:
-            scratchRoot = ''
-            for scratch in self.scratches:
-                if os.path.isdir(scratch):
-                    scratchRoot = scratch
-                    break
-                pass
-
-            if scratchRoot == '':
-                print "Unable to find suitable scratch space."
-                print self.scratches
-                sys.exit(1)
 
-            ## Create scratch directory
-            jobid = 'PID-'+str(os.getpid())
-            if os.environ.get('LSB_JOBID') != None: jobid='LSF-'+os.environ['LSB_JOBID']
-            self.scratch = os.path.join(scratchRoot,self.projectName,str(jobid))
-            print 'defined self.scratch = ',self.scratch
-            os.makedirs(self.scratch)
-            pass
 
-        ## Create 'work' directory in scratch area
+    def getScratch(self):
+        ## Set up scratch space
+        self.scratch = self.scratchObj.getScratch()
+        self.scratchObj.statScratch()
+
+        ## Create phoSim 'work' directory in scratch area
         self.locWork = os.path.join(self.scratch,'work')
-        if not os.access(self.locWork,os.W_OK):os.makedirs(self.locWork)
+        if not os.access(self.locWork,os.W_OK):os.makedirs(self.locWork,0o3755)
         return
 
 
+
     def protectScratch(self):
-        ## Protect scratch directory: rwxr-sr-t
+        ## Protect scratch directory and everything in it: rwxr-sr-t
         if trace: print 'Entering protectScratch().'
         cmd = 'chmod -R 3755 '+self.scratch
         print 'Protect scratch directory\n',cmd
@@ -203,6 +189,12 @@ def makeSEDtree(self):
         rc = os.system(cmd)
         if rc != 0: sys.exit(1)
 
+        ## Change SEDs directory permissions
+        cmd = 'chmod -R 3755 '+self.locSEDs
+        print 'cmd = ',cmd
+        rc = os.system(cmd)
+        if rc != 0: sys.exit(1)
+
         ## Add link to special SEDs for just this run
         if self.locSpecialSEDs != None and not os.path.islink(self.locSpecialSEDs):
             print 'Create sym link to custom SED file directory'
@@ -296,7 +288,8 @@ def clean(self):
         ## Now obliterate the scratch space
         if self.cleanupFlag:
             print "Cleaning up scratch area"
-            shutil.rmtree(self.scratch)
+            self.scratchObj.cleanScratch()
+            #shutil.rmtree(self.scratch)
         else:
             print "Retaining scratch area"
             pass

workflows/TW-phoSim-r3/setupVisit.py

@@ -200,7 +200,7 @@
 ## Create working directory structure for this invocation of phoSim
 
 ### Check if directory already exists from previous execution and, if not, create it.
-## Directory structure: $TW_ROOT/output/<stream>/{work,output}
+## Directory structure: $TW_ROOT/phosim_output/<stream>/{work,output}
 outDir = os.path.join(os.environ['TW_ROOT'],phoSimOutputRoot,os.environ['TW_SIXDIGSTREAM'])
 log.info('PhoSim output directory = \n\t%s',outDir)