Merge pull request #28 from lsst-camera-dh/develop

Develop

bot-data.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env ccs-script
 import sys
+import time
 #sys.path.insert(0,"/gpfs/slac/lsst/fs1/g/data/youtsumi/ts8/fp-scripts/lib")
 from optparse import OptionParser
 from org.lsst.ccs.scripting import CCS
 from ccs import aliases
 from ccs import proxies
+from ccs import versions
 from java.time import Duration
 
 # Temporary work around for problems with CCS responsiveness
@@ -17,6 +19,9 @@
 parser.add_option("-9","--ds9", action="store_true", dest="ds9")
 parser.add_option("--run", dest="run")
 parser.add_option("--symlink", dest="symlink")
+parser.add_option("--skip", dest="skip")
+parser.add_option("--limit", dest="limit")
+
 (options, args) = parser.parse_args()
 
 if len(args)!=1:
@@ -25,9 +30,14 @@
 
 #CCS.aliases = {'focal-plane': 'focal-plane-sim', 'bot-bench': 'bot-bench-sim'}
 #CCS.aliases = {'focal-plane': 'ts8-fp', 'bot-bench': 'ts8-bench' }
-#ccs_sub.write_versions()
+
+# Assume if run is set we are running under eTraveler
+if options.run:
+  fp = CCS.attachProxy('focal-plane')
+  time.sleep(10.0)
+  versions.write_versions(fp)
 
 import config
 
 cfg = config.parseConfig(args[0])
-config.execute(cfg, {"dry_run": options.dry_run, "run": options.run, "symlink": options.symlink})
+config.execute(cfg, {"dry_run": options.dry_run, "run": options.run, "symlink": options.symlink, "skip": options.skip, "limit": options.limit})

lib/acquire.py

@@ -1,12 +1,13 @@
 import os
 import time
 import config
+import sys
 import fp
 #import ccob
 import bot
 from pd import PhotodiodeReadout
 from org.lsst.ccs.utilities.location import LocationSet
-import jarray 
+import jarray
 from java.lang import String
 from org.lsst.ccs.scripting import CCS
 from ccs import aliases
@@ -27,6 +28,9 @@ class TestCoordinator(object):
     def __init__(self, options, test_type, image_type):
         self.run = options['run']
         self.symlink = options['symlink']
+        self.skip = options.getInt('skip', 0)
+        self.limit = self.skip + options.getInt('limit', 10000000)
+        self.noop = self.skip > 0
         self.test_type = test_type
         self.image_type = image_type
         self.annotation = options.get('annotation','')
@@ -61,12 +65,21 @@ def take_image(self, exposure, expose_command, image_type=None, symlink_image_ty
     def __take_image(self, exposure, expose_command, image_type=None, symlink_image_type=None):
         image_type = image_type if image_type else self.image_type
         symlink_image_type = symlink_image_type if symlink_image_type else self.symlink_image_type(image_type)
-        fits_header_data = self.create_fits_header_data(exposure, image_type)
-        image_name, file_list = fp.takeExposure(expose_command, fits_header_data, self.annotation, self.locations, self.clears)
-        self.create_symlink(file_list, self.symlink_test_type(self.test_type), symlink_image_type)
         global test_seq_num
-        test_seq_num = test_seq_num + 1
-        return (image_name, file_list)
+        if test_seq_num >= self.limit:
+            print "Stopping since --limit reached before tSeqNum = %d" % test_seq_num
+            sys.exit()
+
+        if not self.noop:
+            fits_header_data = self.create_fits_header_data(exposure, image_type)
+            image_name, file_list = fp.takeExposure(expose_command, fits_header_data, self.annotation, self.locations, self.clears)
+            self.create_symlink(file_list, self.symlink_test_type(self.test_type), symlink_image_type)
+            test_seq_num += 1
+            return (image_name, file_list)
+        else:
+            test_seq_num += 1
+            self.noop = test_seq_num < self.skip
+            return (None, None)
 
     def create_symlink(self, file_list, test_type, image_type):
         ''' Create symlinks for created FITS files, based on the specification here:
@@ -135,11 +148,11 @@ def set_filters(self, nd_filter, wl_filter):
         bot_bench.setColorFilter(wl_filter)
 
     def take_image(self, exposure, expose_command, image_type=None, symlink_image_type=None):
-        if self.use_photodiodes:
+        if self.use_photodiodes and not self.noop:
             pd_readout = PhotodiodeReadout(exposure)
             pd_readout.start_accumulation()
         image_name, file_list = super(FlatFieldTestCoordinator, self).take_image(exposure, expose_command, image_type, symlink_image_type)
-        if self.use_photodiodes:
+        if self.use_photodiodes and not self.noop:
             # TODO: Why does this need the last argument - in fact it is not used?
             pd_readout.write_readings(file_list.getCommonParentDirectory().toString(),image_name.toString().split('_')[-1],image_name.toString().split('_')[-2])
         return (image_name, file_list)
@@ -173,7 +186,8 @@ def take_images(self):
             exposure = self.compute_exposure_time(nd_filter, self.wl_filter, e_per_pixel)
             expose_command = lambda: bot_bench.openShutter(exposure)
             print "exp %s filter %s" % (exposure, nd_filter)
-            self.set_filters(nd_filter, self.wl_filter)
+            if not self.noop or self.skip - test_seq_num < self.bcount + 2:
+                self.set_filters(nd_filter, self.wl_filter)
             self.take_bias_images(self.bcount)
             for pair in range(2):
                 self.take_image(exposure, expose_command, symlink_image_type='%s_%s_%s_flat%d' % (nd_filter, self.wl_filter, e_per_pixel, pair))
@@ -201,7 +215,8 @@ def take_images(self):
             e_per_pixel = float(e_per_pixel)
             exposure = self.compute_exposure_time(nd_filter, wl_filter, e_per_pixel)
             expose_command = lambda: bot_bench.openShutter(exposure)
-            self.set_filters(nd_filter, wl_filter)
+            if not self.noop or self.skip - test_seq_num < count*(self.bcount + 1):
+                self.set_filters(nd_filter, wl_filter)
             for c in range(count):
                 self.take_bias_plus_image(exposure, expose_command, symlink_image_type='flat_%s_%s'% (wl_filter, self.low_or_high(e_per_pixel)))
 
@@ -216,7 +231,8 @@ def take_images(self):
             wl_filter, e_per_pixel, nd_filter = lamb.split()
             exposure = self.compute_exposure_time(nd_filter, wl_filter, e_per_pixel)
             expose_command = lambda: bot_bench.openShutter(exposure)
-            self.set_filters(nd_filter, wl_filter)
+            if not self.noop or self.skip - test_seq_num < self.bcount + 1:
+                self.set_filters(nd_filter, wl_filter)
             self.take_bias_plus_image(exposure, expose_command, symlink_image_type='flat_%s_%s' % (wl_filter, e_per_pixel))
 
 class PersistenceTestCoordinator(FlatFieldTestCoordinator):
@@ -244,7 +260,8 @@ def take_images(self):
         # dark acquisition
         self.use_photodiodes = False
         for i in range(int(n_of_dark)):
-            time.sleep(float(t_btw_darks))
+            if not self.noop:
+                time.sleep(float(t_btw_darks))
             super(PersistenceTestCoordinator, self).take_image(float(exp_of_dark), lambda: time.sleep(float(exp_of_dark)), image_type="DARK")
         return (image_name, file_list)
 
@@ -273,7 +290,8 @@ def expose_command():
                 if exposure>0:
                     bot_bench.openShutter(exposure) # Flat
                 bot_bench.openFe55Shutter(self.fe55exposure) # Fe55
-            self.set_filters(self.nd_filter, wl_filter)
+            if not self.noop or self.skip - test_seq_num < self.fe55count*(self.bcount + 1):
+                self.set_filters(self.nd_filter, wl_filter)
             for i in range (self.fe55count):
                 self.take_bias_plus_image(exposure, expose_command, symlink_image_type='%s_flat_%s' % (wl_filter, e_per_pixel))
 
@@ -298,8 +316,9 @@ def create_fits_header_data(self, exposure, image_type):
 
     def take_images(self):
         for point in self.points:
-            (x, y) = [float(x) for x in point.split()]
-            bot.moveTo(x, y)
+            if not self.noop or self.skip - test_seq_num < self.exposures*self.imcount*(self.bcount + 1):
+                (x, y) = [float(x) for x in point.split()]
+                bot.moveTo(x, y)
             for exposure in self.exposures:
                 (self.led, self.current, duration) = exposure.split()
                 self.current = float(self.current)
@@ -323,8 +342,9 @@ def __init__(self, options):
 
     def take_images(self):
         for point in self.points:
-            (x, y) = [float(x) for x in point.split()]
-            bot.moveTo(x, y)
+            if not self.noop or self.skip - test_seq_num < self.exposures*self.imcount*(self.bcount + 1):
+                (x, y) = [float(x) for x in point.split()]
+                bot.moveTo(x, y)
             for exposure in self.exposures:
                 exposure = float(exposure)
                 expose_command = lambda: bot_bench.openShutter(exposure)
@@ -354,8 +374,9 @@ def set_filter(self, mask_filter):
 
     def take_images(self):
         for point in self.points:
-            (x, y) = [float(x) for x in point.split()]
-            bot.moveTo(x, y)
+            if not self.noop or self.skip - test_seq_num < self.exposures*self.imcount*(self.bcount + 1):
+                (x, y) = [float(x) for x in point.split()]
+                bot.moveTo(x, y)
             for exposure in self.exposures:
                 (exposure1, exposure2) = exposure.split()
                 self.exposure1 = float(exposure1)
@@ -389,59 +410,59 @@ def __init__(self, options):
         # TODO: Work about e2v sensors
 
     def take_images(self):
-        preCols = fp.fp.getSequencerParameter("PreCols")
-        readCols = fp.fp.getSequencerParameter("ReadCols")
-        postCols = fp.fp.getSequencerParameter("PostCols")
-        overCols = fp.fp.getSequencerParameter("OverCols")
-        preRows = fp.fp.getSequencerParameter("PreRows")
-        readRows = fp.fp.getSequencerParameter("ReadRows")
-        postRows = fp.fp.getSequencerParameter("PostRows")
-        scanMode = fp.fp.isScanEnabled()
-	print "Initial sequencer parameters"
-
-	print "preCols="  , preCols
-	print "readCols=" , readCols
-	print "postCols=" , postCols
-	print "overCols=" , overCols
-
-	print "preRows="  , preRows
-	print "readRows=" , readRows
-	print "postRows=" , postRows
-
-	print "scanMode=" , scanMode
-
-        # set up scan mode
-        fp.fp.sequencerConfig().submitChanges(
-			{
-			"underCols":self.undercols,
-			"preCols":  self.precols,
-			"readCols": self.readcols,
-			"postCols": self.postcols,
-			"overCols": self.overcols,
-			"preRows":  self.prerows,
-			"readRows": self.readrows,
-			"postRows": self.postrows,
-			"overRows": self.overrows,
-			"scanMode": True
-			}
-		)
-        fp.fp.commitBulkChange()
-
-
-	exposure = 1.0
+        if self.noop or self.skip - test_seq_num < self.scanmode + self.transparent:
+            preCols = fp.fp.getSequencerParameter("PreCols")
+            readCols = fp.fp.getSequencerParameter("ReadCols")
+            postCols = fp.fp.getSequencerParameter("PostCols")
+            overCols = fp.fp.getSequencerParameter("OverCols")
+            preRows = fp.fp.getSequencerParameter("PreRows")
+            readRows = fp.fp.getSequencerParameter("ReadRows")
+            postRows = fp.fp.getSequencerParameter("PostRows")
+            scanMode = fp.fp.isScanEnabled()
+            print "Initial sequencer parameters"
+
+            print "preCols="  , preCols
+            print "readCols=" , readCols
+            print "postCols=" , postCols
+            print "overCols=" , overCols
+
+            print "preRows="  , preRows
+            print "readRows=" , readRows
+            print "postRows=" , postRows
+
+            print "scanMode=" , scanMode
+
+            # set up scan mode
+            fp.fp.sequencerConfig().submitChanges(
+                {
+                "underCols":self.undercols,
+                "preCols":  self.precols,
+                "readCols": self.readcols,
+                "postCols": self.postcols,
+                "overCols": self.overcols,
+                "preRows":  self.prerows,
+                "readRows": self.readrows,
+                "postRows": self.postrows,
+                "overRows": self.overrows,
+                "scanMode": True
+                }
+            )
+            fp.fp.commitBulkChange()
+
+        exposure = 1.0
         expose_command = lambda: time.sleep(exposure)
 
         for i in range(self.scanmode):
            self.take_image(exposure, expose_command, image_type=None, symlink_image_type=None)
 
-
-        fp.fp.sequencerConfig().submitChanges(
-			{
-			"transparentMode": 1
-			}
-		)
-        timeout= Duration.ofSeconds(60*5)
-        fp.fp.commitBulkChange(timeout=timeout)
+        if self.noop or self.skip - test_seq_num < self.transparent:
+            fp.fp.sequencerConfig().submitChanges(
+                {
+                "transparentMode": 1
+                }
+            )
+            timeout= Duration.ofSeconds(60*5)
+            fp.fp.commitBulkChange(timeout=timeout)
 
         for i in range(self.transparent):
            self.take_image(exposure, expose_command, image_type=None, symlink_image_type=None)

lib/ccs/versions.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env ccs-script
+from org.lsst.ccs.scripting import CCS
+
+def write_versions(fp):
+   vList = [];
+   versions = fp.getCCSVersions()
+   group = fp.getAgentProperty("group")
+   has_snapshot = False
+   for agent in versions.getAgents():
+      agentGroup = agent.getAgentProperty("group")
+      if agent.isAgentWorkerOrService() and agentGroup == group:
+         version = versions.getDistributionInfo(agent).getVersion()
+         has_snapshot = has_snapshot or version.endswith("SNAPSHOT")
+         vList.append("%s = %s\n" % (agent.getName(), version))
+   vList.sort()
+   if has_snapshot:
+      print "WaRNING, non released CCS subsystem in use"
+   with open("ccs_versions.txt", "w") as file:
+	for version in vList:
+           file.write(version)

lib/config.py

@@ -12,8 +12,8 @@ def parseConfig(file):
 
   # Eliminate inline # delimited comments
   slines = map(lambda l: re.sub(r"([^#]*)\s#.*", r"\1", l), lines)
-  config = ConfigParser.SafeConfigParser(allow_no_value=True) 
-  config.readfp(StringIO.StringIO("".join(slines))) 
+  config = ConfigParser.SafeConfigParser(allow_no_value=True)
+  config.readfp(StringIO.StringIO("".join(slines)))
   return config
 
 def setvoltages(avoltage):
@@ -53,20 +53,20 @@ class Config(dict):
   def getInt(self, key, defaultValue=None):
      value = self.get(key)
      if not value:
-       if defaultValue:
+       if defaultValue != None:
           return defaultValue
        else:
           raise Exception('Missing config value %s' % key)
-     return int(value)    
+     return int(value)
 
   def getFloat(self, key, defaultValue=None):
      value = self.get(key)
      if not value:
-       if defaultValue:
+       if defaultValue != None:
           return defaultValue
        else:
           raise Exception('Missing config value %s' % key)
-     return float(value)    
+     return float(value)
 
   def getList(self, key):
      return self.get(key).replace('\n','').split(',')