Merge branch 'develop'

README.md

@@ -72,3 +72,7 @@ With important contributions from:
  * Jean-Françoise Sauvage (ONERA): Support of SPARTA files
  * Dino Mesa (INAF/OAPD): IFS pre-processing
  * ESO personnel in general: SPHERE pipeline and calibration procedures
+
+And bug reports with suggested fix from:
+ * Wolfgang Brandner (@Rumpelstil)
+ * Tomas Stolker (@tomasstolker)

setup.py

@@ -9,7 +9,7 @@
 # setup
 setup(
     name='vlt-sphere',
-    version='1.4.3',
+    version='1.5',
     description='Reduction and analysis code for the VLT/SPHERE instrument',
     long_description=long_description,
     long_description_content_type='text/markdown',

sphere/IFS.py

@@ -985,8 +985,8 @@ def sort_files(self):
                     files_info.loc[f, sk] = v_begin if v_begin else v_start
                 elif k == 'HIERARCH ESO INS4 DROT3 BEGIN':
                     # in June 2021 ESO changed INS4 DROT3 BEGIN to INS4 DROT3 START
-                    v_begin = hdr.get('HIERARCH ESO INS3 DROT2 BEGIN')
-                    v_start = hdr.get('HIERARCH ESO INS3 DROT2 START')
+                    v_begin = hdr.get('HIERARCH ESO INS4 DROT3 BEGIN')
+                    v_start = hdr.get('HIERARCH ESO INS4 DROT3 START')
                     files_info.loc[f, sk] = v_begin if v_begin else v_start
                 else:
                     files_info.loc[f, sk] = hdr.get(k)
@@ -1367,7 +1367,7 @@ def check_files_association(self):
                         (calibs['DET SEQ1 DIT'].round(2) == 1.65)]
         if len(cfiles) == 0:
             error_flag += 1
-            self._logger.info(' * Error: there is no dark/background for the basic calibrations (DIT=1.65 sec). It is mandatory to include one to obtain the best data reduction. A single dark/background file is sufficient, and it can easily be downloaded from the ESO archive')
+            self._logger.error(' * there is no dark/background for the basic calibrations (DIT=1.65 sec). It is mandatory to include one to obtain the best data reduction. A single dark/background file is sufficient, and it can easily be downloaded from the ESO archive')
 
         ##################################################
         # static calibrations that depend on science DIT
@@ -1818,7 +1818,7 @@ def sph_ifs_cal_wave(self, silent=True):
                     '--no-checksum=TRUE',
                     '--no-datamd5=TRUE',
                     'sph_ifs_wave_calib',
-                    '--ifs.wave_calib.number_lines=3',
+                    '--ifs.wave_calib.number_lines=4',
                     '--ifs.wave_calib.wavelength_line1=0.9877',
                     '--ifs.wave_calib.wavelength_line2=1.1237',
                     '--ifs.wave_calib.wavelength_line3=1.3094',
@@ -2721,12 +2721,12 @@ def sph_ifs_wavelength_recalibration(self, high_pass=False, offset=(0, 0), box_w
             wave_lasers = self._wave_cal_lasers[0:4]
 
         self._logger.debug('> fit new wavelenth solution')
-        res = optim.minimize(wavelength_optimisation, 0.9, method='Nelder-Mead',
+        res = optim.minimize(wavelength_optimisation, 950.0, method='Nelder-Mead',
                              args=(wave_scale, wave_lasers, peak_position_lasers))
 
         wave_final = np.full(nwave, res.x) * wave_scale
 
-        wave_diff = np.abs(wave_final - wave_drh)*1000
+        wave_diff = np.abs(wave_final - wave_drh)
         self._logger.info('   ==> difference with calibrated wavelength: min={0:.1f} nm, max={1:.1f} nm'.format(wave_diff.min(), wave_diff.max()))
 
         # save

sphere/IRDIS/ImagingReduction.py

@@ -608,8 +608,8 @@ def sort_files(self):
                     files_info.loc[f, sk] = v_begin if v_begin else v_start
                 elif k == 'HIERARCH ESO INS4 DROT3 BEGIN':
                     # in June 2021 ESO changed INS4 DROT3 BEGIN to INS4 DROT3 START
-                    v_begin = hdr.get('HIERARCH ESO INS3 DROT2 BEGIN')
-                    v_start = hdr.get('HIERARCH ESO INS3 DROT2 START')
+                    v_begin = hdr.get('HIERARCH ESO INS4 DROT3 BEGIN')
+                    v_start = hdr.get('HIERARCH ESO INS4 DROT3 START')
                     files_info.loc[f, sk] = v_begin if v_begin else v_start
                 else:
                     files_info.loc[f, sk] = hdr.get(k)

sphere/IRDIS/SpectroReduction.py

@@ -665,8 +665,8 @@ def sort_files(self):
                     files_info.loc[f, sk] = v_begin if v_begin else v_start
                 elif k == 'HIERARCH ESO INS4 DROT3 BEGIN':
                     # in June 2021 ESO changed INS4 DROT3 BEGIN to INS4 DROT3 START
-                    v_begin = hdr.get('HIERARCH ESO INS3 DROT2 BEGIN')
-                    v_start = hdr.get('HIERARCH ESO INS3 DROT2 START')
+                    v_begin = hdr.get('HIERARCH ESO INS4 DROT3 BEGIN')
+                    v_start = hdr.get('HIERARCH ESO INS4 DROT3 START')
                     files_info.loc[f, sk] = v_begin if v_begin else v_start
                 else:
                     files_info.loc[f, sk] = hdr.get(k)
@@ -916,6 +916,16 @@ def check_files_association(self):
             # drop the others
             files_info.drop(time_delta[1:].index, inplace=True)
 
+        wavecal_DIT = files_info.loc[(files_info['DPR TYPE'] == 'LAMP,WAVE') & (files_info['INS COMB IFLT'] == filter_comb), 'DET SEQ1 DIT'].values[0]
+
+        # calibs dark file
+        self._logger.debug('> check wavelength calibration dark requirements')
+        cfiles = calibs[((calibs['DPR TYPE'] == 'DARK') | (calibs['DPR TYPE'] == 'DARK,BACKGROUND')) &
+                        (calibs['DET SEQ1 DIT'].round(2) == wavecal_DIT)]
+        if len(cfiles) == 0:
+            error_flag += 1
+            self._logger.warning(' * there is no dark/background for the wavelength calibration (DIT={0:.1f} sec). It is *highly recommended* to include one to obtain the best data reduction. A single dark/background file is sufficient, and it can easily be downloaded from the ESO archive'.format(wavecal_DIT))
+
         ##################################################
         # static calibrations that depend on science DIT
         ##################################################

sphere/__init__.py

@@ -1,7 +1,7 @@
 __author__    = 'Arthur Vigan'
 __copyright__ = 'Copyright (C) 2017-2021 Arthur Vigan'
 __license__   = 'MIT'
-__version__   = '1.4.3'
+__version__   = '1.5'
 
 import logging
 import enum

sphere/instruments/keywords_irdifs.dat

@@ -45,6 +45,7 @@ HIERARCH ESO OBS NAME
 # standard
 DATE
 HIERARCH ESO DET FRAM UTC
+HIERARCH ESO DET DITDELAY
 
 # coordinates
 HIERARCH ESO INS4 DROT2 RA

sphere/toolbox.py

@@ -131,12 +131,13 @@ def compute_times(frames_info, logger=_log):
         time_end   = frames_info['DET FRAM UTC'].values
         time_delta = (time_end - time_start) / frames_info['DET NDIT'].values.astype(np.int)
         DIT        = np.array(frames_info['DET SEQ1 DIT'].values.astype(np.float)*1000, dtype='timedelta64[ms]')
-
+        DITDELAY   = np.array(frames_info['DET DITDELAY'].values.astype(np.float)*1000, dtype='timedelta64[ms]')
+
         # calculate UTC time stamps
         idx = frames_info.index.get_level_values(1).values
-        ts_start = time_start + time_delta * idx
-        ts       = time_start + time_delta * idx + DIT/2
-        ts_end   = time_start + time_delta * idx + DIT
+        ts_start = time_start + time_delta * idx + DITDELAY
+        ts       = time_start + time_delta * idx + DITDELAY + DIT/2
+        ts_end   = time_start + time_delta * idx + DITDELAY + DIT
 
         # mjd
         utc = Time(ts_start.astype(str), scale='utc', location=sphere.location)
@@ -214,7 +215,7 @@ def compute_angles(frames_info, true_north, logger=_log):
     ra_drot_s = ra_drot - ra_drot_h*1e4 - ra_drot_m*1e2
     ra_hour = coordinates.Angle((ra_drot_h, ra_drot_m, ra_drot_s), units.hour)
     ra_deg  = ra_hour*15
-    frames_info['RA'] = ra_deg
+    frames_info['RA'] = ra_deg.value
 
     dec_drot = frames_info['INS4 DROT2 DEC'].values.astype(np.float)
     sign = np.sign(dec_drot)
@@ -224,17 +225,17 @@ def compute_angles(frames_info, true_north, logger=_log):
     dec_drot_s = udec_drot - dec_drot_d*1e4 - dec_drot_m*1e2
     dec_drot_d *= sign
     dec = coordinates.Angle((dec_drot_d, dec_drot_m, dec_drot_s), units.degree)
-    frames_info['DEC'] = dec
+    frames_info['DEC'] = dec.value
 
     # calculate parallactic angles
     utc = Time(frames_info['TIME'].values.astype(str), scale='utc', location=sphere.location)
     lst = utc.sidereal_time('apparent')
     ha  = lst - ra_hour
     pa  = parallatic_angle(ha, dec[0], sphere.latitude)
     frames_info['PARANG'] = pa.value + pa_correction
-    frames_info['HOUR ANGLE'] = ha
-    frames_info['LST'] = lst
-
+    frames_info['HOUR ANGLE'] = ha.value
+    frames_info['LST'] = lst.value
+    
     # Altitude and airmass
     j2000 = coordinates.SkyCoord(ra=ra_hour, dec=dec, frame='icrs', obstime=utc)
     altaz = j2000.transform_to(coordinates.AltAz(location=sphere.location))
@@ -250,16 +251,16 @@ def compute_angles(frames_info, true_north, logger=_log):
         ha  = lst - ra_hour
         pa  = parallatic_angle(ha, dec[0], sphere.latitude)
         frames_info['PARANG START'] = pa.value + pa_correction
-        frames_info['HOUR ANGLE START'] = ha
-        frames_info['LST START'] = lst
+        frames_info['HOUR ANGLE START'] = ha.value
+        frames_info['LST START'] = lst.value
 
         utc = Time(frames_info['TIME END'].values.astype(str), scale='utc', location=sphere.location)
         lst = utc.sidereal_time('apparent')
         ha  = lst - ra_hour
         pa  = parallatic_angle(ha, dec[0], sphere.latitude)
         frames_info['PARANG END'] = pa.value + pa_correction
-        frames_info['HOUR ANGLE END'] = ha
-        frames_info['LST END'] = lst
+        frames_info['HOUR ANGLE END'] = ha.value
+        frames_info['LST END'] = lst.value
 
     #
     # Derotation angles