working verion

Author: BaptisteVandecrux

setup.py

@@ -44,6 +44,7 @@
         'get_promice_data = pypromice.get.get_promice_data:get_promice_data',
         'get_l0tx = pypromice.tx.get_l0tx:get_l0tx',
         'join_l2 = pypromice.process.join_l2:join_l2',
+        'join_l3 = pypromice.process.join_l3:join_l3',
         'get_l2 = pypromice.process.get_l2:get_l2',
         'get_l3 = pypromice.process.get_l3:get_l3',
         'get_l2tol3 = pypromice.process.get_l2tol3:get_l2tol3',

src/pypromice/process/join_l3.py

@@ -165,16 +165,12 @@ def gcnet_postprocessing(l3):
 def join_l3():
     args = parse_arguments_joinl3()
 
+                      
     config_file = os.path.join(args.config_folder, args.site+'.toml')
     conf = toml.load(config_file)
-    
-    if not site:
-        site_list=conf['list_station_id']
-    else:
-        site_list=site
 
     l3m = xr.Dataset()
-    for stid in site_list:
+    for stid in conf['stations']:
         print(stid)
 
         is_promice = False
@@ -221,8 +217,10 @@ def join_l3():
             for v in l3.data_vars:
                 if v not in l3m.data_vars:
                     if v != 'z_stake':
-                        print(v)
-                        l3 = l3.drop(v)
+                       print(v)
+                       l3 = l3.drop(v)
+                    else:
+                       l3m[v] = ('time', l3m.t_u.data*np.nan)
 
             # saving attributes of station under an attribute called $stid
             l3m = l3m.assign_attrs({stid : l3.attrs.copy()})
@@ -243,7 +241,7 @@ def join_l3():
 
     # Assign site id
     l3m.attrs['site_id'] = args.site
-    l3m.attrs['list_station_id'] = conf['list_station_id']
+    l3m.attrs['stations'] = conf['stations']
     v = getVars()
     m = getMeta()
     if args.outpath is not None:

src/pypromice/process/utilities.py

@@ -41,7 +41,12 @@ def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']):
     '''Switch gps_lon to negative values (degrees_east). We do this here, and 
     NOT in addMeta, otherwise we switch back to positive when calling getMeta 
     in joinL2'''
-    if dataset.attrs['station_id'] not in exempt:
+    if 'station_id' in dataset.attrs.keys():
+        id = dataset.attrs['station_id']
+    else:
+        id = dataset.attrs['site_id']
+
+    if id not in exempt:
         dataset['gps_lon'] = dataset['gps_lon'] * -1
     return dataset
 
@@ -163,7 +168,10 @@ def addMeta(ds, meta):
     #             ds[k].attrs['units'] = 'degrees_C'
 
     # https://wiki.esipfed.org/Attribute_Convention_for_Data_Discovery_1-3#geospatial_bounds
-    ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['station_id']))
+    if 'station_id' in ds.attrs.keys():
+        ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['station_id']))
+    else:
+        ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['site_id']))
     ds.attrs['history'] = 'Generated on ' + datetime.datetime.utcnow().isoformat()
     ds.attrs['date_created'] = str(datetime.datetime.now().isoformat())
     ds.attrs['date_modified'] = ds.attrs['date_created']
@@ -230,4 +238,4 @@ def _addAttr(ds, key, value):
             pass
             # logger.info(f'Unable to add metadata to {key.split(".")[0]}')
     else:
-        ds.attrs[key] = value
+        ds.attrs[key] = value

src/pypromice/process/write.py

@@ -46,34 +46,36 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp
 
     # Round all values to specified decimals places
     d2 = utilities.roundValues(d2, vars_df)
-    
-    # Create out directory
-    outdir = os.path.join(outpath, d2.attrs['station_id'])
-    if not os.path.isdir(outdir):
-        os.mkdir(outdir)
-    
+
     # Get variable names to write out
     col_names = write.getColNames(
         vars_df,
-        d2.attrs['number_of_booms'],
-        d2.attrs['format'],
-        d2.attrs['bedrock'],
-    )
-    
+        d2)
+
     # Define filename based on resample rate
     t = int(pd.Timedelta((d2['time'][1] - d2['time'][0]).values).total_seconds())
+    if 'station_id' in d2.attrs.keys():
+        name = d2.attrs['station_id']
+    else:
+        name = d2.attrs['site_id']
+
+    # Create out directory
+    outdir = os.path.join(outpath, name)
+    if not os.path.isdir(outdir):
+        os.mkdir(outdir)
+
     if t == 600:
-        out_csv = os.path.join(outdir, d2.attrs['station_id']+'_10min.csv')
-        out_nc = os.path.join(outdir, d2.attrs['station_id']+'_10min.nc')
+        out_csv = os.path.join(outdir, name+'_10min.csv')
+        out_nc = os.path.join(outdir, name+'_10min.nc')
     elif t == 3600:
-        out_csv = os.path.join(outdir, d2.attrs['station_id']+'_hour.csv')
-        out_nc = os.path.join(outdir, d2.attrs['station_id']+'_hour.nc')
+        out_csv = os.path.join(outdir, name+'_hour.csv')
+        out_nc = os.path.join(outdir, name+'_hour.nc')
     elif t == 86400:
-        out_csv = os.path.join(outdir, d2.attrs['station_id']+'_day.csv')
-        out_nc = os.path.join(outdir, d2.attrs['station_id']+'_day.nc')
+        out_csv = os.path.join(outdir, name+'_day.csv')
+        out_nc = os.path.join(outdir, name+'_day.nc')
     else:
-        out_csv = os.path.join(outdir, d2.attrs['station_id']+'_month.csv')
-        out_nc = os.path.join(outdir, d2.attrs['station_id']+'_month.nc')
+        out_csv = os.path.join(outdir, name+'_month.csv')
+        out_nc = os.path.join(outdir, name+'_month.nc')
     if not os.path.isdir(outdir):
         os.mkdir(outdir)
     # Write to csv file
@@ -150,7 +152,38 @@ def writeNC(outfile, Lx, col_names=None):
         names = list(Lx.keys())
     Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True)
 
-def getColNames(vars_df, booms=None, data_type=None, bedrock=False):
+def getColNames(vars_df, ds):
+    '''Get all variable names for a given data type, based on a variables
+    look-up table. This is mainly for exporting purposes
+
+    Parameters
+    ----------
+    vars_df : pd.DataFrame
+        Variables look-up table
+    ds: xr.dataset
+        Dataset to write
+    Returns
+    -------
+    list
+        Variable names
+    '''
+    if 'data_type' in ds.attrs.keys():
+        if ds.attrs['data_type']=='TX':
+            vars_df = vars_df.loc[vars_df['data_type'].isin(['TX','all'])]
+        elif ds.attrs['data_type']=='STM' or ds.attrs['data_type']=='raw':
+            vars_df = vars_df.loc[vars_df['data_type'].isin(['raw','all'])]
+
+    var_list = list(vars_df.index)
+    for v in var_list:
+         if v not in ds.keys():
+             var_list.remove(v)
+             continue
+         if ds[v].isnull().all():
+             var_list.remove(v)
+    return var_list
+
+
+def getColNames_old(vars_df, booms=None, data_type=None, bedrock=False):
     '''Get all variable names for a given data type, based on a variables
     look-up table. This is mainly for exporting purposes