creating folder before writing files, writing hourly daily monthly fi…

…les out in L2toL3, trying not to re-write sorted tx file if already sorted

src/pypromice/process/get_l2.py

@@ -57,6 +57,8 @@ def get_l2():
 
     # Write out level 2
     if args.outpath is not None:
+        if not os.pat.isdir(arg.outpath):
+            os.mkdir(arg.outpath)
         if aws.L2.attrs['format'] == 'raw':
             prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '10min')
         prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '60min')

src/pypromice/process/get_l2tol3.py

@@ -20,8 +20,6 @@ def parse_arguments_l2tol3(debug_args=None):
                         required=False, help='File path to variables look-up table')
     parser.add_argument('-m', '--metadata', default=None, type=str, 
                         required=False, help='File path to metadata')
-    parser.add_argument('-t', '--time', default='60min', type=str, 
-                        required=False, help='Time interval to resample dataset. The default is "60min"')
     parser.add_argument('-g', '--gcnet_historical', default=None, type=str, 
                         required=False, help='File path to historical GC-Net data file')
 
@@ -54,7 +52,9 @@ def get_l2tol3():
 
     # Write Level 3 dataset to file if output directory given
     if args.outpath is not None:
-        prepare_and_write(l3, args.outpath, v, m, args.time)
+        prepare_and_write(l3, args.outpath, v, m, '60min')
+        prepare_and_write(l3, args.outpath, v, m, 'D')
+        prepare_and_write(l3, args.outpath, v, m, 'M')
 
 if __name__ == "__main__":  
     get_l2tol3()

src/pypromice/process/write.py

@@ -68,7 +68,8 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min'):
     else:
         out_csv = os.path.join(outdir, d2.attrs['station_id']+'_month.csv')
         out_nc = os.path.join(outdir, d2.attrs['station_id']+'_month.nc')
-
+    if not os.path.isdir(outdir):
+        os.mkdir(outdir)
     # Write to csv file
     logger.info('Writing to files...')
     write.writeCSV(out_csv, d2, col_names)
@@ -183,4 +184,4 @@ def getColNames(vars_df, booms=None, data_type=None, bedrock=False):
                 col_names.remove(v)
             except:
                 pass
-    return col_names
+    return col_names

src/pypromice/tx/tx.py

@@ -886,13 +886,13 @@ def sortLines(in_file, out_file, replace_unsorted=True):                       #
         lines = in_f.readlines()
 
     # Remove duplicate lines and sort
-    unique_lines = findDuplicates(lines)
+    unique_lines = findDuplicates(lines.copy())
     unique_lines.sort()
-
-    # Write sorted file
-    with open(out_file, 'w') as out_f:
-        # out_f.write(headers)
-        out_f.writelines(unique_lines)
+    if lines != unique_lines:
+        # Write sorted file
+        with open(out_file, 'w') as out_f:
+            # out_f.write(headers)
+            out_f.writelines(unique_lines)
 
     # Replace input file with new sorted file
     if replace_unsorted: