diff --git a/DOSPORTAL/forms.py b/DOSPORTAL/forms.py index a4b3322..cd80b85 100644 --- a/DOSPORTAL/forms.py +++ b/DOSPORTAL/forms.py @@ -101,7 +101,7 @@ def __init__(self,*args, user=None, **kwargs): class Meta: model = Record exclude = ("time_end", "measurement", "log_original_filename", "metadata", "duration", "record_duration", "author", 'data_file', - "created", "detector") + "metadata_file", "created", "detector") class DetectorEditForm(forms.ModelForm): diff --git a/DOSPORTAL/migrations/0017_record_metadata_file.py b/DOSPORTAL/migrations/0017_record_metadata_file.py new file mode 100644 index 0000000..4a39925 --- /dev/null +++ b/DOSPORTAL/migrations/0017_record_metadata_file.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.11 on 2024-04-05 16:38 + +import DOSPORTAL.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('DOSPORTAL', '0016_alter_record_time_start'), + ] + + operations = [ + migrations.AddField( + model_name='record', + name='metadata_file', + field=models.FileField(blank=True, help_text='Processed metadata file', null=True, upload_to=DOSPORTAL.models.Record.user_directory_path_data, verbose_name='Metadata file'), + ), + ] diff --git a/DOSPORTAL/models.py b/DOSPORTAL/models.py index b8a25cc..42978cc 100644 --- a/DOSPORTAL/models.py +++ b/DOSPORTAL/models.py @@ -507,9 +507,9 @@ def user_directory_path(instance, filename): print("USER FILENAME", filename) return "user_records/record_{0}".format(instance.pk) - def user_directory_path_data(instance, filename): - print("USER FILENAME", filename) - return "user_records/record_{0}.pk".format(instance.pk) + def user_directory_path_data(instance, extension='pk'): + print("USER FILENAME", extension) + return "user_records/record_{0}.{1}".format(instance.pk, extension) log_file = models.FileField( verbose_name=_("File log"), @@ -538,6 +538,15 @@ def user_directory_path_data(instance, filename): blank=True ) + + metadata_file = models.FileField( + verbose_name=_("Metadata file"), + help_text=_("Processed metadata file"), + upload_to=user_directory_path_data, + null=True, + blank=True + ) + time_tracked = models.BooleanField( verbose_name = _("Is time tracked?"), default = False, diff --git a/DOSPORTAL/signals.py b/DOSPORTAL/signals.py index 833289d..2c7af36 100644 --- a/DOSPORTAL/signals.py +++ b/DOSPORTAL/signals.py @@ -84,9 +84,11 @@ def save_record(sender, instance, created = None, **kwargs): 'eeprom': parts[3].strip() } - df_spectrum = pd.read_csv(instance.log_file.path, sep = ',', header = None, names=range(max_size)) + df_log = pd.read_csv(instance.log_file.path, sep = ',', header = None, names=range(max_size)) - df_spectrum = df_spectrum [df_spectrum[0] == '$HIST'] + data_types = df_log[0].unique().tolist() + + df_spectrum = df_log [df_log[0] == '$HIST'] df_spectrum = df_spectrum.drop(columns=[0, 1, 3, 4, 5, 6, 7, 8]) new_columns = ['time'] + list(range(df_spectrum.shape[1] - 1)) @@ -94,11 +96,22 @@ def save_record(sender, instance, created = None, **kwargs): df_spectrum['time'] = df_spectrum['time'].astype(float) duration = df_spectrum['time'].max() - df_spectrum['time'].min() + + metadata['log_info'] = {} + metadata['log_info']['internat_time_min'] = df_spectrum['time'].min() + metadata['log_info']['internat_time_max'] = df_spectrum['time'].max() + metadata['log_info']['log_duration'] = float(duration) + metadata['log_info']['spectral_count'] = df_spectrum.shape[0] + metadata['log_info']['types'] = data_types + + df_spectrum['time'] = df_spectrum['time'] - df_spectrum['time'].min() instance.record_duration = datetime.timedelta(seconds=float(duration)) new_name = instance.user_directory_path_data('pk') df_spectrum.to_pickle('data/media/'+new_name) + del df_spectrum + instance.data_file.name = new_name try: @@ -110,6 +123,46 @@ def save_record(sender, instance, created = None, **kwargs): except Exception as e: print(e) + df_metadata = pd.DataFrame() + + try: + for index, row in df_log.iterrows(): + first_column_value = row[0] + row_as_list = row.tolist()[2:] + + match first_column_value: + case '$BAT': + keys = ['time', 'voltage', 'current', 'capacity_remaining', 'capacity_full', 'temperature'] + bat = { k:float(v) for (k,v) in zip(keys, row_as_list[0:len(keys)])} + bat['current'] /= 1000.0 + bat['voltage'] /= 1000.0 + df_metadata = pd.concat([df_metadata, pd.DataFrame([bat])], ignore_index=True) + del bat + case '$ENV': + keys = ['time', 'temperature_0', 'humidity_0', 'temperature_1', 'humidity_1', 'temperature_2', 'pressure_3'] + env = { k:float(v) for (k,v) in zip(keys, row_as_list[0:len(keys)])} + df_metadata = pd.concat([df_metadata, pd.DataFrame([env])], ignore_index=True) + del env + case '$HIST': + pass + case _: + print('Unknown row', first_column_value) + + print(df_metadata) + + df_metadata = df_metadata.sort_values(by=['time']).reset_index(drop=True) + df_metadata['time'] -= metadata['log_info']['internat_time_min'] + new_name = instance.user_directory_path_data('metadata.pk') + instance.metadata_file.name = new_name + print("NAME", new_name) + print(instance.metadata_file) + df_metadata.to_pickle('data/media/'+new_name) + + + except Exception as e: + print(e) + + print(instance.data_file)