Skip to content

Commit

Permalink
picke type as raw input
Browse files Browse the repository at this point in the history
  • Loading branch information
@ramintoosi
ramintoosi committed Jul 5, 2023
1 parent 2f2bc3f commit 2380c0a
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 43 deletions.
2 changes: 1 addition & 1 deletion ross_backend/resources/browse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ def get(self):
list_of_folders = [x for x in os.listdir(root) if
os.path.isdir(os.path.join(root, x)) and not x.startswith('.')]
list_of_files = [x for x in os.listdir(root) if os.path.isfile(os.path.join(root, x)) and not x.startswith('.')
and x.endswith(('.mat', '.csv', '.tdms'))]
and x.endswith(('.mat', '.pkl'))]
return {'folders': list_of_folders, 'files': list_of_files, 'root': root}
32 changes: 25 additions & 7 deletions ross_backend/rutils/io.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from uuid import uuid4

from scipy.io import loadmat
import numpy as np

Raw_data_path = os.path.join(Path(__file__).parent, '../ross_data/Raw_Data')
Path(Raw_data_path).mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -31,14 +32,31 @@ def read_file_in_server(request_data: dict):

temp = file_raw[variable].flatten()

# ------------------ save raw data as pkl file in data_set folder -----------------------
address = os.path.join(Raw_data_path, str(uuid4()) + '.pkl')
elif file_extension == '.pkl':
with open(filename, 'rb') as f:
file_raw = pickle.load(f)
variables = list(file_raw.keys())

if len(variables) > 1:
if 'varname' in request_data:
variable = request_data['varname']
else:
raise ValueError("More than one variable exists")
else:
variable = variables[0]

temp = np.array(file_raw[variable]).flatten()

with open(address, 'wb') as f:
pickle.dump(temp, f)
# ----------------------------------------------------------------------------------------
return address, temp
else:
raise TypeError("File not supported")
raise TypeError("Type not supported")

# ------------------ save raw data as pkl file in data_set folder -----------------------
address = os.path.join(Raw_data_path, str(uuid4()) + '.pkl')

with open(address, 'wb') as f:
pickle.dump(temp, f)
# ----------------------------------------------------------------------------------------
return address, temp

else:
raise ValueError("request data is incorrect")
90 changes: 55 additions & 35 deletions ross_ui/controller/mainWindow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def onImportRaw(self):
filename, filetype = QtWidgets.QFileDialog.getOpenFileName(self,
self.tr("Open file"),
os.getcwd(),
self.tr("Raw Files(*.mat *.csv *.tdms)")
self.tr("Raw Files(*.mat *.pkl)")
)

if not filename:
Expand All @@ -184,53 +184,73 @@ def onImportRaw(self):
else:
variable = variables[0]

# nd.array with shape (N,)
temp = file_raw[variable].flatten()

self.raw = temp
elif file_extension == '.pkl':
with open(filename, 'rb') as f:
file_raw = pickle.load(f)

# ------------------ save raw data as pkl file in data_set folder ---------------------------------------
address = os.path.join(self.Raw_data_path, str(uuid4()) + '.pkl')

with open(address, 'wb') as f:
pickle.dump(temp, f)

# -----------------------------------------------------------------------------------------------------

elif file_extension == '.csv':
df = pd.read_csv(filename, skiprows=1)
temp = df.to_numpy()
address = os.path.join(self.Raw_data_path, str(uuid4()) + '.pkl')
self.raw = temp
with open(address, 'wb') as f:
pickle.dump(temp, f)
elif file_extension == '.tdms':
tdms_file = TdmsFile.read(filename)
i = 0
for group in tdms_file.groups():
df = tdms_file.object(group).as_dataframe()
variables = list(df.keys())
i = i + 1
variables = list(file_raw.keys())

if len(variables) > 1:
variable = self.open_raw_dialog(variables)
if not variable:
self.statusBar().showMessage('')
self.statusBar().showMessage(self.tr("Nothing selected"))
return
else:
variable = variables[0]
# group = tdms_file['group name']
# channel = group['channel name']
# channel_data = channel[:]
# channel_properties = channel.properties
temp = np.array(df[variable]).flatten()
self.raw = temp

address = os.path.join(self.Raw_data_path, os.path.split(filename)[-1][:-5] + '.pkl')
with open(address, 'wb') as f:
pickle.dump(temp, f)
# nd.array with shape (N,)
temp = np.array(file_raw[variable]).flatten()

# elif file_extension == '.csv':
# df = pd.read_csv(filename, skiprows=1)
# temp = df.to_numpy()
# address = os.path.join(self.Raw_data_path, str(uuid4()) + '.pkl')
# self.raw = temp
# with open(address, 'wb') as f:
# pickle.dump(temp, f)
# elif file_extension == '.tdms':
# tdms_file = TdmsFile.read(filename)
# i = 0
# for group in tdms_file.groups():
# df = tdms_file.object(group).as_dataframe()
# variables = list(df.keys())
# i = i + 1
#
# if len(variables) > 1:
# variable = self.open_raw_dialog(variables)
# if not variable:
# self.statusBar().showMessage('')
# return
# else:
# variable = variables[0]
# # group = tdms_file['group name']
# # channel = group['channel name']
# # channel_data = channel[:]
# # channel_properties = channel.properties
# temp = np.array(df[variable]).flatten()
# self.raw = temp
#
# address = os.path.join(self.Raw_data_path, os.path.split(filename)[-1][:-5] + '.pkl')
# with open(address, 'wb') as f:
# pickle.dump(temp, f)

else:
raise TypeError(f'File type {file_extension} is not supported!')
QtWidgets.QMessageBox.critical(self, 'Error', 'Type is not supported')
return

# check tmp
if temp.ndim != 1:
QtWidgets.QMessageBox.critical(self, 'Error', 'Variable must be a vector')
return

self.raw = temp
address = os.path.join(self.Raw_data_path, str(uuid4()) + '.pkl')

with open(address, 'wb') as f:
pickle.dump(temp, f)

self.refreshAct.setEnabled(True)
self.statusBar().showMessage(self.tr("Successfully loaded file"), 2500)
Expand Down

0 comments on commit 2380c0a

Please sign in to comment.