From cd6afc8c57573d00e99c4d4365e6d0666af83c63 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Thu, 11 Nov 2021 23:14:05 +0000 Subject: [PATCH 01/19] created dialogs.py added browsing remote filesystem --- src/idvc/dialogs.py | 169 ++++++++++++++++++++++++++++++++++++++ src/idvc/dvc_interface.py | 168 ++++++++++++------------------------- 2 files changed, 221 insertions(+), 116 deletions(-) create mode 100644 src/idvc/dialogs.py diff --git a/src/idvc/dialogs.py b/src/idvc/dialogs.py new file mode 100644 index 00000000..3933c495 --- /dev/null +++ b/src/idvc/dialogs.py @@ -0,0 +1,169 @@ +from PySide2 import QtCore, QtGui, QtWidgets +from PySide2.QtWidgets import QCheckBox, QLabel, QDoubleSpinBox, QFrame, QVBoxLayout, QDialogButtonBox, QPushButton, QDialog +from PySide2.QtCore import Qt +import vtk +from brem.ui import RemoteServerSettingDialog +from eqt.ui import UIFormFactory + +class SettingsWindow(QDialog): + + def __init__(self, parent): + super(SettingsWindow, self).__init__(parent) + + self.parent = parent + + self.setWindowTitle("Settings") + + self.dark_checkbox = QCheckBox("Dark Mode") + + self.copy_files_checkbox = QCheckBox("Allow a copy of the image files to be stored. ") + self.vis_size_label = QLabel("Maximum downsampled image size (GB): ") + self.vis_size_entry = QDoubleSpinBox() + + self.vis_size_entry.setMaximum(64.0) + self.vis_size_entry.setMinimum(0.01) + self.vis_size_entry.setSingleStep(0.01) + + if self.parent.settings.value("vis_size") is not None: + self.vis_size_entry.setValue(float(self.parent.settings.value("vis_size"))) + + else: + self.vis_size_entry.setValue(1.0) + + + if self.parent.settings.value("dark_mode") is not None: + if self.parent.settings.value("dark_mode") == "true": + self.dark_checkbox.setChecked(True) + else: + self.dark_checkbox.setChecked(False) + else: + self.dark_checkbox.setChecked(True) + + separator = QFrame() + separator.setFrameShape(QFrame.HLine) + separator.setFrameShadow(QFrame.Raised) + self.adv_settings_label = QLabel("Advanced") + + + self.gpu_label = QLabel("Please set the size of your GPU memory.") + self.gpu_size_label = QLabel("GPU Memory (GB): ") + self.gpu_size_entry = QDoubleSpinBox() + + + if self.parent.settings.value("gpu_size") is not None: + self.gpu_size_entry.setValue(float(self.parent.settings.value("gpu_size"))) + + else: + self.gpu_size_entry.setValue(1.0) + + self.gpu_size_entry.setMaximum(64.0) + self.gpu_size_entry.setMinimum(0.00) + self.gpu_size_entry.setSingleStep(0.01) + self.gpu_checkbox = QCheckBox("Use GPU for volume render. (Recommended) ") + self.gpu_checkbox.setChecked(True) #gpu is default + if self.parent.settings.value("volume_mapper") == "cpu": + self.gpu_checkbox.setChecked(False) + + if hasattr(self.parent, 'copy_files'): + self.copy_files_checkbox.setChecked(self.parent.copy_files) + + self.layout = QVBoxLayout(self) + self.layout.addWidget(self.dark_checkbox) + self.layout.addWidget(self.copy_files_checkbox) + self.layout.addWidget(self.vis_size_label) + self.layout.addWidget(self.vis_size_entry) + self.layout.addWidget(separator) + self.layout.addWidget(self.adv_settings_label) + self.layout.addWidget(self.gpu_checkbox) + self.layout.addWidget(self.gpu_label) + self.layout.addWidget(self.gpu_size_label) + self.layout.addWidget(self.gpu_size_entry) + + # configure remote server settings + remote_separator = QFrame() + remote_separator.setFrameShape(QFrame.HLine) + remote_separator.setFrameShadow(QFrame.Raised) + fw = UIFormFactory.getQWidget(parent=self) + + self.remote_button_entry = QPushButton(self) + self.remote_button_entry.setText("Open Preferences") + self.remote_button_entry.clicked.connect(self.openConfigRemote) + fw.addWidget(self.remote_button_entry, 'Configure remote settings', 'remote_preferences') + fw.addWidget(QCheckBox(self), 'Connect to remote server', 'connect_to_remote') + self.fw = fw + for k,v in fw.widgets.items(): + print ("fw", k) + # add to layout + self.layout.addWidget(remote_separator) + self.layout.addWidget(fw) + + self.buttons = QDialogButtonBox( + QDialogButtonBox.Save | QDialogButtonBox.Cancel, + Qt.Horizontal, self) + self.layout.addWidget(self.buttons) + self.buttons.accepted.connect(self.accept) + self.buttons.rejected.connect(self.quit) + + def accept(self): + #self.parent.settings.setValue("settings_chosen", 1) + if self.dark_checkbox.isChecked(): + self.parent.settings.setValue("dark_mode", True) + else: + self.parent.settings.setValue("dark_mode", False) + self.parent.SetAppStyle() + + if self.copy_files_checkbox.isChecked(): + self.parent.copy_files = 1 # save for this session + self.parent.settings.setValue("copy_files", 1) #save for next time we open app + else: + self.parent.copy_files = 0 + self.parent.settings.setValue("copy_files", 0) + + if self.gpu_checkbox.isChecked(): + self.parent.settings.setValue("volume_mapper", "gpu") + self.parent.vis_widget_3D.volume_mapper = vtk.vtkSmartVolumeMapper() + else: + self.parent.settings.setValue("volume_mapper", "cpu") + + self.parent.settings.setValue("gpu_size", float(self.gpu_size_entry.value())) + self.parent.settings.setValue("vis_size", float(self.vis_size_entry.value())) + + if self.parent.settings.value("first_app_load") != "False": + self.parent.CreateSessionSelector("new window") + self.parent.settings.setValue("first_app_load", "False") + + # if remote is checked + statusBar = self.parent.statusBar() + if self.fw.widgets['connect_to_remote_field'].isChecked(): + self.parent.connection_details = self.connection_details + statusBar.showMessage("Connected to {}@{}:{}".format( + self.connection_details['username'], + self.connection_details['server_name'], + self.connection_details['server_port']) + ) + else: + statusBar.clearMessage() + self.parent.connection_details = None + self.close() + + + #print(self.parent.settings.value("copy_files")) + def quit(self): + if self.parent.settings.value("first_app_load") != "False": + self.parent.CreateSessionSelector("new window") + self.parent.settings.setValue("first_app_load", "False") + self.close() + + def openConfigRemote(self): + + dialog = RemoteServerSettingDialog(self,port=None, + host=None, + username=None, + private_key=None) + dialog.Ok.clicked.connect(lambda: self.getConnectionDetails(dialog)) + dialog.exec() + + def getConnectionDetails(self, dialog): + for k,v in dialog.connection_details.items(): + print (k,v) + self.connection_details = dialog.connection_details \ No newline at end of file diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 78502fbb..8d32d609 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -79,6 +79,10 @@ from qdarkstyle.light.palette import LightPalette from idvc import version as gui_version +from idvc.dialogs import SettingsWindow + +from brem.ui import RemoteFileDialog +from brem import AsyncCopyFromSSH __version__ = gui_version.version @@ -223,6 +227,7 @@ def InitialiseSessionVars(self): self.dvc_input_image_in_session_folder = False if hasattr(self, 'ref_image_data'): del self.ref_image_data + self.connection_details = None #Loading the DockWidgets: @@ -499,7 +504,54 @@ def view_and_load_images(self): self.view_image() self.resetRegistration() + def SelectImage(self, image_var, image, label=None, next_button=None): + if self.connection_details is None: + return self.SelectImageLocal(image_var, image, label, next_button) + else: + return self.SelectImageRemote(image_var, image, label, next_button) + + def SelectImageRemote(self, image_var, image, label=None, next_button=None): + # start the RemoteFileBrowser + logfile = os.path.join(os.getcwd(), "RemoteFileDialog.log") + dialog = RemoteFileDialog(self, logfile=logfile, port=self.connection_details['server_port'], + host=self.connection_details['server_name'], + username=self.connection_details['username'], + private_key=self.connection_details['private_key'], + remote_os=self.connection_details['remote_os']) + dialog.Ok.clicked.connect(lambda: self.getSelected(dialog)) + if hasattr(self, 'files_to_get'): + try: + dialog.widgets['lineEdit'].setText(self.files_to_get[0][0]) + except: + pass + dialog.exec() + def getSelected(self, dialog): + if hasattr(dialog, 'selected'): + print (type(dialog.selected)) + for el in dialog.selected: + print ("Return from dialogue", el) + self.files_to_get = list (dialog.selected) + def ResampleAndGetFileFromRemote(self): + # 1 download self.files_to_get + if len(self.files_to_get) == 1: + self.asyncCopy = AsyncCopyFromSSH() + if not hasattr(self, 'connection_details'): + self.statusBar().showMessage("define the connection") + return + username = self.connection_details['username'] + port = self.connection_details['server_port'] + host = self.connection_details['server_name'] + private_key = self.connection_details['private_key'] + + self.asyncCopy.setRemoteConnectionSettings(username=username, + port=port, host=host, private_key=private_key) + self.asyncCopy.SetRemoteFileName(dirname=self.files_to_get[0][0], filename=self.files_to_get[0][1]) + self.asyncCopy.SetDestinationDir(os.path.abspath(self.tempdir.name)) + self.asyncCopy.signals.finished.connect(lambda: self.visualise()) + self.asyncCopy.GetFile() + + def SelectImageLocal(self, image_var, image, label=None, next_button=None): #print("In select image") dialogue = QFileDialog() files = dialogue.getOpenFileNames(self,"Load Images")[0] @@ -5021,123 +5073,7 @@ def progress(self, value): -class SettingsWindow(QDialog): - - def __init__(self, parent): - super(SettingsWindow, self).__init__(parent) - - self.parent = parent - self.setWindowTitle("Settings") - - self.dark_checkbox = QCheckBox("Dark Mode") - - self.copy_files_checkbox = QCheckBox("Allow a copy of the image files to be stored. ") - self.vis_size_label = QLabel("Maximum downsampled image size (GB): ") - self.vis_size_entry = QDoubleSpinBox() - - self.vis_size_entry.setMaximum(64.0) - self.vis_size_entry.setMinimum(0.01) - self.vis_size_entry.setSingleStep(0.01) - - if self.parent.settings.value("vis_size") is not None: - self.vis_size_entry.setValue(float(self.parent.settings.value("vis_size"))) - - else: - self.vis_size_entry.setValue(1.0) - - - if self.parent.settings.value("dark_mode") is not None: - if self.parent.settings.value("dark_mode") == "true": - self.dark_checkbox.setChecked(True) - else: - self.dark_checkbox.setChecked(False) - else: - self.dark_checkbox.setChecked(True) - - separator = QFrame() - separator.setFrameShape(QFrame.HLine) - separator.setFrameShadow(QFrame.Raised) - self.adv_settings_label = QLabel("Advanced") - - - self.gpu_label = QLabel("Please set the size of your GPU memory.") - self.gpu_size_label = QLabel("GPU Memory (GB): ") - self.gpu_size_entry = QDoubleSpinBox() - - - if self.parent.settings.value("gpu_size") is not None: - self.gpu_size_entry.setValue(float(self.parent.settings.value("gpu_size"))) - - else: - self.gpu_size_entry.setValue(1.0) - - self.gpu_size_entry.setMaximum(64.0) - self.gpu_size_entry.setMinimum(0.00) - self.gpu_size_entry.setSingleStep(0.01) - self.gpu_checkbox = QCheckBox("Use GPU for volume render. (Recommended) ") - self.gpu_checkbox.setChecked(True) #gpu is default - if self.parent.settings.value("volume_mapper") == "cpu": - self.gpu_checkbox.setChecked(False) - - if hasattr(self.parent, 'copy_files'): - self.copy_files_checkbox.setChecked(self.parent.copy_files) - - self.layout = QVBoxLayout(self) - self.layout.addWidget(self.dark_checkbox) - self.layout.addWidget(self.copy_files_checkbox) - self.layout.addWidget(self.vis_size_label) - self.layout.addWidget(self.vis_size_entry) - self.layout.addWidget(separator) - self.layout.addWidget(self.adv_settings_label) - self.layout.addWidget(self.gpu_checkbox) - self.layout.addWidget(self.gpu_label) - self.layout.addWidget(self.gpu_size_label) - self.layout.addWidget(self.gpu_size_entry) - self.buttons = QDialogButtonBox( - QDialogButtonBox.Save | QDialogButtonBox.Cancel, - Qt.Horizontal, self) - self.layout.addWidget(self.buttons) - self.buttons.accepted.connect(self.accept) - self.buttons.rejected.connect(self.quit) - - def accept(self): - #self.parent.settings.setValue("settings_chosen", 1) - if self.dark_checkbox.isChecked(): - self.parent.settings.setValue("dark_mode", True) - else: - self.parent.settings.setValue("dark_mode", False) - self.parent.SetAppStyle() - - if self.copy_files_checkbox.isChecked(): - self.parent.copy_files = 1 # save for this session - self.parent.settings.setValue("copy_files", 1) #save for next time we open app - else: - self.parent.copy_files = 0 - self.parent.settings.setValue("copy_files", 0) - - if self.gpu_checkbox.isChecked(): - self.parent.settings.setValue("volume_mapper", "gpu") - self.parent.vis_widget_3D.volume_mapper = vtk.vtkSmartVolumeMapper() - else: - self.parent.settings.setValue("volume_mapper", "cpu") - - self.parent.settings.setValue("gpu_size", float(self.gpu_size_entry.value())) - self.parent.settings.setValue("vis_size", float(self.vis_size_entry.value())) - - if self.parent.settings.value("first_app_load") != "False": - self.parent.CreateSessionSelector("new window") - self.parent.settings.setValue("first_app_load", "False") - - self.close() - - - #print(self.parent.settings.value("copy_files")) - def quit(self): - if self.parent.settings.value("first_app_load") != "False": - self.parent.CreateSessionSelector("new window") - self.parent.settings.setValue("first_app_load", "False") - self.close() From 2f963dbe6db094f620fac77d3b73566e59816f91 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Thu, 11 Nov 2021 23:29:45 +0000 Subject: [PATCH 02/19] reset the remote settings checkbox --- src/idvc/dialogs.py | 4 +++- src/idvc/dvc_interface.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/idvc/dialogs.py b/src/idvc/dialogs.py index 3933c495..4221cae8 100644 --- a/src/idvc/dialogs.py +++ b/src/idvc/dialogs.py @@ -89,7 +89,9 @@ def __init__(self, parent): self.remote_button_entry.setText("Open Preferences") self.remote_button_entry.clicked.connect(self.openConfigRemote) fw.addWidget(self.remote_button_entry, 'Configure remote settings', 'remote_preferences') - fw.addWidget(QCheckBox(self), 'Connect to remote server', 'connect_to_remote') + cb = QCheckBox(self) + cb.setChecked(self.parent.connection_details is not None) + fw.addWidget(cb, 'Connect to remote server', 'connect_to_remote') self.fw = fw for k,v in fw.widgets.items(): print ("fw", k) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 8d32d609..4aaae121 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -513,7 +513,7 @@ def SelectImage(self, image_var, image, label=None, next_button=None): def SelectImageRemote(self, image_var, image, label=None, next_button=None): # start the RemoteFileBrowser - logfile = os.path.join(os.getcwd(), "RemoteFileDialog.log") + logfile = os.path.join(os.getcwd(), '..','..',"RemoteFileDialog.log") dialog = RemoteFileDialog(self, logfile=logfile, port=self.connection_details['server_port'], host=self.connection_details['server_name'], username=self.connection_details['username'], From 4e9d2004a04357c8d88e0494c80a411aaa111890 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Tue, 30 Nov 2021 14:24:59 +0000 Subject: [PATCH 03/19] allows copy from the remote to local --- recipe/meta.yaml | 1 + src/idvc/dvc_interface.py | 193 ++++++++++++++++++++++++-------------- 2 files changed, 124 insertions(+), 70 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index ad233525..37b6523f 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -48,6 +48,7 @@ requirements: - matplotlib - openmp # [osx] - qdarkstyle + - brem about: home: http://www.ccpi.ac.uk diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 4aaae121..761c9284 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -55,13 +55,9 @@ import json import shutil import zipfile -import zlib - -import csv +from time import sleep from functools import reduce -import subprocess - import copy from distutils.dir_util import copy_tree @@ -82,7 +78,7 @@ from idvc.dialogs import SettingsWindow from brem.ui import RemoteFileDialog -from brem import AsyncCopyFromSSH +from brem import AsyncCopyOverSSH __version__ = gui_version.version @@ -190,9 +186,9 @@ def CreateWorkingTempFolder(self): temp_folder = os.path.join(working_directory, "DVC_Sessions") self.temp_folder = os.path.abspath(temp_folder) - tempfile.tempdir = tempfile.mkdtemp(dir = self.temp_folder) + self.tempdir = tempfile.mkdtemp(dir = self.temp_folder) - os.chdir(tempfile.tempdir) + os.chdir(self.tempdir) # Creates folder in tempdir to save mask files in os.mkdir("Masks") @@ -519,23 +515,34 @@ def SelectImageRemote(self, image_var, image, label=None, next_button=None): username=self.connection_details['username'], private_key=self.connection_details['private_key'], remote_os=self.connection_details['remote_os']) - dialog.Ok.clicked.connect(lambda: self.getSelected(dialog)) + dialog.Ok.clicked.connect( + lambda: self.getSelectedDownloadAndUpdateUI(dialog, image_var, image, label, next_button) + ) if hasattr(self, 'files_to_get'): try: dialog.widgets['lineEdit'].setText(self.files_to_get[0][0]) except: pass dialog.exec() - def getSelected(self, dialog): + def getSelectedDownloadAndUpdateUI(self, dialog, image_var, image, label, next_button): if hasattr(dialog, 'selected'): print (type(dialog.selected)) for el in dialog.selected: print ("Return from dialogue", el) self.files_to_get = list (dialog.selected) - def ResampleAndGetFileFromRemote(self): + if len(self.files_to_get) == 1: + self.GetFileFromRemote(image_var, image, label, next_button) + else: + self.warningDialog("Sorry, currently we can only get one file.", + "Error: cannot handle multiple files") + + + + + def GetFileFromRemote(self, image_var, image, label, next_button): # 1 download self.files_to_get if len(self.files_to_get) == 1: - self.asyncCopy = AsyncCopyFromSSH() + self.asyncCopy = AsyncCopyOverSSH() if not hasattr(self, 'connection_details'): self.statusBar().showMessage("define the connection") return @@ -546,16 +553,62 @@ def ResampleAndGetFileFromRemote(self): self.asyncCopy.setRemoteConnectionSettings(username=username, port=port, host=host, private_key=private_key) - self.asyncCopy.SetRemoteFileName(dirname=self.files_to_get[0][0], filename=self.files_to_get[0][1]) - self.asyncCopy.SetDestinationDir(os.path.abspath(self.tempdir.name)) - self.asyncCopy.signals.finished.connect(lambda: self.visualise()) - self.asyncCopy.GetFile() + + + + files = [os.path.join(self.tempdir, self.files_to_get[0][1])] + remotepath = self.asyncCopy.remotepath.join(self.files_to_get[0][0], self.files_to_get[0][1]) + + # this shouldn't be necessary, however the signals and the workers are created before the async copy + # object is created and then the local dir is not set in the worker. + self.asyncCopy.SetRemoteDir(self.files_to_get[0][0]) + self.asyncCopy.SetCopyFromRemote() + self.asyncCopy.SetLocalDir(self.tempdir) + self.asyncCopy.SetRemoteDir(self.asyncCopy.remotepath.dirname(remotepath)) + self.asyncCopy.SetFileName(self.asyncCopy.remotepath.basename(remotepath)) + + self.asyncCopy.signals.finished.connect( + lambda: self._UpdateSelectFileUI(files, image_var, image, label, next_button) + ) + # this should also not be done like this. + self.asyncCopy.threadpool.start(self.asyncCopy.worker) + + sleep(1) + + self.updateUnknownProgressDialog = Worker(self.UnknownProgressUpdateDialog) + self.updateUnknownProgressDialog.signals.finished.connect(self.StopUnknownProgressUpdate) + self.threadpool.start(self.updateUnknownProgressDialog) + + + # self.asyncCopy.GetFile( + # self.asyncCopy.remotepath.join(self.files_to_get[0][0], self.files_to_get[0][1]), + # self.tempdir + # ) + + def UnknownProgressUpdateDialog(self, **kwargs): + + t0 = time.time() + while True: + tc = self.asyncCopy.threadpool.activeThreadCount() + + if tc == 0: + break + # print (tc) + sleep(1) + self.statusBar().showMessage("Copying {} ... {}s".format(self.files_to_get[0][1], time.time()-t0)) + + + def StopUnknownProgressUpdate(self): + self.statusBar().showMessage("File copied.") + def SelectImageLocal(self, image_var, image, label=None, next_button=None): #print("In select image") dialogue = QFileDialog() files = dialogue.getOpenFileNames(self,"Load Images")[0] + self._UpdateSelectFileUI(files, image_var, image, label, next_button) + def _UpdateSelectFileUI(self, files, image_var, image, label=None, next_button=None): if len(files) > 0: if self.copy_files: self.image_copied[image_var] = True @@ -1575,13 +1628,13 @@ def LoadImagesAndCompleteRegistration(self): #print("About to create image") self.unsampled_ref_image_data = vtk.vtkImageData() ImageDataCreator.createImageData(self, self.image[0], self.unsampled_ref_image_data, info_var=self.unsampled_image_info, crop_image=True, origin=origin, - target_z_extent=target_z_extent, output_dir=os.path.abspath(tempfile.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) + target_z_extent=target_z_extent, output_dir=os.path.abspath(self.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) #TODO: move to doing both image data creators simultaneously - would this work? return if previous_reg_box_extent != reg_box_extent: ImageDataCreator.createImageData(self, self.image[0], self.unsampled_ref_image_data, info_var=self.unsampled_image_info, crop_image=True, origin=origin, - target_z_extent=target_z_extent, output_dir=os.path.abspath(tempfile.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) + target_z_extent=target_z_extent, output_dir=os.path.abspath(self.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) else: self.completeRegistration() @@ -1599,7 +1652,7 @@ def LoadCorrImageForReg(self,resample_corr_image= False, crop_corr_image = False self.unsampled_corr_image_data = vtk.vtkImageData() ImageDataCreator.createImageData(self, self.image[1], self.unsampled_corr_image_data, info_var=self.unsampled_image_info, resample=resample_corr_image, - crop_image=crop_corr_image, origin=origin, target_z_extent=z_extent, finish_fn=self.completeRegistration, output_dir=os.path.abspath(tempfile.tempdir)) + crop_image=crop_corr_image, origin=origin, target_z_extent=z_extent, finish_fn=self.completeRegistration, output_dir=os.path.abspath(self.tempdir)) def completeRegistration(self): self.updatePoint0Display() @@ -2163,7 +2216,7 @@ def select_mask(self): if mask: if ".mha" in mask: filename = os.path.basename(mask) - shutil.copyfile(mask, os.path.join(tempfile.tempdir, "Masks", filename)) + shutil.copyfile(mask, os.path.join(self.tempdir, "Masks", filename)) self.mask_parameters["masksList"].addItem(filename) self.mask_parameters["masksList"].setCurrentText(filename) self.clearMask() @@ -2698,8 +2751,8 @@ def select_pointcloud(self): #, label): if self.copy_files: filename = os.path.basename(self.roi) - shutil.copyfile(self.roi, os.path.join(tempfile.tempdir, filename)) - self.roi = os.path.abspath(os.path.join(tempfile.tempdir, filename)) + shutil.copyfile(self.roi, os.path.join(self.tempdir, filename)) + self.roi = os.path.abspath(os.path.join(self.tempdir, filename)) self.pointcloud_parameters['pointcloudList'].addItem(filename) self.pointcloud_parameters['pointcloudList'].setCurrentText(filename) @@ -2712,7 +2765,7 @@ def PointCloudWorker(self, type, filename = None, disp_file = None, vector_dim = self.pointcloud_worker.signals.finished.connect(self.DisplayPointCloud) elif type == "load selection": self.clearPointCloud() - self.pointcloud_worker = Worker(self.loadPointCloud, os.path.join(tempfile.tempdir, self.pointcloud_parameters['pointcloudList'].currentText())) + self.pointcloud_worker = Worker(self.loadPointCloud, os.path.join(self.tempdir, self.pointcloud_parameters['pointcloudList'].currentText())) self.pointcloud_worker.signals.finished.connect(self.DisplayLoadedPointCloud) elif type == "load pointcloud file": self.clearPointCloud() @@ -2727,7 +2780,7 @@ def PointCloudWorker(self, type, filename = None, disp_file = None, vector_dim = self.create_progress_window("Loading", "Loading Pointcloud") self.pointcloud_worker.signals.progress.connect(self.progress) self.progress_window.setValue(10) - os.chdir(tempfile.tempdir) + os.chdir(self.tempdir) self.threadpool.start(self.pointcloud_worker) # Show error and allow re-selection of pointcloud if it can't be loaded: @@ -3028,7 +3081,7 @@ def createPointCloud(self, **kwargs): array.append((count, *pp)) count += 1 - np.savetxt(tempfile.tempdir + "/" + filename, array, '%d\t%.3f\t%.3f\t%.3f', delimiter=';') + np.savetxt(self.tempdir + "/" + filename, array, '%d\t%.3f\t%.3f\t%.3f', delimiter=';') self.roi = filename return(True) @@ -3884,7 +3937,7 @@ def create_config_worker(self): folder_name = "_" + self.rdvc_widgets['name_entry'].text() - results_folder = os.path.join(tempfile.tempdir, "Results") + results_folder = os.path.join(self.tempdir, "Results") new_folder = os.path.join(results_folder, folder_name) @@ -3903,12 +3956,12 @@ def create_config_worker(self): def create_run_config(self, **kwargs): - os.chdir(tempfile.tempdir) + os.chdir(self.tempdir) progress_callback = kwargs.get('progress_callback', None) try: folder_name = "_" + self.rdvc_widgets['name_entry'].text() - results_folder = os.path.join(tempfile.tempdir, "Results") + results_folder = os.path.join(self.tempdir, "Results") os.mkdir(os.path.join(results_folder, folder_name)) if self.singleRun_groupBox.isVisible(): @@ -4012,7 +4065,7 @@ def create_run_config(self, **kwargs): run_config['point0'] = self.getPoint0ImageCoords() suffix_text = "run_config" - self.run_config_file = os.path.join(tempfile.tempdir, "Results", folder_name, "_" + suffix_text + ".json") + self.run_config_file = os.path.join(self.tempdir, "Results", folder_name, "_" + suffix_text + ".json") with open(self.run_config_file, "w+") as tmp: json.dump(run_config, tmp) @@ -4054,7 +4107,7 @@ def run_external_code(self, error = None): # this command will call DVC_runner to create the directories self.dvc_runner = DVC_runner(self, os.path.abspath(self.run_config_file), - self.finished_run, self.run_succeeded, tempfile.tempdir) + self.finished_run, self.run_succeeded, self.tempdir) self.dvc_runner.run_dvc() @@ -4190,7 +4243,7 @@ def show_run_pcs(self): self.result_widgets['pc_entry'].clear() self.result_widgets['subvol_entry'].clear() - directory = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + directory = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) self.results_folder = directory file_list=[] @@ -4236,7 +4289,7 @@ def LoadResultsOnViewer(self): self.warningDialog("An error occurred with this run so the results could not be displayed.", "Error") else: - results_folder = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + results_folder = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) self.roi = os.path.join(results_folder ,"_" + str(subvol_size) + ".roi") #print("New roi is", self.roi) self.results_folder = results_folder @@ -4267,7 +4320,7 @@ def LoadResultsOnViewer(self): def CreateGraphsWindow(self): #print("Create graphs") if self.result_widgets['run_entry'].currentText() is not "": - self.results_folder = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + self.results_folder = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) else: self.results_folder = None @@ -4403,11 +4456,11 @@ def SaveSession(self, text_value, compress, event): self.config['dvc_input_image_in_session_folder'] = self.dvc_input_image_in_session_folder # print("ROI: ", self.roi) - # print("temp", tempfile.tempdir) + # print("temp", self.tempdir) if (self.roi): self.roi = os.path.abspath(self.roi) - if os.path.abspath(tempfile.tempdir) in self.roi: - self.config['roi_file'] = self.roi[len(os.path.abspath(tempfile.tempdir))+1:] + if os.path.abspath(self.tempdir) in self.roi: + self.config['roi_file'] = self.roi[len(os.path.abspath(self.tempdir))+1:] self.config['roi_ext'] = False else: self.config['roi_file'] = self.roi @@ -4419,8 +4472,8 @@ def SaveSession(self, text_value, compress, event): if hasattr(self, 'mask_file'): self.config['mask_details']=self.mask_details - if tempfile.tempdir in os.path.abspath(self.mask_file): - self.config['mask_file']=self.mask_file[len(os.path.abspath(tempfile.tempdir))+1:] + if self.tempdir in os.path.abspath(self.mask_file): + self.config['mask_file']=self.mask_file[len(os.path.abspath(self.tempdir))+1:] self.config['mask_ext'] = False else: self.config['mask_file']=self.mask_file @@ -4491,10 +4544,10 @@ def SaveSession(self, text_value, compress, event): suffix_text = "_" + user_string + "_" + now_string os.chdir(self.temp_folder) - tempdir = shutil.move(tempfile.tempdir, suffix_text) - tempfile.tempdir = os.path.abspath(tempdir) + tempdir = shutil.move(self.tempdir, suffix_text) + self.tempdir = os.path.abspath(tempdir) - fd, f = tempfile.mkstemp(suffix=suffix_text + ".json", dir = tempfile.tempdir) #could not delete this using rmtree? + fd, f = tempfile.mkstemp(suffix=suffix_text + ".json", dir = self.tempdir) #could not delete this using rmtree? with open(f, "w+") as tmp: json.dump(self.config, tmp) @@ -4504,7 +4557,7 @@ def SaveSession(self, text_value, compress, event): self.create_progress_window("Saving","Saving") - zip_worker = Worker(self.ZipDirectory, tempfile.tempdir, compress) + zip_worker = Worker(self.ZipDirectory, self.tempdir, compress) if type(event) == QCloseEvent: zip_worker.signals.finished.connect(lambda: self.RemoveTemp(event)) else: @@ -4513,47 +4566,47 @@ def SaveSession(self, text_value, compress, event): self.threadpool.start(zip_worker) if compress: - self.ShowZipProgress(tempfile.tempdir, tempfile.tempdir +'.zip', 0.7) + self.ShowZipProgress(self.tempdir, self.tempdir +'.zip', 0.7) else: - self.ShowZipProgress(tempfile.tempdir, tempfile.tempdir +'.zip', 1) + self.ShowZipProgress(self.tempdir, self.tempdir +'.zip', 1) #give variables filepath including new name of temp folder: - # print("temp", tempfile.tempdir) + # print("temp", self.tempdir) # print("roi ext", self.config['roi_ext']) # print(self.roi, self.config['roi_file']) if (self.roi and not self.config['roi_ext']): - self.roi = os.path.join(os.path.abspath(tempfile.tempdir), self.config['roi_file']) + self.roi = os.path.join(os.path.abspath(self.tempdir), self.config['roi_file']) #print(self.roi) if hasattr(self, 'mask_file'): if 'mask_file' in self.config: - self.mask_file = os.path.join(os.path.abspath(tempfile.tempdir), self.config['mask_file']) + self.mask_file = os.path.join(os.path.abspath(self.tempdir), self.config['mask_file']) count = 0 for i in self.image[0]: if self.image_copied[0]: - # print(os.path.join(os.path.abspath(tempfile.tempdir), self.config['image'][0][count]) ) - self.image[0][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['image'][0][count]) + # print(os.path.join(os.path.abspath(self.tempdir), self.config['image'][0][count]) ) + self.image[0][count] = os.path.join(os.path.abspath(self.tempdir), self.config['image'][0][count]) count +=1 count = 0 for j in self.image[1]: if self.image_copied[1]: - self.image[1][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['image'][1][count]) + self.image[1][count] = os.path.join(os.path.abspath(self.tempdir), self.config['image'][1][count]) count += 1 count = 0 for i in self.dvc_input_image[0]: if self.image_copied[0] or self.dvc_input_image_in_session_folder: - self.dvc_input_image[0][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['dvc_input_image'][0][count]) + self.dvc_input_image[0][count] = os.path.join(os.path.abspath(self.tempdir), self.config['dvc_input_image'][0][count]) count+=1 count=0 for j in self.dvc_input_image[1]: if self.image_copied[1] or self.dvc_input_image_in_session_folder: - self.dvc_input_image[1][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['dvc_input_image'][1][count]) + self.dvc_input_image[1][count] = os.path.join(os.path.abspath(self.tempdir), self.config['dvc_input_image'][1][count]) count+=1 - results_folder = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + results_folder = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) self.results_folder = results_folder def CloseSaveWindow(self): @@ -4588,8 +4641,8 @@ def RemoveTemp(self, event): self.progress_window.setLabelText("Closing") self.progress_window.setMaximum(100) self.progress_window.setValue(98) - #print("removed temp", tempfile.tempdir) - shutil.rmtree(tempfile.tempdir) + #print("removed temp", self.tempdir) + shutil.rmtree(self.tempdir) if hasattr(self, 'progress_window'): self.progress_window.setValue(100) @@ -4670,11 +4723,11 @@ def ExportSession(self): export_worker = Worker(self.exporter, export_location) export_worker.signals.finished.connect(self.progress_complete) self.threadpool.start(export_worker) - self.ShowExportProgress(tempfile.tempdir, export_location) + self.ShowExportProgress(self.tempdir, export_location) def exporter(self, *args, **kwargs): export_location = args[0] - shutil.copytree(tempfile.tempdir, export_location) + shutil.copytree(self.tempdir, export_location) #Dealing with loading sessions: @@ -4786,17 +4839,17 @@ def LoadConfigWorker(self, **kwargs): if not results_folder_exists: os.mkdir(os.path.join(loaded_tempdir, "Results")) - #print(tempfile.tempdir) + #print(self.tempdir) - # if tempfile.tempdir != loaded_tempdir: # if we are not loading the same session that we already had open - # shutil.rmtree(tempfile.tempdir) + # if self.tempdir != loaded_tempdir: # if we are not loading the same session that we already had open + # shutil.rmtree(self.tempdir) if progress_callback is not None: progress_callback.emit(90) - tempfile.tempdir = loaded_tempdir + self.tempdir = loaded_tempdir #print("working tempdir") - #print(tempfile.tempdir) + #print(self.tempdir) #selected_file = "" json_filename = date_and_time + ".json" @@ -4814,7 +4867,7 @@ def LoadConfigWorker(self, **kwargs): progress_callback.emit(100) def LoadSession(self): - os.chdir(tempfile.tempdir) + os.chdir(self.tempdir) self.resetRegistration() self.loading_session = True @@ -4836,12 +4889,12 @@ def LoadSession(self): # if 'roi_ext' in self.config: # if not self.config['roi_ext']: # self.roi = os.path.abspath( - # os.path.join(tempfile.tempdir, self.roi)) + # os.path.join(self.tempdir, self.roi)) #pointcloud files could still exist even if there wasn't a pointcloud displayed when the session was saved. pointcloud_files = [] #get list of pointcloud files: - for r, d, f in os.walk(tempfile.tempdir): + for r, d, f in os.walk(self.tempdir): for _file in f: if '.roi' in _file: pointcloud_files.append(_file) @@ -4883,7 +4936,7 @@ def LoadSession(self): #save paths to images to variable if self.config['image_copied'][j]: - path = os.path.abspath(os.path.join(tempfile.tempdir, i)) + path = os.path.abspath(os.path.join(self.tempdir, i)) # print("The DVC input path is") # print(path) self.dvc_input_image[j].append(path) @@ -5004,7 +5057,7 @@ def LoadSession(self): if 'mask_file' in self.config: self.mask_parameters['loadButton'].setEnabled(True) - mask_folder = os.path.join(tempfile.tempdir, "Masks") + mask_folder = os.path.join(self.tempdir, "Masks") mask_files = [] #get list of mask files: #print(mask_folder) @@ -5021,7 +5074,7 @@ def LoadSession(self): self.mask_parameters['loadButton'].setEnabled(False) - results_directory = os.path.join(tempfile.tempdir, "Results") + results_directory = os.path.join(self.tempdir, "Results") for i in range(self.result_widgets['run_entry'].count()): self.result_widgets['run_entry'].removeItem(i) @@ -5126,7 +5179,7 @@ def save(self, save_only): #Load Saved Session #print("Write mask to file, then carry on") filename = self.textbox.text() + ".mha" - shutil.copyfile(os.path.join(tempfile.tempdir, self.parent.mask_file), os.path.join(tempfile.tempdir, "Masks", filename)) + shutil.copyfile(os.path.join(self.tempdir, self.parent.mask_file), os.path.join(self.tempdir, "Masks", filename)) self.parent.mask_parameters['masksList'].addItem(filename) self.parent.mask_details[filename] = self.parent.mask_details['current'] #print(self.parent.mask_details) @@ -5147,7 +5200,7 @@ def save(self, save_only): if self.object == "pointcloud": filename = self.textbox.text() + ".roi" - shutil.copyfile(os.path.join(tempfile.tempdir, "latest_pointcloud.roi"), os.path.join(tempfile.tempdir, filename)) + shutil.copyfile(os.path.join(self.tempdir, "latest_pointcloud.roi"), os.path.join(self.tempdir, filename)) self.parent.pointcloud_parameters['loadButton'].setEnabled(True) self.parent.pointcloud_parameters['pointcloudList'].setEnabled(True) From cf3fb235f0ebd04fb28fc81101710975d8d015d6 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Tue, 30 Nov 2021 15:06:08 +0000 Subject: [PATCH 04/19] update progress dialog --- src/idvc/dvc_interface.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 761c9284..fe4e4b91 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -575,6 +575,7 @@ def GetFileFromRemote(self, image_var, image, label, next_button): sleep(1) + self.create_progress_window("Getting files from remote", "", 0, None, False) self.updateUnknownProgressDialog = Worker(self.UnknownProgressUpdateDialog) self.updateUnknownProgressDialog.signals.finished.connect(self.StopUnknownProgressUpdate) self.threadpool.start(self.updateUnknownProgressDialog) @@ -586,20 +587,20 @@ def GetFileFromRemote(self, image_var, image, label, next_button): # ) def UnknownProgressUpdateDialog(self, **kwargs): - t0 = time.time() while True: tc = self.asyncCopy.threadpool.activeThreadCount() - if tc == 0: break # print (tc) - sleep(1) - self.statusBar().showMessage("Copying {} ... {}s".format(self.files_to_get[0][1], time.time()-t0)) - + sleep(0.25) + self.progress_window.setLabelText( + "Copying {} ... {:.1f} s".format(self.files_to_get[0][1], time.time()-t0) + ) + def StopUnknownProgressUpdate(self): - self.statusBar().showMessage("File copied.") + self.progress_window.close() def SelectImageLocal(self, image_var, image, label=None, next_button=None): @@ -881,7 +882,7 @@ def visualise(self): #bring image loading panel to front if it isnt already: self.select_image_dock.raise_() - def create_progress_window(self, title, text, max = 100, cancel = None): + def create_progress_window(self, title, text, max = 100, cancel = None, autoClose = True): self.progress_window = QProgressDialog(text, "Cancel", 0,max, self, QtCore.Qt.Window) self.progress_window.setWindowTitle(title) @@ -889,7 +890,7 @@ def create_progress_window(self, title, text, max = 100, cancel = None): self.progress_window.setMinimumDuration(0.01) self.progress_window.setWindowFlag(QtCore.Qt.WindowCloseButtonHint, True) self.progress_window.setWindowFlag(QtCore.Qt.WindowMaximizeButtonHint, False) - self.progress_window.setAutoClose(True) + self.progress_window.setAutoClose(autoClose) if cancel is None: self.progress_window.setCancelButton(None) else: From 61b3c080a3e3618cbb10f9e6cc0b874fb2e9f543 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Tue, 30 Nov 2021 15:33:36 +0000 Subject: [PATCH 05/19] revert tempfile.tempdir change --- src/idvc/dvc_interface.py | 122 +++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 62 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index fe4e4b91..ae6cad19 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -186,9 +186,9 @@ def CreateWorkingTempFolder(self): temp_folder = os.path.join(working_directory, "DVC_Sessions") self.temp_folder = os.path.abspath(temp_folder) - self.tempdir = tempfile.mkdtemp(dir = self.temp_folder) + tempfile.tempdir = tempfile.mkdtemp(dir = self.temp_folder) - os.chdir(self.tempdir) + os.chdir(tempfile.tempdir) # Creates folder in tempdir to save mask files in os.mkdir("Masks") @@ -540,6 +540,7 @@ def getSelectedDownloadAndUpdateUI(self, dialog, image_var, image, label, next_b def GetFileFromRemote(self, image_var, image, label, next_button): + '''Downloads a file from remote''' # 1 download self.files_to_get if len(self.files_to_get) == 1: self.asyncCopy = AsyncCopyOverSSH() @@ -556,14 +557,14 @@ def GetFileFromRemote(self, image_var, image, label, next_button): - files = [os.path.join(self.tempdir, self.files_to_get[0][1])] + files = [os.path.join(tempfile.tempdir, self.files_to_get[0][1])] remotepath = self.asyncCopy.remotepath.join(self.files_to_get[0][0], self.files_to_get[0][1]) # this shouldn't be necessary, however the signals and the workers are created before the async copy # object is created and then the local dir is not set in the worker. self.asyncCopy.SetRemoteDir(self.files_to_get[0][0]) self.asyncCopy.SetCopyFromRemote() - self.asyncCopy.SetLocalDir(self.tempdir) + self.asyncCopy.SetLocalDir(tempfile.tempdir) self.asyncCopy.SetRemoteDir(self.asyncCopy.remotepath.dirname(remotepath)) self.asyncCopy.SetFileName(self.asyncCopy.remotepath.basename(remotepath)) @@ -581,12 +582,9 @@ def GetFileFromRemote(self, image_var, image, label, next_button): self.threadpool.start(self.updateUnknownProgressDialog) - # self.asyncCopy.GetFile( - # self.asyncCopy.remotepath.join(self.files_to_get[0][0], self.files_to_get[0][1]), - # self.tempdir - # ) def UnknownProgressUpdateDialog(self, **kwargs): + '''Update the progress dialog where we don't know at what stage we are''' t0 = time.time() while True: tc = self.asyncCopy.threadpool.activeThreadCount() @@ -1629,13 +1627,13 @@ def LoadImagesAndCompleteRegistration(self): #print("About to create image") self.unsampled_ref_image_data = vtk.vtkImageData() ImageDataCreator.createImageData(self, self.image[0], self.unsampled_ref_image_data, info_var=self.unsampled_image_info, crop_image=True, origin=origin, - target_z_extent=target_z_extent, output_dir=os.path.abspath(self.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) + target_z_extent=target_z_extent, output_dir=os.path.abspath(tempfile.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) #TODO: move to doing both image data creators simultaneously - would this work? return if previous_reg_box_extent != reg_box_extent: ImageDataCreator.createImageData(self, self.image[0], self.unsampled_ref_image_data, info_var=self.unsampled_image_info, crop_image=True, origin=origin, - target_z_extent=target_z_extent, output_dir=os.path.abspath(self.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) + target_z_extent=target_z_extent, output_dir=os.path.abspath(tempfile.tempdir), finish_fn=self.LoadCorrImageForReg, crop_corr_image=True) else: self.completeRegistration() @@ -1653,7 +1651,7 @@ def LoadCorrImageForReg(self,resample_corr_image= False, crop_corr_image = False self.unsampled_corr_image_data = vtk.vtkImageData() ImageDataCreator.createImageData(self, self.image[1], self.unsampled_corr_image_data, info_var=self.unsampled_image_info, resample=resample_corr_image, - crop_image=crop_corr_image, origin=origin, target_z_extent=z_extent, finish_fn=self.completeRegistration, output_dir=os.path.abspath(self.tempdir)) + crop_image=crop_corr_image, origin=origin, target_z_extent=z_extent, finish_fn=self.completeRegistration, output_dir=os.path.abspath(tempfile.tempdir)) def completeRegistration(self): self.updatePoint0Display() @@ -2217,7 +2215,7 @@ def select_mask(self): if mask: if ".mha" in mask: filename = os.path.basename(mask) - shutil.copyfile(mask, os.path.join(self.tempdir, "Masks", filename)) + shutil.copyfile(mask, os.path.join(tempfile.tempdir, "Masks", filename)) self.mask_parameters["masksList"].addItem(filename) self.mask_parameters["masksList"].setCurrentText(filename) self.clearMask() @@ -2752,8 +2750,8 @@ def select_pointcloud(self): #, label): if self.copy_files: filename = os.path.basename(self.roi) - shutil.copyfile(self.roi, os.path.join(self.tempdir, filename)) - self.roi = os.path.abspath(os.path.join(self.tempdir, filename)) + shutil.copyfile(self.roi, os.path.join(tempfile.tempdir, filename)) + self.roi = os.path.abspath(os.path.join(tempfile.tempdir, filename)) self.pointcloud_parameters['pointcloudList'].addItem(filename) self.pointcloud_parameters['pointcloudList'].setCurrentText(filename) @@ -2766,7 +2764,7 @@ def PointCloudWorker(self, type, filename = None, disp_file = None, vector_dim = self.pointcloud_worker.signals.finished.connect(self.DisplayPointCloud) elif type == "load selection": self.clearPointCloud() - self.pointcloud_worker = Worker(self.loadPointCloud, os.path.join(self.tempdir, self.pointcloud_parameters['pointcloudList'].currentText())) + self.pointcloud_worker = Worker(self.loadPointCloud, os.path.join(tempfile.tempdir, self.pointcloud_parameters['pointcloudList'].currentText())) self.pointcloud_worker.signals.finished.connect(self.DisplayLoadedPointCloud) elif type == "load pointcloud file": self.clearPointCloud() @@ -2781,7 +2779,7 @@ def PointCloudWorker(self, type, filename = None, disp_file = None, vector_dim = self.create_progress_window("Loading", "Loading Pointcloud") self.pointcloud_worker.signals.progress.connect(self.progress) self.progress_window.setValue(10) - os.chdir(self.tempdir) + os.chdir(tempfile.tempdir) self.threadpool.start(self.pointcloud_worker) # Show error and allow re-selection of pointcloud if it can't be loaded: @@ -3082,7 +3080,7 @@ def createPointCloud(self, **kwargs): array.append((count, *pp)) count += 1 - np.savetxt(self.tempdir + "/" + filename, array, '%d\t%.3f\t%.3f\t%.3f', delimiter=';') + np.savetxt(tempfile.tempdir + "/" + filename, array, '%d\t%.3f\t%.3f\t%.3f', delimiter=';') self.roi = filename return(True) @@ -3938,7 +3936,7 @@ def create_config_worker(self): folder_name = "_" + self.rdvc_widgets['name_entry'].text() - results_folder = os.path.join(self.tempdir, "Results") + results_folder = os.path.join(tempfile.tempdir, "Results") new_folder = os.path.join(results_folder, folder_name) @@ -3957,12 +3955,12 @@ def create_config_worker(self): def create_run_config(self, **kwargs): - os.chdir(self.tempdir) + os.chdir(tempfile.tempdir) progress_callback = kwargs.get('progress_callback', None) try: folder_name = "_" + self.rdvc_widgets['name_entry'].text() - results_folder = os.path.join(self.tempdir, "Results") + results_folder = os.path.join(tempfile.tempdir, "Results") os.mkdir(os.path.join(results_folder, folder_name)) if self.singleRun_groupBox.isVisible(): @@ -4066,7 +4064,7 @@ def create_run_config(self, **kwargs): run_config['point0'] = self.getPoint0ImageCoords() suffix_text = "run_config" - self.run_config_file = os.path.join(self.tempdir, "Results", folder_name, "_" + suffix_text + ".json") + self.run_config_file = os.path.join(tempfile.tempdir, "Results", folder_name, "_" + suffix_text + ".json") with open(self.run_config_file, "w+") as tmp: json.dump(run_config, tmp) @@ -4108,7 +4106,7 @@ def run_external_code(self, error = None): # this command will call DVC_runner to create the directories self.dvc_runner = DVC_runner(self, os.path.abspath(self.run_config_file), - self.finished_run, self.run_succeeded, self.tempdir) + self.finished_run, self.run_succeeded, tempfile.tempdir) self.dvc_runner.run_dvc() @@ -4244,7 +4242,7 @@ def show_run_pcs(self): self.result_widgets['pc_entry'].clear() self.result_widgets['subvol_entry'].clear() - directory = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + directory = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) self.results_folder = directory file_list=[] @@ -4290,7 +4288,7 @@ def LoadResultsOnViewer(self): self.warningDialog("An error occurred with this run so the results could not be displayed.", "Error") else: - results_folder = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + results_folder = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) self.roi = os.path.join(results_folder ,"_" + str(subvol_size) + ".roi") #print("New roi is", self.roi) self.results_folder = results_folder @@ -4321,7 +4319,7 @@ def LoadResultsOnViewer(self): def CreateGraphsWindow(self): #print("Create graphs") if self.result_widgets['run_entry'].currentText() is not "": - self.results_folder = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + self.results_folder = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) else: self.results_folder = None @@ -4457,11 +4455,11 @@ def SaveSession(self, text_value, compress, event): self.config['dvc_input_image_in_session_folder'] = self.dvc_input_image_in_session_folder # print("ROI: ", self.roi) - # print("temp", self.tempdir) + # print("temp", tempfile.tempdir) if (self.roi): self.roi = os.path.abspath(self.roi) - if os.path.abspath(self.tempdir) in self.roi: - self.config['roi_file'] = self.roi[len(os.path.abspath(self.tempdir))+1:] + if os.path.abspath(tempfile.tempdir) in self.roi: + self.config['roi_file'] = self.roi[len(os.path.abspath(tempfile.tempdir))+1:] self.config['roi_ext'] = False else: self.config['roi_file'] = self.roi @@ -4473,8 +4471,8 @@ def SaveSession(self, text_value, compress, event): if hasattr(self, 'mask_file'): self.config['mask_details']=self.mask_details - if self.tempdir in os.path.abspath(self.mask_file): - self.config['mask_file']=self.mask_file[len(os.path.abspath(self.tempdir))+1:] + if tempfile.tempdir in os.path.abspath(self.mask_file): + self.config['mask_file']=self.mask_file[len(os.path.abspath(tempfile.tempdir))+1:] self.config['mask_ext'] = False else: self.config['mask_file']=self.mask_file @@ -4545,10 +4543,10 @@ def SaveSession(self, text_value, compress, event): suffix_text = "_" + user_string + "_" + now_string os.chdir(self.temp_folder) - tempdir = shutil.move(self.tempdir, suffix_text) - self.tempdir = os.path.abspath(tempdir) + tempdir = shutil.move(tempfile.tempdir, suffix_text) + tempfile.tempdir = os.path.abspath(tempdir) - fd, f = tempfile.mkstemp(suffix=suffix_text + ".json", dir = self.tempdir) #could not delete this using rmtree? + fd, f = tempfile.mkstemp(suffix=suffix_text + ".json", dir = tempfile.tempdir) #could not delete this using rmtree? with open(f, "w+") as tmp: json.dump(self.config, tmp) @@ -4558,7 +4556,7 @@ def SaveSession(self, text_value, compress, event): self.create_progress_window("Saving","Saving") - zip_worker = Worker(self.ZipDirectory, self.tempdir, compress) + zip_worker = Worker(self.ZipDirectory, tempfile.tempdir, compress) if type(event) == QCloseEvent: zip_worker.signals.finished.connect(lambda: self.RemoveTemp(event)) else: @@ -4567,47 +4565,47 @@ def SaveSession(self, text_value, compress, event): self.threadpool.start(zip_worker) if compress: - self.ShowZipProgress(self.tempdir, self.tempdir +'.zip', 0.7) + self.ShowZipProgress(tempfile.tempdir, tempfile.tempdir +'.zip', 0.7) else: - self.ShowZipProgress(self.tempdir, self.tempdir +'.zip', 1) + self.ShowZipProgress(tempfile.tempdir, tempfile.tempdir +'.zip', 1) #give variables filepath including new name of temp folder: - # print("temp", self.tempdir) + # print("temp", tempfile.tempdir) # print("roi ext", self.config['roi_ext']) # print(self.roi, self.config['roi_file']) if (self.roi and not self.config['roi_ext']): - self.roi = os.path.join(os.path.abspath(self.tempdir), self.config['roi_file']) + self.roi = os.path.join(os.path.abspath(tempfile.tempdir), self.config['roi_file']) #print(self.roi) if hasattr(self, 'mask_file'): if 'mask_file' in self.config: - self.mask_file = os.path.join(os.path.abspath(self.tempdir), self.config['mask_file']) + self.mask_file = os.path.join(os.path.abspath(tempfile.tempdir), self.config['mask_file']) count = 0 for i in self.image[0]: if self.image_copied[0]: - # print(os.path.join(os.path.abspath(self.tempdir), self.config['image'][0][count]) ) - self.image[0][count] = os.path.join(os.path.abspath(self.tempdir), self.config['image'][0][count]) + # print(os.path.join(os.path.abspath(tempfile.tempdir), self.config['image'][0][count]) ) + self.image[0][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['image'][0][count]) count +=1 count = 0 for j in self.image[1]: if self.image_copied[1]: - self.image[1][count] = os.path.join(os.path.abspath(self.tempdir), self.config['image'][1][count]) + self.image[1][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['image'][1][count]) count += 1 count = 0 for i in self.dvc_input_image[0]: if self.image_copied[0] or self.dvc_input_image_in_session_folder: - self.dvc_input_image[0][count] = os.path.join(os.path.abspath(self.tempdir), self.config['dvc_input_image'][0][count]) + self.dvc_input_image[0][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['dvc_input_image'][0][count]) count+=1 count=0 for j in self.dvc_input_image[1]: if self.image_copied[1] or self.dvc_input_image_in_session_folder: - self.dvc_input_image[1][count] = os.path.join(os.path.abspath(self.tempdir), self.config['dvc_input_image'][1][count]) + self.dvc_input_image[1][count] = os.path.join(os.path.abspath(tempfile.tempdir), self.config['dvc_input_image'][1][count]) count+=1 - results_folder = os.path.join(self.tempdir, "Results", self.result_widgets['run_entry'].currentText()) + results_folder = os.path.join(tempfile.tempdir, "Results", self.result_widgets['run_entry'].currentText()) self.results_folder = results_folder def CloseSaveWindow(self): @@ -4642,8 +4640,8 @@ def RemoveTemp(self, event): self.progress_window.setLabelText("Closing") self.progress_window.setMaximum(100) self.progress_window.setValue(98) - #print("removed temp", self.tempdir) - shutil.rmtree(self.tempdir) + #print("removed temp", tempfile.tempdir) + shutil.rmtree(tempfile.tempdir) if hasattr(self, 'progress_window'): self.progress_window.setValue(100) @@ -4724,11 +4722,11 @@ def ExportSession(self): export_worker = Worker(self.exporter, export_location) export_worker.signals.finished.connect(self.progress_complete) self.threadpool.start(export_worker) - self.ShowExportProgress(self.tempdir, export_location) + self.ShowExportProgress(tempfile.tempdir, export_location) def exporter(self, *args, **kwargs): export_location = args[0] - shutil.copytree(self.tempdir, export_location) + shutil.copytree(tempfile.tempdir, export_location) #Dealing with loading sessions: @@ -4840,17 +4838,17 @@ def LoadConfigWorker(self, **kwargs): if not results_folder_exists: os.mkdir(os.path.join(loaded_tempdir, "Results")) - #print(self.tempdir) + #print(tempfile.tempdir) - # if self.tempdir != loaded_tempdir: # if we are not loading the same session that we already had open - # shutil.rmtree(self.tempdir) + # if tempfile.tempdir != loaded_tempdir: # if we are not loading the same session that we already had open + # shutil.rmtree(tempfile.tempdir) if progress_callback is not None: progress_callback.emit(90) - self.tempdir = loaded_tempdir + tempfile.tempdir = loaded_tempdir #print("working tempdir") - #print(self.tempdir) + #print(tempfile.tempdir) #selected_file = "" json_filename = date_and_time + ".json" @@ -4868,7 +4866,7 @@ def LoadConfigWorker(self, **kwargs): progress_callback.emit(100) def LoadSession(self): - os.chdir(self.tempdir) + os.chdir(tempfile.tempdir) self.resetRegistration() self.loading_session = True @@ -4890,12 +4888,12 @@ def LoadSession(self): # if 'roi_ext' in self.config: # if not self.config['roi_ext']: # self.roi = os.path.abspath( - # os.path.join(self.tempdir, self.roi)) + # os.path.join(tempfile.tempdir, self.roi)) #pointcloud files could still exist even if there wasn't a pointcloud displayed when the session was saved. pointcloud_files = [] #get list of pointcloud files: - for r, d, f in os.walk(self.tempdir): + for r, d, f in os.walk(tempfile.tempdir): for _file in f: if '.roi' in _file: pointcloud_files.append(_file) @@ -4937,7 +4935,7 @@ def LoadSession(self): #save paths to images to variable if self.config['image_copied'][j]: - path = os.path.abspath(os.path.join(self.tempdir, i)) + path = os.path.abspath(os.path.join(tempfile.tempdir, i)) # print("The DVC input path is") # print(path) self.dvc_input_image[j].append(path) @@ -5058,7 +5056,7 @@ def LoadSession(self): if 'mask_file' in self.config: self.mask_parameters['loadButton'].setEnabled(True) - mask_folder = os.path.join(self.tempdir, "Masks") + mask_folder = os.path.join(tempfile.tempdir, "Masks") mask_files = [] #get list of mask files: #print(mask_folder) @@ -5075,7 +5073,7 @@ def LoadSession(self): self.mask_parameters['loadButton'].setEnabled(False) - results_directory = os.path.join(self.tempdir, "Results") + results_directory = os.path.join(tempfile.tempdir, "Results") for i in range(self.result_widgets['run_entry'].count()): self.result_widgets['run_entry'].removeItem(i) @@ -5180,7 +5178,7 @@ def save(self, save_only): #Load Saved Session #print("Write mask to file, then carry on") filename = self.textbox.text() + ".mha" - shutil.copyfile(os.path.join(self.tempdir, self.parent.mask_file), os.path.join(self.tempdir, "Masks", filename)) + shutil.copyfile(os.path.join(tempfile.tempdir, self.parent.mask_file), os.path.join(tempfile.tempdir, "Masks", filename)) self.parent.mask_parameters['masksList'].addItem(filename) self.parent.mask_details[filename] = self.parent.mask_details['current'] #print(self.parent.mask_details) @@ -5201,7 +5199,7 @@ def save(self, save_only): if self.object == "pointcloud": filename = self.textbox.text() + ".roi" - shutil.copyfile(os.path.join(self.tempdir, "latest_pointcloud.roi"), os.path.join(self.tempdir, filename)) + shutil.copyfile(os.path.join(tempfile.tempdir, "latest_pointcloud.roi"), os.path.join(tempfile.tempdir, filename)) self.parent.pointcloud_parameters['loadButton'].setEnabled(True) self.parent.pointcloud_parameters['pointcloudList'].setEnabled(True) From cd35b9c7d44abd67a71176c5f729575e8628439e Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Tue, 30 Nov 2021 23:44:40 +0000 Subject: [PATCH 06/19] added remote workdir selection --- src/idvc/dialogs.py | 46 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/src/idvc/dialogs.py b/src/idvc/dialogs.py index 4221cae8..2a9877b2 100644 --- a/src/idvc/dialogs.py +++ b/src/idvc/dialogs.py @@ -1,10 +1,13 @@ from PySide2 import QtCore, QtGui, QtWidgets -from PySide2.QtWidgets import QCheckBox, QLabel, QDoubleSpinBox, QFrame, QVBoxLayout, QDialogButtonBox, QPushButton, QDialog +from PySide2.QtWidgets import QCheckBox, QLabel, QDoubleSpinBox, QFrame, QVBoxLayout,\ + QDialogButtonBox, QPushButton, QDialog, QLineEdit from PySide2.QtCore import Qt import vtk -from brem.ui import RemoteServerSettingDialog +from brem.ui import RemoteServerSettingDialog, RemoteFileDialog from eqt.ui import UIFormFactory +import os, posixpath + class SettingsWindow(QDialog): def __init__(self, parent): @@ -92,6 +95,13 @@ def __init__(self, parent): cb = QCheckBox(self) cb.setChecked(self.parent.connection_details is not None) fw.addWidget(cb, 'Connect to remote server', 'connect_to_remote') + select_remote_workdir = QPushButton(self) + select_remote_workdir.setText('Browse') + select_remote_workdir.clicked.connect(self.browseRemote) + fw.addWidget(select_remote_workdir, 'Select remote workdir', 'select_remote_workdir') + remote_workdir = QLineEdit(self) + fw.addWidget(remote_workdir, 'Remote workdir', 'remote_workdir') + self.fw = fw for k,v in fw.widgets.items(): print ("fw", k) @@ -168,4 +178,34 @@ def openConfigRemote(self): def getConnectionDetails(self, dialog): for k,v in dialog.connection_details.items(): print (k,v) - self.connection_details = dialog.connection_details \ No newline at end of file + self.connection_details = dialog.connection_details + + + def browseRemote(self): + # start the RemoteFileBrowser + logfile = os.path.join(os.getcwd(), '..','..',"RemoteFileDialog.log") + # logfile = None + dialog = RemoteFileDialog(self, logfile=logfile, port=self.connection_details['server_port'], + host=self.connection_details['server_name'], + username=self.connection_details['username'], + private_key=self.connection_details['private_key'], + remote_os=self.connection_details['remote_os']) + dialog.Ok.clicked.connect( + lambda: self.getSelectedRemoteWorkdir(dialog) + ) + if hasattr(self, 'files_to_get'): + try: + dialog.widgets['lineEdit'].setText(self.files_to_get[0][0]) + except: + pass + dialog.exec() + + + def getSelectedRemoteWorkdir(self, dialog): + if hasattr(dialog, 'selected'): + print (type(dialog.selected)) + for el in dialog.selected: + print ("Return from dialogue", el) + self.files_to_get = list (dialog.selected) + remote_workdir = posixpath.join(self.files_to_get[0][0], self.files_to_get[0][1]) + self.fw.widgets['remote_workdir_field'].setText(remote_workdir) From e6c47e7168186a89922d1de51ce72b992ba2e81f Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Tue, 30 Nov 2021 23:46:54 +0000 Subject: [PATCH 07/19] work in progress to push dvc config to remote --- src/idvc/dvc_interface.py | 37 ++++++++++++++++++++++++++++--------- src/idvc/dvc_remote.py | 25 +++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 src/idvc/dvc_remote.py diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index ae6cad19..eddfbdb6 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -574,6 +574,12 @@ def GetFileFromRemote(self, image_var, image, label, next_button): # this should also not be done like this. self.asyncCopy.threadpool.start(self.asyncCopy.worker) + # save into these variables for the remote run in create_run_config + if image_var == 0: + self.reference_file = remotepath + elif image_var == 1: + self.correlate_file = remotepath + sleep(1) self.create_progress_window("Getting files from remote", "", 0, None, False) @@ -3876,6 +3882,7 @@ def CreateRunDVCPanel(self): #Add button functionality: rdvc_widgets['run_type_entry'].currentIndexChanged.connect(self.show_run_groupbox) + # if connected to remote do something else. rdvc_widgets['run_button'].clicked.connect(self.create_config_worker) self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, dockWidget) @@ -3949,10 +3956,22 @@ def create_config_worker(self): self.create_progress_window("Loading", "Generating Run Config") self.config_worker.signals.progress.connect(self.progress) # if single or bulk use the line below, if remote develop new functionality - self.config_worker.signals.result.connect(partial (self.run_external_code)) + if not self.settings_window.fw['connect_to_remote'].isChecked(): + self.config_worker.signals.result.connect(partial (self.run_external_code)) + else: + # do not run the dvc locally but + # 1 zip and + # 2 upload the config to remote and then + # 3 run the code on the remote + self.config_worker.signals.result.connect(partial (self.ZipAndUploadConfigToRemote)) + pass + self.threadpool.start(self.config_worker) self.progress_window.setValue(10) + def ZipAndUploadConfigToRemote(self, **kwargs): + pass + def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) @@ -4021,12 +4040,13 @@ def create_run_config(self, **kwargs): progress_callback.emit(subvol_size_count/len(self.subvol_sizes)*90) #print("finished making pointclouds") - #print(self.roi_files) - - #print("DVC in: ", self.dvc_input_image) - - self.reference_file = self.dvc_input_image[0][0] - self.correlate_file = self.dvc_input_image[1][0] + # if remote mode this should not be the local copy + if self.settings_window.fw['connect_to_remote'].isChecked(): + # this should point to the remote files set at the time of download + pass + else: + self.reference_file = self.dvc_input_image[0][0] + self.correlate_file = self.dvc_input_image[1][0] #print("REF: ", self.reference_file) @@ -4100,8 +4120,6 @@ def run_external_code(self, error = None): self.cancelled = True return - - self.run_succeeded = True # this command will call DVC_runner to create the directories @@ -4110,6 +4128,7 @@ def run_external_code(self, error = None): self.dvc_runner.run_dvc() + def update_progress(self, exe = None): if exe: line_b = self.process.readLine() diff --git a/src/idvc/dvc_remote.py b/src/idvc/dvc_remote.py new file mode 100644 index 00000000..658c0fb4 --- /dev/null +++ b/src/idvc/dvc_remote.py @@ -0,0 +1,25 @@ +import os + +class PrepareDVCRemote(object): + def __init__(self, parent): + self._config_json = None + self._remote_workdir = None + self.parent = parent + + + @property + def config_json(self): + return self._config_json + + + def set_config_json(self, value): + self._config_json = os.path.abspath(value) + + + @property + def remote_workdir(self): + return self._remote_workdir + + + def set_remote_workdir(self): + self._remote_workdir = self.parent.settings_window \ No newline at end of file From effa4177046bf61c64d541cc3ba6dee25300db54 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Wed, 1 Dec 2021 18:16:11 +0000 Subject: [PATCH 08/19] about ok --- src/idvc/dvc_interface.py | 84 +++++++++++++++++++++++++++++++-------- src/idvc/dvc_runner.py | 56 +++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 18 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index eddfbdb6..181eaeeb 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -1,3 +1,4 @@ +import pysnooper import os import sys from PySide2 import QtCore, QtGui, QtWidgets @@ -542,6 +543,15 @@ def getSelectedDownloadAndUpdateUI(self, dialog, image_var, image, label, next_b def GetFileFromRemote(self, image_var, image, label, next_button): '''Downloads a file from remote''' # 1 download self.files_to_get + if image_var == 1: + # let's not download the correlate image + if next_button is not None: + try: + for el in next_button: + el.setEnabled(True) + except: + next_button.setEnabled(True) + if len(self.files_to_get) == 1: self.asyncCopy = AsyncCopyOverSSH() if not hasattr(self, 'connection_details'): @@ -556,13 +566,18 @@ def GetFileFromRemote(self, image_var, image, label, next_button): port=port, host=host, private_key=private_key) - - files = [os.path.join(tempfile.tempdir, self.files_to_get[0][1])] remotepath = self.asyncCopy.remotepath.join(self.files_to_get[0][0], self.files_to_get[0][1]) + if image_var == 1: + self.remote_correlate_image_fname = remotepath + return + else: + self.remote_reference_image_fname = remotepath + + files = [os.path.join(tempfile.tempdir, self.files_to_get[0][1])] + # this shouldn't be necessary, however the signals and the workers are created before the async copy # object is created and then the local dir is not set in the worker. - self.asyncCopy.SetRemoteDir(self.files_to_get[0][0]) self.asyncCopy.SetCopyFromRemote() self.asyncCopy.SetLocalDir(tempfile.tempdir) self.asyncCopy.SetRemoteDir(self.asyncCopy.remotepath.dirname(remotepath)) @@ -586,6 +601,8 @@ def GetFileFromRemote(self, image_var, image, label, next_button): self.updateUnknownProgressDialog = Worker(self.UnknownProgressUpdateDialog) self.updateUnknownProgressDialog.signals.finished.connect(self.StopUnknownProgressUpdate) self.threadpool.start(self.updateUnknownProgressDialog) + # make the progress dialog start after minimum 0.5 s + self.progress_window.setMinimumDuration(0.5) @@ -643,7 +660,7 @@ def _UpdateSelectFileUI(self, files, image_var, image, label=None, next_button=N else: image[image_var].append(files[0]) if label is not None: - label.setText(os.path.basename(files[0])) + label.setText(files[0]) else: # Make sure that the files are sorted 0 - end @@ -664,7 +681,12 @@ def _UpdateSelectFileUI(self, files, image_var, image, label=None, next_button=N label.setText(os.path.basename(self.image[image_var][0]) + " + " + str(len(files)) + " more files.") if next_button is not None: - next_button.setEnabled(True) + try: + for el in next_button: + el.setEnabled(True) + except: + next_button.setEnabled(True) + def copy_file(self, **kwargs): @@ -1159,7 +1181,7 @@ def CreateRegistrationPanel(self): rp['translate_X_entry'].setValidator(validatorint) rp['translate_X_entry'].setText("0") rp['translate_X_entry'].setToolTip(translation_tooltip_text) - #rp['translate_X_entry'].setEnabled(False) + rp['translate_X_entry'].textEdited.connect(self._updateTranslateObject) formLayout.setWidget(widgetno, QFormLayout.FieldRole, rp['translate_X_entry']) widgetno += 1 # Translate Y field @@ -1171,7 +1193,7 @@ def CreateRegistrationPanel(self): rp['translate_Y_entry'].setValidator(validatorint) rp['translate_Y_entry'].setText("0") rp['translate_Y_entry'].setToolTip(translation_tooltip_text) - #rp['translate_Y_entry'].setEnabled(False) + rp['translate_Y_entry'].textEdited.connect(self._updateTranslateObject) formLayout.setWidget(widgetno, QFormLayout.FieldRole, rp['translate_Y_entry']) widgetno += 1 # Translate Z field @@ -1183,10 +1205,12 @@ def CreateRegistrationPanel(self): rp['translate_Z_entry'].setValidator(validatorint) rp['translate_Z_entry'].setText("0") rp['translate_Z_entry'].setToolTip(translation_tooltip_text) - #rp['translate_Z_entry'].setEnabled(False) + rp['translate_Y_entry'].textEdited.connect(self._updateTranslateObject) formLayout.setWidget(widgetno, QFormLayout.FieldRole, rp['translate_Z_entry']) widgetno += 1 + # self.translate.SetTranslation(-int(rp['translate_X_entry'].text()),-int(rp['translate_Y_entry'].text()),-int(rp['translate_Z_entry'].text())) + # Add submit button rp['start_registration_button'] = QPushButton(groupBox) rp['start_registration_button'].setText("Start Registration") @@ -1201,6 +1225,24 @@ def CreateRegistrationPanel(self): # save to instance self.registration_parameters = rp + + def _updateTranslateObject(self, text, **kwargs): + rp = self.registration_parameters + + + # setup the appropriate stuff to run the registration + if not hasattr(self, 'translate'): + self.translate = vtk.vtkImageTranslateExtent() + elif self.translate is None: + self.translate = vtk.vtkImageTranslateExtent() + + self.translate.SetTranslation(-int(rp['translate_X_entry'].text()), + -int(rp['translate_Y_entry'].text()), + -int(rp['translate_Z_entry'].text()) + ) + + + def createRegistrationViewer(self): # print("Create reg viewer") #Get current orientation and slice of 2D viewer, registration viewer will be set up to have these @@ -3956,23 +3998,32 @@ def create_config_worker(self): self.create_progress_window("Loading", "Generating Run Config") self.config_worker.signals.progress.connect(self.progress) # if single or bulk use the line below, if remote develop new functionality - if not self.settings_window.fw['connect_to_remote'].isChecked(): + if not self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): self.config_worker.signals.result.connect(partial (self.run_external_code)) else: # do not run the dvc locally but # 1 zip and # 2 upload the config to remote and then # 3 run the code on the remote - self.config_worker.signals.result.connect(partial (self.ZipAndUploadConfigToRemote)) + self.config_worker.signals.finished.connect(partial (self.ZipAndUploadConfigToRemote)) pass self.threadpool.start(self.config_worker) self.progress_window.setValue(10) - - def ZipAndUploadConfigToRemote(self, **kwargs): - pass - + @pysnooper.snoop() + def ZipAndUploadConfigToRemote(self): + # this command will call DVC_runner to create the directories + self.run_succeeded = True + self.dvc_runner = DVC_runner(self, os.path.abspath(self.run_config_file), + self.finished_run, self.run_succeeded, tempfile.tempdir) + self.config_worker = Worker(self.dvc_runner.zip_workdir_and_upload) + self.create_progress_window("Zipping and uploading ", "Generating Run Config") + # self.config_worker.signals.progress.connect(self.progress) + self.threadpool.start(self.config_worker) + + + def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) progress_callback = kwargs.get('progress_callback', None) @@ -4041,9 +4092,10 @@ def create_run_config(self, **kwargs): #print("finished making pointclouds") # if remote mode this should not be the local copy - if self.settings_window.fw['connect_to_remote'].isChecked(): + if self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): # this should point to the remote files set at the time of download - pass + self.reference_file = self.remote_reference_image_fname + self.correlate_file = self.remote_correlate_image_fname else: self.reference_file = self.dvc_input_image[0][0] self.correlate_file = self.dvc_input_image[1][0] diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index 53f58daf..e7594a65 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -2,11 +2,13 @@ import numpy as np from PySide2 import QtCore from datetime import datetime -from PySide2.QtWidgets import QMessageBox +from PySide2.QtWidgets import QMessageBox, QProgressDialog import json import time import shutil import platform +import pysnooper +from brem import AsyncCopyOverSSH, BasicRemoteExecutionManager count = 0 runs_completed = 0 @@ -97,7 +99,7 @@ def create_progress_window(main_window, title, text, max = 100, cancel = None): main_window.progress_window = QProgressDialog(text, "Cancel", 0,max, main_window, QtCore.Qt.Window) main_window.progress_window.setWindowTitle(title) main_window.progress_window.setWindowModality(QtCore.Qt.ApplicationModal) #This means the other windows can't be used while this is open - main_window.progress_window.setMinimumDuration(0.1) + main_window.progress_window.setMinimumDuration(100) main_window.progress_window.setWindowFlag(QtCore.Qt.WindowCloseButtonHint, False) main_window.progress_window.setWindowFlag(QtCore.Qt.WindowMaximizeButtonHint, False) if cancel is None: @@ -150,6 +152,7 @@ def finished_run(main_window, exitCode, exitStatus, process = None, required_run # print("did") class DVC_runner(object): + def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_folder): # print("The session folder is", session_folder) self.main_window = main_window @@ -329,7 +332,56 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo self.processes.append( (exe_file, [ config_filename ], required_runs, total_points) ) + + @pysnooper.snoop() + def zip_workdir_and_upload(self, **kwargs): + #param_file is a list with at least 1 item but we are interested in the first + # because we want to know the path to it and all files will be in the same directory + exe_file, param_file, required_runs,\ + total_points = self.processes[0] + # config is in the directory + + configdir = os.path.join(os.path.dirname(param_file[0]),'..') + zipped = shutil.make_archive(os.path.join(configdir, '..', 'remote_run'), 'zip', configdir) + + self.asyncCopy = AsyncCopyOverSSH() + if not hasattr(self.main_window, 'connection_details'): + self.main_window.statusBar().showMessage("define the connection") + return + username = self.main_window.connection_details['username'] + port = self.main_window.connection_details['server_port'] + host = self.main_window.connection_details['server_name'] + private_key = self.main_window.connection_details['private_key'] + + self.asyncCopy.setRemoteConnectionSettings(username=username, + port=port, host=host, private_key=private_key) + + + remote_dir = self.main_window.settings_window.fw.widgets['remote_workdir_field'].text() + # this shouldn't be necessary, however the signals and the workers are created before the async copy + # object is created and then the local dir is not set in the worker. + self.asyncCopy.SetRemoteDir(remote_dir) + self.asyncCopy.SetCopyToRemote() + self.asyncCopy.SetLocalDir(os.path.dirname(zipped)) + self.asyncCopy.SetFileName(os.path.basename(zipped)) + + self.asyncCopy.signals.finished.connect( + lambda: self._unzip_on_remote(remote_dir, os.path.basename(zipped)) + ) + + self.asyncCopy.threadpool.start(self.asyncCopy.worker) + + + def _unzip_on_remote(self, workdir, filename, **kwargs): + + # 1 create a BasicRemoteExecutionManager + # 2 go to workdir + # 2 run 'unzip filename' + print("_unzip_on_remote") + + + def run_dvc(self): main_window = self.main_window input_file = self.input_file From a2f563eff0206eb297b4f60762fd8e9aecf36271 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Wed, 1 Dec 2021 21:01:49 +0000 Subject: [PATCH 09/19] up to zip dvc config and transfer to remote --- src/idvc/dvc_interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 181eaeeb..1d18dfcd 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -196,7 +196,8 @@ def CreateWorkingTempFolder(self): os.mkdir("Results") def OpenSettings(self): - self.settings_window = SettingsWindow(self) + if not hasattr(self, 'settings_window'): + self.settings_window = SettingsWindow(self) self.settings_window.show() def InitialiseSessionVars(self): From 7fc1624da91f7af262501f7095b12e44149c3dbb Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Fri, 3 Dec 2021 13:58:49 +0000 Subject: [PATCH 10/19] up to unzip on remote --- src/idvc/dvc_interface.py | 24 +++-- src/idvc/dvc_remote.py | 217 +++++++++++++++++++++++++++++++++++++- src/idvc/dvc_runner.py | 31 +++++- 3 files changed, 260 insertions(+), 12 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 1d18dfcd..5a9bb04d 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -603,7 +603,7 @@ def GetFileFromRemote(self, image_var, image, label, next_button): self.updateUnknownProgressDialog.signals.finished.connect(self.StopUnknownProgressUpdate) self.threadpool.start(self.updateUnknownProgressDialog) # make the progress dialog start after minimum 0.5 s - self.progress_window.setMinimumDuration(0.5) + self.progress_window.setMinimumDuration(0) @@ -4017,13 +4017,25 @@ def ZipAndUploadConfigToRemote(self): # this command will call DVC_runner to create the directories self.run_succeeded = True self.dvc_runner = DVC_runner(self, os.path.abspath(self.run_config_file), - self.finished_run, self.run_succeeded, tempfile.tempdir) + self.finished_run, self.run_succeeded, tempfile.tempdir, remote_os=self.connection_details['remote_os']) self.config_worker = Worker(self.dvc_runner.zip_workdir_and_upload) - self.create_progress_window("Zipping and uploading ", "Generating Run Config") - # self.config_worker.signals.progress.connect(self.progress) + # self.create_progress_window("Connecting with remote", "Zipping and uploading", 0, None, False) + self.config_worker.signals.finished.connect( self.run_code_remote ) self.threadpool.start(self.config_worker) - - + + def run_code_remote(self): + print ("run_code_remote") + while True: + tc = self.dvc_runner.asyncCopy.threadpool.activeThreadCount() + if tc == 0: + break + # print (tc) + sleep(0.25) + self.unzip_worker = Worker(self.dvc_runner._unzip_on_remote, self.dvc_runner.asyncCopy.remotedir, self.dvc_runner.asyncCopy.filename) + self.create_progress_window("Connecting with remote", "Unzipping", 0, None, False) + self.unzip_worker.signals.finished.connect( self.progress_window.close ) + self.threadpool.start(self.unzip_worker) + def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) diff --git a/src/idvc/dvc_remote.py b/src/idvc/dvc_remote.py index 658c0fb4..fb6bd8c0 100644 --- a/src/idvc/dvc_remote.py +++ b/src/idvc/dvc_remote.py @@ -1,4 +1,5 @@ import os +from brem import RemoteRunControl class PrepareDVCRemote(object): def __init__(self, parent): @@ -22,4 +23,218 @@ def remote_workdir(self): def set_remote_workdir(self): - self._remote_workdir = self.parent.settings_window \ No newline at end of file + self._remote_workdir = self.parent.settings_window + +class DVCRemoteRunControl(RemoteRunControl): + def __init__(self, connection_details=None, + reference_filename=None, correlate_filename=None, + dvclog_filename=None, + dev_config=None): + + super(DVCRemoteRunControl, self).__init__(connection_details=connection_details) + self.reference_fname = reference_filename + self.correlate_fname = correlate_filename + self.dvclog_fname = dvclog_filename + + # try to create a worker + self.create_job(self.run_worker, + reference_fname=self.reference_fname, + correlate_fname=self.correlate_fname, + update_delay=10, logfile=self.dvclog_fname) + + # Not required for base class + @property + def reference_fname(self): + return self._reference_fname + @reference_fname.setter + def reference_fname(self, value): + '''setter for reference file name.''' + self._reference_fname = value + + @property + def correlate_fname(self): + return self._correlate_fname + @correlate_fname.setter + def correlate_fname(self, value): + '''setter for correlate file name.''' + self._correlate_fname = value + + @property + def dvclog_fname(self): + return self._dvclog_fname + @dvclog_fname.setter + def dvclog_fname(self, value): + '''setter for dvclog file name.''' + self._dvclog_fname = value + + + + + # @pysnooper.snoop() + def run_worker(self, **kwargs): + # retrieve the appropriate parameters from the kwargs + host = kwargs.get('host', None) + username = kwargs.get('username', None) + port = kwargs.get('port', None) + private_key = kwargs.get('private_key', None) + logfile = kwargs.get('logfile', None) + update_delay = kwargs.get('update_delay', None) + # get the callbacks + message_callback = kwargs.get('message_callback', None) + progress_callback = kwargs.get('progress_callback', None) + status_callback = kwargs.get('status_callback', None) + + reference_fname = kwargs.get('reference_fname', None) + correlate_fname = kwargs.get('correlate_fname', None) + + + from time import sleep + + a = brem.BasicRemoteExecutionManager( host=host, + username=username, + port=22, + private_key=private_key) + + a.login(passphrase=False) + + inp="input.dvc" + # folder="/work3/cse/dvc/test-edo" + folder = dpath.dirname(logfile) + datafolder="/work3/cse/dvc/test_data" + + with open(inp,'w', newline='\n') as f: + print("""############################################################################### +# +# +# example dvc process control file +# +# +############################################################################### + +# all lines beginning with a # character are ignored +# some parameters are conditionally required, depending on the setting of other parameters +# for example, if subvol_thresh is off, the threshold description parameters are not required + +### file names + +reference_filename\t{0}/frame_000_f.npy\t### reference tomography image volume +correlate_filename\t{0}/frame_010_f.npy\t### correlation tomography image volume + +point_cloud_filename\t{1}/medium_grid.roi\t### file of search point locations +output_filename\t{1}/medium_grid\t### base name for output files + +### description of the image data files, all must be the same size and structure + +vol_bit_depth 8 ### 8 or 16 +vol_hdr_lngth 96 ### fixed-length header size, may be zero +vol_wide 1520 ### width in pixels of each slice +vol_high 1257 ### height in pixels of each slice +vol_tall 1260 ### number of slices in the stack + +### parameters defining the subvolumes that will be created at each search point + +subvol_geom sphere ### cube, sphere +subvol_size 80 ### side length or diameter, in voxels +subvol_npts 8000 ### number of points to distribute within the subvol + +subvol_thresh off ### on or off, evaluate subvolumes based on threshold +# gray_thresh_min 27 ### lower limit of a gray threshold range if subvol_thresh is on +# gray_thresh_max 127 ### upper limit of a gray threshold range if subvol_thresh is on +# min_vol_fract 0.2 ### only search if subvol fraction is greater than + +### required parameters defining the basic the search process + +disp_max 38 ### in voxels, used for range checking and global search limits +num_srch_dof 6 ### 3, 6, or 12 +obj_function znssd ### sad, ssd, zssd, nssd, znssd +interp_type tricubic ### trilinear, tricubic + +### optional parameters tuning and refining the search process + +rigid_trans 34.0 4.0 0.0 ### rigid body offset of target volume, in voxels +basin_radius 0.0 ### coarse-search resolution, in voxels, 0.0 = none +subvol_aspect 1.0 1.0 1.0 ### subvolume aspect ratio + + + +""".format(datafolder,folder),file=f) + + a.put_file(inp, remote_filename=dpath.join(folder, inp)) + + + job=""" + +module purge +module load AMDmodules foss/2019b + +/work3/cse/dvc/codes/CCPi-DVC/build-amd/Core/dvc {0} > {1} 2>&1 +#{0} + """.format(inp, logfile) + + + + jobid = a.submit_job(folder,job) + self.job_id = jobid + print(jobid) + status = a.job_status(jobid) + print(status) + i = 0 + start_at = 0 + while status in [b'PENDING',b'RUNNING']: + i+=1 + # widgets['jobid'].setText("Job id: {} {}".format(jobid, str(status))) + status_callback.emit((jobid, status.decode('utf-8'))) + self.internalsignals.status.emit((jobid, status.decode('utf-8'))) + if status == b'PENDING': + print("job is queueing") + # message_callback.emit("Job {} queueing".format(jobid)) + else: + print("job is running") + # widgets['buttonBox'].button(QtWidgets.QDialogButtonBox.Apply).setText('Running') + + # tails the output of dvc + tail = self.pytail(a, logfile, start_at) + # count the number of newlines + for i in tail: + if i == "\n": + start_at+=1 + message_callback.emit("{}".format(tail.decode('utf-8'))) + # try to infer the progress + def progress(line): + import re + try: + match = re.search('^([0-9]*)/([0-9]*)', line.decode('utf-8')) + if match is not None: + return eval(match.group(0)) + except Exception as err: + print (err) + + line = tail.splitlines() + if len(line) >= 2: + line = line[-2] + + curr_progress = progress(line) + if curr_progress is not None: + # widgets['progressBar'].setValue(int(progress(line)*100)) + progress_callback.emit(int(progress(line)*100)) + print ("attempt evaluate progress ", progress(line)) + + sleep(update_delay) + status = a.job_status(jobid) + + + # dvc computation is finished, we get the last part of the output + tail = self.pytail(a, logfile, start_at) + message_callback.emit("{}".format(tail.decode('utf-8'))) + # set the progress to 100 + progress_callback.emit(100) + + a.changedir(folder) + a.get_file("slurm-{}.out".format(jobid)) + a.get_file("dvc.out".format(jobid)) + # here we should fetch also all the output files defined at + # output_filename\t{1}/small_grid\t### base name for output files + + a.logout() + self.internalsignals.status.emit((jobid, 'FINISHED')) + diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index e7594a65..9c9c3b09 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -9,6 +9,8 @@ import platform import pysnooper from brem import AsyncCopyOverSSH, BasicRemoteExecutionManager +import tempfile +import ntpath, posixpath count = 0 runs_completed = 0 @@ -153,7 +155,7 @@ def finished_run(main_window, exitCode, exitStatus, process = None, required_run class DVC_runner(object): - def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_folder): + def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_folder, remote_os=None): # print("The session folder is", session_folder) self.main_window = main_window self.input_file = input_file @@ -289,6 +291,15 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo selected_central_grid.write(line) + if remote_os is not None: + if remote_os == 'Windows': + dpath = ntpath + elif remote_os == 'POSIX': + dpath = posixpath + for f in [grid_roi_fname, output_filename]: + f = dpath.abspath(f) + + config = blank_config.format( reference_filename= reference_file, # reference tomography image volume correlate_filename= correlate_file, # correlation tomography image volume @@ -333,7 +344,6 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo (exe_file, [ config_filename ], required_runs, total_points) ) - @pysnooper.snoop() def zip_workdir_and_upload(self, **kwargs): #param_file is a list with at least 1 item but we are interested in the first # because we want to know the path to it and all files will be in the same directory @@ -366,19 +376,30 @@ def zip_workdir_and_upload(self, **kwargs): self.asyncCopy.SetLocalDir(os.path.dirname(zipped)) self.asyncCopy.SetFileName(os.path.basename(zipped)) + self.asyncCopy.signals.status.connect(print) self.asyncCopy.signals.finished.connect( lambda: self._unzip_on_remote(remote_dir, os.path.basename(zipped)) ) - - self.asyncCopy.threadpool.start(self.asyncCopy.worker) + self.asyncCopy.threadpool.start(self.asyncCopy.worker) + @pysnooper.snoop() def _unzip_on_remote(self, workdir, filename, **kwargs): # 1 create a BasicRemoteExecutionManager + username = self.main_window.connection_details['username'] + port = self.main_window.connection_details['server_port'] + host = self.main_window.connection_details['server_name'] + private_key = self.main_window.connection_details['private_key'] + remote_os = self.main_window.connection_details['remote_os'] + logfile = os.path.join(tempfile.tempdir, 'ssh.log') + conn = BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) + conn.login(passphrase=False) # 2 go to workdir + conn.changedir(workdir) # 2 run 'unzip filename' - print("_unzip_on_remote") + stdout, stderr = conn.run('cd {} && unzip {}'.format(workdir, filename)) + From 6dc482335a98bca249e247ddc5d213f44511bfe5 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Fri, 3 Dec 2021 15:14:46 +0000 Subject: [PATCH 11/19] first working run on the remote --- src/idvc/dvc_interface.py | 14 ++++++++--- src/idvc/dvc_runner.py | 49 ++++++++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 5a9bb04d..f9cf8d35 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -4020,10 +4020,10 @@ def ZipAndUploadConfigToRemote(self): self.finished_run, self.run_succeeded, tempfile.tempdir, remote_os=self.connection_details['remote_os']) self.config_worker = Worker(self.dvc_runner.zip_workdir_and_upload) # self.create_progress_window("Connecting with remote", "Zipping and uploading", 0, None, False) - self.config_worker.signals.finished.connect( self.run_code_remote ) + self.config_worker.signals.finished.connect( self.unzip_on_remote ) self.threadpool.start(self.config_worker) - def run_code_remote(self): + def unzip_on_remote(self): print ("run_code_remote") while True: tc = self.dvc_runner.asyncCopy.threadpool.activeThreadCount() @@ -4033,9 +4033,17 @@ def run_code_remote(self): sleep(0.25) self.unzip_worker = Worker(self.dvc_runner._unzip_on_remote, self.dvc_runner.asyncCopy.remotedir, self.dvc_runner.asyncCopy.filename) self.create_progress_window("Connecting with remote", "Unzipping", 0, None, False) - self.unzip_worker.signals.finished.connect( self.progress_window.close ) + self.unzip_worker.signals.finished.connect( self.run_code_on_remote ) self.threadpool.start(self.unzip_worker) + @pysnooper.snoop() + def run_code_on_remote(self): + self.progress_window.close() + self.dvc_worker = Worker(self.dvc_runner.run_dvc_on_remote, self.dvc_runner.asyncCopy.remotedir) + self.create_progress_window("Connecting with remote", "Running DVC remote", 0, None, False) + self.dvc_worker.signals.finished.connect( self.progress_window.close ) + self.threadpool.start(self.dvc_worker) + def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index 9c9c3b09..582866e1 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -290,14 +290,14 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo if i in lines_to_write: selected_central_grid.write(line) - + newline = None if remote_os is not None: - if remote_os == 'Windows': - dpath = ntpath - elif remote_os == 'POSIX': - dpath = posixpath - for f in [grid_roi_fname, output_filename]: - f = dpath.abspath(f) + if remote_os in ['Windows', 'POSIX'] : + # on remote we aim at running in the directory + grid_roi_fname = os.path.basename(grid_roi_fname) + output_filename = os.path.basename(output_filename) + if remote_os == 'POSIX': + newline = "\n" config = blank_config.format( @@ -326,7 +326,9 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo basin_radius='0.0', subvol_aspect='1.0 1.0 1.0') # image spacing time.sleep(1) - with open(config_filename,"w") as config_file: + + + with open(config_filename,"w", newline=newline) as config_file: config_file.write(config) #if run_count == len( subvolume_points): @@ -400,7 +402,36 @@ def _unzip_on_remote(self, workdir, filename, **kwargs): # 2 run 'unzip filename' stdout, stderr = conn.run('cd {} && unzip {}'.format(workdir, filename)) - + @pysnooper.snoop() + def run_dvc_on_remote(self, workdir, **kwargs): + # 1 create a BasicRemoteExecutionManager + username = self.main_window.connection_details['username'] + port = self.main_window.connection_details['server_port'] + host = self.main_window.connection_details['server_name'] + private_key = self.main_window.connection_details['private_key'] + remote_os = self.main_window.connection_details['remote_os'] + logfile = os.path.join(tempfile.tempdir, 'ssh.log') + + progress_callback = kwargs.get('progress_callback', None) + + if remote_os == 'POSIX': + dpath = posixpath + else: + dpath = ntpath + + conn = BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) + conn.login(passphrase=False) + # 2 go to workdir + conn.changedir(workdir) + for i,el in enumerate(self.processes): + if progress_callback is not None: + progress_callback.emit(i) + + param_file = el[1][0] + + wdir = dpath.join(workdir, 'dvc_result_{}'.format(i)) + # 2 run 'unzip filename' + stdout, stderr = conn.run('cd {} && . ~/condarc && conda activate dvc && dvc dvc_config.txt'.format(wdir)) def run_dvc(self): From 654e1111a7d58f4950b87f5fce9aa541ff73982c Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Fri, 3 Dec 2021 15:19:45 +0000 Subject: [PATCH 12/19] fix indent --- src/idvc/dvc_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index 582866e1..c9657106 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -415,7 +415,7 @@ def run_dvc_on_remote(self, workdir, **kwargs): progress_callback = kwargs.get('progress_callback', None) if remote_os == 'POSIX': - dpath = posixpath + dpath = posixpath else: dpath = ntpath From a5dde6c82caf9670a3659e4280f6edb1f6ec7308 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Fri, 3 Dec 2021 16:30:45 +0000 Subject: [PATCH 13/19] some updates --- src/idvc/dvc_interface.py | 9 +++++- src/idvc/dvc_remote.py | 68 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index f9cf8d35..99c853f6 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -80,6 +80,7 @@ from brem.ui import RemoteFileDialog from brem import AsyncCopyOverSSH +from idvc.dvc_remote import DVCRemoteRunControl __version__ = gui_version.version @@ -4039,7 +4040,13 @@ def unzip_on_remote(self): @pysnooper.snoop() def run_code_on_remote(self): self.progress_window.close() - self.dvc_worker = Worker(self.dvc_runner.run_dvc_on_remote, self.dvc_runner.asyncCopy.remotedir) + + self.dvc_remote_controller = DVCRemoteRunControl(self.connection_details) + self.dvc_remote_controller.set_workdir(self.dvc_runner.asyncCopy.remotedir) + self.dvc_remote_controller.set_num_runs(len(self.dvc_runner.processes)) + + # self.dvc_worker = Worker(self.dvc_runner.run_dvc_on_remote, self.dvc_runner.asyncCopy.remotedir) + self.dvc_worker = Worker(self.dvc_remote_controller.run_dvc_on_remote) self.create_progress_window("Connecting with remote", "Running DVC remote", 0, None, False) self.dvc_worker.signals.finished.connect( self.progress_window.close ) self.threadpool.start(self.dvc_worker) diff --git a/src/idvc/dvc_remote.py b/src/idvc/dvc_remote.py index fb6bd8c0..f008e9db 100644 --- a/src/idvc/dvc_remote.py +++ b/src/idvc/dvc_remote.py @@ -1,5 +1,8 @@ import os from brem import RemoteRunControl +import brem +from PySide2 import QtCore +import ntpath, posixpath class PrepareDVCRemote(object): def __init__(self, parent): @@ -25,13 +28,76 @@ def remote_workdir(self): def set_remote_workdir(self): self._remote_workdir = self.parent.settings_window + +class DVCRemoteRunControlSignals(QtCore.QObject): + status = QtCore.Signal(tuple) + job_id = QtCore.Signal(int) + progress = QtCore.Signal(int) + class DVCRemoteRunControl(RemoteRunControl): + def __init__(self, connection_details): + + super(DVCRemoteRunControl, self).__init__(connection_details=connection_details) + self.signals = DVCRemoteRunControlSignals() + self._num_runs = 0 + self._workdir = None + + + def set_num_runs(self, value): + self._num_runs = value + + + def set_workdir(self, value): + self._workdir = value + + + @property + def num_runs(self): + return self._num_runs + + + @property + def workir(self): + return self._workdir + + + @pysnooper.snoop() + def run_dvc_on_remote(self, **kwargs): + # 1 create a BasicRemoteExecutionManager + username = self.connection_details['username'] + port = self.connection_details['server_port'] + host = self.connection_details['server_name'] + private_key = self.connection_details['private_key'] + remote_os = self.connection_details['remote_os'] + logfile = os.path.join('ssh.log') + + progress_callback = kwargs.get('progress_callback', None) + + if remote_os == 'POSIX': + dpath = posixpath + else: + dpath = ntpath + + conn = brem.BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) + conn.login(passphrase=False) + # 2 go to workdir + conn.changedir(self.workdir) + for i in range(self.num_runs): + if progress_callback is not None: + progress_callback.emit(i) + + wdir = dpath.join(self.workdir, 'dvc_result_{}'.format(i)) + # 2 run 'unzip filename' + stdout, stderr = conn.run('cd {} && . ~/condarc && conda activate dvc && dvc dvc_config.txt'.format(wdir)) + + +class DVCSLURMRemoteRunControl(RemoteRunControl): def __init__(self, connection_details=None, reference_filename=None, correlate_filename=None, dvclog_filename=None, dev_config=None): - super(DVCRemoteRunControl, self).__init__(connection_details=connection_details) + super(DVCSLURMRemoteRunControl, self).__init__(connection_details=connection_details) self.reference_fname = reference_filename self.correlate_fname = correlate_filename self.dvclog_fname = dvclog_filename From 8272d5345ba9ac7e12de403dc620e37bd82de978 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Fri, 10 Dec 2021 23:47:05 +0000 Subject: [PATCH 14/19] retrieves the results from remote --- setup.py | 4 +-- src/idvc/dvc_interface.py | 48 ++++++++++++++++++++++++---- src/idvc/dvc_remote.py | 67 ++++++++++++++++++++++++++++++++------- src/idvc/dvc_runner.py | 10 +++++- 4 files changed, 107 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 92ec52bf..1986524c 100644 --- a/setup.py +++ b/setup.py @@ -38,8 +38,8 @@ setup( name = "idvc", description = 'CCPi DVC Configurator', - version = dversion, - packages = {'idvc'}, + version = dversion, + packages = {'idvc'}, package_dir = {'idvc': os.path.join('src','idvc')}, package_data = {'idvc':['DVCIconSquare.png']}, # metadata for upload to PyPI diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 99c853f6..f5e7c782 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -599,13 +599,10 @@ def GetFileFromRemote(self, image_var, image, label, next_button): sleep(1) - self.create_progress_window("Getting files from remote", "", 0, None, False) + self.create_progress_window("Getting files from remote", "", 0, None, False, 0) self.updateUnknownProgressDialog = Worker(self.UnknownProgressUpdateDialog) self.updateUnknownProgressDialog.signals.finished.connect(self.StopUnknownProgressUpdate) self.threadpool.start(self.updateUnknownProgressDialog) - # make the progress dialog start after minimum 0.5 s - self.progress_window.setMinimumDuration(0) - def UnknownProgressUpdateDialog(self, **kwargs): @@ -910,12 +907,21 @@ def visualise(self): #bring image loading panel to front if it isnt already: self.select_image_dock.raise_() - def create_progress_window(self, title, text, max = 100, cancel = None, autoClose = True): + def create_progress_window(self, title, text, max = 100, cancel = None, autoClose = True, minimumDuration=4000): + '''Creates a QProgressDialog + + :param title: title + :param text: text in the dialog + :param max: max value of the progress, default 100. Minimum is set to 0. + :param cancel: if to show a cancel button, default None hence Cancel not shown. + :autoClose: if the dialog should close when max is reached + :minimumDuration: This property holds the time that must pass before the dialog appears, default 4000 ms + ''' self.progress_window = QProgressDialog(text, "Cancel", 0,max, self, QtCore.Qt.Window) self.progress_window.setWindowTitle(title) self.progress_window.setWindowModality(QtCore.Qt.ApplicationModal) #This means the other windows can't be used while this is open - self.progress_window.setMinimumDuration(0.01) + self.progress_window.setMinimumDuration(int(minimumDuration)) self.progress_window.setWindowFlag(QtCore.Qt.WindowCloseButtonHint, True) self.progress_window.setWindowFlag(QtCore.Qt.WindowMaximizeButtonHint, False) self.progress_window.setAutoClose(autoClose) @@ -923,6 +929,7 @@ def create_progress_window(self, title, text, max = 100, cancel = None, autoClos self.progress_window.setCancelButton(None) else: self.progress_window.canceled.connect(cancel) + self.progress_window.show() def setup2DPointCloudPipeline(self): @@ -4024,6 +4031,7 @@ def ZipAndUploadConfigToRemote(self): self.config_worker.signals.finished.connect( self.unzip_on_remote ) self.threadpool.start(self.config_worker) + def unzip_on_remote(self): print ("run_code_remote") while True: @@ -4035,8 +4043,29 @@ def unzip_on_remote(self): self.unzip_worker = Worker(self.dvc_runner._unzip_on_remote, self.dvc_runner.asyncCopy.remotedir, self.dvc_runner.asyncCopy.filename) self.create_progress_window("Connecting with remote", "Unzipping", 0, None, False) self.unzip_worker.signals.finished.connect( self.run_code_on_remote ) + self.unzip_worker.signals.status.connect( self.update_status) + self.unzip_worker.signals.error.connect( self.update_on_error) + self.threadpool.start(self.unzip_worker) + + def update_status(self, data): + print ("STDOUT", data[0]) + print ("STDERR", data[1]) + + + def update_on_error(self, data): + # traceback.print_exc() + # exctype, value = sys.exc_info()[:2] + # self.signals.error.emit((exctype, value, traceback.format_exc())) + print(data) + msg = QMessageBox(self) + msg.setIcon(QMessageBox.Critical) + msg.setWindowTitle("Remote execution Error") + msg.setText("exectype {}, value {}".format(data[0], data[1])) + msg.setDetailedText(data[2]) + msg.exec_() + @pysnooper.snoop() def run_code_on_remote(self): self.progress_window.close() @@ -4048,9 +4077,14 @@ def run_code_on_remote(self): # self.dvc_worker = Worker(self.dvc_runner.run_dvc_on_remote, self.dvc_runner.asyncCopy.remotedir) self.dvc_worker = Worker(self.dvc_remote_controller.run_dvc_on_remote) self.create_progress_window("Connecting with remote", "Running DVC remote", 0, None, False) - self.dvc_worker.signals.finished.connect( self.progress_window.close ) + self.dvc_worker.signals.finished.connect( self.retrieve_results_and_close_progress ) self.threadpool.start(self.dvc_worker) + def retrieve_results_and_close_progress(self): + self.dvc_worker = Worker(self.dvc_remote_controller.retrieve_results, os.path.abspath(self.run_config_file)) + self.dvc_worker.signals.finished.connect( self.progress_window.close ) + self.threadpool.start(self.dvc_worker) + def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) diff --git a/src/idvc/dvc_remote.py b/src/idvc/dvc_remote.py index f008e9db..f4aea974 100644 --- a/src/idvc/dvc_remote.py +++ b/src/idvc/dvc_remote.py @@ -3,6 +3,8 @@ import brem from PySide2 import QtCore import ntpath, posixpath +import pysnooper +import json class PrepareDVCRemote(object): def __init__(self, parent): @@ -31,16 +33,17 @@ def set_remote_workdir(self): class DVCRemoteRunControlSignals(QtCore.QObject): status = QtCore.Signal(tuple) - job_id = QtCore.Signal(int) progress = QtCore.Signal(int) -class DVCRemoteRunControl(RemoteRunControl): +class DVCRemoteRunControl(object): def __init__(self, connection_details): - super(DVCRemoteRunControl, self).__init__(connection_details=connection_details) + super(DVCRemoteRunControl, self).__init__() + # self.signals is a property of RemoteRunControl self.signals = DVCRemoteRunControlSignals() self._num_runs = 0 self._workdir = None + self.connection_details = connection_details def set_num_runs(self, value): @@ -57,12 +60,10 @@ def num_runs(self): @property - def workir(self): + def workdir(self): return self._workdir - - @pysnooper.snoop() - def run_dvc_on_remote(self, **kwargs): + def _create_connection(self): # 1 create a BasicRemoteExecutionManager username = self.connection_details['username'] port = self.connection_details['server_port'] @@ -71,17 +72,27 @@ def run_dvc_on_remote(self, **kwargs): remote_os = self.connection_details['remote_os'] logfile = os.path.join('ssh.log') + conn = brem.BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) + conn.login(passphrase=False) + # 2 go to workdir + conn.changedir(self.workdir) + + return conn + + @pysnooper.snoop() + def run_dvc_on_remote(self, **kwargs): + # 1 create a BasicRemoteExecutionManager + + conn = self._create_connection() + remote_os = conn.remote_os + progress_callback = kwargs.get('progress_callback', None) if remote_os == 'POSIX': dpath = posixpath else: dpath = ntpath - - conn = brem.BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) - conn.login(passphrase=False) - # 2 go to workdir - conn.changedir(self.workdir) + for i in range(self.num_runs): if progress_callback is not None: progress_callback.emit(i) @@ -90,6 +101,38 @@ def run_dvc_on_remote(self, **kwargs): # 2 run 'unzip filename' stdout, stderr = conn.run('cd {} && . ~/condarc && conda activate dvc && dvc dvc_config.txt'.format(wdir)) + @pysnooper.snoop() + def retrieve_results(self, config_file, **kwargs): + + # created in dvc_interface create_run_config + with open(config_file) as tmp: + config = json.load(tmp) + run_folder = config['run_folder'] + + conn = self._create_connection() + remote_os = conn.remote_os + + progress_callback = kwargs.get('progress_callback', None) + + if remote_os == 'POSIX': + dpath = posixpath + else: + dpath = ntpath + + # retrieve the results in each directory and store it locally + for i in range(self.num_runs): + if progress_callback is not None: + progress_callback.emit(i) + fname = 'dvc_result_{}'.format(i) + + localdir = os.path.join(run_folder, "dvc_result_{}".format(i)) + + for extension in ['stat', 'disp']: + path_to_file = dpath.join(self.workdir, fname) + file_to_get = '{}.{}'.format(fname, extension) + conn.changedir(path_to_file) + conn.get_file(file_to_get, localdir) + class DVCSLURMRemoteRunControl(RemoteRunControl): def __init__(self, connection_details=None, diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index c9657106..0371972b 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -400,7 +400,15 @@ def _unzip_on_remote(self, workdir, filename, **kwargs): # 2 go to workdir conn.changedir(workdir) # 2 run 'unzip filename' - stdout, stderr = conn.run('cd {} && unzip {}'.format(workdir, filename)) + # -o forces to overwrite the files being unzipped + # unzip should be substituted with some python like + # import zipfile + # with zipfile.ZipFile("remote_run.zip", 'r') as mz: + # mz.extractall() + stdout, stderr = conn.run('cd {} && unzip -o {}'.format(workdir, filename)) + status_callback = kwargs.get('status_callback', None) + if status_callback is not None: + status_callback.emit((stdout, stderr)) @pysnooper.snoop() def run_dvc_on_remote(self, workdir, **kwargs): From d14e18eac33a905cecf8ffc53a7e41b8efa2152f Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Sat, 11 Dec 2021 00:07:21 +0000 Subject: [PATCH 15/19] graph window partially populated --- src/idvc/dvc_interface.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index f5e7c782..d491d509 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -4042,7 +4042,7 @@ def unzip_on_remote(self): sleep(0.25) self.unzip_worker = Worker(self.dvc_runner._unzip_on_remote, self.dvc_runner.asyncCopy.remotedir, self.dvc_runner.asyncCopy.filename) self.create_progress_window("Connecting with remote", "Unzipping", 0, None, False) - self.unzip_worker.signals.finished.connect( self.run_code_on_remote ) + self.unzip_worker.signals.finished.connect( self.remote_run_code ) self.unzip_worker.signals.status.connect( self.update_status) self.unzip_worker.signals.error.connect( self.update_on_error) @@ -4067,7 +4067,7 @@ def update_on_error(self, data): msg.exec_() @pysnooper.snoop() - def run_code_on_remote(self): + def remote_run_code(self): self.progress_window.close() self.dvc_remote_controller = DVCRemoteRunControl(self.connection_details) @@ -4077,14 +4077,18 @@ def run_code_on_remote(self): # self.dvc_worker = Worker(self.dvc_runner.run_dvc_on_remote, self.dvc_runner.asyncCopy.remotedir) self.dvc_worker = Worker(self.dvc_remote_controller.run_dvc_on_remote) self.create_progress_window("Connecting with remote", "Running DVC remote", 0, None, False) - self.dvc_worker.signals.finished.connect( self.retrieve_results_and_close_progress ) + self.dvc_worker.signals.finished.connect( self.remote_retrieve_results ) self.threadpool.start(self.dvc_worker) - def retrieve_results_and_close_progress(self): + def remote_retrieve_results(self): self.dvc_worker = Worker(self.dvc_remote_controller.retrieve_results, os.path.abspath(self.run_config_file)) - self.dvc_worker.signals.finished.connect( self.progress_window.close ) + self.dvc_worker.signals.finished.connect( self.remote_update_result_panel ) self.threadpool.start(self.dvc_worker) - + + def remote_update_result_panel(self): + self.run_succeeded = True + self.progress_window.close() + self.finished_run() def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) @@ -4191,7 +4195,7 @@ def create_run_config(self, **kwargs): else: run_config['rigid_trans']= "0.0 0.0 0.0" - self.run_folder = "Results/" + folder_name + self.run_folder = os.path.join("Results", folder_name) run_config['run_folder'] = self.run_folder #where is point0 @@ -4369,7 +4373,7 @@ def CreateViewDVCResultsPanel(self): self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, dockWidget) self.result_widgets = result_widgets - + @pysnooper.snoop() def show_run_pcs(self): #show pointcloud files in list self.result_widgets['pc_entry'].clear() From ac2124cea7ae7a8191dc67e404ecbf1ba94072e4 Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Wed, 15 Dec 2021 08:07:35 +0000 Subject: [PATCH 16/19] added comments on remote transfer of files --- src/idvc/dvc_interface.py | 1 + src/idvc/dvc_runner.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index e85b5440..4d251df4 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -4033,6 +4033,7 @@ def ZipAndUploadConfigToRemote(self): def unzip_on_remote(self): + # TODO: keep this here and remove the async copy in dvc_runner print ("run_code_remote") while True: tc = self.dvc_runner.asyncCopy.threadpool.activeThreadCount() diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index 0371972b..53a235cb 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -347,6 +347,8 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo ) def zip_workdir_and_upload(self, **kwargs): + # TODO this is already in a worker and does not need to run the AsyncCopyOverSSH + #param_file is a list with at least 1 item but we are interested in the first # because we want to know the path to it and all files will be in the same directory exe_file, param_file, required_runs,\ From e1fd2b6b18df34e35d50a0b11cc359dd4985109e Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Wed, 15 Dec 2021 14:32:22 +0000 Subject: [PATCH 17/19] fix check for local run see https://github.com/TomographicImaging/iDVC/pull/78 --- src/idvc/dvc_interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 323c090c..2580de65 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -3985,7 +3985,8 @@ def create_config_worker(self): self.create_progress_window("Loading", "Generating Run Config") self.config_worker.signals.progress.connect(self.progress) # if single or bulk use the line below, if remote develop new functionality - if not self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): + if not hasattr(self, 'settings_window') and\ + not self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): self.config_worker.signals.result.connect(partial (self.run_external_code)) else: # do not run the dvc locally but From 43d680e2cac221fc57f9500bafe009b7965295de Mon Sep 17 00:00:00 2001 From: Laura <60604372+lauramurgatroyd@users.noreply.github.com> Date: Wed, 15 Dec 2021 14:33:33 +0000 Subject: [PATCH 18/19] Fixes error in displaying DVC results (#78) * Remove underscore in result folder name * reinstate imports --- src/idvc/dvc_interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index e85b5440..3c75a0bc 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -3992,7 +3992,7 @@ def create_config_worker(self): message="Load a mask on the viewer first" ) return - folder_name = "_" + self.rdvc_widgets['name_entry'].text() + folder_name = self.rdvc_widgets['name_entry'].text() results_folder = os.path.join(tempfile.tempdir, "Results") @@ -4094,7 +4094,7 @@ def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) progress_callback = kwargs.get('progress_callback', None) try: - folder_name = "_" + self.rdvc_widgets['name_entry'].text() + folder_name = self.rdvc_widgets['name_entry'].text() results_folder = os.path.join(tempfile.tempdir, "Results") os.mkdir(os.path.join(results_folder, folder_name)) From 0807b9280787910cc7365c979dcae6c0724fbe1a Mon Sep 17 00:00:00 2001 From: Edoardo Pasca Date: Wed, 15 Dec 2021 16:23:05 +0000 Subject: [PATCH 19/19] make local dvc analysis work again --- src/idvc/dvc_interface.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index d3880468..e6b82890 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -3950,6 +3950,7 @@ def select_roi(self, label, next_button): if self.roi: next_button.setEnabled(True) + def create_config_worker(self): if hasattr(self, 'translate'): if self.translate is None: @@ -3985,16 +3986,20 @@ def create_config_worker(self): self.create_progress_window("Loading", "Generating Run Config") self.config_worker.signals.progress.connect(self.progress) # if single or bulk use the line below, if remote develop new functionality - if not hasattr(self, 'settings_window') and\ - not self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): + run_local = True + if hasattr(self, 'settings_window'): + if not self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): + run_local = False + + if run_local: self.config_worker.signals.result.connect(partial (self.run_external_code)) else: # do not run the dvc locally but # 1 zip and # 2 upload the config to remote and then # 3 run the code on the remote - self.config_worker.signals.finished.connect(partial (self.ZipAndUploadConfigToRemote)) - pass + self.config_worker.signals.finished.connect(self.ZipAndUploadConfigToRemote) + self.threadpool.start(self.config_worker) self.progress_window.setValue(10) @@ -4046,7 +4051,7 @@ def update_on_error(self, data): msg.setDetailedText(data[2]) msg.exec_() - @pysnooper.snoop() + def remote_run_code(self): self.progress_window.close() @@ -4070,6 +4075,7 @@ def remote_update_result_panel(self): self.progress_window.close() self.finished_run() + def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) progress_callback = kwargs.get('progress_callback', None) @@ -4138,7 +4144,7 @@ def create_run_config(self, **kwargs): #print("finished making pointclouds") # if remote mode this should not be the local copy - if self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): + if hasattr(self, 'settings_window') and self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): # this should point to the remote files set at the time of download self.reference_file = self.remote_reference_image_fname self.correlate_file = self.remote_correlate_image_fname