Merge branch 'master' of https://github.com/AcademySoftwareFoundation/OpenCue into rqd-cuda

Author: n-jay

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java

@@ -44,7 +44,6 @@ public class DispatchQuery {
                 "AND folder.pk_dept         = point.pk_dept " +
                 "AND folder.pk_show         = point.pk_show " +
                 "AND job.pk_job             = layer.pk_job " +
-                "AND job_resource.pk_job    = job.pk_job " +
                 "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN layer_stat.pk_layer ELSE NULL END) = layer.pk_layer " +
                 "AND " +
                     "(" +
@@ -139,7 +138,6 @@ public class DispatchQuery {
                 "AND folder.pk_dept         = point.pk_dept " +
                 "AND folder.pk_show         = point.pk_show " +
                 "AND job.pk_job             = layer.pk_job " +
-                "AND job_resource.pk_job    = job.pk_job " +
                 "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN layer_stat.pk_layer ELSE NULL END) = layer.pk_layer " +
                 "AND " +
                     "(" +

cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java

@@ -23,7 +23,7 @@
 
 import io.grpc.stub.StreamObserver;
 
-import com.imageworks.spcue.ServiceEntity;
+import com.imageworks.spcue.ServiceOverrideEntity;
 import com.imageworks.spcue.grpc.service.Service;
 import com.imageworks.spcue.grpc.service.ServiceOverrideDeleteRequest;
 import com.imageworks.spcue.grpc.service.ServiceOverrideDeleteResponse;
@@ -39,15 +39,17 @@ public class ManageServiceOverride extends ServiceOverrideInterfaceGrpc.ServiceO
     @Override
     public void delete(ServiceOverrideDeleteRequest request,
                        StreamObserver<ServiceOverrideDeleteResponse> responseObserver) {
-        serviceManager.deleteService(toServiceEntity(request.getService()));
+        // Passing null on showId as the interface doesn't require a showId in this situation
+        serviceManager.deleteService(toServiceOverrideEntity(request.getService(), null));
         responseObserver.onNext(ServiceOverrideDeleteResponse.newBuilder().build());
         responseObserver.onCompleted();
     }
 
     @Override
     public void update(ServiceOverrideUpdateRequest request,
                        StreamObserver<ServiceOverrideUpdateResponse> responseObserver) {
-        serviceManager.updateService(toServiceEntity(request.getService()));
+        // Passing null on showId as the interface doesn't require a showId in this situation
+        serviceManager.updateService(toServiceOverrideEntity(request.getService(), null));
         responseObserver.onNext(ServiceOverrideUpdateResponse.newBuilder().build());
         responseObserver.onCompleted();
     }
@@ -60,8 +62,8 @@ public void setServiceManager(ServiceManager serviceManager) {
         this.serviceManager = serviceManager;
     }
 
-    private ServiceEntity toServiceEntity(Service service) {
-        ServiceEntity entity = new ServiceEntity();
+    private ServiceOverrideEntity toServiceOverrideEntity(Service service, String showId){
+        ServiceOverrideEntity entity = new ServiceOverrideEntity();
         entity.id = service.getId();
         entity.name = service.getName();
         entity.minCores = service.getMinCores();
@@ -72,6 +74,7 @@ private ServiceEntity toServiceEntity(Service service) {
         entity.minGpuMemory = service.getMinGpuMemory();
         entity.tags = new LinkedHashSet<>(service.getTagsList());
         entity.threadable = service.getThreadable();
+        entity.showId = showId;
         entity.timeout = service.getTimeout();
         entity.timeout_llu = service.getTimeoutLlu();
         entity.minMemoryIncrease = service.getMinMemoryIncrease();

cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes

@@ -0,0 +1,29 @@
+
+--Performance issue, Created new index on column int_gpus_min
+
+
+CREATE INDEX IF NOT EXISTS i_layer_int_gpu_mem_min
+    ON public.layer USING btree
+    (int_gpus_min ASC NULLS LAST)
+    TABLESPACE pg_default;
+    
+
+CREATE INDEX IF NOT EXISTS i_layer_int_gpu_mem_min_1
+    ON public.layer USING btree
+    (int_gpu_min ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+
+create index concurrently i_layer_int_cores_max on layer(int_cores_max);
+
+create index concurrently i_job_resource_int_priority on job_resource(int_priority);
+
+create index concurrently i_job_int_min_cores on job(int_min_cores);
+
+create index concurrently  i_layer_limit_pk_layer on layer_limit(pk_layer);
+
+create index concurrently i_folder_resource_int_cores on folder_resource(int_cores);
+
+create index concurrently  i_job_ts_updated on job(ts_updated);
+
+create index concurrently i_layer_str_tags on layer(str_tags);

cuegui/cuegui/JobMonitorTree.py

@@ -494,7 +494,7 @@ def _getUpdate(self):
                 # an empty list for the id argument!
                 if not ids:
                     continue
-                tmp = opencue.api.getJobs(id=ids, all=True)
+                tmp = opencue.api.getJobs(id=ids, include_finished=True)
                 self.__dependentJobs[job] = tmp
 
             if self.__loadMine:

cuegui/cuegui/Main.py

@@ -92,7 +92,6 @@ def startup(app_name, app_version, argv):
     app.aboutToQuit.connect(closingTime)  # pylint: disable=no-member
     app.exec_()
 
-
 def closingTime():
     """Window close callback."""
     logger.info("Closing all threads...")

cuegui/cuegui/ServiceDialog.py

@@ -31,6 +31,7 @@
 import cuegui.Constants
 import cuegui.TagsWidget
 import cuegui.Utils
+from opencue.wrappers.service import ServiceOverride
 
 
 class ServiceForm(QtWidgets.QWidget):
@@ -124,13 +125,13 @@ def setService(self, service):
         """
         self.__service = service
         self.__buttons.setDisabled(False)
-        self.name.setText(service.data.name)
-        self.threadable.setChecked(service.data.threadable)
-        self.min_cores.setValue(service.data.min_cores)
-        self.max_cores.setValue(service.data.max_cores)
-        self.min_memory.setValue(service.data.min_memory // 1024)
+        self.name.setText(service.name())
+        self.threadable.setChecked(service.threadable())
+        self.min_cores.setValue(service.minCores())
+        self.max_cores.setValue(service.maxCores())
         self.min_gpu_memory.setValue(service.data.min_gpu_memory // 1024)
-        self._tags_w.set_tags(service.data.tags)
+        self.min_memory.setValue(service.minMemory() // 1024)
+        self._tags_w.set_tags(service.tags())
         self.timeout.setValue(service.data.timeout)
         self.timeout_llu.setValue(service.data.timeout_llu)
         self.min_memory_increase.setValue(service.data.min_memory_increase // 1024)
@@ -263,11 +264,16 @@ def saved(self, service):
 
         if self.__new_service:
             if self.__show:
-                self.__show.createServiceOverride(service.data)
+                serviceOverride = self.__show.createServiceOverride(service.data)
             else:
                 opencue.api.createService(service.data)
         else:
-            service.update()
+            if self.__show:
+                serviceOverride = ServiceOverride(service)
+                serviceOverride.id = service.id()
+                serviceOverride.update()
+            else:
+                service.update()
 
         self.refresh()
         self.__new_service = False

cuegui/cuegui/plugins/LogViewPlugin.py

@@ -25,7 +25,9 @@
 import os
 import re
 import string
+import sys
 import time
+import traceback
 
 from qtpy import QtGui
 from qtpy import QtCore
@@ -292,10 +294,40 @@ def line_number_area_paint_event(self, event):
             bottom = top + self.blockBoundingRect(block).height()
             block_number += 1
 
+class LogLoadSignals(QtCore.QObject):
+    """Signals for the LoadLog action"""
+    SIG_LOG_LOAD_ERROR = QtCore.Signal(tuple)
+    SIG_LOG_LOAD_RESULT = QtCore.Signal(str, str)
+    SIG_LOG_LOAD_FINISHED = QtCore.Signal()
+
+class LogLoader(QtCore.QRunnable):
+    """A thread to load logs"""
+    def __init__(self, fn, *args, **kwargs):
+        super(LogLoader, self).__init__()
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+        self.signals = LogLoadSignals()
+
+    @QtCore.Slot()
+    def run(self):
+        # pylint: disable=bare-except
+        try:
+            content, log_mtime = self.fn(*self.args, **self.kwargs)
+        except:
+            exctype, value = sys.exc_info()[:2]
+            self.signals.SIG_LOG_LOAD_ERROR.emit(
+                (exctype, value, traceback.format_exc()))
+        else:
+            self.signals.SIG_LOG_LOAD_RESULT.emit(content, log_mtime)
+        finally:
+            self.signals.SIG_LOG_LOAD_FINISHED.emit()
 
 class LogViewWidget(QtWidgets.QWidget):
-    """Displays the log file for the selected frame."""
-
+    """
+    Displays the log file for the selected frame
+    """
+    SIG_CONTENT_UPDATED = QtCore.Signal(str, str)
     def __init__(self, parent=None):
         """
         Create the UI elements
@@ -453,6 +485,9 @@ def __init__(self, parent=None):
         self._current_match = 0
         self._content_box.mousePressedSignal.connect(self._on_mouse_pressed)
 
+        self.SIG_CONTENT_UPDATED.connect(self._update_log_content)
+        self.log_thread_pool = QtCore.QThreadPool()
+
     def _on_mouse_pressed(self, pos):
         """
         Mouse press event, to be called when the user scrolls by hand or moves
@@ -788,12 +823,56 @@ def _display_log_content(self):
         """
 
         try:
-            self._update_log()
-            self._new_log = False
+            if not os.path.exists(self._log_file):
+                self._log_file_exists = False
+                content = 'Log file does not exist: %s' % self._log_file
+                self._content_timestamp = time.time()
+                self._update_log_content(content, self._log_mtime)
+            else:
+                # Creating the load logs process as qrunnables so
+                # that they don't block the ui while loading
+                log_loader = LogLoader(self._load_log, self._log_file,
+                    self._new_log, self._log_mtime)
+                log_loader.signals.SIG_LOG_LOAD_RESULT.connect(
+                    self._receive_log_results)
+                log_loader.setAutoDelete(True)
+                self.log_thread_pool.start(log_loader)
+                self.log_thread_pool.waitForDone()
+                self._new_log = False
         finally:
             QtCore.QTimer.singleShot(5000, self._display_log_content)
 
-    def _update_log(self):
+    # pylint: disable=no-self-use
+    @QtCore.Slot()
+    def _load_log(self, log_file, new_log, curr_log_mtime):
+        content = None
+        log_size = int(os.stat(log_file).st_size)
+        if log_size > 1 * 1e6:
+            content = ('Log file size (%0.1f MB) exceeds the size '
+                        'threshold (1.0 MB).'
+                        % float(log_size / (1024 * 1024)))
+        elif not new_log and os.path.exists(log_file):
+            log_mtime = os.path.getmtime(log_file)
+            if log_mtime > curr_log_mtime:
+                curr_log_mtime = log_mtime  # no new updates
+                content = ''
+
+        if content is None:
+            content = ''
+            try:
+                with open(log_file, 'r') as f:
+                    content = f.read()
+            except IOError:
+                content = 'Can not access log file: %s' % log_file
+
+        return content, curr_log_mtime
+
+    @QtCore.Slot()
+    def _receive_log_results(self, content, log_mtime):
+        self.SIG_CONTENT_UPDATED.emit(content, log_mtime)
+
+    @QtCore.Slot(str, str)
+    def _update_log_content(self, content, log_mtime):
         """
         Updates the content of the content box with the content of the log
         file, if necessary. The full path to the log file will be populated in
@@ -813,49 +892,23 @@ def _update_log(self):
                         (if necessary)
         """
 
-        # Get the content of the log file
-        if not self._log_file:
-            return  # There's no log file, nothing to do here!
-        self._path.setText(self._log_file)
-        content = None
-        if not os.path.exists(self._log_file):
-            self._log_file_exists = False
-            content = 'Log file does not exist: %s' % self._log_file
-            self._content_timestamp = time.time()
-        else:
-            log_size = int(os.stat(self._log_file).st_size)
-            if log_size > 5 * 1e6:
-                content = ('Log file size (%0.1f MB) exceeds the size '
-                           'threshold (5.0 MB).'
-                           % float(log_size / (1024 * 1024)))
-            elif not self._new_log and os.path.exists(self._log_file):
-                log_mtime = os.path.getmtime(self._log_file)
-                if log_mtime > self._log_mtime:
-                    self._log_mtime = log_mtime  # no new updates
-                    content = ''
-
-        if content is None:
-            content = ''
-            try:
-                with open(self._log_file, 'r') as f:
-                    content = f.read()
-            except IOError:
-                content = 'Can not access log file: %s' % self._log_file
+        self._log_mtime = log_mtime
 
-        # Do we need to scroll to the end?
-        scroll_to_end = (self._scrollbar_max == self._scrollbar_value
-                         or self._new_log)
+        self.app.processEvents()
 
         # Update the content in the gui (if necessary)
-        current_text = (self._content_box.toPlainText() or '')
-        new_text = content.lstrip(str(current_text))
-        if new_text:
-            if self._new_log:
-                self._content_box.setPlainText(content)
-            else:
+        if self._new_log:
+            self._content_box.setPlainText(content)
+        else:
+            current_text = (self._content_box.toPlainText() or '')
+            new_text = content.lstrip(str(current_text))
+            if new_text:
                 self._content_box.appendPlainText(new_text)
-            self._content_timestamp = time.time()
-        self.app.processEvents()
+        self._content_timestamp = time.time()
+        self._path.setText(self._log_file)
+
+        scroll_to_end = (self._scrollbar_max == self._scrollbar_value
+                         or self._new_log)
 
         # Adjust scrollbar value (if necessary)
         self._scrollbar_max = self._log_scrollbar.maximum()

cuegui/cuegui/plugins/MonitorJobsPlugin.py

@@ -197,17 +197,17 @@ def _regexLoadJobsHandle(self):
 
         self.jobMonitor.removeAllItems()
 
-        if cuegui.Utils.isStringId(substring):
-            # If a uuid is provided, load it
-            self.jobMonitor.addJob(substring)
-        elif load_finished_jobs or re.search(
+        if substring:
+            if cuegui.Utils.isStringId(substring):
+                # If a uuid is provided, load it
+                self.jobMonitor.addJob(substring)
+            elif load_finished_jobs or re.search(
                 r"^([a-z0-9_]+)\-([a-z0-9\.]+)\-", substring, re.IGNORECASE):
-            # If show and shot is provided, or if "load finished" checkbox is checked, load all jobs
-            for job in opencue.api.getJobs(regex=[substring], include_finished=True):
-                self.jobMonitor.addJob(job)
-        else:
-            # Otherwise, just load current matching jobs (except for the empty string)
-            if substring:
+                # Load all ff show and shot is provided or if "load finished" checkbox is checked
+                for job in opencue.api.getJobs(regex=[substring], include_finished=True):
+                    self.jobMonitor.addJob(job)
+            else:
+                # Otherwise, just load current matching jobs (except for the empty string)
                 for job in opencue.api.getJobs(regex=[substring]):
                     self.jobMonitor.addJob(job)