[rqd] [cuegui] Add support for Loki for frame logs

VERSION.in

@@ -1 +1 @@
-1.4
+1.5

ci/run_python_lint.sh

@@ -51,4 +51,5 @@ echo "Running lint for rqd/..."
 cd rqd
 python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto
 python -m pylint --rcfile=../ci/pylintrc_test tests
+python -m pylint --rcfile=../ci/pylintrc_test pytests
 cd ..

ci/run_python_tests.sh

@@ -27,6 +27,7 @@ PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p
 PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
 PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
 python -m pytest rqd/tests
+python -m pytest rqd/pytests
 
 # Xvfb no longer supports Python 2.
 if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then

cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java

@@ -72,5 +72,9 @@ public void setMinMemory(long minMemory) {
     public long getMinMemory() {
         return this.minMemory;
     }
+
+    // Parameters to tell rqd whether or not to use Loki for frame logs and which base url to use
+    public boolean lokiEnabled;
+    public String lokiURL;
 }
 

cuebot/src/main/java/com/imageworks/spcue/JobDetail.java

@@ -59,5 +59,10 @@ public class JobDetail extends JobEntity implements JobInterface, DepartmentInte
     public String getDepartmentId() {
         return deptId;
     }
+
+    // Parameters to tell cuebot whether or not to Loki is used for frame logs of the job and which
+    // base url to use for querying them
+    public Boolean logLokiEnabled;
+    public String logLokiURL;
 }
 

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java

@@ -531,6 +531,8 @@ private static final String replaceQueryForFifo(String query) {
         "int_uid, " +
         "str_log_dir, " +
         "COALESCE(str_os, '') AS str_os, " +
+        "b_loki_enabled, " +
+        "str_loki_url, " +
         "frame_name, " +
         "frame_state, " +
         "pk_frame, " +
@@ -572,6 +574,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +
@@ -659,6 +663,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +
@@ -747,6 +753,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +
@@ -828,6 +836,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +
@@ -912,6 +922,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +
@@ -999,6 +1011,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +
@@ -1087,6 +1101,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +
@@ -1168,6 +1184,8 @@ private static final String replaceQueryForFifo(String query) {
                 "job.int_uid, " +
                 "job.str_log_dir, " +
                 "job.str_os, " +
+                "job.b_loki_enabled, " +
+                "job.str_loki_url, " +
                 "frame.str_name AS frame_name, " +
                 "frame.str_state AS frame_state, " +
                 "frame.pk_frame, " +

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java

@@ -332,6 +332,8 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException {
             frame.version = rs.getInt("int_version");
             frame.services = rs.getString("str_services");
             frame.os = rs.getString("str_os");
+            frame.lokiEnabled = rs.getBoolean("b_loki_enabled");
+            frame.lokiURL = rs.getString("str_loki_url");
             return frame;
         }
     };
@@ -349,6 +351,8 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException {
             "job.int_uid,"+
             "job.str_log_dir,"+
             "COALESCE(str_os, '') AS str_os, " +
+            "job.b_loki_enabled,"+
+            "job.str_loki_url,"+
             "frame.str_name AS frame_name, "+
             "frame.str_state AS frame_state, "+
             "frame.pk_frame, "+

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java

@@ -137,6 +137,8 @@ public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException {
                 job.showName = rs.getString("show_name");
                 job.facilityName = rs.getString("facility_name");
                 job.deptName = rs.getString("dept_name");
+                job.logLokiEnabled = rs.getBoolean("b_loki_enabled");
+                job.logLokiURL = rs.getString("str_loki_url");
                 return job;
             }
         };
@@ -206,6 +208,8 @@ public boolean isJobComplete(JobInterface job) {
              "job.pk_dept,"+
              "job.pk_folder,"+
              "job.str_log_dir,"+
+             "job.b_loki_enabled,"+
+             "job.str_loki_url,"+
              "job.str_name,"+
              "job.str_shot,"+
              "job.str_state,"+
@@ -473,20 +477,25 @@ public boolean updateJobFinished(JobInterface job) {
             "int_uid," +
             "b_paused," +
             "b_autoeat,"+
-            "int_max_retries " +
+            "int_max_retries," +
+            "b_loki_enabled," +
+            "str_loki_url " +
         ") " +
-        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
 
     @Override
     public void insertJob(JobDetail j, JobLogUtil jobLogUtil) {
         j.id = SqlUtil.genKeyRandom();
         j.logDir = jobLogUtil.getJobLogPath(j);
+        j.logLokiEnabled = jobLogUtil.getLokiIsEnabled();
+        j.logLokiURL = jobLogUtil.getLokiURL();
         if (j.minCoreUnits < 100) { j.minCoreUnits = 100; }
 
         getJdbcTemplate().update(INSERT_JOB,
                 j.id, j.showId, j.groupId, j.facilityId, j.deptId,
                 j.name, j.name, j.showName, j.shot, j.user, j.email, j.state.toString(),
-                j.logDir, j.os, j.uid.orElse(null), j.isPaused, j.isAutoEat, j.maxRetries);
+                j.logDir, j.os, j.uid.orElse(null), j.isPaused, j.isAutoEat, j.maxRetries,
+                j.logLokiEnabled, j.logLokiURL);
     }
 
     private static final String JOB_EXISTS =

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java

@@ -1190,7 +1190,9 @@ public Job mapRow(ResultSet rs, int rowNum) throws SQLException {
                             .setHasComment(rs.getBoolean("b_comment"))
                             .setAutoEat(rs.getBoolean("b_autoeat"))
                             .setStartTime((int) (rs.getTimestamp("ts_started").getTime() / 1000))
-                            .setOs(SqlUtil.getString(rs,"str_os"));
+                            .setOs(SqlUtil.getString(rs,"str_os"))
+                            .setLokiEnabled(rs.getBoolean("b_loki_enabled"))
+                            .setLokiUrl(SqlUtil.getString(rs, "str_loki_url"));
 
                     int uid = rs.getInt("int_uid");
                     if (!rs.wasNull()) {
@@ -1935,6 +1937,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException {
         "SELECT " +
             "job.pk_job,"+
             "job.str_log_dir," +
+            "job.b_loki_enabled," +
+            "job.str_loki_url," +
             "job_resource.int_max_cores," +
             "job_resource.int_min_cores," +
             "job_resource.int_max_gpus," +

cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java

@@ -385,6 +385,8 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) {
                 .setShow(frame.show)
                 .setUserName(frame.owner)
                 .setLogDir(frame.logDir)
+                .setLokiEnabled(frame.lokiEnabled)
+                .setLokiUrl(frame.lokiURL)
                 .setJobId(frame.jobId)
                 .setJobName(frame.jobName)
                 .setFrameId(frame.id)

cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java

@@ -66,4 +66,12 @@ public String getJobLogRootDir(String os) {
             return env.getRequiredProperty("log.frame-log-root.default_os", String.class);
         }
     }
-}
+
+    public Boolean getLokiIsEnabled() {
+        return env.getRequiredProperty("log.loki.enabled", Boolean.class);
+    }
+
+    public String getLokiURL() {
+        return env.getRequiredProperty("log.loki.url", String.class);
+    }
+}

cuebot/src/main/resources/conf/ddl/postgres/migrations/V33__Add_loki_job_fields.sql

@@ -0,0 +1,6 @@
+alter table job
+    add b_loki_enabled bool;
+
+alter table job
+    add str_loki_url varchar(256);
+

cuebot/src/main/resources/opencue.properties

@@ -61,6 +61,14 @@ log.frame-log-root.default_os=${CUE_FRAME_LOG_DIR:/shots}
 # are planning to use a folder in the root, use:
 #   - log.frame-log-root.Windows=${S:}
 
+# Loki
+# To enable rqd frame logs to Loki enable this flag and configure the url as shown below. When Loki
+# rqd frame logs are enable all frame logs will be streamed to the loki server instead of using the
+# CUE_FRAME_LOG_DIR filesystem path. Refer to the documentaion on how to configure Loki.
+log.loki.enabled=false
+# This is the base url of the Loki server. If the url is not reachable, rqd will fail running frames.
+log.loki.url=http://localhost/loki/api
+
 # Maximum number of jobs to query.
 dispatcher.job_query_max=20
 # Number of seconds before waiting to book the same job from a different host.

cuebot/src/test/resources/opencue.properties

@@ -93,4 +93,8 @@ dispatcher.memory.mem_reserved_min = 262144
 dispatcher.memory.mem_reserved_system = 524288
 dispatcher.memory.mem_gpu_reserved_default = 0
 dispatcher.memory.mem_gpu_reserved_min = 0
-dispatcher.memory.mem_gpu_reserved_max = 104857600
+dispatcher.memory.mem_gpu_reserved_max = 104857600
+
+# Loki
+log.loki.enabled = false
+log.loki.url = http://localhost/loki/api

cuegui/cuegui/App.py

@@ -30,6 +30,7 @@ class CueGuiApplication(QtWidgets.QApplication):
 
     # Global signals
     display_log_file_content = QtCore.Signal(object)
+    select_frame = QtCore.Signal(object, object)
     double_click = QtCore.Signal(object)
     facility_changed = QtCore.Signal()
     single_click = QtCore.Signal(object)

cuegui/cuegui/FrameMonitorTree.py

@@ -359,6 +359,7 @@ def __itemSingleClickedViewLog(self, item, col):
             old_log_files = []
 
         self.app.display_log_file_content.emit([current_log_file] + old_log_files)
+        self.app.select_frame.emit(self.__job, item.rpcObject)
 
     def __itemDoubleClickedViewLog(self, item, col):
         """Called when a frame is double clicked, views the frame log in a popup