From 3a00739179193f884b657e2253052d1015e9ae66 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 4 Nov 2024 22:49:46 +0100 Subject: [PATCH 01/48] Remove org.slf4j:slf4j-log4j12 from build.gradle which is already provided by ch.qos.logback:logback-classic and causing conflict --- cuebot/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/cuebot/build.gradle b/cuebot/build.gradle index 0a6d08c1a..ce6d2a3b7 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -46,7 +46,6 @@ dependencies { implementation group: 'org.postgresql', name: 'postgresql', version: '42.2.2' implementation group: 'com.google.protobuf', name: 'protobuf-java', version: "${protobufVersion}" implementation group: 'log4j', name: 'log4j', version: '1.2.17' - implementation group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.26' implementation group: 'io.sentry', name: 'sentry-log4j2', version: '7.11.0' implementation group: 'io.prometheus', name: 'simpleclient', version: '0.16.0' implementation group: 'io.prometheus', name: 'simpleclient_servlet', version: '0.16.0' From 67d93a4f530115d14deb31c9719bfc9e44ac415f Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 5 Nov 2024 23:05:35 +0100 Subject: [PATCH 02/48] Add loki properties for cuebot Add loki details to jobs in database using extra fields --- VERSION.in | 2 +- .../main/java/com/imageworks/spcue/JobDetail.java | 2 ++ .../imageworks/spcue/dao/postgres/JobDaoJdbc.java | 15 ++++++++++++--- .../com/imageworks/spcue/util/JobLogUtil.java | 10 +++++++++- .../migrations/V31__Add_loki_job_fields.sql | 6 ++++++ cuebot/src/main/resources/opencue.properties | 4 ++++ proto/rqd.proto | 2 ++ 7 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V31__Add_loki_job_fields.sql diff --git a/VERSION.in b/VERSION.in index d3827e75a..9459d4ba2 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.0 +1.1 diff --git a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java index dad6f8a6d..8a339c0f9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java @@ -30,6 +30,8 @@ public class JobDetail extends JobEntity implements JobInterface, DepartmentInte public String email; public Optional uid; public String logDir; + public Boolean logLokiEnabled; + public String logLokiURL; public boolean isPaused; public boolean isAutoEat; public int totalFrames; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 872ab41d7..250b9f9b2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -114,6 +114,8 @@ public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException { job.deptId = rs.getString("pk_dept"); job.groupId = rs.getString("pk_folder"); job.logDir = rs.getString("str_log_dir"); + job.logLokiEnabled = rs.getBoolean("b_loki_enabled"); + job.logLokiURL = rs.getString("str_loki_url"); job.maxCoreUnits = rs.getInt("int_max_cores"); job.minCoreUnits = rs.getInt("int_min_cores"); job.maxGpuUnits = rs.getInt("int_max_gpus"); @@ -206,6 +208,8 @@ public boolean isJobComplete(JobInterface job) { "job.pk_dept,"+ "job.pk_folder,"+ "job.str_log_dir,"+ + "job.b_loki_enabled,"+ + "job.str_loki_url,"+ "job.str_name,"+ "job.str_shot,"+ "job.str_state,"+ @@ -473,20 +477,25 @@ public boolean updateJobFinished(JobInterface job) { "int_uid," + "b_paused," + "b_autoeat,"+ - "int_max_retries " + + "int_max_retries," + + "b_loki_enabled," + + "str_loki_url" + ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insertJob(JobDetail j, JobLogUtil jobLogUtil) { j.id = SqlUtil.genKeyRandom(); j.logDir = jobLogUtil.getJobLogPath(j); + j.logLokiEnabled = jobLogUtil.getLokiIsEnabled(); + j.logLokiURL = jobLogUtil.getLokiURL(); if (j.minCoreUnits < 100) { j.minCoreUnits = 100; } getJdbcTemplate().update(INSERT_JOB, j.id, j.showId, j.groupId, j.facilityId, j.deptId, j.name, j.name, j.showName, j.shot, j.user, j.email, j.state.toString(), - j.logDir, j.os, j.uid.orElse(null), j.isPaused, j.isAutoEat, j.maxRetries); + j.logDir, j.os, j.uid.orElse(null), j.isPaused, j.isAutoEat, j.maxRetries, + j.logLokiEnabled, j.logLokiURL); } private static final String JOB_EXISTS = diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java index c223ebbc0..5f96eaeb6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java @@ -66,4 +66,12 @@ public String getJobLogRootDir(String os) { return env.getRequiredProperty("log.frame-log-root.default_os", String.class); } } -} \ No newline at end of file + + public Boolean getLokiIsEnabled() { + return env.getRequiredProperty("log.loki.enabled", Boolean.class); + } + + public String getLokiURL() { + return env.getRequiredProperty("log.loki.url", String.class); + } +} diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V31__Add_loki_job_fields.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V31__Add_loki_job_fields.sql new file mode 100644 index 000000000..7d5409cc7 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V31__Add_loki_job_fields.sql @@ -0,0 +1,6 @@ +alter table job + add b_loki_enabled bool; + +alter table job + add str_loki_url varchar(256); + diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 0340da60e..4ebea1214 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -61,6 +61,10 @@ log.frame-log-root.default_os=${CUE_FRAME_LOG_DIR:/shots} # are planning to use a folder in the root, use: # - log.frame-log-root.Windows=${S:} +# Loki +log.loki.enabled=true +log.loki.url=http://localhost/loki/api + # Maximum number of jobs to query. dispatcher.job_query_max=20 # Number of seconds before waiting to book the same job from a different host. diff --git a/proto/rqd.proto b/proto/rqd.proto index f6e0d8790..fd0690c23 100644 --- a/proto/rqd.proto +++ b/proto/rqd.proto @@ -112,6 +112,8 @@ message RunFrame { map attributes = 22; int32 num_gpus = 23; report.ChildrenProcStats children = 24; + bool loki_enabled = 25; + string loki_url = 26; } message RunFrameSeq { From ed66ee5a4113405244ffea69ece8027c25483140 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 6 Nov 2024 22:12:22 +0100 Subject: [PATCH 03/48] Disable loki by default --- cuebot/src/main/resources/opencue.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 4ebea1214..ba0fbc130 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -62,7 +62,7 @@ log.frame-log-root.default_os=${CUE_FRAME_LOG_DIR:/shots} # - log.frame-log-root.Windows=${S:} # Loki -log.loki.enabled=true +log.loki.enabled=false log.loki.url=http://localhost/loki/api # Maximum number of jobs to query. From ab8f16c27bf08c53d61d0fbd91a04f94316ad4da Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 6 Nov 2024 22:45:33 +0100 Subject: [PATCH 04/48] Add loki details to runFrame object --- .../com/imageworks/spcue/DispatchFrame.java | 2 ++ .../spcue/dao/postgres/DispatchQuery.java | 32 +++++++++++++++++++ .../spcue/dao/postgres/FrameDaoJdbc.java | 4 +++ .../dispatcher/DispatchSupportService.java | 2 ++ 4 files changed, 40 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java index faa1a9c04..83e67bf4c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java @@ -32,6 +32,8 @@ public class DispatchFrame extends FrameEntity implements FrameInterface { public String owner; public Optional uid; public String logDir; + public boolean lokiEnabled; + public String lokiURL; public String command; public String range; public int chunkSize; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index e36f97999..611869df4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -534,6 +534,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -569,6 +571,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + @@ -649,6 +653,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -684,6 +690,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + @@ -765,6 +773,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -800,6 +810,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + @@ -874,6 +886,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -909,6 +923,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + @@ -986,6 +1002,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -1021,6 +1039,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + @@ -1101,6 +1121,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -1136,6 +1158,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + @@ -1217,6 +1241,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -1252,6 +1278,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + @@ -1326,6 +1354,8 @@ private static final String replaceQueryForFifo(String query) { "str_user, " + "int_uid, " + "str_log_dir, " + + "b_loki_enabled, " + + "str_loki_url, " + "frame_name, " + "frame_state, " + "pk_frame, " + @@ -1361,6 +1391,8 @@ private static final String replaceQueryForFifo(String query) { "job.str_user, " + "job.int_uid, " + "job.str_log_dir, " + + "job.b_loki_enabled, " + + "job.str_loki_url, " + "frame.str_name AS frame_name, " + "frame.str_state AS frame_state, " + "frame.pk_frame, " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 0546d4558..7a2948f4a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -316,6 +316,8 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { frame.chunkSize = rs.getInt("int_chunk_size"); frame.range = rs.getString("str_range"); frame.logDir = rs.getString("str_log_dir"); + frame.lokiEnabled = rs.getBoolean("b_loki_enabled"); + frame.lokiURL = rs.getString("str_loki_url"); frame.shot = rs.getString("str_shot"); frame.show = rs.getString("show_name"); frame.owner = rs.getString("str_user"); @@ -347,6 +349,8 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { "job.str_user,"+ "job.int_uid,"+ "job.str_log_dir,"+ + "job.b_loki_enabled,"+ + "job.str_loki_url,"+ "frame.str_name AS frame_name, "+ "frame.str_state AS frame_state, "+ "frame.pk_frame, "+ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index f60b2c1e6..fa18b2372 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -385,6 +385,8 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { .setShow(frame.show) .setUserName(frame.owner) .setLogDir(frame.logDir) + .setLokiEnabled(frame.lokiEnabled) + .setLokiUrl(frame.lokiURL) .setJobId(frame.jobId) .setJobName(frame.jobName) .setFrameId(frame.id) From 6abc9b023a6a00df2b5d1cfd9f260832ce087d78 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sat, 9 Nov 2024 23:19:35 +0100 Subject: [PATCH 05/48] Add loki_client module for logging to Loki --- rqd/rqd/loki_client.py | 575 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 575 insertions(+) create mode 100644 rqd/rqd/loki_client.py diff --git a/rqd/rqd/loki_client.py b/rqd/rqd/loki_client.py new file mode 100644 index 000000000..e42b8ba97 --- /dev/null +++ b/rqd/rqd/loki_client.py @@ -0,0 +1,575 @@ +from urllib.parse import urlparse, urlencode +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +import datetime +import json +from typing import Dict, List + +# Support Loki version:2.4.2 +MAX_REQUEST_RETRIES = 3 +RETRY_BACKOFF_FACTOR =1 +RETRY_ON_STATUS = [408, 429, 500, 502, 503, 504] +SUPPORTED_DIRECTION = ["backward", "forward"] +# the duration before end time when get context +CONTEXT_HOURS_DELTA = 1 +DEFAULT_HOURS_DELTA = 2 * 24 + + +class LokiClient(object): + """ + Loki client for Python to communicate with Loki server. + Ref: https://grafana.com/docs/loki/v2.4/api/ + """ + def __init__(self, + url: str = "http://127.0.0.1:3100", + headers: dict = None, + disable_ssl: bool = True, + retry: Retry = None, + hours_delta = DEFAULT_HOURS_DELTA): + """ + constructor + :param url: + :param headers: + :param disable_ssl: + :param retry: + :param hours_delta: + :return: + """ + if url is None: + raise TypeError("Url can not be empty!") + + self.headers = headers + self.url = url + self.loki_host = urlparse(self.url).netloc + self._all_metrics = None + self.ssl_verification = not disable_ssl + # the days between start and end time + self.hours_delta = hours_delta + # the time range when searching context for one key line + self.context_timedelta = int(CONTEXT_HOURS_DELTA * 3600 * 10 ** 9) + + if retry is None: + retry = Retry(total=MAX_REQUEST_RETRIES, backoff_factor=RETRY_BACKOFF_FACTOR, status_forcelist=RETRY_ON_STATUS) + + self.__session = requests.Session() + self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) + self.__session.keep_alive = False + + def ready(self) -> bool: + """ + Check whether Loki host is ready to accept traffic. + Ref: https://grafana.com/docs/loki/v2.4/api/#get-ready + :return: + bool: True if Loki is ready, False otherwise. + """ + try: + response = self.__session.get( + url="{}/ready".format(self.url), + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok + except Exception as ex: + return False + + def labels(self, + start: datetime = None, + end: datetime = None, + params: dict = None) -> tuple: + """ + Get the list of known labels within a given time span, corresponding labels. + Ref: GET /loki/api/v1/labels + :param start: + :param end: + :param params: + :return: + """ + params = params or {} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def label_values(self, + label: str, + start: datetime = None, + end: datetime = None, + params : dict = None) -> tuple: + """ + Get the list of known values for a given label within a given time span, corresponding values. + Ref: GET /loki/api/v1/label//values + :param label: + :param start: + :param end: + :param params: + :return: + """ + params = params or {} + + if label: + if not isinstance(label, str): + return False, {'message': 'Incorrect label type {}, should be type {}.'.format(type(label), str)} + else: + return False, {'message':'Param label can not be empty.'} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( + hours=self.hours_delta)).timestamp() * 10 ** 9) + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def query(self, + query: str, + limit: int = 100, + time: datetime = None, + direction: str = SUPPORTED_DIRECTION[0], + params: dict = None) -> tuple: + """ + Query logs from Loki, corresponding query. + Ref: GET /loki/api/v1/query + :param query: + :param limit: + :param time: + :param direction: + :param params: + :return: + """ + params = params or {} + + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message':'Param query can not be empty.'} + + if limit: + params['limit'] = limit + else: + return False, {'message': 'The value of limit is not correct.'} + + if time: + if not isinstance(time, datetime): + return False, {'message': 'Incorrect time type {}, should be type {}.'.format(type(time), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['time'] = int(time.timestamp() * 10 ** 9) + else: + params['time'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if direction not in SUPPORTED_DIRECTION: + return False, {'message': 'Invalid direction value: {}.'.format(direction)} + params['direction'] = direction + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/query?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def query_range(self, + query: str, + limit: int = 100, + start: datetime = None, + end: datetime = None, + direction: str = SUPPORTED_DIRECTION[0], + params: dict = None) -> tuple: + """ + Query logs from Loki, corresponding query_range. + Ref: GET /loki/api/v1/query_range + :param query: + :param limit: + :param start: + :param end: + :param direction: + :param params: + :return: + """ + params = params or {} + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message': 'Param query can not be empty.'} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + + if limit: + params['limit'] = limit + else: + return False, {'message': 'The value of limit is not correct.'} + + if direction not in SUPPORTED_DIRECTION: + return False, {'message': 'Invalid direction value: {}.'.format(direction)} + params['direction'] = direction + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def query_range_with_context(self, + query: str, + limit: int = 100, + context_before: int = 5, + context_after: int = 3, + start: datetime = None, + end: datetime = None, + direction: str = SUPPORTED_DIRECTION[1], + params: dict = None) -> tuple: + """ + Query key logs from Loki with contexts. + Ref: GET /loki/api/v1/query_range + :param query: + :param limit: + :param context_before: + :param context_after: + :param start: + :param end: + :param direction: + :param params: + :return: + """ + params = params or {} + + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message': 'Param query can not be empty.'} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( + hours=self.hours_delta)).timestamp() * 10 ** 9) + + if limit: + params['limit'] = limit + else: + return False, {'message': 'The value of limit is not correct.'} + + if direction not in SUPPORTED_DIRECTION: + return False, {'message': 'Invalid direction value: {}.'.format(direction)} + params['direction'] = direction + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + except Exception as ex: + return False, {'message': repr(ex)} + + if not response.ok: + return False, {'message': response.reason} + + key_log_result = response.json() + + if key_log_result['status'] != 'success': + return False, {'message': 'Query from Loki unsuccessfully.'} + + context_results = [] + + for result_item in key_log_result['data']['result']: + labels = result_item['stream'] + file_name = result_item['stream']['filename'] + file_datas = [] + for value_item in result_item['values']: + cur_ts = int(value_item[0]) + key_line = value_item[1].strip() + + start_ts = cur_ts - self.context_timedelta + end_ts = cur_ts + self.context_timedelta + + lbs = [r'%s="%s"' % (i, labels[i]) for i in labels.keys()] + q = ','.join(lbs) + query = '{' + q + '}' + + # context before + params_ctx_before = { + 'query': query, + 'limit': context_before, + 'start': start_ts, + 'end': cur_ts, + 'direction': SUPPORTED_DIRECTION[0] + } + + enc_query = urlencode(params_ctx_before) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_before = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + # context after + params_ctx_after = { + 'query': query, + 'limit': context_after + 1, + 'start': cur_ts, + 'end': end_ts, + 'direction': SUPPORTED_DIRECTION[1] + } + + enc_query = urlencode(params_ctx_after) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_after = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + key_line_ctx = [] + for value_item in reversed(ctx_before['data']['result'][0]['values']): + key_line_ctx.append(value_item[1].strip()) + + # context_after result including the key line + for value_item in ctx_after['data']['result'][0]['values']: + key_line_ctx.append(value_item[1].strip()) + + data_item = { + 'key_line': key_line, + 'context': key_line_ctx + } + + file_datas.append(data_item) + + result_item = { + 'file_name': file_name, + 'datas': file_datas + } + + context_results.append(result_item) + + return True, {'results': context_results} + + def query_context_by_timestamp(self, + query: str, + cur_ts: int, + context_before: int = 5, + context_after: int = 3, + params: dict = None) -> tuple: + """ + Query contexts by the given query and timestamp. + Ref: GET /loki/api/v1/query_range + :param query: + :param cur_ts: + :param context_before: + :param context_after: + :param params: + :return: + """ + params = params or {} + + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message': 'Param query can not be empty.'} + + start_ts = cur_ts - self.context_timedelta + end_ts = cur_ts + self.context_timedelta + + # context before + params_ctx_before = { + 'query': query, + 'limit': context_before, + 'start': start_ts, + 'end': cur_ts, + 'direction': SUPPORTED_DIRECTION[0] + } + + enc_query = urlencode(params_ctx_before) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_before = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + # context after + params_ctx_after = { + 'query': query, + 'limit': context_after + 1, + 'start': cur_ts, + 'end': end_ts, + 'direction': SUPPORTED_DIRECTION[1] + } + + enc_query = urlencode(params_ctx_after) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_after = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + key_line_ctx = [] + for value_item in reversed(ctx_before['data']['result'][0]['values']): + key_line_ctx.append(value_item[1].strip()) + + # context_after result including the key line + for value_item in ctx_after['data']['result'][0]['values']: + key_line_ctx.append(value_item[1].strip()) + + data_item = { + 'key_line': ctx_after['data']['result'][0]['values'][0][1].strip(), + 'context': key_line_ctx + } + + return True, data_item + + def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: + """ + Post logs to Loki, with given labels. + Ref: POST /loki/api/v1/push + :param labels: + :param logs: + :return: + """ + headers = { + 'Content-type': 'application/json' + } + + cur_ts = int(datetime.datetime.now().timestamp() * 10 ** 9) + logs_ts = [[str(cur_ts), log] for log in logs] + + payload = { + 'streams': [ + { + 'stream': labels, + 'values': logs_ts + } + ] + } + + payload_json = json.dumps(payload) + target_url = '{}/loki/api/v1/push'.format(self.url) + + try: + response = self.__session.post( + url=target_url, + verify=self.ssl_verification, + data=payload_json, + headers=headers + ) + return response.ok, response.reason + except Exception as ex: + return False, {'message': repr(ex)} From 7c847e4e662510fedcd06d51b1e9af073861ce39 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sat, 9 Nov 2024 23:20:23 +0100 Subject: [PATCH 06/48] Add LokiLogger for loki logging and switch to using it if it's enabled in the job --- rqd/rqd/rqcore.py | 8 ++++++-- rqd/rqd/rqlogging.py | 49 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index abff2aed0..8537d3e19 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -491,8 +491,12 @@ def run(self): # Setup frame logging try: - self.rqlog = rqd.rqlogging.RqdLogger(runFrame.log_dir_file) - self.rqlog.waitForFile() + if self.runFrame.loki_enabled: + self.rqlog = rqd.rqlogging.LokiLogger(self.runFrame.loki_url, runFrame) + self.rqlog.waitForFile() + else: + self.rqlog = rqd.rqlogging.RqdLogger(runFrame.log_dir_file) + self.rqlog.waitForFile() # pylint: disable=broad-except except Exception as e: err = "Unable to write to %s due to %s" % (runFrame.log_dir_file, e) diff --git a/rqd/rqd/rqlogging.py b/rqd/rqd/rqlogging.py index e8878c0d9..9a91ca917 100644 --- a/rqd/rqd/rqlogging.py +++ b/rqd/rqd/rqlogging.py @@ -22,6 +22,7 @@ import datetime import platform +from rqd.loki_client import LokiClient import rqd.rqconstants log = logging.getLogger(__name__) @@ -131,3 +132,51 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): pass + +class LokiLogger(object): + """Class for logging to a loki server. It mimics a file object as much as possible""" + def __init__(self, lokiURL, runFrame): + self.client = LokiClient(url=lokiURL) + self.runFrame = runFrame + self.sessionStartTime = datetime.datetime.now().timestamp() + self.defaultLogData = { + 'host': 'test', + 'frame_id': self.runFrame.frame_id, + 'session_start_time': str(self.sessionStartTime) + } + + def waitForFile(self, maxTries=5): + """Waits for the connection to be ready before continuing""" + tries = 0 + while tries < maxTries: + if self.client.ready() is True: + print("Loki is ready") + return + tries += 1 + time.sleep(0.5 * tries) + raise IOError("Failed to create loki stream") + + def write(self, data, prependTimestamp=False): + if len(data.strip()) == 0: + return + if isinstance(data, bytes): + data = data.decode('utf-8', errors='ignore') + requestStatus, requestCode = self.client.post(self.defaultLogData, [data.strip()]) + if requestStatus is not True: + raise IOError(f"Failed to write log to loki server with error : {requestCode}") + + def writelines(self, __lines): + """Provides support for writing mutliple lines at a time""" + for line in __lines: + self.write(line) + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + From c47d92666a89dc18be594f94487395baaae514c4 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sat, 9 Nov 2024 23:31:35 +0100 Subject: [PATCH 07/48] Add more labels --- rqd/rqd/rqlogging.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqlogging.py b/rqd/rqd/rqlogging.py index 9a91ca917..b40e06ed2 100644 --- a/rqd/rqd/rqlogging.py +++ b/rqd/rqd/rqlogging.py @@ -140,7 +140,10 @@ def __init__(self, lokiURL, runFrame): self.runFrame = runFrame self.sessionStartTime = datetime.datetime.now().timestamp() self.defaultLogData = { - 'host': 'test', + 'host': platform.node(), + 'job_name': self.runFrame.job_name, + 'frame_name': self.runFrame.frame_name, + 'username': self.runFrame.user_name, 'frame_id': self.runFrame.frame_id, 'session_start_time': str(self.sessionStartTime) } @@ -157,6 +160,10 @@ def waitForFile(self, maxTries=5): raise IOError("Failed to create loki stream") def write(self, data, prependTimestamp=False): + """ + Provides write function for writing to loki server. + Ignores prepentTimeStamp which is redundant with Loki + """ if len(data.strip()) == 0: return if isinstance(data, bytes): From 7ca5065a92c8eee527f52f1df1d6b058a0fda1ea Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sun, 10 Nov 2024 00:29:06 +0100 Subject: [PATCH 08/48] Add loki details to proto file --- proto/job.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto/job.proto b/proto/job.proto index 2149148b6..9ac505d19 100644 --- a/proto/job.proto +++ b/proto/job.proto @@ -639,6 +639,8 @@ message Job { JobStats job_stats = 20; float min_gpus = 21; float max_gpus = 22; + bool loki_enabled = 23; + string loki_url = 24; } // Use to filter the job search. Please note that by searching for non-pending jobs, the output is limited to 200 jobs From ea935174e887647dcc23176def7db71d5bb64b0c Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sun, 10 Nov 2024 01:49:46 +0100 Subject: [PATCH 09/48] Add loki details to job object --- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 4 ++++ pycue/opencue/wrappers/job.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 1bc6bed59..3b2f6412e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -1174,6 +1174,8 @@ public Job mapRow(ResultSet rs, int rowNum) throws SQLException { Job.Builder jobBuilder = Job.newBuilder() .setId(SqlUtil.getString(rs, "pk_job")) .setLogDir(SqlUtil.getString(rs, "str_log_dir")) + .setLokiEnabled(rs.getBoolean("b_loki_enabled")) + .setLokiUrl(SqlUtil.getString(rs, "str_loki_url")) .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) .setMaxGpus(rs.getInt("int_max_gpus")) @@ -1935,6 +1937,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "SELECT " + "job.pk_job,"+ "job.str_log_dir," + + "job.b_loki_enabled," + + "job.str_loki_url," + "job_resource.int_max_cores," + "job_resource.int_min_cores," + "job_resource.int_max_gpus," + diff --git a/pycue/opencue/wrappers/job.py b/pycue/opencue/wrappers/job.py index e582a91bf..43001d236 100644 --- a/pycue/opencue/wrappers/job.py +++ b/pycue/opencue/wrappers/job.py @@ -814,6 +814,21 @@ def shutdownIfCompleted(self): self.stub.ShutdownIfCompleted(job_pb2.JobShutdownIfCompletedRequest(job=self.data), timeout=Cuebot.Timeout) + def lokiEnabled(self): + """Returns whether or now loki si enabled + + :rtype: bool + :return: Return true if loki ei enabled + """ + return self.data.loki_enabled + + def lokiURL(self): + """Returns url for loki server on the job + + :rtype: str + "return: Return URL of loki server of the job + """ + return self.data.loki_url class NestedJob(Job): """This class contains information and actions related to a nested job.""" From 5f31cf63d0159ea7a1a55fe8fa3d96064d20898f Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sun, 10 Nov 2024 01:50:19 +0100 Subject: [PATCH 10/48] Add new LokiView plugin for viewing loki logs --- cuegui/cuegui/App.py | 1 + cuegui/cuegui/FrameMonitorTree.py | 1 + cuegui/cuegui/loki_client.py | 575 ++++++++++++++++++++++++ cuegui/cuegui/plugins/LokiViewPlugin.py | 159 +++++++ 4 files changed, 736 insertions(+) create mode 100644 cuegui/cuegui/loki_client.py create mode 100644 cuegui/cuegui/plugins/LokiViewPlugin.py diff --git a/cuegui/cuegui/App.py b/cuegui/cuegui/App.py index 8e5409520..db188958c 100644 --- a/cuegui/cuegui/App.py +++ b/cuegui/cuegui/App.py @@ -30,6 +30,7 @@ class CueGuiApplication(QtWidgets.QApplication): # Global signals display_log_file_content = QtCore.Signal(object) + display_frame_log_content = QtCore.Signal(object, object) double_click = QtCore.Signal(object) facility_changed = QtCore.Signal() single_click = QtCore.Signal(object) diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index 46da7ee30..7af37ac9b 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -359,6 +359,7 @@ def __itemSingleClickedViewLog(self, item, col): old_log_files = [] self.app.display_log_file_content.emit([current_log_file] + old_log_files) + self.app.display_frame_log_content.emit(self.__job, item.rpcObject) def __itemDoubleClickedViewLog(self, item, col): """Called when a frame is double clicked, views the frame log in a popup diff --git a/cuegui/cuegui/loki_client.py b/cuegui/cuegui/loki_client.py new file mode 100644 index 000000000..e42b8ba97 --- /dev/null +++ b/cuegui/cuegui/loki_client.py @@ -0,0 +1,575 @@ +from urllib.parse import urlparse, urlencode +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +import datetime +import json +from typing import Dict, List + +# Support Loki version:2.4.2 +MAX_REQUEST_RETRIES = 3 +RETRY_BACKOFF_FACTOR =1 +RETRY_ON_STATUS = [408, 429, 500, 502, 503, 504] +SUPPORTED_DIRECTION = ["backward", "forward"] +# the duration before end time when get context +CONTEXT_HOURS_DELTA = 1 +DEFAULT_HOURS_DELTA = 2 * 24 + + +class LokiClient(object): + """ + Loki client for Python to communicate with Loki server. + Ref: https://grafana.com/docs/loki/v2.4/api/ + """ + def __init__(self, + url: str = "http://127.0.0.1:3100", + headers: dict = None, + disable_ssl: bool = True, + retry: Retry = None, + hours_delta = DEFAULT_HOURS_DELTA): + """ + constructor + :param url: + :param headers: + :param disable_ssl: + :param retry: + :param hours_delta: + :return: + """ + if url is None: + raise TypeError("Url can not be empty!") + + self.headers = headers + self.url = url + self.loki_host = urlparse(self.url).netloc + self._all_metrics = None + self.ssl_verification = not disable_ssl + # the days between start and end time + self.hours_delta = hours_delta + # the time range when searching context for one key line + self.context_timedelta = int(CONTEXT_HOURS_DELTA * 3600 * 10 ** 9) + + if retry is None: + retry = Retry(total=MAX_REQUEST_RETRIES, backoff_factor=RETRY_BACKOFF_FACTOR, status_forcelist=RETRY_ON_STATUS) + + self.__session = requests.Session() + self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) + self.__session.keep_alive = False + + def ready(self) -> bool: + """ + Check whether Loki host is ready to accept traffic. + Ref: https://grafana.com/docs/loki/v2.4/api/#get-ready + :return: + bool: True if Loki is ready, False otherwise. + """ + try: + response = self.__session.get( + url="{}/ready".format(self.url), + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok + except Exception as ex: + return False + + def labels(self, + start: datetime = None, + end: datetime = None, + params: dict = None) -> tuple: + """ + Get the list of known labels within a given time span, corresponding labels. + Ref: GET /loki/api/v1/labels + :param start: + :param end: + :param params: + :return: + """ + params = params or {} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def label_values(self, + label: str, + start: datetime = None, + end: datetime = None, + params : dict = None) -> tuple: + """ + Get the list of known values for a given label within a given time span, corresponding values. + Ref: GET /loki/api/v1/label//values + :param label: + :param start: + :param end: + :param params: + :return: + """ + params = params or {} + + if label: + if not isinstance(label, str): + return False, {'message': 'Incorrect label type {}, should be type {}.'.format(type(label), str)} + else: + return False, {'message':'Param label can not be empty.'} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( + hours=self.hours_delta)).timestamp() * 10 ** 9) + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def query(self, + query: str, + limit: int = 100, + time: datetime = None, + direction: str = SUPPORTED_DIRECTION[0], + params: dict = None) -> tuple: + """ + Query logs from Loki, corresponding query. + Ref: GET /loki/api/v1/query + :param query: + :param limit: + :param time: + :param direction: + :param params: + :return: + """ + params = params or {} + + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message':'Param query can not be empty.'} + + if limit: + params['limit'] = limit + else: + return False, {'message': 'The value of limit is not correct.'} + + if time: + if not isinstance(time, datetime): + return False, {'message': 'Incorrect time type {}, should be type {}.'.format(type(time), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['time'] = int(time.timestamp() * 10 ** 9) + else: + params['time'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if direction not in SUPPORTED_DIRECTION: + return False, {'message': 'Invalid direction value: {}.'.format(direction)} + params['direction'] = direction + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/query?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def query_range(self, + query: str, + limit: int = 100, + start: datetime = None, + end: datetime = None, + direction: str = SUPPORTED_DIRECTION[0], + params: dict = None) -> tuple: + """ + Query logs from Loki, corresponding query_range. + Ref: GET /loki/api/v1/query_range + :param query: + :param limit: + :param start: + :param end: + :param direction: + :param params: + :return: + """ + params = params or {} + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message': 'Param query can not be empty.'} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + + if limit: + params['limit'] = limit + else: + return False, {'message': 'The value of limit is not correct.'} + + if direction not in SUPPORTED_DIRECTION: + return False, {'message': 'Invalid direction value: {}.'.format(direction)} + params['direction'] = direction + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + return response.ok, response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + def query_range_with_context(self, + query: str, + limit: int = 100, + context_before: int = 5, + context_after: int = 3, + start: datetime = None, + end: datetime = None, + direction: str = SUPPORTED_DIRECTION[1], + params: dict = None) -> tuple: + """ + Query key logs from Loki with contexts. + Ref: GET /loki/api/v1/query_range + :param query: + :param limit: + :param context_before: + :param context_after: + :param start: + :param end: + :param direction: + :param params: + :return: + """ + params = params or {} + + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message': 'Param query can not be empty.'} + + if end: + if not isinstance(end, datetime): + return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['end'] = int(end.timestamp() * 10 ** 9) + else: + params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) + + if start: + if not isinstance(start, datetime): + return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + # Convert to int, or will be scientific notation, which will result in request exception + params['start'] = int(start.timestamp() * 10 ** 9) + else: + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( + hours=self.hours_delta)).timestamp() * 10 ** 9) + + if limit: + params['limit'] = limit + else: + return False, {'message': 'The value of limit is not correct.'} + + if direction not in SUPPORTED_DIRECTION: + return False, {'message': 'Invalid direction value: {}.'.format(direction)} + params['direction'] = direction + + enc_query = urlencode(params) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + except Exception as ex: + return False, {'message': repr(ex)} + + if not response.ok: + return False, {'message': response.reason} + + key_log_result = response.json() + + if key_log_result['status'] != 'success': + return False, {'message': 'Query from Loki unsuccessfully.'} + + context_results = [] + + for result_item in key_log_result['data']['result']: + labels = result_item['stream'] + file_name = result_item['stream']['filename'] + file_datas = [] + for value_item in result_item['values']: + cur_ts = int(value_item[0]) + key_line = value_item[1].strip() + + start_ts = cur_ts - self.context_timedelta + end_ts = cur_ts + self.context_timedelta + + lbs = [r'%s="%s"' % (i, labels[i]) for i in labels.keys()] + q = ','.join(lbs) + query = '{' + q + '}' + + # context before + params_ctx_before = { + 'query': query, + 'limit': context_before, + 'start': start_ts, + 'end': cur_ts, + 'direction': SUPPORTED_DIRECTION[0] + } + + enc_query = urlencode(params_ctx_before) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_before = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + # context after + params_ctx_after = { + 'query': query, + 'limit': context_after + 1, + 'start': cur_ts, + 'end': end_ts, + 'direction': SUPPORTED_DIRECTION[1] + } + + enc_query = urlencode(params_ctx_after) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_after = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + key_line_ctx = [] + for value_item in reversed(ctx_before['data']['result'][0]['values']): + key_line_ctx.append(value_item[1].strip()) + + # context_after result including the key line + for value_item in ctx_after['data']['result'][0]['values']: + key_line_ctx.append(value_item[1].strip()) + + data_item = { + 'key_line': key_line, + 'context': key_line_ctx + } + + file_datas.append(data_item) + + result_item = { + 'file_name': file_name, + 'datas': file_datas + } + + context_results.append(result_item) + + return True, {'results': context_results} + + def query_context_by_timestamp(self, + query: str, + cur_ts: int, + context_before: int = 5, + context_after: int = 3, + params: dict = None) -> tuple: + """ + Query contexts by the given query and timestamp. + Ref: GET /loki/api/v1/query_range + :param query: + :param cur_ts: + :param context_before: + :param context_after: + :param params: + :return: + """ + params = params or {} + + if query: + if not isinstance(query, str): + return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + params['query'] = query + else: + return False, {'message': 'Param query can not be empty.'} + + start_ts = cur_ts - self.context_timedelta + end_ts = cur_ts + self.context_timedelta + + # context before + params_ctx_before = { + 'query': query, + 'limit': context_before, + 'start': start_ts, + 'end': cur_ts, + 'direction': SUPPORTED_DIRECTION[0] + } + + enc_query = urlencode(params_ctx_before) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_before = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + # context after + params_ctx_after = { + 'query': query, + 'limit': context_after + 1, + 'start': cur_ts, + 'end': end_ts, + 'direction': SUPPORTED_DIRECTION[1] + } + + enc_query = urlencode(params_ctx_after) + target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) + + try: + response = self.__session.get( + url=target_url, + verify=self.ssl_verification, + headers=self.headers + ) + ctx_after = response.json() + except Exception as ex: + return False, {'message': repr(ex)} + + key_line_ctx = [] + for value_item in reversed(ctx_before['data']['result'][0]['values']): + key_line_ctx.append(value_item[1].strip()) + + # context_after result including the key line + for value_item in ctx_after['data']['result'][0]['values']: + key_line_ctx.append(value_item[1].strip()) + + data_item = { + 'key_line': ctx_after['data']['result'][0]['values'][0][1].strip(), + 'context': key_line_ctx + } + + return True, data_item + + def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: + """ + Post logs to Loki, with given labels. + Ref: POST /loki/api/v1/push + :param labels: + :param logs: + :return: + """ + headers = { + 'Content-type': 'application/json' + } + + cur_ts = int(datetime.datetime.now().timestamp() * 10 ** 9) + logs_ts = [[str(cur_ts), log] for log in logs] + + payload = { + 'streams': [ + { + 'stream': labels, + 'values': logs_ts + } + ] + } + + payload_json = json.dumps(payload) + target_url = '{}/loki/api/v1/push'.format(self.url) + + try: + response = self.__session.post( + url=target_url, + verify=self.ssl_verification, + data=payload_json, + headers=headers + ) + return response.ok, response.reason + except Exception as ex: + return False, {'message': repr(ex)} diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py new file mode 100644 index 000000000..6512061d1 --- /dev/null +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -0,0 +1,159 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Plugin for viewing loki logs.""" + +import string +import time +import datetime + +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets + +from opencue.wrappers import job, frame + +from cuegui.loki_client import LokiClient + +import cuegui.Constants +import cuegui.AbstractDockWidget + +PLUGIN_NAME = 'LokiView' +PLUGIN_CATEGORY = 'Other' +PLUGIN_DESCRIPTION = 'Displays Frame Log from Loki' +PLUGIN_PROVIDES = 'LokiViewPlugin' +PRINTABLE = set(string.printable) + +class LokiViewWidget(QtWidgets.QWidget): + """ + Displays the log file for the selected frame + """ + SIG_CONTENT_UPDATED = QtCore.Signal(str, str) + def __init__(self, parent=None): + super().__init__(parent) + self.app = cuegui.app() + self.setupUi() + + self.app.display_frame_log_content.connect(self._display_frame_log) + + def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): + jobName = jobObj.name() + frameName = frameObj.name() + frameId = frameObj.id() + self.frameLogCombo.clear() + if jobObj.lokiEnabled(): + self.frameNameLabel.setText(f"{jobName}.{frameName}") + client = LokiClient(jobObj.lokiURL()) + maxTries = 5 + tries = 0 + while tries < maxTries: + if client.ready() is True: + print("Loki is ready") + break + tries += 1 + time.sleep(0.5 * tries) + success, result = client.label_values("session_start_time", params={'query': '{frame_id="'+frameId+'"}'}) + if success is True: + labelValues = result.get('data', []) + for unix_timestamp in sorted(labelValues, reverse=True): + self.frameLogCombo.addItem(unix_to_datetime(int(float(unix_timestamp)))) + self.frameLogCombo.adjustSize() + else: + pass + # print(jobName, frameName, frameId, lokiEnabled, lokiURL) + + def setupUi(self): + # self.setObjectName("self") + # self.resize(958, 663) + self.verticalLayout = QtWidgets.QVBoxLayout(self) + self.verticalLayout.setObjectName("verticalLayout") + self.horizontalLayout = QtWidgets.QHBoxLayout() + self.horizontalLayout.setObjectName("horizontalLayout") + self.frameNameLabel = QtWidgets.QLabel(self) + self.frameNameLabel.setObjectName("frameNameLabel") + self.horizontalLayout.addWidget(self.frameNameLabel) + self.frameLogCombo = QtWidgets.QComboBox(self) + self.frameLogCombo.setSizeAdjustPolicy(QtWidgets.QComboBox.AdjustToContents) + self.frameLogCombo.setObjectName("frameLogCombo") + self.horizontalLayout.addWidget(self.frameLogCombo) + self.wordWrapCheck = QtWidgets.QCheckBox(self) + self.wordWrapCheck.setObjectName("wordWrapCheck") + self.horizontalLayout.addWidget(self.wordWrapCheck) + self.refreshButton = QtWidgets.QPushButton(self) + self.refreshButton.setObjectName("refreshButton") + self.horizontalLayout.addWidget(self.refreshButton) + self.horizontalLayout.setStretch(0, 1) + self.verticalLayout.addLayout(self.horizontalLayout) + self.textEdit = QtWidgets.QTextEdit(self) + self.textEdit.setReadOnly(True) + self.textEdit.setObjectName("textEdit") + self.verticalLayout.addWidget(self.textEdit) + self.horizontalLayout_2 = QtWidgets.QHBoxLayout() + self.horizontalLayout_2.setObjectName("horizontalLayout_2") + self.caseCheck = QtWidgets.QCheckBox(self) + self.caseCheck.setObjectName("caseCheck") + self.horizontalLayout_2.addWidget(self.caseCheck) + self.searchLine = QtWidgets.QLineEdit(self) + self.searchLine.setText("") + self.searchLine.setClearButtonEnabled(True) + self.searchLine.setObjectName("searchLine") + self.horizontalLayout_2.addWidget(self.searchLine) + self.findButton = QtWidgets.QPushButton(self) + self.findButton.setObjectName("findButton") + self.horizontalLayout_2.addWidget(self.findButton) + self.nextButton = QtWidgets.QPushButton(self) + self.nextButton.setObjectName("nextButton") + self.horizontalLayout_2.addWidget(self.nextButton) + self.prevButton = QtWidgets.QPushButton(self) + self.prevButton.setObjectName("prevButton") + self.horizontalLayout_2.addWidget(self.prevButton) + self.verticalLayout.addLayout(self.horizontalLayout_2) + + self.retranslateUi() + QtCore.QMetaObject.connectSlotsByName(self) + + def retranslateUi(self): + _translate = QtCore.QCoreApplication.translate + self.setWindowTitle(_translate("self", "self")) + self.wordWrapCheck.setText(_translate("self", "Word Wrap")) + self.refreshButton.setText(_translate("self", "Refresh")) + self.caseCheck.setText(_translate("self", "Aa")) + self.searchLine.setPlaceholderText(_translate("self", "Search log..")) + self.findButton.setText(_translate("self", "Find")) + self.nextButton.setText(_translate("self", "Next")) + self.prevButton.setText(_translate("self", "Prev")) + + +def unix_to_datetime(unix_timestamp): + return datetime.datetime.fromtimestamp(int(unix_timestamp)).strftime('%Y-%m-%d %H:%M:%S') + + +class LokiViewPlugin(cuegui.AbstractDockWidget.AbstractDockWidget): + """ + Plugin for displaying the log file content for the selected frame with + the ability to perself regex-based search. + """ + + def __init__(self, parent=None): + """ + Create a LogViewPlugin instance + + @param parent: The parent widget + @type parent: QtWidgets.QWidget or None + """ + cuegui.AbstractDockWidget.AbstractDockWidget.__init__( + self, parent, PLUGIN_NAME, QtCore.Qt.BottomDockWidgetArea) + self.logview_widget = LokiViewWidget(self) + self.layout().addWidget(self.logview_widget) From f72c19452afe83f16299e8f95cdf04f9f3792d69 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sun, 10 Nov 2024 18:55:31 +0100 Subject: [PATCH 11/48] Read logs from the server when session is selected --- cuegui/cuegui/plugins/LokiViewPlugin.py | 32 ++++++++++++++++--------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 6512061d1..cbdd812a1 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -45,7 +45,7 @@ def __init__(self, parent=None): super().__init__(parent) self.app = cuegui.app() self.setupUi() - + self.frameLogCombo.currentIndexChanged.connect(self._selectLog) self.app.display_frame_log_content.connect(self._display_frame_log) def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): @@ -55,24 +55,23 @@ def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): self.frameLogCombo.clear() if jobObj.lokiEnabled(): self.frameNameLabel.setText(f"{jobName}.{frameName}") - client = LokiClient(jobObj.lokiURL()) + self.client = LokiClient(jobObj.lokiURL()) maxTries = 5 tries = 0 while tries < maxTries: - if client.ready() is True: - print("Loki is ready") + if self.client.ready() is True: break tries += 1 time.sleep(0.5 * tries) - success, result = client.label_values("session_start_time", params={'query': '{frame_id="'+frameId+'"}'}) + success, result = self.client.label_values("session_start_time", params={'query': f'{{frame_id="{frameId}"}}'}) if success is True: labelValues = result.get('data', []) for unix_timestamp in sorted(labelValues, reverse=True): - self.frameLogCombo.addItem(unix_to_datetime(int(float(unix_timestamp)))) + query = f'{{session_start_time="{unix_timestamp}", frame_id="{frameId}"}}' + self.frameLogCombo.addItem(unix_to_datetime(int(float(unix_timestamp))), userData=query) self.frameLogCombo.adjustSize() else: pass - # print(jobName, frameName, frameId, lokiEnabled, lokiURL) def setupUi(self): # self.setObjectName("self") @@ -96,10 +95,12 @@ def setupUi(self): self.horizontalLayout.addWidget(self.refreshButton) self.horizontalLayout.setStretch(0, 1) self.verticalLayout.addLayout(self.horizontalLayout) - self.textEdit = QtWidgets.QTextEdit(self) - self.textEdit.setReadOnly(True) - self.textEdit.setObjectName("textEdit") - self.verticalLayout.addWidget(self.textEdit) + self.frameText = QtWidgets.QTextEdit(self) + self.frameText.setStyleSheet("pre {display: inline;}") + self.frameText.setLineWrapMode(QtWidgets.QTextEdit.NoWrap) + self.frameText.setReadOnly(True) + self.frameText.setObjectName("frameText") + self.verticalLayout.addWidget(self.frameText) self.horizontalLayout_2 = QtWidgets.QHBoxLayout() self.horizontalLayout_2.setObjectName("horizontalLayout_2") self.caseCheck = QtWidgets.QCheckBox(self) @@ -135,6 +136,15 @@ def retranslateUi(self): self.nextButton.setText(_translate("self", "Next")) self.prevButton.setText(_translate("self", "Prev")) + def _selectLog(self, index): + self.frameText.clear() + query = self.frameLogCombo.currentData() + success, result = self.client.query_range(query=query, direction='forward', limit=1000) + if success is True: + for res in result.get('data', {}).get('result', []): + for timestamp, line in res.get('values'): + self.frameText.append(f"
{line}
") + def unix_to_datetime(unix_timestamp): return datetime.datetime.fromtimestamp(int(unix_timestamp)).strftime('%Y-%m-%d %H:%M:%S') From 154f45c7774aaed5b67aa787748aecbdd79a7a7e Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sun, 10 Nov 2024 18:57:38 +0100 Subject: [PATCH 12/48] Some small cleanup --- cuegui/cuegui/plugins/LokiViewPlugin.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index cbdd812a1..95f07f0fb 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -19,7 +19,6 @@ import time import datetime -from qtpy import QtGui from qtpy import QtCore from qtpy import QtWidgets @@ -122,20 +121,8 @@ def setupUi(self): self.horizontalLayout_2.addWidget(self.prevButton) self.verticalLayout.addLayout(self.horizontalLayout_2) - self.retranslateUi() QtCore.QMetaObject.connectSlotsByName(self) - def retranslateUi(self): - _translate = QtCore.QCoreApplication.translate - self.setWindowTitle(_translate("self", "self")) - self.wordWrapCheck.setText(_translate("self", "Word Wrap")) - self.refreshButton.setText(_translate("self", "Refresh")) - self.caseCheck.setText(_translate("self", "Aa")) - self.searchLine.setPlaceholderText(_translate("self", "Search log..")) - self.findButton.setText(_translate("self", "Find")) - self.nextButton.setText(_translate("self", "Next")) - self.prevButton.setText(_translate("self", "Prev")) - def _selectLog(self, index): self.frameText.clear() query = self.frameLogCombo.currentData() From c53673a45a41e6f04152650ee167b0f5f2184872 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 11 Nov 2024 21:29:55 +0100 Subject: [PATCH 13/48] Move to end of class --- cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java index 83e67bf4c..e590195a5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java @@ -32,8 +32,6 @@ public class DispatchFrame extends FrameEntity implements FrameInterface { public String owner; public Optional uid; public String logDir; - public boolean lokiEnabled; - public String lokiURL; public String command; public String range; public int chunkSize; @@ -51,5 +49,8 @@ public class DispatchFrame extends FrameEntity implements FrameInterface { // A comma separated list of services public String services; + + public boolean lokiEnabled; + public String lokiURL; } From 7e5bfc85a13e86c80f8d8f7b6e018e40ffe8776e Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 11 Nov 2024 21:52:40 +0100 Subject: [PATCH 14/48] Small spelling fix --- pycue/opencue/wrappers/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycue/opencue/wrappers/job.py b/pycue/opencue/wrappers/job.py index 43001d236..b5dd3c005 100644 --- a/pycue/opencue/wrappers/job.py +++ b/pycue/opencue/wrappers/job.py @@ -818,7 +818,7 @@ def lokiEnabled(self): """Returns whether or now loki si enabled :rtype: bool - :return: Return true if loki ei enabled + :return: Return true if loki is enabled """ return self.data.loki_enabled From a72b1160dc6603a962e7f4ca693f990d2bd9f233 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 11 Nov 2024 22:32:43 +0100 Subject: [PATCH 15/48] Move to end of class --- .../java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java | 4 ++-- .../imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 250b9f9b2..7c3064594 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -114,8 +114,6 @@ public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException { job.deptId = rs.getString("pk_dept"); job.groupId = rs.getString("pk_folder"); job.logDir = rs.getString("str_log_dir"); - job.logLokiEnabled = rs.getBoolean("b_loki_enabled"); - job.logLokiURL = rs.getString("str_loki_url"); job.maxCoreUnits = rs.getInt("int_max_cores"); job.minCoreUnits = rs.getInt("int_min_cores"); job.maxGpuUnits = rs.getInt("int_max_gpus"); @@ -139,6 +137,8 @@ public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException { job.showName = rs.getString("show_name"); job.facilityName = rs.getString("facility_name"); job.deptName = rs.getString("dept_name"); + job.logLokiEnabled = rs.getBoolean("b_loki_enabled"); + job.logLokiURL = rs.getString("str_loki_url"); return job; } }; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 3b2f6412e..33b0c4d33 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -1174,8 +1174,6 @@ public Job mapRow(ResultSet rs, int rowNum) throws SQLException { Job.Builder jobBuilder = Job.newBuilder() .setId(SqlUtil.getString(rs, "pk_job")) .setLogDir(SqlUtil.getString(rs, "str_log_dir")) - .setLokiEnabled(rs.getBoolean("b_loki_enabled")) - .setLokiUrl(SqlUtil.getString(rs, "str_loki_url")) .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) .setMaxGpus(rs.getInt("int_max_gpus")) @@ -1192,7 +1190,9 @@ public Job mapRow(ResultSet rs, int rowNum) throws SQLException { .setHasComment(rs.getBoolean("b_comment")) .setAutoEat(rs.getBoolean("b_autoeat")) .setStartTime((int) (rs.getTimestamp("ts_started").getTime() / 1000)) - .setOs(SqlUtil.getString(rs,"str_os")); + .setOs(SqlUtil.getString(rs,"str_os")) + .setLokiEnabled(rs.getBoolean("b_loki_enabled")) + .setLokiUrl(SqlUtil.getString(rs, "str_loki_url")); int uid = rs.getInt("int_uid"); if (!rs.wasNull()) { From a1327afe5cbba1b681d1861ee71da50df3c46d62 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 11 Nov 2024 22:38:49 +0100 Subject: [PATCH 16/48] Add loki options to test config --- cuebot/src/test/resources/opencue.properties | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 5ec6fba06..b04585bb9 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -93,4 +93,8 @@ dispatcher.memory.mem_reserved_min = 262144 dispatcher.memory.mem_reserved_system = 524288 dispatcher.memory.mem_gpu_reserved_default = 0 dispatcher.memory.mem_gpu_reserved_min = 0 -dispatcher.memory.mem_gpu_reserved_max = 104857600 \ No newline at end of file +dispatcher.memory.mem_gpu_reserved_max = 104857600 + +# Loki +log.loki.enabled = false +log.loki.url = http://localhost/loki/api From ee25f0f988ac971adfcbea967d6cb44b62ff471f Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 11 Nov 2024 23:30:18 +0100 Subject: [PATCH 17/48] Remove requests as a dependency and fix various linting --- rqd/rqd/loki_client.py | 326 +++++------------------------------------ 1 file changed, 37 insertions(+), 289 deletions(-) diff --git a/rqd/rqd/loki_client.py b/rqd/rqd/loki_client.py index e42b8ba97..c6269d20e 100644 --- a/rqd/rqd/loki_client.py +++ b/rqd/rqd/loki_client.py @@ -1,9 +1,9 @@ -from urllib.parse import urlparse, urlencode -import requests -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry +import urllib3 import datetime import json + +from urllib.parse import urlparse, urlencode +from urllib3.util.retry import Retry from typing import Dict, List # Support Loki version:2.4.2 @@ -24,14 +24,12 @@ class LokiClient(object): def __init__(self, url: str = "http://127.0.0.1:3100", headers: dict = None, - disable_ssl: bool = True, retry: Retry = None, hours_delta = DEFAULT_HOURS_DELTA): """ constructor :param url: :param headers: - :param disable_ssl: :param retry: :param hours_delta: :return: @@ -43,7 +41,6 @@ def __init__(self, self.url = url self.loki_host = urlparse(self.url).netloc self._all_metrics = None - self.ssl_verification = not disable_ssl # the days between start and end time self.hours_delta = hours_delta # the time range when searching context for one key line @@ -52,8 +49,8 @@ def __init__(self, if retry is None: retry = Retry(total=MAX_REQUEST_RETRIES, backoff_factor=RETRY_BACKOFF_FACTOR, status_forcelist=RETRY_ON_STATUS) - self.__session = requests.Session() - self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) + self.__session = urllib3.PoolManager() + # self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) self.__session.keep_alive = False def ready(self) -> bool: @@ -64,18 +61,18 @@ def ready(self) -> bool: bool: True if Loki is ready, False otherwise. """ try: - response = self.__session.get( + response = self.__session.request( + "GET", url="{}/ready".format(self.url), - verify=self.ssl_verification, headers=self.headers ) - return response.ok + return True if response.status == 200 else False except Exception as ex: return False def labels(self, - start: datetime = None, - end: datetime = None, + start: datetime.datetime = None, + end: datetime.datetime = None, params: dict = None) -> tuple: """ Get the list of known labels within a given time span, corresponding labels. @@ -88,7 +85,7 @@ def labels(self, params = params or {} if end: - if not isinstance(end, datetime): + if end is not type(datetime.datetime): return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) @@ -96,7 +93,7 @@ def labels(self, params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) if start: - if not isinstance(start, datetime): + if not isinstance(start, datetime.datetime): return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) @@ -107,19 +104,19 @@ def labels(self, target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) - return response.ok, response.json() + return True if response.status == 200 else False, response.json() except Exception as ex: return False, {'message': repr(ex)} def label_values(self, label: str, - start: datetime = None, - end: datetime = None, + start: datetime.datetime = None, + end: datetime.datetime = None, params : dict = None) -> tuple: """ Get the list of known values for a given label within a given time span, corresponding values. @@ -139,7 +136,7 @@ def label_values(self, return False, {'message':'Param label can not be empty.'} if end: - if not isinstance(end, datetime): + if not isinstance(end, datetime.datetime): return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) @@ -147,7 +144,7 @@ def label_values(self, params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) if start: - if not isinstance(start, datetime): + if not isinstance(start, datetime.datetime): return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) @@ -159,12 +156,12 @@ def label_values(self, target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) - return response.ok, response.json() + return True if response.status == 200 else False, response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -199,7 +196,7 @@ def query(self, return False, {'message': 'The value of limit is not correct.'} if time: - if not isinstance(time, datetime): + if not isinstance(time, datetime.datetime): return False, {'message': 'Incorrect time type {}, should be type {}.'.format(type(time), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['time'] = int(time.timestamp() * 10 ** 9) @@ -214,20 +211,20 @@ def query(self, target_url = '{}/loki/api/v1/query?{}'.format(self.url, enc_query) try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) - return response.ok, response.json() + return True if response.status == 200 else False, response.json() except Exception as ex: return False, {'message': repr(ex)} def query_range(self, query: str, limit: int = 100, - start: datetime = None, - end: datetime = None, + start: datetime.datetime = None, + end: datetime.datetime = None, direction: str = SUPPORTED_DIRECTION[0], params: dict = None) -> tuple: """ @@ -250,7 +247,7 @@ def query_range(self, return False, {'message': 'Param query can not be empty.'} if end: - if not isinstance(end, datetime): + if not isinstance(end, datetime.datetime): return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) @@ -258,7 +255,7 @@ def query_range(self, params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) if start: - if not isinstance(start, datetime): + if not isinstance(start, datetime.datetime): return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) @@ -278,264 +275,15 @@ def query_range(self, target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - return response.ok, response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def query_range_with_context(self, - query: str, - limit: int = 100, - context_before: int = 5, - context_after: int = 3, - start: datetime = None, - end: datetime = None, - direction: str = SUPPORTED_DIRECTION[1], - params: dict = None) -> tuple: - """ - Query key logs from Loki with contexts. - Ref: GET /loki/api/v1/query_range - :param query: - :param limit: - :param context_before: - :param context_after: - :param start: - :param end: - :param direction: - :param params: - :return: - """ - params = params or {} - - if query: - if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} - params['query'] = query - else: - return False, {'message': 'Param query can not be empty.'} - - if end: - if not isinstance(end, datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( - hours=self.hours_delta)).timestamp() * 10 ** 9) - - if limit: - params['limit'] = limit - else: - return False, {'message': 'The value of limit is not correct.'} - - if direction not in SUPPORTED_DIRECTION: - return False, {'message': 'Invalid direction value: {}.'.format(direction)} - params['direction'] = direction - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - except Exception as ex: - return False, {'message': repr(ex)} - - if not response.ok: - return False, {'message': response.reason} - - key_log_result = response.json() - - if key_log_result['status'] != 'success': - return False, {'message': 'Query from Loki unsuccessfully.'} - - context_results = [] - - for result_item in key_log_result['data']['result']: - labels = result_item['stream'] - file_name = result_item['stream']['filename'] - file_datas = [] - for value_item in result_item['values']: - cur_ts = int(value_item[0]) - key_line = value_item[1].strip() - - start_ts = cur_ts - self.context_timedelta - end_ts = cur_ts + self.context_timedelta - - lbs = [r'%s="%s"' % (i, labels[i]) for i in labels.keys()] - q = ','.join(lbs) - query = '{' + q + '}' - - # context before - params_ctx_before = { - 'query': query, - 'limit': context_before, - 'start': start_ts, - 'end': cur_ts, - 'direction': SUPPORTED_DIRECTION[0] - } - - enc_query = urlencode(params_ctx_before) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - ctx_before = response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - # context after - params_ctx_after = { - 'query': query, - 'limit': context_after + 1, - 'start': cur_ts, - 'end': end_ts, - 'direction': SUPPORTED_DIRECTION[1] - } - - enc_query = urlencode(params_ctx_after) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - ctx_after = response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - key_line_ctx = [] - for value_item in reversed(ctx_before['data']['result'][0]['values']): - key_line_ctx.append(value_item[1].strip()) - - # context_after result including the key line - for value_item in ctx_after['data']['result'][0]['values']: - key_line_ctx.append(value_item[1].strip()) - - data_item = { - 'key_line': key_line, - 'context': key_line_ctx - } - - file_datas.append(data_item) - - result_item = { - 'file_name': file_name, - 'datas': file_datas - } - - context_results.append(result_item) - - return True, {'results': context_results} - - def query_context_by_timestamp(self, - query: str, - cur_ts: int, - context_before: int = 5, - context_after: int = 3, - params: dict = None) -> tuple: - """ - Query contexts by the given query and timestamp. - Ref: GET /loki/api/v1/query_range - :param query: - :param cur_ts: - :param context_before: - :param context_after: - :param params: - :return: - """ - params = params or {} - - if query: - if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} - params['query'] = query - else: - return False, {'message': 'Param query can not be empty.'} - - start_ts = cur_ts - self.context_timedelta - end_ts = cur_ts + self.context_timedelta - - # context before - params_ctx_before = { - 'query': query, - 'limit': context_before, - 'start': start_ts, - 'end': cur_ts, - 'direction': SUPPORTED_DIRECTION[0] - } - - enc_query = urlencode(params_ctx_before) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) - ctx_before = response.json() + return True if response.status == 200 else False, response.json() except Exception as ex: return False, {'message': repr(ex)} - # context after - params_ctx_after = { - 'query': query, - 'limit': context_after + 1, - 'start': cur_ts, - 'end': end_ts, - 'direction': SUPPORTED_DIRECTION[1] - } - - enc_query = urlencode(params_ctx_after) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - ctx_after = response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - key_line_ctx = [] - for value_item in reversed(ctx_before['data']['result'][0]['values']): - key_line_ctx.append(value_item[1].strip()) - - # context_after result including the key line - for value_item in ctx_after['data']['result'][0]['values']: - key_line_ctx.append(value_item[1].strip()) - - data_item = { - 'key_line': ctx_after['data']['result'][0]['values'][0][1].strip(), - 'context': key_line_ctx - } - - return True, data_item - def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: """ Post logs to Loki, with given labels. @@ -564,12 +312,12 @@ def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: target_url = '{}/loki/api/v1/push'.format(self.url) try: - response = self.__session.post( + response = self.__session.request( + "POST", url=target_url, - verify=self.ssl_verification, data=payload_json, headers=headers ) - return response.ok, response.reason + return True if response.status == 200 else False, response.reason except Exception as ex: return False, {'message': repr(ex)} From fd7f111ae777275b11c79e0512d4de30e986f12e Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 00:29:22 +0100 Subject: [PATCH 18/48] Remove requests as a dependency and fix various linting --- cuegui/cuegui/loki_client.py | 335 +++++------------------------------ rqd/rqd/loki_client.py | 4 +- 2 files changed, 45 insertions(+), 294 deletions(-) diff --git a/cuegui/cuegui/loki_client.py b/cuegui/cuegui/loki_client.py index e42b8ba97..c0b72c3ad 100644 --- a/cuegui/cuegui/loki_client.py +++ b/cuegui/cuegui/loki_client.py @@ -1,9 +1,9 @@ -from urllib.parse import urlparse, urlencode -import requests -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry +import urllib3 import datetime import json + +from urllib.parse import urlparse, urlencode +from urllib3.util.retry import Retry from typing import Dict, List # Support Loki version:2.4.2 @@ -24,14 +24,12 @@ class LokiClient(object): def __init__(self, url: str = "http://127.0.0.1:3100", headers: dict = None, - disable_ssl: bool = True, retry: Retry = None, hours_delta = DEFAULT_HOURS_DELTA): """ constructor :param url: :param headers: - :param disable_ssl: :param retry: :param hours_delta: :return: @@ -43,7 +41,6 @@ def __init__(self, self.url = url self.loki_host = urlparse(self.url).netloc self._all_metrics = None - self.ssl_verification = not disable_ssl # the days between start and end time self.hours_delta = hours_delta # the time range when searching context for one key line @@ -52,8 +49,8 @@ def __init__(self, if retry is None: retry = Retry(total=MAX_REQUEST_RETRIES, backoff_factor=RETRY_BACKOFF_FACTOR, status_forcelist=RETRY_ON_STATUS) - self.__session = requests.Session() - self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) + self.__session = urllib3.PoolManager() + # self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) self.__session.keep_alive = False def ready(self) -> bool: @@ -64,18 +61,18 @@ def ready(self) -> bool: bool: True if Loki is ready, False otherwise. """ try: - response = self.__session.get( + response = self.__session.request( + "GET", url="{}/ready".format(self.url), - verify=self.ssl_verification, headers=self.headers ) - return response.ok + return True if response.status == 200 else False except Exception as ex: return False def labels(self, - start: datetime = None, - end: datetime = None, + start: datetime.datetime = None, + end: datetime.datetime = None, params: dict = None) -> tuple: """ Get the list of known labels within a given time span, corresponding labels. @@ -88,7 +85,7 @@ def labels(self, params = params or {} if end: - if not isinstance(end, datetime): + if end is not type(datetime.datetime): return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) @@ -96,7 +93,7 @@ def labels(self, params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) if start: - if not isinstance(start, datetime): + if not isinstance(start, datetime.datetime): return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) @@ -107,19 +104,19 @@ def labels(self, target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) - return response.ok, response.json() + return True if response.status == 200 else False, response.json() except Exception as ex: return False, {'message': repr(ex)} def label_values(self, label: str, - start: datetime = None, - end: datetime = None, + start: datetime.datetime = None, + end: datetime.datetime = None, params : dict = None) -> tuple: """ Get the list of known values for a given label within a given time span, corresponding values. @@ -139,7 +136,7 @@ def label_values(self, return False, {'message':'Param label can not be empty.'} if end: - if not isinstance(end, datetime): + if not isinstance(end, datetime.datetime): return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) @@ -147,7 +144,7 @@ def label_values(self, params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) if start: - if not isinstance(start, datetime): + if not isinstance(start, datetime.datetime): return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) @@ -159,12 +156,12 @@ def label_values(self, target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) - return response.ok, response.json() + return True if response.status == 200 else False, response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -199,7 +196,7 @@ def query(self, return False, {'message': 'The value of limit is not correct.'} if time: - if not isinstance(time, datetime): + if not isinstance(time, datetime.datetime): return False, {'message': 'Incorrect time type {}, should be type {}.'.format(type(time), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['time'] = int(time.timestamp() * 10 ** 9) @@ -214,20 +211,20 @@ def query(self, target_url = '{}/loki/api/v1/query?{}'.format(self.url, enc_query) try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) - return response.ok, response.json() + return True if response.status == 200 else False, response.json() except Exception as ex: return False, {'message': repr(ex)} def query_range(self, query: str, limit: int = 100, - start: datetime = None, - end: datetime = None, + start: datetime.datetime = None, + end: datetime.datetime = None, direction: str = SUPPORTED_DIRECTION[0], params: dict = None) -> tuple: """ @@ -250,7 +247,7 @@ def query_range(self, return False, {'message': 'Param query can not be empty.'} if end: - if not isinstance(end, datetime): + if not isinstance(end, datetime.datetime): return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) @@ -258,12 +255,13 @@ def query_range(self, params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) if start: - if not isinstance(start, datetime): + if not isinstance(start, datetime.datetime): return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) else: params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + print(params['start']) if limit: params['limit'] = limit @@ -275,267 +273,20 @@ def query_range(self, params['direction'] = direction enc_query = urlencode(params) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - return response.ok, response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def query_range_with_context(self, - query: str, - limit: int = 100, - context_before: int = 5, - context_after: int = 3, - start: datetime = None, - end: datetime = None, - direction: str = SUPPORTED_DIRECTION[1], - params: dict = None) -> tuple: - """ - Query key logs from Loki with contexts. - Ref: GET /loki/api/v1/query_range - :param query: - :param limit: - :param context_before: - :param context_after: - :param start: - :param end: - :param direction: - :param params: - :return: - """ - params = params or {} - - if query: - if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} - params['query'] = query - else: - return False, {'message': 'Param query can not be empty.'} - - if end: - if not isinstance(end, datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( - hours=self.hours_delta)).timestamp() * 10 ** 9) - - if limit: - params['limit'] = limit - else: - return False, {'message': 'The value of limit is not correct.'} - - if direction not in SUPPORTED_DIRECTION: - return False, {'message': 'Invalid direction value: {}.'.format(direction)} - params['direction'] = direction - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - + target_url = f'{self.url}/loki/api/v1/query_range?{enc_query}' try: - response = self.__session.get( + response = self.__session.request( + "GET", url=target_url, - verify=self.ssl_verification, headers=self.headers ) + if response.status == 200: + return True, response.json() + else: + return False, response.data except Exception as ex: return False, {'message': repr(ex)} - if not response.ok: - return False, {'message': response.reason} - - key_log_result = response.json() - - if key_log_result['status'] != 'success': - return False, {'message': 'Query from Loki unsuccessfully.'} - - context_results = [] - - for result_item in key_log_result['data']['result']: - labels = result_item['stream'] - file_name = result_item['stream']['filename'] - file_datas = [] - for value_item in result_item['values']: - cur_ts = int(value_item[0]) - key_line = value_item[1].strip() - - start_ts = cur_ts - self.context_timedelta - end_ts = cur_ts + self.context_timedelta - - lbs = [r'%s="%s"' % (i, labels[i]) for i in labels.keys()] - q = ','.join(lbs) - query = '{' + q + '}' - - # context before - params_ctx_before = { - 'query': query, - 'limit': context_before, - 'start': start_ts, - 'end': cur_ts, - 'direction': SUPPORTED_DIRECTION[0] - } - - enc_query = urlencode(params_ctx_before) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - ctx_before = response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - # context after - params_ctx_after = { - 'query': query, - 'limit': context_after + 1, - 'start': cur_ts, - 'end': end_ts, - 'direction': SUPPORTED_DIRECTION[1] - } - - enc_query = urlencode(params_ctx_after) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - ctx_after = response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - key_line_ctx = [] - for value_item in reversed(ctx_before['data']['result'][0]['values']): - key_line_ctx.append(value_item[1].strip()) - - # context_after result including the key line - for value_item in ctx_after['data']['result'][0]['values']: - key_line_ctx.append(value_item[1].strip()) - - data_item = { - 'key_line': key_line, - 'context': key_line_ctx - } - - file_datas.append(data_item) - - result_item = { - 'file_name': file_name, - 'datas': file_datas - } - - context_results.append(result_item) - - return True, {'results': context_results} - - def query_context_by_timestamp(self, - query: str, - cur_ts: int, - context_before: int = 5, - context_after: int = 3, - params: dict = None) -> tuple: - """ - Query contexts by the given query and timestamp. - Ref: GET /loki/api/v1/query_range - :param query: - :param cur_ts: - :param context_before: - :param context_after: - :param params: - :return: - """ - params = params or {} - - if query: - if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} - params['query'] = query - else: - return False, {'message': 'Param query can not be empty.'} - - start_ts = cur_ts - self.context_timedelta - end_ts = cur_ts + self.context_timedelta - - # context before - params_ctx_before = { - 'query': query, - 'limit': context_before, - 'start': start_ts, - 'end': cur_ts, - 'direction': SUPPORTED_DIRECTION[0] - } - - enc_query = urlencode(params_ctx_before) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - ctx_before = response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - # context after - params_ctx_after = { - 'query': query, - 'limit': context_after + 1, - 'start': cur_ts, - 'end': end_ts, - 'direction': SUPPORTED_DIRECTION[1] - } - - enc_query = urlencode(params_ctx_after) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - - try: - response = self.__session.get( - url=target_url, - verify=self.ssl_verification, - headers=self.headers - ) - ctx_after = response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - key_line_ctx = [] - for value_item in reversed(ctx_before['data']['result'][0]['values']): - key_line_ctx.append(value_item[1].strip()) - - # context_after result including the key line - for value_item in ctx_after['data']['result'][0]['values']: - key_line_ctx.append(value_item[1].strip()) - - data_item = { - 'key_line': ctx_after['data']['result'][0]['values'][0][1].strip(), - 'context': key_line_ctx - } - - return True, data_item - def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: """ Post logs to Loki, with given labels. @@ -564,12 +315,12 @@ def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: target_url = '{}/loki/api/v1/push'.format(self.url) try: - response = self.__session.post( + response = self.__session.request( + "POST", url=target_url, - verify=self.ssl_verification, - data=payload_json, + body=payload_json, headers=headers ) - return response.ok, response.reason + return True if response.status == 204 else False, response.reason except Exception as ex: return False, {'message': repr(ex)} diff --git a/rqd/rqd/loki_client.py b/rqd/rqd/loki_client.py index c6269d20e..6505c2724 100644 --- a/rqd/rqd/loki_client.py +++ b/rqd/rqd/loki_client.py @@ -315,9 +315,9 @@ def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: response = self.__session.request( "POST", url=target_url, - data=payload_json, + body=payload_json, headers=headers ) - return True if response.status == 200 else False, response.reason + return True if response.status == 204 else False, response.reason except Exception as ex: return False, {'message': repr(ex)} From a957703d971f4287c1e942d2e1dac921bcf85b0f Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 00:31:23 +0100 Subject: [PATCH 19/48] Remove debug print statement --- rqd/rqd/rqlogging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rqd/rqd/rqlogging.py b/rqd/rqd/rqlogging.py index b40e06ed2..4d9569d8d 100644 --- a/rqd/rqd/rqlogging.py +++ b/rqd/rqd/rqlogging.py @@ -153,7 +153,6 @@ def waitForFile(self, maxTries=5): tries = 0 while tries < maxTries: if self.client.ready() is True: - print("Loki is ready") return tries += 1 time.sleep(0.5 * tries) From 424687ed12f77319c7b488be243ddbd14e193905 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 00:36:59 +0100 Subject: [PATCH 20/48] Use timestamp to set start range --- cuegui/cuegui/plugins/LokiViewPlugin.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 95f07f0fb..01c9deedf 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -67,7 +67,8 @@ def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): labelValues = result.get('data', []) for unix_timestamp in sorted(labelValues, reverse=True): query = f'{{session_start_time="{unix_timestamp}", frame_id="{frameId}"}}' - self.frameLogCombo.addItem(unix_to_datetime(int(float(unix_timestamp))), userData=query) + data = [unix_timestamp, query] + self.frameLogCombo.addItem(unix_to_datetime(int(float(unix_timestamp))), userData=data) self.frameLogCombo.adjustSize() else: pass @@ -125,12 +126,17 @@ def setupUi(self): def _selectLog(self, index): self.frameText.clear() - query = self.frameLogCombo.currentData() - success, result = self.client.query_range(query=query, direction='forward', limit=1000) + timestamp, query = self.frameLogCombo.currentData() + # start = datetime.datetime.now() - datetime.timedelta(days=30) + start = datetime.datetime.fromtimestamp(float(timestamp)) + end = datetime.datetime.now() + success, result = self.client.query_range(query=query, direction='forward', limit=1000, start=start, end=end) if success is True: for res in result.get('data', {}).get('result', []): for timestamp, line in res.get('values'): self.frameText.append(f"
{line}
") + else: + print(success, result) def unix_to_datetime(unix_timestamp): From ec68dcdf80e86ee344c1d616cdfa13afe46e440d Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 21:56:49 +0100 Subject: [PATCH 21/48] Re-add the translation setup --- cuegui/cuegui/plugins/LokiViewPlugin.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 01c9deedf..3a2ae72ab 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -122,8 +122,20 @@ def setupUi(self): self.horizontalLayout_2.addWidget(self.prevButton) self.verticalLayout.addLayout(self.horizontalLayout_2) + self.retranslateUi() QtCore.QMetaObject.connectSlotsByName(self) + def retranslateUi(self): + _translate = QtCore.QCoreApplication.translate + self.setWindowTitle(_translate("self", "self")) + self.wordWrapCheck.setText(_translate("self", "Word Wrap")) + self.refreshButton.setText(_translate("self", "Refresh")) + self.caseCheck.setText(_translate("self", "Aa")) + self.searchLine.setPlaceholderText(_translate("self", "Search log..")) + self.findButton.setText(_translate("self", "Find")) + self.nextButton.setText(_translate("self", "Next")) + self.prevButton.setText(_translate("self", "Prev")) + def _selectLog(self, index): self.frameText.clear() timestamp, query = self.frameLogCombo.currentData() From 3b7f47971d8ccf1c52b4bd79ccc3e8a0738ff418 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 22:04:53 +0100 Subject: [PATCH 22/48] Move new attributes down --- cuebot/src/main/java/com/imageworks/spcue/JobDetail.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java index 8a339c0f9..d07f77b34 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java @@ -30,8 +30,6 @@ public class JobDetail extends JobEntity implements JobInterface, DepartmentInte public String email; public Optional uid; public String logDir; - public Boolean logLokiEnabled; - public String logLokiURL; public boolean isPaused; public boolean isAutoEat; public int totalFrames; @@ -61,5 +59,8 @@ public class JobDetail extends JobEntity implements JobInterface, DepartmentInte public String getDepartmentId() { return deptId; } + + public Boolean logLokiEnabled; + public String logLokiURL; } From eeb80ca7ab58b2ae94ace3b58a19792f7cc95028 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 22:05:43 +0100 Subject: [PATCH 23/48] Move new attributes down --- .../java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 7a2948f4a..722e9aac9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -316,8 +316,6 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { frame.chunkSize = rs.getInt("int_chunk_size"); frame.range = rs.getString("str_range"); frame.logDir = rs.getString("str_log_dir"); - frame.lokiEnabled = rs.getBoolean("b_loki_enabled"); - frame.lokiURL = rs.getString("str_loki_url"); frame.shot = rs.getString("str_shot"); frame.show = rs.getString("show_name"); frame.owner = rs.getString("str_user"); @@ -333,6 +331,8 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { frame.minGpuMemory = rs.getLong("int_gpu_mem_min"); frame.version = rs.getInt("int_version"); frame.services = rs.getString("str_services"); + frame.lokiEnabled = rs.getBoolean("b_loki_enabled"); + frame.lokiURL = rs.getString("str_loki_url"); return frame; } }; From 495f1341c138f06f2ff198901917a8561ee36b8c Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 22:26:30 +0100 Subject: [PATCH 24/48] Small docstring fix --- pycue/opencue/wrappers/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycue/opencue/wrappers/job.py b/pycue/opencue/wrappers/job.py index b5dd3c005..c87d19628 100644 --- a/pycue/opencue/wrappers/job.py +++ b/pycue/opencue/wrappers/job.py @@ -826,7 +826,7 @@ def lokiURL(self): """Returns url for loki server on the job :rtype: str - "return: Return URL of loki server of the job + :return: Return URL of loki server of the job """ return self.data.loki_url From daf023229556150959f203586fb6fc684b5c8c96 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 23:27:50 +0100 Subject: [PATCH 25/48] Fix pylint errors --- cuegui/cuegui/loki_client.py | 100 +++++++++++++++-------- cuegui/cuegui/plugins/LokiViewPlugin.py | 20 +++-- rqd/rqd/loki_client.py | 103 ++++++++++++++++-------- 3 files changed, 151 insertions(+), 72 deletions(-) diff --git a/cuegui/cuegui/loki_client.py b/cuegui/cuegui/loki_client.py index c0b72c3ad..94364034e 100644 --- a/cuegui/cuegui/loki_client.py +++ b/cuegui/cuegui/loki_client.py @@ -1,10 +1,27 @@ -import urllib3 +#! /usr/local/bin/python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for communicating with Loki""" + import datetime import json - -from urllib.parse import urlparse, urlencode -from urllib3.util.retry import Retry from typing import Dict, List +from urllib.parse import urlparse, urlencode +import urllib3 + # Support Loki version:2.4.2 MAX_REQUEST_RETRIES = 3 @@ -24,7 +41,6 @@ class LokiClient(object): def __init__(self, url: str = "http://127.0.0.1:3100", headers: dict = None, - retry: Retry = None, hours_delta = DEFAULT_HOURS_DELTA): """ constructor @@ -46,11 +62,7 @@ def __init__(self, # the time range when searching context for one key line self.context_timedelta = int(CONTEXT_HOURS_DELTA * 3600 * 10 ** 9) - if retry is None: - retry = Retry(total=MAX_REQUEST_RETRIES, backoff_factor=RETRY_BACKOFF_FACTOR, status_forcelist=RETRY_ON_STATUS) - self.__session = urllib3.PoolManager() - # self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) self.__session.keep_alive = False def ready(self) -> bool: @@ -66,8 +78,9 @@ def ready(self) -> bool: url="{}/ready".format(self.url), headers=self.headers ) - return True if response.status == 200 else False - except Exception as ex: + return bool(response.status == 200) + # pylint: disable=bare-except + except: return False def labels(self, @@ -86,7 +99,9 @@ def labels(self, if end: if end is not type(datetime.datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + return False, { + 'message': + f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) else: @@ -94,11 +109,15 @@ def labels(self, if start: if not isinstance(start, datetime.datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + return False, { + 'message': + f'Incorrect end type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) + - datetime.timedelta(hours=self.hours_delta)).timestamp() + * 10 ** 9) enc_query = urlencode(params) target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) @@ -109,7 +128,7 @@ def labels(self, url=target_url, headers=self.headers ) - return True if response.status == 200 else False, response.json() + return bool(response.status == 200), response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -119,7 +138,8 @@ def label_values(self, end: datetime.datetime = None, params : dict = None) -> tuple: """ - Get the list of known values for a given label within a given time span, corresponding values. + Get the list of known values for a given label within a given time span and + corresponding values. Ref: GET /loki/api/v1/label//values :param label: :param start: @@ -131,13 +151,16 @@ def label_values(self, if label: if not isinstance(label, str): - return False, {'message': 'Incorrect label type {}, should be type {}.'.format(type(label), str)} + return False, {'message': + f'Incorrect label type {type(label)}, should be type {str}.'} else: return False, {'message':'Param label can not be empty.'} if end: if not isinstance(end, datetime.datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + return False, { + 'message': + f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) else: @@ -145,12 +168,15 @@ def label_values(self, if start: if not isinstance(start, datetime.datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + return False, { + 'message': + f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( - hours=self.hours_delta)).timestamp() * 10 ** 9) + params['start'] = int( + (datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) + - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) enc_query = urlencode(params) target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) @@ -161,7 +187,7 @@ def label_values(self, url=target_url, headers=self.headers ) - return True if response.status == 200 else False, response.json() + return bool(response.status == 200), response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -185,7 +211,8 @@ def query(self, if query: if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + return False, {'message': + f'Incorrect query type {type(query)}, should be type {str}.'} params['query'] = query else: return False, {'message':'Param query can not be empty.'} @@ -197,7 +224,9 @@ def query(self, if time: if not isinstance(time, datetime.datetime): - return False, {'message': 'Incorrect time type {}, should be type {}.'.format(type(time), datetime)} + return False, { + 'message': + f'Incorrect start type {type(time)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['time'] = int(time.timestamp() * 10 ** 9) else: @@ -216,7 +245,7 @@ def query(self, url=target_url, headers=self.headers ) - return True if response.status == 200 else False, response.json() + return bool(response.status == 200), response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -241,14 +270,17 @@ def query_range(self, params = params or {} if query: if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + return False, {'message': + f'Incorrect query type {type(query)}, should be type {str}.'} params['query'] = query else: return False, {'message': 'Param query can not be empty.'} if end: if not isinstance(end, datetime.datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + return False, { + 'message': + f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) else: @@ -256,12 +288,15 @@ def query_range(self, if start: if not isinstance(start, datetime.datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + return False, { + 'message': + f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) - print(params['start']) + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) + - datetime.timedelta(hours=self.hours_delta)).timestamp() + * 10 ** 9) if limit: params['limit'] = limit @@ -282,8 +317,7 @@ def query_range(self, ) if response.status == 200: return True, response.json() - else: - return False, response.data + return False, response.data except Exception as ex: return False, {'message': repr(ex)} @@ -321,6 +355,6 @@ def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: body=payload_json, headers=headers ) - return True if response.status == 204 else False, response.reason + return bool(response.status == 204), response.reason except Exception as ex: return False, {'message': repr(ex)} diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 3a2ae72ab..c0ee2fddf 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -39,7 +39,7 @@ class LokiViewWidget(QtWidgets.QWidget): """ Displays the log file for the selected frame """ - SIG_CONTENT_UPDATED = QtCore.Signal(str, str) + client = None def __init__(self, parent=None): super().__init__(parent) self.app = cuegui.app() @@ -62,18 +62,23 @@ def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): break tries += 1 time.sleep(0.5 * tries) - success, result = self.client.label_values("session_start_time", params={'query': f'{{frame_id="{frameId}"}}'}) + success, result = self.client.label_values( + "session_start_time", params={'query': f'{{frame_id="{frameId}"}}'} + ) if success is True: labelValues = result.get('data', []) for unix_timestamp in sorted(labelValues, reverse=True): query = f'{{session_start_time="{unix_timestamp}", frame_id="{frameId}"}}' data = [unix_timestamp, query] - self.frameLogCombo.addItem(unix_to_datetime(int(float(unix_timestamp))), userData=data) + self.frameLogCombo.addItem( + _unix_to_datetime(int(float(unix_timestamp))), userData=data + ) self.frameLogCombo.adjustSize() else: pass def setupUi(self): + """Function for setting up the UI widgets""" # self.setObjectName("self") # self.resize(958, 663) self.verticalLayout = QtWidgets.QVBoxLayout(self) @@ -126,6 +131,7 @@ def setupUi(self): QtCore.QMetaObject.connectSlotsByName(self) def retranslateUi(self): + """ Add text to the widgets""" _translate = QtCore.QCoreApplication.translate self.setWindowTitle(_translate("self", "self")) self.wordWrapCheck.setText(_translate("self", "Word Wrap")) @@ -136,13 +142,14 @@ def retranslateUi(self): self.nextButton.setText(_translate("self", "Next")) self.prevButton.setText(_translate("self", "Prev")) + # pylint: disable=unused-argument def _selectLog(self, index): self.frameText.clear() timestamp, query = self.frameLogCombo.currentData() - # start = datetime.datetime.now() - datetime.timedelta(days=30) start = datetime.datetime.fromtimestamp(float(timestamp)) end = datetime.datetime.now() - success, result = self.client.query_range(query=query, direction='forward', limit=1000, start=start, end=end) + success, result = self.client.query_range(query=query, direction='forward', + limit=1000, start=start, end=end) if success is True: for res in result.get('data', {}).get('result', []): for timestamp, line in res.get('values'): @@ -151,7 +158,8 @@ def _selectLog(self, index): print(success, result) -def unix_to_datetime(unix_timestamp): +def _unix_to_datetime(unix_timestamp): + """Simple function to convert from timestamp to human readable string""" return datetime.datetime.fromtimestamp(int(unix_timestamp)).strftime('%Y-%m-%d %H:%M:%S') diff --git a/rqd/rqd/loki_client.py b/rqd/rqd/loki_client.py index 6505c2724..94364034e 100644 --- a/rqd/rqd/loki_client.py +++ b/rqd/rqd/loki_client.py @@ -1,10 +1,27 @@ -import urllib3 +#! /usr/local/bin/python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for communicating with Loki""" + import datetime import json - -from urllib.parse import urlparse, urlencode -from urllib3.util.retry import Retry from typing import Dict, List +from urllib.parse import urlparse, urlencode +import urllib3 + # Support Loki version:2.4.2 MAX_REQUEST_RETRIES = 3 @@ -24,7 +41,6 @@ class LokiClient(object): def __init__(self, url: str = "http://127.0.0.1:3100", headers: dict = None, - retry: Retry = None, hours_delta = DEFAULT_HOURS_DELTA): """ constructor @@ -46,11 +62,7 @@ def __init__(self, # the time range when searching context for one key line self.context_timedelta = int(CONTEXT_HOURS_DELTA * 3600 * 10 ** 9) - if retry is None: - retry = Retry(total=MAX_REQUEST_RETRIES, backoff_factor=RETRY_BACKOFF_FACTOR, status_forcelist=RETRY_ON_STATUS) - self.__session = urllib3.PoolManager() - # self.__session.mount(self.url, HTTPAdapter(max_retries=retry)) self.__session.keep_alive = False def ready(self) -> bool: @@ -66,8 +78,9 @@ def ready(self) -> bool: url="{}/ready".format(self.url), headers=self.headers ) - return True if response.status == 200 else False - except Exception as ex: + return bool(response.status == 200) + # pylint: disable=bare-except + except: return False def labels(self, @@ -86,7 +99,9 @@ def labels(self, if end: if end is not type(datetime.datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + return False, { + 'message': + f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) else: @@ -94,11 +109,15 @@ def labels(self, if start: if not isinstance(start, datetime.datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + return False, { + 'message': + f'Incorrect end type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) + - datetime.timedelta(hours=self.hours_delta)).timestamp() + * 10 ** 9) enc_query = urlencode(params) target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) @@ -109,7 +128,7 @@ def labels(self, url=target_url, headers=self.headers ) - return True if response.status == 200 else False, response.json() + return bool(response.status == 200), response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -119,7 +138,8 @@ def label_values(self, end: datetime.datetime = None, params : dict = None) -> tuple: """ - Get the list of known values for a given label within a given time span, corresponding values. + Get the list of known values for a given label within a given time span and + corresponding values. Ref: GET /loki/api/v1/label//values :param label: :param start: @@ -131,13 +151,16 @@ def label_values(self, if label: if not isinstance(label, str): - return False, {'message': 'Incorrect label type {}, should be type {}.'.format(type(label), str)} + return False, {'message': + f'Incorrect label type {type(label)}, should be type {str}.'} else: return False, {'message':'Param label can not be empty.'} if end: if not isinstance(end, datetime.datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + return False, { + 'message': + f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) else: @@ -145,12 +168,15 @@ def label_values(self, if start: if not isinstance(start, datetime.datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + return False, { + 'message': + f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta( - hours=self.hours_delta)).timestamp() * 10 ** 9) + params['start'] = int( + (datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) + - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) enc_query = urlencode(params) target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) @@ -161,7 +187,7 @@ def label_values(self, url=target_url, headers=self.headers ) - return True if response.status == 200 else False, response.json() + return bool(response.status == 200), response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -185,7 +211,8 @@ def query(self, if query: if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + return False, {'message': + f'Incorrect query type {type(query)}, should be type {str}.'} params['query'] = query else: return False, {'message':'Param query can not be empty.'} @@ -197,7 +224,9 @@ def query(self, if time: if not isinstance(time, datetime.datetime): - return False, {'message': 'Incorrect time type {}, should be type {}.'.format(type(time), datetime)} + return False, { + 'message': + f'Incorrect start type {type(time)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['time'] = int(time.timestamp() * 10 ** 9) else: @@ -216,7 +245,7 @@ def query(self, url=target_url, headers=self.headers ) - return True if response.status == 200 else False, response.json() + return bool(response.status == 200), response.json() except Exception as ex: return False, {'message': repr(ex)} @@ -241,14 +270,17 @@ def query_range(self, params = params or {} if query: if not isinstance(query, str): - return False, {'message': 'Incorrect query type {}, should be type {}.'.format(type(query), str)} + return False, {'message': + f'Incorrect query type {type(query)}, should be type {str}.'} params['query'] = query else: return False, {'message': 'Param query can not be empty.'} if end: if not isinstance(end, datetime.datetime): - return False, {'message': 'Incorrect end type {}, should be type {}.'.format(type(end), datetime)} + return False, { + 'message': + f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['end'] = int(end.timestamp() * 10 ** 9) else: @@ -256,11 +288,15 @@ def query_range(self, if start: if not isinstance(start, datetime.datetime): - return False, {'message': 'Incorrect start type {}, should be type {}.'.format(type(start), datetime)} + return False, { + 'message': + f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} # Convert to int, or will be scientific notation, which will result in request exception params['start'] = int(start.timestamp() * 10 ** 9) else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) + params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) + - datetime.timedelta(hours=self.hours_delta)).timestamp() + * 10 ** 9) if limit: params['limit'] = limit @@ -272,15 +308,16 @@ def query_range(self, params['direction'] = direction enc_query = urlencode(params) - target_url = '{}/loki/api/v1/query_range?{}'.format(self.url, enc_query) - + target_url = f'{self.url}/loki/api/v1/query_range?{enc_query}' try: response = self.__session.request( "GET", url=target_url, headers=self.headers ) - return True if response.status == 200 else False, response.json() + if response.status == 200: + return True, response.json() + return False, response.data except Exception as ex: return False, {'message': repr(ex)} @@ -318,6 +355,6 @@ def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: body=payload_json, headers=headers ) - return True if response.status == 204 else False, response.reason + return bool(response.status == 204), response.reason except Exception as ex: return False, {'message': repr(ex)} From ab06d5ba4a835c5b17e9e0529811c3f23daba83d Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 23:32:23 +0100 Subject: [PATCH 26/48] Small pylint fixes --- rqd/rqd/rqlogging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rqd/rqd/rqlogging.py b/rqd/rqd/rqlogging.py index 4d9569d8d..264cdd62e 100644 --- a/rqd/rqd/rqlogging.py +++ b/rqd/rqd/rqlogging.py @@ -158,6 +158,7 @@ def waitForFile(self, maxTries=5): time.sleep(0.5 * tries) raise IOError("Failed to create loki stream") + # pylint: disable=unused-argument def write(self, data, prependTimestamp=False): """ Provides write function for writing to loki server. @@ -177,6 +178,7 @@ def writelines(self, __lines): self.write(line) def close(self): + """Dummy function since cloasing it not necessary for the http connection""" pass def __enter__(self): @@ -184,5 +186,3 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): pass - - From b313e55442d1d15e3a6a784165d997bcce82c69c Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 12 Nov 2024 23:34:39 +0100 Subject: [PATCH 27/48] Small pylint fix --- rqd/rqd/rqlogging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rqd/rqd/rqlogging.py b/rqd/rqd/rqlogging.py index 264cdd62e..18c4dc9a1 100644 --- a/rqd/rqd/rqlogging.py +++ b/rqd/rqd/rqlogging.py @@ -179,7 +179,6 @@ def writelines(self, __lines): def close(self): """Dummy function since cloasing it not necessary for the http connection""" - pass def __enter__(self): return self From 6070bfdd37d0ae2923c58d90667b4ca933202dc6 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 13 Nov 2024 23:30:10 +0100 Subject: [PATCH 28/48] Force older version of urllib3 for python 3.9 and below --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4fc14043b..e905d23d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ pynput==1.7.6 PyYAML==5.1 six==1.16.0 pytest==8.3.3 +urllib3==1.26.20;python_version<"3.10" # Optional requirements # Sentry support for rqd From d6d882a437988faa41618eca0ea1044636a85d6b Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 13 Nov 2024 23:31:59 +0100 Subject: [PATCH 29/48] Specify urllib3 for other python versions --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e905d23d4..699523e8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,8 @@ pynput==1.7.6 PyYAML==5.1 six==1.16.0 pytest==8.3.3 -urllib3==1.26.20;python_version<"3.10" +urllib3==1.26.20;python_version<="3.9" +urllib3==2.2.3;python_version>"3.9" # Optional requirements # Sentry support for rqd From c9c3b26aff7815cb493d6d75e45777587ecc7567 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 14 Nov 2024 22:45:41 +0100 Subject: [PATCH 30/48] Add some basic pytests for the rqdlogging.LokiLogger and add to CI --- ci/run_python_lint.sh | 1 + ci/run_python_tests.sh | 1 + rqd/pytests/__init__.py | 0 rqd/pytests/test_rqdlogging.py | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 rqd/pytests/__init__.py create mode 100644 rqd/pytests/test_rqdlogging.py diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index ba98d2b6c..042da2ef0 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -51,4 +51,5 @@ echo "Running lint for rqd/..." cd rqd python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto python -m pylint --rcfile=../ci/pylintrc_test tests +python -m pylint --rcfile=../ci/pylintrc_test pytests cd .. diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 101d75288..902cf029d 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -27,6 +27,7 @@ PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" python -m pytest rqd/tests +python -m pytest rqd/pytests # Xvfb no longer supports Python 2. diff --git a/rqd/pytests/__init__.py b/rqd/pytests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/rqd/pytests/test_rqdlogging.py b/rqd/pytests/test_rqdlogging.py new file mode 100644 index 000000000..416aa30fc --- /dev/null +++ b/rqd/pytests/test_rqdlogging.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Pytests for rqd.rqdlogging""" + + +import mock +import pytest +import rqd.compiled_proto.rqd_pb2 +from rqd.rqlogging import LokiLogger + +@pytest.fixture +@mock.patch('rqd.compiled_proto.rqd_pb2_grpc.RunningFrameStub') +@mock.patch('rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceStub') +@mock.patch('grpc.insecure_channel', new=mock.MagicMock()) +def runFrame(stubMock, frameStubMock): + rf = rqd.compiled_proto.rqd_pb2.RunFrame() + rf.job_id = "SD6F3S72DJ26236KFS" + rf.job_name = "edu-trn_job-name" + rf.frame_id = "FD1S3I154O646UGSNN" + return rf + +# pylint: disable=redefined-outer-name +def test_LokiLogger(runFrame): + ll = LokiLogger("http://localhost:3100", runFrame) + assert isinstance(ll, LokiLogger) + +def test_LokiLogger_invalid_runFrame(): + rf = None + + with pytest.raises(AttributeError) as excinfo: + LokiLogger("http://localhost:3100", rf) + assert excinfo.type == AttributeError From f7e79e6ae1ecce650773866c63f797f9cc98e36f Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sun, 17 Nov 2024 23:14:49 +0100 Subject: [PATCH 31/48] Switch to use published loki-urllib3-client --- cuegui/cuegui/loki_client.py | 360 ------------------------ cuegui/cuegui/plugins/LokiViewPlugin.py | 2 +- requirements.txt | 4 +- rqd/rqd/loki_client.py | 360 ------------------------ rqd/rqd/rqlogging.py | 2 +- 5 files changed, 5 insertions(+), 723 deletions(-) delete mode 100644 cuegui/cuegui/loki_client.py delete mode 100644 rqd/rqd/loki_client.py diff --git a/cuegui/cuegui/loki_client.py b/cuegui/cuegui/loki_client.py deleted file mode 100644 index 94364034e..000000000 --- a/cuegui/cuegui/loki_client.py +++ /dev/null @@ -1,360 +0,0 @@ -#! /usr/local/bin/python - -# Copyright Contributors to the OpenCue Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module for communicating with Loki""" - -import datetime -import json -from typing import Dict, List -from urllib.parse import urlparse, urlencode -import urllib3 - - -# Support Loki version:2.4.2 -MAX_REQUEST_RETRIES = 3 -RETRY_BACKOFF_FACTOR =1 -RETRY_ON_STATUS = [408, 429, 500, 502, 503, 504] -SUPPORTED_DIRECTION = ["backward", "forward"] -# the duration before end time when get context -CONTEXT_HOURS_DELTA = 1 -DEFAULT_HOURS_DELTA = 2 * 24 - - -class LokiClient(object): - """ - Loki client for Python to communicate with Loki server. - Ref: https://grafana.com/docs/loki/v2.4/api/ - """ - def __init__(self, - url: str = "http://127.0.0.1:3100", - headers: dict = None, - hours_delta = DEFAULT_HOURS_DELTA): - """ - constructor - :param url: - :param headers: - :param retry: - :param hours_delta: - :return: - """ - if url is None: - raise TypeError("Url can not be empty!") - - self.headers = headers - self.url = url - self.loki_host = urlparse(self.url).netloc - self._all_metrics = None - # the days between start and end time - self.hours_delta = hours_delta - # the time range when searching context for one key line - self.context_timedelta = int(CONTEXT_HOURS_DELTA * 3600 * 10 ** 9) - - self.__session = urllib3.PoolManager() - self.__session.keep_alive = False - - def ready(self) -> bool: - """ - Check whether Loki host is ready to accept traffic. - Ref: https://grafana.com/docs/loki/v2.4/api/#get-ready - :return: - bool: True if Loki is ready, False otherwise. - """ - try: - response = self.__session.request( - "GET", - url="{}/ready".format(self.url), - headers=self.headers - ) - return bool(response.status == 200) - # pylint: disable=bare-except - except: - return False - - def labels(self, - start: datetime.datetime = None, - end: datetime.datetime = None, - params: dict = None) -> tuple: - """ - Get the list of known labels within a given time span, corresponding labels. - Ref: GET /loki/api/v1/labels - :param start: - :param end: - :param params: - :return: - """ - params = params or {} - - if end: - if end is not type(datetime.datetime): - return False, { - 'message': - f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime.datetime): - return False, { - 'message': - f'Incorrect end type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - - datetime.timedelta(hours=self.hours_delta)).timestamp() - * 10 ** 9) - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) - - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - return bool(response.status == 200), response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def label_values(self, - label: str, - start: datetime.datetime = None, - end: datetime.datetime = None, - params : dict = None) -> tuple: - """ - Get the list of known values for a given label within a given time span and - corresponding values. - Ref: GET /loki/api/v1/label//values - :param label: - :param start: - :param end: - :param params: - :return: - """ - params = params or {} - - if label: - if not isinstance(label, str): - return False, {'message': - f'Incorrect label type {type(label)}, should be type {str}.'} - else: - return False, {'message':'Param label can not be empty.'} - - if end: - if not isinstance(end, datetime.datetime): - return False, { - 'message': - f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int( - (datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) - - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - return bool(response.status == 200), response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def query(self, - query: str, - limit: int = 100, - time: datetime = None, - direction: str = SUPPORTED_DIRECTION[0], - params: dict = None) -> tuple: - """ - Query logs from Loki, corresponding query. - Ref: GET /loki/api/v1/query - :param query: - :param limit: - :param time: - :param direction: - :param params: - :return: - """ - params = params or {} - - if query: - if not isinstance(query, str): - return False, {'message': - f'Incorrect query type {type(query)}, should be type {str}.'} - params['query'] = query - else: - return False, {'message':'Param query can not be empty.'} - - if limit: - params['limit'] = limit - else: - return False, {'message': 'The value of limit is not correct.'} - - if time: - if not isinstance(time, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(time)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['time'] = int(time.timestamp() * 10 ** 9) - else: - params['time'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if direction not in SUPPORTED_DIRECTION: - return False, {'message': 'Invalid direction value: {}.'.format(direction)} - params['direction'] = direction - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/query?{}'.format(self.url, enc_query) - - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - return bool(response.status == 200), response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def query_range(self, - query: str, - limit: int = 100, - start: datetime.datetime = None, - end: datetime.datetime = None, - direction: str = SUPPORTED_DIRECTION[0], - params: dict = None) -> tuple: - """ - Query logs from Loki, corresponding query_range. - Ref: GET /loki/api/v1/query_range - :param query: - :param limit: - :param start: - :param end: - :param direction: - :param params: - :return: - """ - params = params or {} - if query: - if not isinstance(query, str): - return False, {'message': - f'Incorrect query type {type(query)}, should be type {str}.'} - params['query'] = query - else: - return False, {'message': 'Param query can not be empty.'} - - if end: - if not isinstance(end, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - - datetime.timedelta(hours=self.hours_delta)).timestamp() - * 10 ** 9) - - if limit: - params['limit'] = limit - else: - return False, {'message': 'The value of limit is not correct.'} - - if direction not in SUPPORTED_DIRECTION: - return False, {'message': 'Invalid direction value: {}.'.format(direction)} - params['direction'] = direction - - enc_query = urlencode(params) - target_url = f'{self.url}/loki/api/v1/query_range?{enc_query}' - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - if response.status == 200: - return True, response.json() - return False, response.data - except Exception as ex: - return False, {'message': repr(ex)} - - def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: - """ - Post logs to Loki, with given labels. - Ref: POST /loki/api/v1/push - :param labels: - :param logs: - :return: - """ - headers = { - 'Content-type': 'application/json' - } - - cur_ts = int(datetime.datetime.now().timestamp() * 10 ** 9) - logs_ts = [[str(cur_ts), log] for log in logs] - - payload = { - 'streams': [ - { - 'stream': labels, - 'values': logs_ts - } - ] - } - - payload_json = json.dumps(payload) - target_url = '{}/loki/api/v1/push'.format(self.url) - - try: - response = self.__session.request( - "POST", - url=target_url, - body=payload_json, - headers=headers - ) - return bool(response.status == 204), response.reason - except Exception as ex: - return False, {'message': repr(ex)} diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index c0ee2fddf..85a034336 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -24,7 +24,7 @@ from opencue.wrappers import job, frame -from cuegui.loki_client import LokiClient +from loki_urllib3_client import LokiClient import cuegui.Constants import cuegui.AbstractDockWidget diff --git a/requirements.txt b/requirements.txt index 699523e8d..dc8ca029b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,6 @@ urllib3==2.2.3;python_version>"3.9" # Optional requirements # Sentry support for rqd -sentry-sdk==2.11.0 \ No newline at end of file +sentry-sdk==2.11.0 + +loki-urllib3-client==0.2.2 diff --git a/rqd/rqd/loki_client.py b/rqd/rqd/loki_client.py deleted file mode 100644 index 94364034e..000000000 --- a/rqd/rqd/loki_client.py +++ /dev/null @@ -1,360 +0,0 @@ -#! /usr/local/bin/python - -# Copyright Contributors to the OpenCue Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module for communicating with Loki""" - -import datetime -import json -from typing import Dict, List -from urllib.parse import urlparse, urlencode -import urllib3 - - -# Support Loki version:2.4.2 -MAX_REQUEST_RETRIES = 3 -RETRY_BACKOFF_FACTOR =1 -RETRY_ON_STATUS = [408, 429, 500, 502, 503, 504] -SUPPORTED_DIRECTION = ["backward", "forward"] -# the duration before end time when get context -CONTEXT_HOURS_DELTA = 1 -DEFAULT_HOURS_DELTA = 2 * 24 - - -class LokiClient(object): - """ - Loki client for Python to communicate with Loki server. - Ref: https://grafana.com/docs/loki/v2.4/api/ - """ - def __init__(self, - url: str = "http://127.0.0.1:3100", - headers: dict = None, - hours_delta = DEFAULT_HOURS_DELTA): - """ - constructor - :param url: - :param headers: - :param retry: - :param hours_delta: - :return: - """ - if url is None: - raise TypeError("Url can not be empty!") - - self.headers = headers - self.url = url - self.loki_host = urlparse(self.url).netloc - self._all_metrics = None - # the days between start and end time - self.hours_delta = hours_delta - # the time range when searching context for one key line - self.context_timedelta = int(CONTEXT_HOURS_DELTA * 3600 * 10 ** 9) - - self.__session = urllib3.PoolManager() - self.__session.keep_alive = False - - def ready(self) -> bool: - """ - Check whether Loki host is ready to accept traffic. - Ref: https://grafana.com/docs/loki/v2.4/api/#get-ready - :return: - bool: True if Loki is ready, False otherwise. - """ - try: - response = self.__session.request( - "GET", - url="{}/ready".format(self.url), - headers=self.headers - ) - return bool(response.status == 200) - # pylint: disable=bare-except - except: - return False - - def labels(self, - start: datetime.datetime = None, - end: datetime.datetime = None, - params: dict = None) -> tuple: - """ - Get the list of known labels within a given time span, corresponding labels. - Ref: GET /loki/api/v1/labels - :param start: - :param end: - :param params: - :return: - """ - params = params or {} - - if end: - if end is not type(datetime.datetime): - return False, { - 'message': - f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime.datetime): - return False, { - 'message': - f'Incorrect end type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - - datetime.timedelta(hours=self.hours_delta)).timestamp() - * 10 ** 9) - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/labels?{}'.format(self.url, enc_query) - - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - return bool(response.status == 200), response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def label_values(self, - label: str, - start: datetime.datetime = None, - end: datetime.datetime = None, - params : dict = None) -> tuple: - """ - Get the list of known values for a given label within a given time span and - corresponding values. - Ref: GET /loki/api/v1/label//values - :param label: - :param start: - :param end: - :param params: - :return: - """ - params = params or {} - - if label: - if not isinstance(label, str): - return False, {'message': - f'Incorrect label type {type(label)}, should be type {str}.'} - else: - return False, {'message':'Param label can not be empty.'} - - if end: - if not isinstance(end, datetime.datetime): - return False, { - 'message': - f'Incorrect end type {type(end)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int( - (datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - - datetime.timedelta(hours=self.hours_delta)).timestamp() * 10 ** 9) - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/label/{}/values?{}'.format(self.url, label, enc_query) - - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - return bool(response.status == 200), response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def query(self, - query: str, - limit: int = 100, - time: datetime = None, - direction: str = SUPPORTED_DIRECTION[0], - params: dict = None) -> tuple: - """ - Query logs from Loki, corresponding query. - Ref: GET /loki/api/v1/query - :param query: - :param limit: - :param time: - :param direction: - :param params: - :return: - """ - params = params or {} - - if query: - if not isinstance(query, str): - return False, {'message': - f'Incorrect query type {type(query)}, should be type {str}.'} - params['query'] = query - else: - return False, {'message':'Param query can not be empty.'} - - if limit: - params['limit'] = limit - else: - return False, {'message': 'The value of limit is not correct.'} - - if time: - if not isinstance(time, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(time)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['time'] = int(time.timestamp() * 10 ** 9) - else: - params['time'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if direction not in SUPPORTED_DIRECTION: - return False, {'message': 'Invalid direction value: {}.'.format(direction)} - params['direction'] = direction - - enc_query = urlencode(params) - target_url = '{}/loki/api/v1/query?{}'.format(self.url, enc_query) - - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - return bool(response.status == 200), response.json() - except Exception as ex: - return False, {'message': repr(ex)} - - def query_range(self, - query: str, - limit: int = 100, - start: datetime.datetime = None, - end: datetime.datetime = None, - direction: str = SUPPORTED_DIRECTION[0], - params: dict = None) -> tuple: - """ - Query logs from Loki, corresponding query_range. - Ref: GET /loki/api/v1/query_range - :param query: - :param limit: - :param start: - :param end: - :param direction: - :param params: - :return: - """ - params = params or {} - if query: - if not isinstance(query, str): - return False, {'message': - f'Incorrect query type {type(query)}, should be type {str}.'} - params['query'] = query - else: - return False, {'message': 'Param query can not be empty.'} - - if end: - if not isinstance(end, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['end'] = int(end.timestamp() * 10 ** 9) - else: - params['end'] = int(datetime.datetime.now().timestamp() * 10 ** 9) - - if start: - if not isinstance(start, datetime.datetime): - return False, { - 'message': - f'Incorrect start type {type(start)}, should be type {datetime.datetime}.'} - # Convert to int, or will be scientific notation, which will result in request exception - params['start'] = int(start.timestamp() * 10 ** 9) - else: - params['start'] = int((datetime.datetime.fromtimestamp(params['end'] / 10 ** 9) - - datetime.timedelta(hours=self.hours_delta)).timestamp() - * 10 ** 9) - - if limit: - params['limit'] = limit - else: - return False, {'message': 'The value of limit is not correct.'} - - if direction not in SUPPORTED_DIRECTION: - return False, {'message': 'Invalid direction value: {}.'.format(direction)} - params['direction'] = direction - - enc_query = urlencode(params) - target_url = f'{self.url}/loki/api/v1/query_range?{enc_query}' - try: - response = self.__session.request( - "GET", - url=target_url, - headers=self.headers - ) - if response.status == 200: - return True, response.json() - return False, response.data - except Exception as ex: - return False, {'message': repr(ex)} - - def post(self, labels: Dict[str, str], logs: List[str]) -> tuple: - """ - Post logs to Loki, with given labels. - Ref: POST /loki/api/v1/push - :param labels: - :param logs: - :return: - """ - headers = { - 'Content-type': 'application/json' - } - - cur_ts = int(datetime.datetime.now().timestamp() * 10 ** 9) - logs_ts = [[str(cur_ts), log] for log in logs] - - payload = { - 'streams': [ - { - 'stream': labels, - 'values': logs_ts - } - ] - } - - payload_json = json.dumps(payload) - target_url = '{}/loki/api/v1/push'.format(self.url) - - try: - response = self.__session.request( - "POST", - url=target_url, - body=payload_json, - headers=headers - ) - return bool(response.status == 204), response.reason - except Exception as ex: - return False, {'message': repr(ex)} diff --git a/rqd/rqd/rqlogging.py b/rqd/rqd/rqlogging.py index 18c4dc9a1..b3843809c 100644 --- a/rqd/rqd/rqlogging.py +++ b/rqd/rqd/rqlogging.py @@ -22,7 +22,7 @@ import datetime import platform -from rqd.loki_client import LokiClient +from loki_urllib3_client import LokiClient import rqd.rqconstants log = logging.getLogger(__name__) From 1066e9715aed7c390af2bd6e65d7bf4f89e5c7c3 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sun, 17 Nov 2024 23:32:02 +0100 Subject: [PATCH 32/48] Use job start date to query frame labels with --- cuegui/cuegui/plugins/LokiViewPlugin.py | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 85a034336..2e6198fd8 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -63,7 +63,9 @@ def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): tries += 1 time.sleep(0.5 * tries) success, result = self.client.label_values( - "session_start_time", params={'query': f'{{frame_id="{frameId}"}}'} + label="session_start_time", + start=datetime.datetime.fromtimestamp(jobObj.startTime()), + params={'query': f'{{frame_id="{frameId}"}}'} ) if success is True: labelValues = result.get('data', []) @@ -145,17 +147,19 @@ def retranslateUi(self): # pylint: disable=unused-argument def _selectLog(self, index): self.frameText.clear() - timestamp, query = self.frameLogCombo.currentData() - start = datetime.datetime.fromtimestamp(float(timestamp)) - end = datetime.datetime.now() - success, result = self.client.query_range(query=query, direction='forward', - limit=1000, start=start, end=end) - if success is True: - for res in result.get('data', {}).get('result', []): - for timestamp, line in res.get('values'): - self.frameText.append(f"
{line}
") - else: - print(success, result) + if self.frameLogCombo.currentData(): + timestamp, query = self.frameLogCombo.currentData() + start = datetime.datetime.fromtimestamp(float(timestamp)) + end = datetime.datetime.now() + success, result = self.client.query_range(query=query, + direction=LokiClient.Direction.forward, + limit=1000, start=start, end=end) + if success is True: + for res in result.get('data', {}).get('result', []): + for timestamp, line in res.get('values'): + self.frameText.append(f"
{line}
") + else: + print(success, result) def _unix_to_datetime(unix_timestamp): From 28f3cdde7fbae024ee88044acb1537e5019c474e Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 18 Nov 2024 22:55:43 +0100 Subject: [PATCH 33/48] Bump number of migration sql script --- ...{V31__Add_loki_job_fields.sql => V32__Add_loki_job_fields.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cuebot/src/main/resources/conf/ddl/postgres/migrations/{V31__Add_loki_job_fields.sql => V32__Add_loki_job_fields.sql} (100%) diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V31__Add_loki_job_fields.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V32__Add_loki_job_fields.sql similarity index 100% rename from cuebot/src/main/resources/conf/ddl/postgres/migrations/V31__Add_loki_job_fields.sql rename to cuebot/src/main/resources/conf/ddl/postgres/migrations/V32__Add_loki_job_fields.sql From 041705136fdebd0906a84a6c2695e456d4bc792c Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 18 Nov 2024 23:43:16 +0100 Subject: [PATCH 34/48] Don't repeat yourself.. --- rqd/rqd/rqcore.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index c734ba865..bb6584022 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -1330,10 +1330,9 @@ def run(self): try: if self.runFrame.loki_enabled: self.rqlog = rqd.rqlogging.LokiLogger(self.runFrame.loki_url, runFrame) - self.rqlog.waitForFile() else: self.rqlog = rqd.rqlogging.RqdLogger(runFrame.log_dir_file) - self.rqlog.waitForFile() + self.rqlog.waitForFile() # pylint: disable=broad-except except Exception as e: err = "Unable to write to %s due to %s" % (runFrame.log_dir_file, e) From 204be817f28b1250a7be216df59bee85013a615d Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 18 Nov 2024 23:43:49 +0100 Subject: [PATCH 35/48] Small tweak for consistency --- .../java/com/imageworks/spcue/dao/postgres/DispatchQuery.java | 2 +- .../main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index 16ac50e03..4a6cfea52 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -72,7 +72,7 @@ public class DispatchQuery { "AND job.pk_facility = ? " + "AND " + "(" + - "job.str_os IS NULL OR job.str_os IN '' " + + "job.str_os IS NULL OR job.str_os = '' " + "OR " + "job.str_os IN ? " + ") " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 7c3064594..423c66ad6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -479,7 +479,7 @@ public boolean updateJobFinished(JobInterface job) { "b_autoeat,"+ "int_max_retries," + "b_loki_enabled," + - "str_loki_url" + + "str_loki_url " + ") " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; From fda4f0d28f3c3a5316934fa1523130e4dce663c3 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 19 Nov 2024 21:37:56 +0100 Subject: [PATCH 36/48] Make sure that the frame dispatch returns a non-null value for job.str_os --- .../java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 9e0f6f80c..67a13c4ee 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -331,7 +331,7 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { frame.minGpuMemory = rs.getLong("int_gpu_mem_min"); frame.version = rs.getInt("int_version"); frame.services = rs.getString("str_services"); - frame.os = rs.getString("str_os"); + frame.os = Optional.ofNullable(rs.getString("str_os")).orElse(""); return frame; } }; From 1539db9f957abadd2d2de1706c557824fb97c74e Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 19 Nov 2024 22:57:13 +0100 Subject: [PATCH 37/48] Make sure that the frame dispatch returns a non-null value for job.str_os --- .../java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java | 3 ++- .../imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java | 3 ++- .../java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 20fe7b1ef..146046e5a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -27,6 +27,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -219,7 +220,7 @@ public DispatchHost mapRow(ResultSet rs, int rowNum) throws SQLException { host.isNimby = rs.getBoolean("b_nimby"); host.threadMode = rs.getInt("int_thread_mode"); host.tags = rs.getString("str_tags"); - host.setOs(rs.getString("str_os")); + host.setOs(Optional.ofNullable(rs.getString("str_os")).orElse("")); host.hardwareState = HardwareState.valueOf(rs.getString("str_state")); return host; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java index 8c920c5a4..5aa972b58 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -326,7 +327,7 @@ private static final NestedJob mapResultSetToJob(ResultSet rs) throws SQLExcepti .setPriority(rs.getInt("int_priority")) .setShot(rs.getString("str_shot")) .setShow(rs.getString("str_show")) - .setOs(rs.getString("str_os")) + .setOs(Optional.ofNullable(rs.getString("str_os")).orElse("")) .setFacility(rs.getString("facility_name")) .setGroup(rs.getString("group_name")) .setState(JobState.valueOf(rs.getString("str_state"))) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 2a597cfeb..42d4603e3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -326,7 +327,7 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { proc.memoryUsed = rs.getLong("int_mem_used"); proc.unbooked = rs.getBoolean("b_unbooked"); proc.isLocalDispatch = rs.getBoolean("b_local"); - proc.os = rs.getString("str_os"); + proc.os = Optional.ofNullable(rs.getString("str_os")).orElse(""); proc.childProcesses = rs.getBytes("bytea_children"); return proc; } From dd8437a8aa77397a8d41fc8ce8a69c48654ca462 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 19 Nov 2024 23:20:44 +0100 Subject: [PATCH 38/48] Remove next and prev buttons --- cuegui/cuegui/plugins/LokiViewPlugin.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 2e6198fd8..d0f0cff2d 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -121,12 +121,6 @@ def setupUi(self): self.findButton = QtWidgets.QPushButton(self) self.findButton.setObjectName("findButton") self.horizontalLayout_2.addWidget(self.findButton) - self.nextButton = QtWidgets.QPushButton(self) - self.nextButton.setObjectName("nextButton") - self.horizontalLayout_2.addWidget(self.nextButton) - self.prevButton = QtWidgets.QPushButton(self) - self.prevButton.setObjectName("prevButton") - self.horizontalLayout_2.addWidget(self.prevButton) self.verticalLayout.addLayout(self.horizontalLayout_2) self.retranslateUi() @@ -141,8 +135,6 @@ def retranslateUi(self): self.caseCheck.setText(_translate("self", "Aa")) self.searchLine.setPlaceholderText(_translate("self", "Search log..")) self.findButton.setText(_translate("self", "Find")) - self.nextButton.setText(_translate("self", "Next")) - self.prevButton.setText(_translate("self", "Prev")) # pylint: disable=unused-argument def _selectLog(self, index): From d5f59038320a517db4a905a1545f19204afa31ed Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 19 Nov 2024 23:26:10 +0100 Subject: [PATCH 39/48] Move element creation to init --- cuegui/cuegui/plugins/LokiViewPlugin.py | 56 +++++++++---------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index d0f0cff2d..1dc5136eb 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -43,8 +43,19 @@ class LokiViewWidget(QtWidgets.QWidget): def __init__(self, parent=None): super().__init__(parent) self.app = cuegui.app() + self.verticalLayout = QtWidgets.QVBoxLayout(self) + self.horizontalLayout = QtWidgets.QHBoxLayout() + self.frameNameLabel = QtWidgets.QLabel(self) + self.frameLogCombo = QtWidgets.QComboBox(self) + self.wordWrapCheck = QtWidgets.QCheckBox(self) + self.refreshButton = QtWidgets.QPushButton(self) + self.frameText = QtWidgets.QTextEdit(self) + self.horizontalLayout_2 = QtWidgets.QHBoxLayout() + self.caseCheck = QtWidgets.QCheckBox(self) + self.searchLine = QtWidgets.QLineEdit(self) + self.findButton = QtWidgets.QPushButton(self) self.setupUi() - self.frameLogCombo.currentIndexChanged.connect(self._selectLog) + self.app.display_frame_log_content.connect(self._display_frame_log) def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): @@ -81,60 +92,31 @@ def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): def setupUi(self): """Function for setting up the UI widgets""" - # self.setObjectName("self") - # self.resize(958, 663) - self.verticalLayout = QtWidgets.QVBoxLayout(self) - self.verticalLayout.setObjectName("verticalLayout") - self.horizontalLayout = QtWidgets.QHBoxLayout() - self.horizontalLayout.setObjectName("horizontalLayout") - self.frameNameLabel = QtWidgets.QLabel(self) - self.frameNameLabel.setObjectName("frameNameLabel") + self.horizontalLayout.addWidget(self.frameNameLabel) - self.frameLogCombo = QtWidgets.QComboBox(self) self.frameLogCombo.setSizeAdjustPolicy(QtWidgets.QComboBox.AdjustToContents) - self.frameLogCombo.setObjectName("frameLogCombo") self.horizontalLayout.addWidget(self.frameLogCombo) - self.wordWrapCheck = QtWidgets.QCheckBox(self) - self.wordWrapCheck.setObjectName("wordWrapCheck") + self.wordWrapCheck.setText("Word Wrap") self.horizontalLayout.addWidget(self.wordWrapCheck) - self.refreshButton = QtWidgets.QPushButton(self) - self.refreshButton.setObjectName("refreshButton") + self.refreshButton.setText("Refresh") self.horizontalLayout.addWidget(self.refreshButton) self.horizontalLayout.setStretch(0, 1) self.verticalLayout.addLayout(self.horizontalLayout) - self.frameText = QtWidgets.QTextEdit(self) self.frameText.setStyleSheet("pre {display: inline;}") self.frameText.setLineWrapMode(QtWidgets.QTextEdit.NoWrap) self.frameText.setReadOnly(True) - self.frameText.setObjectName("frameText") self.verticalLayout.addWidget(self.frameText) - self.horizontalLayout_2 = QtWidgets.QHBoxLayout() self.horizontalLayout_2.setObjectName("horizontalLayout_2") - self.caseCheck = QtWidgets.QCheckBox(self) - self.caseCheck.setObjectName("caseCheck") + self.caseCheck.setText("Aa") self.horizontalLayout_2.addWidget(self.caseCheck) - self.searchLine = QtWidgets.QLineEdit(self) + self.searchLine.setPlaceholderText("Search log..") self.searchLine.setText("") self.searchLine.setClearButtonEnabled(True) - self.searchLine.setObjectName("searchLine") self.horizontalLayout_2.addWidget(self.searchLine) - self.findButton = QtWidgets.QPushButton(self) - self.findButton.setObjectName("findButton") + self.findButton.setText("Find") self.horizontalLayout_2.addWidget(self.findButton) self.verticalLayout.addLayout(self.horizontalLayout_2) - - self.retranslateUi() - QtCore.QMetaObject.connectSlotsByName(self) - - def retranslateUi(self): - """ Add text to the widgets""" - _translate = QtCore.QCoreApplication.translate - self.setWindowTitle(_translate("self", "self")) - self.wordWrapCheck.setText(_translate("self", "Word Wrap")) - self.refreshButton.setText(_translate("self", "Refresh")) - self.caseCheck.setText(_translate("self", "Aa")) - self.searchLine.setPlaceholderText(_translate("self", "Search log..")) - self.findButton.setText(_translate("self", "Find")) + self.frameLogCombo.currentIndexChanged.connect(self._selectLog) # pylint: disable=unused-argument def _selectLog(self, index): From 3df67748070a90a6cd110e588947cdcd7208835e Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 19 Nov 2024 23:27:55 +0100 Subject: [PATCH 40/48] Move everything into init --- cuegui/cuegui/plugins/LokiViewPlugin.py | 54 ++++++++++++------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 1dc5136eb..03b786f6b 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -54,8 +54,32 @@ def __init__(self, parent=None): self.caseCheck = QtWidgets.QCheckBox(self) self.searchLine = QtWidgets.QLineEdit(self) self.findButton = QtWidgets.QPushButton(self) - self.setupUi() + self.horizontalLayout.addWidget(self.frameNameLabel) + self.frameLogCombo.setSizeAdjustPolicy(QtWidgets.QComboBox.AdjustToContents) + self.horizontalLayout.addWidget(self.frameLogCombo) + self.wordWrapCheck.setText("Word Wrap") + self.horizontalLayout.addWidget(self.wordWrapCheck) + self.refreshButton.setText("Refresh") + self.horizontalLayout.addWidget(self.refreshButton) + self.horizontalLayout.setStretch(0, 1) + self.verticalLayout.addLayout(self.horizontalLayout) + self.frameText.setStyleSheet("pre {display: inline;}") + self.frameText.setLineWrapMode(QtWidgets.QTextEdit.NoWrap) + self.frameText.setReadOnly(True) + self.verticalLayout.addWidget(self.frameText) + self.horizontalLayout_2.setObjectName("horizontalLayout_2") + self.caseCheck.setText("Aa") + self.horizontalLayout_2.addWidget(self.caseCheck) + self.searchLine.setPlaceholderText("Search log..") + self.searchLine.setText("") + self.searchLine.setClearButtonEnabled(True) + self.horizontalLayout_2.addWidget(self.searchLine) + self.findButton.setText("Find") + self.horizontalLayout_2.addWidget(self.findButton) + self.verticalLayout.addLayout(self.horizontalLayout_2) + + self.frameLogCombo.currentIndexChanged.connect(self._selectLog) self.app.display_frame_log_content.connect(self._display_frame_log) def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): @@ -90,34 +114,6 @@ def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): else: pass - def setupUi(self): - """Function for setting up the UI widgets""" - - self.horizontalLayout.addWidget(self.frameNameLabel) - self.frameLogCombo.setSizeAdjustPolicy(QtWidgets.QComboBox.AdjustToContents) - self.horizontalLayout.addWidget(self.frameLogCombo) - self.wordWrapCheck.setText("Word Wrap") - self.horizontalLayout.addWidget(self.wordWrapCheck) - self.refreshButton.setText("Refresh") - self.horizontalLayout.addWidget(self.refreshButton) - self.horizontalLayout.setStretch(0, 1) - self.verticalLayout.addLayout(self.horizontalLayout) - self.frameText.setStyleSheet("pre {display: inline;}") - self.frameText.setLineWrapMode(QtWidgets.QTextEdit.NoWrap) - self.frameText.setReadOnly(True) - self.verticalLayout.addWidget(self.frameText) - self.horizontalLayout_2.setObjectName("horizontalLayout_2") - self.caseCheck.setText("Aa") - self.horizontalLayout_2.addWidget(self.caseCheck) - self.searchLine.setPlaceholderText("Search log..") - self.searchLine.setText("") - self.searchLine.setClearButtonEnabled(True) - self.horizontalLayout_2.addWidget(self.searchLine) - self.findButton.setText("Find") - self.horizontalLayout_2.addWidget(self.findButton) - self.verticalLayout.addLayout(self.horizontalLayout_2) - self.frameLogCombo.currentIndexChanged.connect(self._selectLog) - # pylint: disable=unused-argument def _selectLog(self, index): self.frameText.clear() From 83438f04fdea007fd5bdd075331d0e5afd69624a Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 19 Nov 2024 23:35:04 +0100 Subject: [PATCH 41/48] Re-order init elements --- cuegui/cuegui/plugins/LokiViewPlugin.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 03b786f6b..9e0abe888 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -46,35 +46,33 @@ def __init__(self, parent=None): self.verticalLayout = QtWidgets.QVBoxLayout(self) self.horizontalLayout = QtWidgets.QHBoxLayout() self.frameNameLabel = QtWidgets.QLabel(self) - self.frameLogCombo = QtWidgets.QComboBox(self) - self.wordWrapCheck = QtWidgets.QCheckBox(self) - self.refreshButton = QtWidgets.QPushButton(self) - self.frameText = QtWidgets.QTextEdit(self) - self.horizontalLayout_2 = QtWidgets.QHBoxLayout() - self.caseCheck = QtWidgets.QCheckBox(self) - self.searchLine = QtWidgets.QLineEdit(self) - self.findButton = QtWidgets.QPushButton(self) - self.horizontalLayout.addWidget(self.frameNameLabel) + self.frameLogCombo = QtWidgets.QComboBox(self) self.frameLogCombo.setSizeAdjustPolicy(QtWidgets.QComboBox.AdjustToContents) self.horizontalLayout.addWidget(self.frameLogCombo) + self.wordWrapCheck = QtWidgets.QCheckBox(self) self.wordWrapCheck.setText("Word Wrap") self.horizontalLayout.addWidget(self.wordWrapCheck) + self.refreshButton = QtWidgets.QPushButton(self) self.refreshButton.setText("Refresh") self.horizontalLayout.addWidget(self.refreshButton) self.horizontalLayout.setStretch(0, 1) self.verticalLayout.addLayout(self.horizontalLayout) + self.frameText = QtWidgets.QTextEdit(self) self.frameText.setStyleSheet("pre {display: inline;}") self.frameText.setLineWrapMode(QtWidgets.QTextEdit.NoWrap) self.frameText.setReadOnly(True) self.verticalLayout.addWidget(self.frameText) - self.horizontalLayout_2.setObjectName("horizontalLayout_2") + self.horizontalLayout_2 = QtWidgets.QHBoxLayout() + self.caseCheck = QtWidgets.QCheckBox(self) self.caseCheck.setText("Aa") self.horizontalLayout_2.addWidget(self.caseCheck) + self.searchLine = QtWidgets.QLineEdit(self) self.searchLine.setPlaceholderText("Search log..") self.searchLine.setText("") self.searchLine.setClearButtonEnabled(True) self.horizontalLayout_2.addWidget(self.searchLine) + self.findButton = QtWidgets.QPushButton(self) self.findButton.setText("Find") self.horizontalLayout_2.addWidget(self.findButton) self.verticalLayout.addLayout(self.horizontalLayout_2) From 23b4db1d0c2a0212268c8c2995125b1578e90fa8 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 19 Nov 2024 23:41:46 +0100 Subject: [PATCH 42/48] Rename signal to more generic name --- cuegui/cuegui/App.py | 2 +- cuegui/cuegui/FrameMonitorTree.py | 2 +- cuegui/cuegui/plugins/LokiViewPlugin.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/App.py b/cuegui/cuegui/App.py index db188958c..6a48e1ca6 100644 --- a/cuegui/cuegui/App.py +++ b/cuegui/cuegui/App.py @@ -30,7 +30,7 @@ class CueGuiApplication(QtWidgets.QApplication): # Global signals display_log_file_content = QtCore.Signal(object) - display_frame_log_content = QtCore.Signal(object, object) + select_frame = QtCore.Signal(object, object) double_click = QtCore.Signal(object) facility_changed = QtCore.Signal() single_click = QtCore.Signal(object) diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index 7af37ac9b..108e907fc 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -359,7 +359,7 @@ def __itemSingleClickedViewLog(self, item, col): old_log_files = [] self.app.display_log_file_content.emit([current_log_file] + old_log_files) - self.app.display_frame_log_content.emit(self.__job, item.rpcObject) + self.app.select_frame.emit(self.__job, item.rpcObject) def __itemDoubleClickedViewLog(self, item, col): """Called when a frame is double clicked, views the frame log in a popup diff --git a/cuegui/cuegui/plugins/LokiViewPlugin.py b/cuegui/cuegui/plugins/LokiViewPlugin.py index 9e0abe888..c7918434e 100644 --- a/cuegui/cuegui/plugins/LokiViewPlugin.py +++ b/cuegui/cuegui/plugins/LokiViewPlugin.py @@ -78,7 +78,7 @@ def __init__(self, parent=None): self.verticalLayout.addLayout(self.horizontalLayout_2) self.frameLogCombo.currentIndexChanged.connect(self._selectLog) - self.app.display_frame_log_content.connect(self._display_frame_log) + self.app.select_frame.connect(self._display_frame_log) def _display_frame_log(self, jobObj: job.Job, frameObj: frame.Frame): jobName = jobObj.name() From 21f0157cb99fa28e28207dd6eab17d2d360cee3a Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 27 Nov 2024 17:21:24 +0100 Subject: [PATCH 43/48] Revert "Make sure that the frame dispatch returns a non-null value for job.str_os" This reverts commit 1539db9f957abadd2d2de1706c557824fb97c74e. --- .../java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java | 3 +-- .../imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java | 3 +-- .../java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 146046e5a..20fe7b1ef 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -27,7 +27,6 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -220,7 +219,7 @@ public DispatchHost mapRow(ResultSet rs, int rowNum) throws SQLException { host.isNimby = rs.getBoolean("b_nimby"); host.threadMode = rs.getInt("int_thread_mode"); host.tags = rs.getString("str_tags"); - host.setOs(Optional.ofNullable(rs.getString("str_os")).orElse("")); + host.setOs(rs.getString("str_os")); host.hardwareState = HardwareState.valueOf(rs.getString("str_state")); return host; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java index 5aa972b58..8c920c5a4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java @@ -27,7 +27,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -327,7 +326,7 @@ private static final NestedJob mapResultSetToJob(ResultSet rs) throws SQLExcepti .setPriority(rs.getInt("int_priority")) .setShot(rs.getString("str_shot")) .setShow(rs.getString("str_show")) - .setOs(Optional.ofNullable(rs.getString("str_os")).orElse("")) + .setOs(rs.getString("str_os")) .setFacility(rs.getString("facility_name")) .setGroup(rs.getString("group_name")) .setState(JobState.valueOf(rs.getString("str_state"))) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 42d4603e3..2a597cfeb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -327,7 +326,7 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { proc.memoryUsed = rs.getLong("int_mem_used"); proc.unbooked = rs.getBoolean("b_unbooked"); proc.isLocalDispatch = rs.getBoolean("b_local"); - proc.os = Optional.ofNullable(rs.getString("str_os")).orElse(""); + proc.os = rs.getString("str_os"); proc.childProcesses = rs.getBytes("bytea_children"); return proc; } From f1b2ad63fdc886086e81439c9d36f28b9584ad42 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 27 Nov 2024 17:23:02 +0100 Subject: [PATCH 44/48] Revert "Make sure that the frame dispatch returns a non-null value for job.str_os" This reverts commit fda4f0d2 --- .../java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 3f983f776..ac73957f3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -331,7 +331,7 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { frame.minGpuMemory = rs.getLong("int_gpu_mem_min"); frame.version = rs.getInt("int_version"); frame.services = rs.getString("str_services"); - frame.os = Optional.ofNullable(rs.getString("str_os")).orElse(""); + frame.os = rs.getString("str_os"); frame.lokiEnabled = rs.getBoolean("b_loki_enabled"); frame.lokiURL = rs.getString("str_loki_url"); return frame; From 01f3870a0a871011ab5a9ebca53446cfa0b2660b Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 5 Dec 2024 00:01:28 +0100 Subject: [PATCH 45/48] Added small description on how to configure for rqd framelogs to Loki --- cuebot/src/main/resources/opencue.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index ba0fbc130..12a821a4d 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -62,7 +62,11 @@ log.frame-log-root.default_os=${CUE_FRAME_LOG_DIR:/shots} # - log.frame-log-root.Windows=${S:} # Loki +# To enable rqd frame logs to Loki enable this flag and configure the url as shown below. When Loki +# rqd frame logs are enable all frame logs will be streamed to the loki server instead of using the +# CUE_FRAME_LOG_DIR filesystem path. Refer to the documentaion on how to configure Loki. log.loki.enabled=false +# This is the base url of the Loki server. If the url is not reachable, rqd will fail running frames. log.loki.url=http://localhost/loki/api # Maximum number of jobs to query. From f5ec53b099fb38a31a313c6aba2c7fd1cf0c8b3e Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 5 Dec 2024 00:12:07 +0100 Subject: [PATCH 46/48] Added comments regarding new parameters used by rqd and cuegui --- cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java | 1 + cuebot/src/main/java/com/imageworks/spcue/JobDetail.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java index 607bce328..3aa4d60de 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java @@ -73,6 +73,7 @@ public long getMinMemory() { return this.minMemory; } + // Parameters to tell rqd whether or not to use Loki for frame logs and which base url to use public boolean lokiEnabled; public String lokiURL; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java index d07f77b34..2b837b836 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java @@ -60,6 +60,8 @@ public String getDepartmentId() { return deptId; } + // Parameters to tell cuebot whether or not to Loki is used for frame logs of the job and which + // base url to use for querying them public Boolean logLokiEnabled; public String logLokiURL; } From 77629dfd408e13f921f00ea4809736b0571e8541 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 16 Dec 2024 21:56:44 +0100 Subject: [PATCH 47/48] Bump version --- VERSION.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.in b/VERSION.in index c068b2447..c239c60cb 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.4 +1.5 From 58c5d52fd7e862cc1949edf4c3a63579906a1232 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Mon, 16 Dec 2024 22:00:00 +0100 Subject: [PATCH 48/48] Bump version of DB migration script --- ...{V32__Add_loki_job_fields.sql => V33__Add_loki_job_fields.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cuebot/src/main/resources/conf/ddl/postgres/migrations/{V32__Add_loki_job_fields.sql => V33__Add_loki_job_fields.sql} (100%) diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V32__Add_loki_job_fields.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V33__Add_loki_job_fields.sql similarity index 100% rename from cuebot/src/main/resources/conf/ddl/postgres/migrations/V32__Add_loki_job_fields.sql rename to cuebot/src/main/resources/conf/ddl/postgres/migrations/V33__Add_loki_job_fields.sql